diff options
| author | Suraj Kandpal <suraj.kandpal@intel.com> | 2024-10-28 10:22:46 +0530 |
|---|---|---|
| committer | Suraj Kandpal <suraj.kandpal@intel.com> | 2024-10-28 10:22:46 +0530 |
| commit | a092bc531da99903b88e7306ad45ee1f92efef9d (patch) | |
| tree | 73abc4f7c725fc60d58d130cd80ae03c4057377a | |
| parent | 12837c251571546470056761ee0c7b944c89edc3 (diff) | |
2024y-10m-28d-04h-51m-10s UTC: drm-tip rerere cache update
git version 2.34.1
30 files changed, 2462 insertions, 66930 deletions
diff --git a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/postimage.2 b/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/postimage.2 deleted file mode 100644 index a31fae5feedf..000000000000 --- a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/postimage.2 +++ /dev/null @@ -1,2870 +0,0 @@ -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#define SWSMU_CODE_LAYER_L2 - -#include <linux/firmware.h> -#include <linux/pci.h> -#include <linux/i2c.h> -#include "amdgpu.h" -#include "amdgpu_smu.h" -#include "atomfirmware.h" -#include "amdgpu_atomfirmware.h" -#include "amdgpu_atombios.h" -#include "smu_v14_0.h" -#include "smu14_driver_if_v14_0.h" -#include "soc15_common.h" -#include "atom.h" -#include "smu_v14_0_2_ppt.h" -#include "smu_v14_0_2_pptable.h" -#include "smu_v14_0_2_ppsmc.h" -#include "mp/mp_14_0_2_offset.h" -#include "mp/mp_14_0_2_sh_mask.h" - -#include "smu_cmn.h" -#include "amdgpu_ras.h" - -/* - * DO NOT use these for err/warn/info/debug messages. - * Use dev_err, dev_warn, dev_info and dev_dbg instead. - * They are more MGPU friendly. - */ -#undef pr_err -#undef pr_warn -#undef pr_info -#undef pr_debug - -#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) - -#define FEATURE_MASK(feature) (1ULL << feature) -#define SMC_DPM_FEATURE ( \ - FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_LINK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_FCLK_BIT)) - -#define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE 0x4000 -#define DEBUGSMC_MSG_Mode1Reset 2 -#define LINK_SPEED_MAX 3 - -#define PP_OD_FEATURE_GFXCLK_FMIN 0 -#define PP_OD_FEATURE_GFXCLK_FMAX 1 -#define PP_OD_FEATURE_UCLK_FMIN 2 -#define PP_OD_FEATURE_UCLK_FMAX 3 -#define PP_OD_FEATURE_GFX_VF_CURVE 4 -#define PP_OD_FEATURE_FAN_CURVE_TEMP 5 -#define PP_OD_FEATURE_FAN_CURVE_PWM 6 -#define PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT 7 -#define PP_OD_FEATURE_FAN_ACOUSTIC_TARGET 8 -#define PP_OD_FEATURE_FAN_TARGET_TEMPERATURE 9 -#define PP_OD_FEATURE_FAN_MINIMUM_PWM 10 - -static struct cmn2asic_msg_mapping smu_v14_0_2_message_map[SMU_MSG_MAX_COUNT] = { - MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 1), - MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), - MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 1), - MSG_MAP(SetAllowedFeaturesMaskLow, PPSMC_MSG_SetAllowedFeaturesMaskLow, 0), - MSG_MAP(SetAllowedFeaturesMaskHigh, PPSMC_MSG_SetAllowedFeaturesMaskHigh, 0), - MSG_MAP(EnableAllSmuFeatures, PPSMC_MSG_EnableAllSmuFeatures, 0), - MSG_MAP(DisableAllSmuFeatures, PPSMC_MSG_DisableAllSmuFeatures, 0), - MSG_MAP(EnableSmuFeaturesLow, PPSMC_MSG_EnableSmuFeaturesLow, 1), - MSG_MAP(EnableSmuFeaturesHigh, PPSMC_MSG_EnableSmuFeaturesHigh, 1), - MSG_MAP(DisableSmuFeaturesLow, PPSMC_MSG_DisableSmuFeaturesLow, 1), - MSG_MAP(DisableSmuFeaturesHigh, PPSMC_MSG_DisableSmuFeaturesHigh, 1), - MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetRunningSmuFeaturesLow, 1), - MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetRunningSmuFeaturesHigh, 1), - MSG_MAP(SetWorkloadMask, PPSMC_MSG_SetWorkloadMask, 1), - MSG_MAP(SetPptLimit, PPSMC_MSG_SetPptLimit, 0), - MSG_MAP(SetDriverDramAddrHigh, PPSMC_MSG_SetDriverDramAddrHigh, 1), - MSG_MAP(SetDriverDramAddrLow, PPSMC_MSG_SetDriverDramAddrLow, 1), - MSG_MAP(SetToolsDramAddrHigh, PPSMC_MSG_SetToolsDramAddrHigh, 0), - MSG_MAP(SetToolsDramAddrLow, PPSMC_MSG_SetToolsDramAddrLow, 0), - MSG_MAP(TransferTableSmu2Dram, PPSMC_MSG_TransferTableSmu2Dram, 1), - MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), - MSG_MAP(UseDefaultPPTable, PPSMC_MSG_UseDefaultPPTable, 0), - MSG_MAP(RunDcBtc, PPSMC_MSG_RunDcBtc, 0), - MSG_MAP(EnterBaco, PPSMC_MSG_EnterBaco, 0), - MSG_MAP(ExitBaco, PPSMC_MSG_ExitBaco, 0), - MSG_MAP(SetSoftMinByFreq, PPSMC_MSG_SetSoftMinByFreq, 1), - MSG_MAP(SetSoftMaxByFreq, PPSMC_MSG_SetSoftMaxByFreq, 1), - MSG_MAP(SetHardMinByFreq, PPSMC_MSG_SetHardMinByFreq, 1), - MSG_MAP(SetHardMaxByFreq, PPSMC_MSG_SetHardMaxByFreq, 0), - MSG_MAP(GetMinDpmFreq, PPSMC_MSG_GetMinDpmFreq, 1), - MSG_MAP(GetMaxDpmFreq, PPSMC_MSG_GetMaxDpmFreq, 1), - MSG_MAP(GetDpmFreqByIndex, PPSMC_MSG_GetDpmFreqByIndex, 1), - MSG_MAP(PowerUpVcn, PPSMC_MSG_PowerUpVcn, 0), - MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), - MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), - MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), - MSG_MAP(GetDcModeMaxDpmFreq, PPSMC_MSG_GetDcModeMaxDpmFreq, 1), - MSG_MAP(OverridePcieParameters, PPSMC_MSG_OverridePcieParameters, 0), - MSG_MAP(DramLogSetDramAddrHigh, PPSMC_MSG_DramLogSetDramAddrHigh, 0), - MSG_MAP(DramLogSetDramAddrLow, PPSMC_MSG_DramLogSetDramAddrLow, 0), - MSG_MAP(DramLogSetDramSize, PPSMC_MSG_DramLogSetDramSize, 0), - MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), - MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), - MSG_MAP(GetPptLimit, PPSMC_MSG_GetPptLimit, 0), - MSG_MAP(NotifyPowerSource, PPSMC_MSG_NotifyPowerSource, 0), - MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), - MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), - MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), - MSG_MAP(SetNumBadMemoryPagesRetired, PPSMC_MSG_SetNumBadMemoryPagesRetired, 0), - MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel, - PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel, 0), - MSG_MAP(AllowIHHostInterrupt, PPSMC_MSG_AllowIHHostInterrupt, 0), - MSG_MAP(ReenableAcDcInterrupt, PPSMC_MSG_ReenableAcDcInterrupt, 0), -}; - -static struct cmn2asic_mapping smu_v14_0_2_clk_map[SMU_CLK_COUNT] = { - CLK_MAP(GFXCLK, PPCLK_GFXCLK), - CLK_MAP(SCLK, PPCLK_GFXCLK), - CLK_MAP(SOCCLK, PPCLK_SOCCLK), - CLK_MAP(FCLK, PPCLK_FCLK), - CLK_MAP(UCLK, PPCLK_UCLK), - CLK_MAP(MCLK, PPCLK_UCLK), - CLK_MAP(VCLK, PPCLK_VCLK_0), - CLK_MAP(DCLK, PPCLK_DCLK_0), - CLK_MAP(DCEFCLK, PPCLK_DCFCLK), -}; - -static struct cmn2asic_mapping smu_v14_0_2_feature_mask_map[SMU_FEATURE_COUNT] = { - FEA_MAP(FW_DATA_READ), - FEA_MAP(DPM_GFXCLK), - FEA_MAP(DPM_GFX_POWER_OPTIMIZER), - FEA_MAP(DPM_UCLK), - FEA_MAP(DPM_FCLK), - FEA_MAP(DPM_SOCCLK), - FEA_MAP(DPM_LINK), - FEA_MAP(DPM_DCN), - FEA_MAP(VMEMP_SCALING), - FEA_MAP(VDDIO_MEM_SCALING), - FEA_MAP(DS_GFXCLK), - FEA_MAP(DS_SOCCLK), - FEA_MAP(DS_FCLK), - FEA_MAP(DS_LCLK), - FEA_MAP(DS_DCFCLK), - FEA_MAP(DS_UCLK), - FEA_MAP(GFX_ULV), - FEA_MAP(FW_DSTATE), - FEA_MAP(GFXOFF), - FEA_MAP(BACO), - FEA_MAP(MM_DPM), - FEA_MAP(SOC_MPCLK_DS), - FEA_MAP(BACO_MPCLK_DS), - FEA_MAP(THROTTLERS), - FEA_MAP(SMARTSHIFT), - FEA_MAP(GTHR), - FEA_MAP(ACDC), - FEA_MAP(VR0HOT), - FEA_MAP(FW_CTF), - FEA_MAP(FAN_CONTROL), - FEA_MAP(GFX_DCS), - FEA_MAP(GFX_READ_MARGIN), - FEA_MAP(LED_DISPLAY), - FEA_MAP(GFXCLK_SPREAD_SPECTRUM), - FEA_MAP(OUT_OF_BAND_MONITOR), - FEA_MAP(OPTIMIZED_VMIN), - FEA_MAP(GFX_IMU), - FEA_MAP(BOOT_TIME_CAL), - FEA_MAP(GFX_PCC_DFLL), - FEA_MAP(SOC_CG), - FEA_MAP(DF_CSTATE), - FEA_MAP(GFX_EDC), - FEA_MAP(BOOT_POWER_OPT), - FEA_MAP(CLOCK_POWER_DOWN_BYPASS), - FEA_MAP(DS_VCN), - FEA_MAP(BACO_CG), - FEA_MAP(MEM_TEMP_READ), - FEA_MAP(ATHUB_MMHUB_PG), - FEA_MAP(SOC_PCC), - [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_PPT_BIT] = {1, FEATURE_THROTTLERS_BIT}, -}; - -static struct cmn2asic_mapping smu_v14_0_2_table_map[SMU_TABLE_COUNT] = { - TAB_MAP(PPTABLE), - TAB_MAP(WATERMARKS), - TAB_MAP(AVFS_PSM_DEBUG), - TAB_MAP(PMSTATUSLOG), - TAB_MAP(SMU_METRICS), - TAB_MAP(DRIVER_SMU_CONFIG), - TAB_MAP(ACTIVITY_MONITOR_COEFF), - [SMU_TABLE_COMBO_PPTABLE] = {1, TABLE_COMBO_PPTABLE}, - TAB_MAP(I2C_COMMANDS), - TAB_MAP(ECCINFO), - TAB_MAP(OVERDRIVE), -}; - -static struct cmn2asic_mapping smu_v14_0_2_pwr_src_map[SMU_POWER_SOURCE_COUNT] = { - PWR_MAP(AC), - PWR_MAP(DC), -}; - -static struct cmn2asic_mapping smu_v14_0_2_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT, WORKLOAD_PPLIB_DEFAULT_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_POWERSAVING, WORKLOAD_PPLIB_POWER_SAVING_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_WINDOW3D, WORKLOAD_PPLIB_WINDOW_3D_BIT), -}; - -static const uint8_t smu_v14_0_2_throttler_map[] = { - [THROTTLER_PPT0_BIT] = (SMU_THROTTLER_PPT0_BIT), - [THROTTLER_PPT1_BIT] = (SMU_THROTTLER_PPT1_BIT), - [THROTTLER_PPT2_BIT] = (SMU_THROTTLER_PPT2_BIT), - [THROTTLER_PPT3_BIT] = (SMU_THROTTLER_PPT3_BIT), - [THROTTLER_TDC_GFX_BIT] = (SMU_THROTTLER_TDC_GFX_BIT), - [THROTTLER_TDC_SOC_BIT] = (SMU_THROTTLER_TDC_SOC_BIT), - [THROTTLER_TEMP_EDGE_BIT] = (SMU_THROTTLER_TEMP_EDGE_BIT), - [THROTTLER_TEMP_HOTSPOT_BIT] = (SMU_THROTTLER_TEMP_HOTSPOT_BIT), - [THROTTLER_TEMP_MEM_BIT] = (SMU_THROTTLER_TEMP_MEM_BIT), - [THROTTLER_TEMP_VR_GFX_BIT] = (SMU_THROTTLER_TEMP_VR_GFX_BIT), - [THROTTLER_TEMP_VR_SOC_BIT] = (SMU_THROTTLER_TEMP_VR_SOC_BIT), - [THROTTLER_TEMP_VR_MEM0_BIT] = (SMU_THROTTLER_TEMP_VR_MEM0_BIT), - [THROTTLER_TEMP_VR_MEM1_BIT] = (SMU_THROTTLER_TEMP_VR_MEM1_BIT), - [THROTTLER_TEMP_LIQUID0_BIT] = (SMU_THROTTLER_TEMP_LIQUID0_BIT), - [THROTTLER_TEMP_LIQUID1_BIT] = (SMU_THROTTLER_TEMP_LIQUID1_BIT), - [THROTTLER_GFX_APCC_PLUS_BIT] = (SMU_THROTTLER_APCC_BIT), - [THROTTLER_FIT_BIT] = (SMU_THROTTLER_FIT_BIT), -}; - -static int -smu_v14_0_2_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - /*u32 smu_version;*/ - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0xff, sizeof(uint32_t) * num); - - if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT); - } -#if 0 - if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) || - !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); - - if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); - - /* PMFW 78.58 contains a critical fix for gfxoff feature */ - smu_cmn_get_smc_version(smu, NULL, &smu_version); - if ((smu_version < 0x004e3a00) || - !(adev->pm.pp_feature & PP_GFXOFF_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT); - - if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); - } - - if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - - if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT); - } - - if (!(adev->pm.pp_feature & PP_ULV_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT); -#endif - - return 0; -} - -static int smu_v14_0_2_check_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - struct smu_baco_context *smu_baco = &smu->smu_baco; - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC) - smu->dc_controlled_by_gpio = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_BACO) { - smu_baco->platform_support = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_MACO) - smu_baco->maco_support = true; - } - - if (!overdrive_lowerlimits->FeatureCtrlMask || - !overdrive_upperlimits->FeatureCtrlMask) - smu->od_enabled = false; - - table_context->thermal_controller_type = - powerplay_table->thermal_controller_type; - - /* - * Instead of having its own buffer space and get overdrive_table copied, - * smu->od_settings just points to the actual overdrive_table - */ - smu->od_settings = &powerplay_table->overdrive_table; - - smu->adev->pm.no_fan = - !(pptable->PFE_Settings.FeaturesToRun[0] & (1 << FEATURE_FAN_CONTROL_BIT)); - - return 0; -} - -static int smu_v14_0_2_store_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - - memcpy(table_context->driver_pptable, &powerplay_table->smc_pptable, - sizeof(PPTable_t)); - - return 0; -} - -#ifndef atom_smc_dpm_info_table_14_0_0 -struct atom_smc_dpm_info_table_14_0_0 { - struct atom_common_table_header table_header; - BoardTable_t BoardTable; -}; -#endif - -static int smu_v14_0_2_append_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - struct atom_smc_dpm_info_table_14_0_0 *smc_dpm_table; - BoardTable_t *BoardTable = &smc_pptable->BoardTable; - int index, ret; - - index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, - smc_dpm_info); - - ret = amdgpu_atombios_get_data_table(smu->adev, index, NULL, NULL, NULL, - (uint8_t **)&smc_dpm_table); - if (ret) - return ret; - - memcpy(BoardTable, &smc_dpm_table->BoardTable, sizeof(BoardTable_t)); - - return 0; -} - -#if 0 -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_powerplay_table); - - return 0; -} -#endif - -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_2_powerplay_table); - - return 0; -} - -static int smu_v14_0_2_setup_pptable(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct amdgpu_device *adev = smu->adev; - int ret = 0; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!adev->scpm_enabled) - ret = smu_v14_0_setup_pptable(smu); - else - ret = smu_v14_0_2_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - if (ret) - return ret; - - ret = smu_v14_0_2_store_powerplay_table(smu); - if (ret) - return ret; - - /* - * With SCPM enabled, the operation below will be handled - * by PSP. Driver involvment is unnecessary and useless. - */ - if (!adev->scpm_enabled) { - ret = smu_v14_0_2_append_powerplay_table(smu); - if (ret) - return ret; - } - - ret = smu_v14_0_2_check_powerplay_table(smu); - if (ret) - return ret; - - return ret; -} - -static int smu_v14_0_2_tables_init(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *tables = smu_table->tables; - - SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetricsExternal_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_PMSTATUSLOG, SMU14_TOOL_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ACTIVITY_MONITOR_COEFF, - sizeof(DpmActivityMonitorCoeffIntExternal_t), PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_COMBO_PPTABLE, MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ECCINFO, sizeof(EccInfoTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - - smu_table->metrics_table = kzalloc(sizeof(SmuMetricsExternal_t), GFP_KERNEL); - if (!smu_table->metrics_table) - goto err0_out; - smu_table->metrics_time = 0; - - smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3); - smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); - if (!smu_table->gpu_metrics_table) - goto err1_out; - - smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); - if (!smu_table->watermarks_table) - goto err2_out; - - smu_table->ecc_table = kzalloc(tables[SMU_TABLE_ECCINFO].size, GFP_KERNEL); - if (!smu_table->ecc_table) - goto err3_out; - - return 0; - -err3_out: - kfree(smu_table->watermarks_table); -err2_out: - kfree(smu_table->gpu_metrics_table); -err1_out: - kfree(smu_table->metrics_table); -err0_out: - return -ENOMEM; -} - -static int smu_v14_0_2_allocate_dpm_context(struct smu_context *smu) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - - smu_dpm->dpm_context = kzalloc(sizeof(struct smu_14_0_dpm_context), - GFP_KERNEL); - if (!smu_dpm->dpm_context) - return -ENOMEM; - - smu_dpm->dpm_context_size = sizeof(struct smu_14_0_dpm_context); - - return 0; -} - -static int smu_v14_0_2_init_smc_tables(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_v14_0_2_tables_init(smu); - if (ret) - return ret; - - ret = smu_v14_0_2_allocate_dpm_context(smu); - if (ret) - return ret; - - return smu_v14_0_init_smc_tables(smu); -} - -static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - SkuTable_t *skutable = &pptable->SkuTable; - struct smu_14_0_dpm_table *dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t link_level; - int ret = 0; - - /* socclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.soc_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_SOCCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* gfxclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_GFXCLK, - dpm_table); - if (ret) - return ret; - - /* - * Update the reported maximum shader clock to the value - * which can be guarded to be achieved on all cards. This - * is aligned with Window setting. And considering that value - * might be not the peak frequency the card can achieve, it - * is normal some real-time clock frequency can overtake this - * labelled maximum clock frequency(for example in pp_dpm_sclk - * sysfs output). - */ - if (skutable->DriverReportedClocks.GameClockAc && - (dpm_table->dpm_levels[dpm_table->count - 1].value > - skutable->DriverReportedClocks.GameClockAc)) { - dpm_table->dpm_levels[dpm_table->count - 1].value = - skutable->DriverReportedClocks.GameClockAc; - dpm_table->max = skutable->DriverReportedClocks.GameClockAc; - } - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* uclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_UCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* fclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_FCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* vclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_VCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* dclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* lclk dpm table setup */ - pcie_table = &dpm_context->dpm_tables.pcie_table; - pcie_table->num_of_link_levels = 0; - for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) { - if (!skutable->PcieGenSpeed[link_level] && - !skutable->PcieLaneCount[link_level] && - !skutable->LclkFreq[link_level]) - continue; - - pcie_table->pcie_gen[pcie_table->num_of_link_levels] = - skutable->PcieGenSpeed[link_level]; - pcie_table->pcie_lane[pcie_table->num_of_link_levels] = - skutable->PcieLaneCount[link_level]; - pcie_table->clk_freq[pcie_table->num_of_link_levels] = - skutable->LclkFreq[link_level]; - pcie_table->num_of_link_levels++; - } - - /* dcefclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dcef_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCEFCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - return 0; -} - -static bool smu_v14_0_2_is_dpm_running(struct smu_context *smu) -{ - int ret = 0; - uint64_t feature_enabled; - - ret = smu_cmn_get_enabled_mask(smu, &feature_enabled); - if (ret) - return false; - - return !!(feature_enabled & SMC_DPM_FEATURE); -} - -static void smu_v14_0_2_dump_pptable(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - PFE_Settings_t *PFEsettings = &pptable->PFE_Settings; - - dev_info(smu->adev->dev, "Dumped PPTable:\n"); - - dev_info(smu->adev->dev, "Version = 0x%08x\n", PFEsettings->Version); - dev_info(smu->adev->dev, "FeaturesToRun[0] = 0x%08x\n", PFEsettings->FeaturesToRun[0]); - dev_info(smu->adev->dev, "FeaturesToRun[1] = 0x%08x\n", PFEsettings->FeaturesToRun[1]); -} - -static uint32_t smu_v14_0_2_get_throttler_status(SmuMetrics_t *metrics) -{ - uint32_t throttler_status = 0; - int i; - - for (i = 0; i < THROTTLER_COUNT; i++) - throttler_status |= - (metrics->ThrottlingPercentage[i] ? 1U << i : 0); - - return throttler_status; -} - -#define SMU_14_0_2_BUSY_THRESHOLD 5 -static int smu_v14_0_2_get_smu_metrics_data(struct smu_context *smu, - MetricsMember_t member, - uint32_t *value) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - NULL, - false); - if (ret) - return ret; - - switch (member) { - case METRICS_CURR_GFXCLK: - *value = metrics->CurrClock[PPCLK_GFXCLK]; - break; - case METRICS_CURR_SOCCLK: - *value = metrics->CurrClock[PPCLK_SOCCLK]; - break; - case METRICS_CURR_UCLK: - *value = metrics->CurrClock[PPCLK_UCLK]; - break; - case METRICS_CURR_VCLK: - *value = metrics->CurrClock[PPCLK_VCLK_0]; - break; - case METRICS_CURR_DCLK: - *value = metrics->CurrClock[PPCLK_DCLK_0]; - break; - case METRICS_CURR_FCLK: - *value = metrics->CurrClock[PPCLK_FCLK]; - break; - case METRICS_CURR_DCEFCLK: - *value = metrics->CurrClock[PPCLK_DCFCLK]; - break; - case METRICS_AVERAGE_GFXCLK: - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageGfxclkFrequencyPostDs; - else - *value = metrics->AverageGfxclkFrequencyPreDs; - break; - case METRICS_AVERAGE_FCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageFclkFrequencyPostDs; - else - *value = metrics->AverageFclkFrequencyPreDs; - break; - case METRICS_AVERAGE_UCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageMemclkFrequencyPostDs; - else - *value = metrics->AverageMemclkFrequencyPreDs; - break; - case METRICS_AVERAGE_VCLK: - *value = metrics->AverageVclk0Frequency; - break; - case METRICS_AVERAGE_DCLK: - *value = metrics->AverageDclk0Frequency; - break; - case METRICS_AVERAGE_VCLK1: - *value = metrics->AverageVclk1Frequency; - break; - case METRICS_AVERAGE_DCLK1: - *value = metrics->AverageDclk1Frequency; - break; - case METRICS_AVERAGE_GFXACTIVITY: - *value = metrics->AverageGfxActivity; - break; - case METRICS_AVERAGE_MEMACTIVITY: - *value = metrics->AverageUclkActivity; - break; - case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->AverageSocketPower << 8; - break; - case METRICS_TEMPERATURE_EDGE: - *value = metrics->AvgTemperature[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_HOTSPOT: - *value = metrics->AvgTemperature[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_MEM: - *value = metrics->AvgTemperature[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRGFX: - *value = metrics->AvgTemperature[TEMP_VR_GFX] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRSOC: - *value = metrics->AvgTemperature[TEMP_VR_SOC] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_THROTTLER_STATUS: - *value = smu_v14_0_2_get_throttler_status(metrics); - break; - case METRICS_CURR_FANSPEED: - *value = metrics->AvgFanRpm; - break; - case METRICS_CURR_FANPWM: - *value = metrics->AvgFanPwm; - break; - case METRICS_VOLTAGE_VDDGFX: - *value = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - break; - case METRICS_PCIE_RATE: - *value = metrics->PcieRate; - break; - case METRICS_PCIE_WIDTH: - *value = metrics->PcieWidth; - break; - default: - *value = UINT_MAX; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *min, - uint32_t *max) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *dpm_table; - - switch (clk_type) { - case SMU_MCLK: - case SMU_UCLK: - /* uclk dpm table */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - break; - case SMU_GFXCLK: - case SMU_SCLK: - /* gfxclk dpm table */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - break; - case SMU_SOCCLK: - /* socclk dpm table */ - dpm_table = &dpm_context->dpm_tables.soc_table; - break; - case SMU_FCLK: - /* fclk dpm table */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - break; - case SMU_VCLK: - case SMU_VCLK1: - /* vclk dpm table */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - break; - case SMU_DCLK: - case SMU_DCLK1: - /* dclk dpm table */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - break; - default: - dev_err(smu->adev->dev, "Unsupported clock type!\n"); - return -EINVAL; - } - - if (min) - *min = dpm_table->min; - if (max) - *max = dpm_table->max; - - return 0; -} - -static int smu_v14_0_2_read_sensor(struct smu_context *smu, - enum amd_pp_sensors sensor, - void *data, - uint32_t *size) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - int ret = 0; - - switch (sensor) { - case AMDGPU_PP_SENSOR_MAX_FAN_RPM: - *(uint16_t *)data = smc_pptable->CustomSkuTable.FanMaximumRpm; - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_MEMACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_AVG_POWER: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_SOCKETPOWER, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_HOTSPOT, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_EDGE_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_EDGE, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_MEM, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_MCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_CURR_UCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_VDDGFX: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_VOLTAGE_VDDGFX, - (uint32_t *)data); - *size = 4; - break; - default: - ret = -EOPNOTSUPP; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_current_clk_freq_by_table(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *value) -{ - MetricsMember_t member_type; - int clk_id = 0; - - clk_id = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_CLK, - clk_type); - if (clk_id < 0) - return -EINVAL; - - switch (clk_id) { - case PPCLK_GFXCLK: - member_type = METRICS_AVERAGE_GFXCLK; - break; - case PPCLK_UCLK: - member_type = METRICS_CURR_UCLK; - break; - case PPCLK_FCLK: - member_type = METRICS_CURR_FCLK; - break; - case PPCLK_SOCCLK: - member_type = METRICS_CURR_SOCCLK; - break; - case PPCLK_VCLK_0: - member_type = METRICS_AVERAGE_VCLK; - break; - case PPCLK_DCLK_0: - member_type = METRICS_AVERAGE_DCLK; - break; - case PPCLK_DCFCLK: - member_type = METRICS_CURR_DCEFCLK; - break; - default: - return -EINVAL; - } - - return smu_v14_0_2_get_smu_metrics_data(smu, - member_type, - value); -} - -static bool smu_v14_0_2_is_od_feature_supported(struct smu_context *smu, - int od_feature_bit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); -} - -static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, - int od_feature_bit, - int32_t *min, - int32_t *max) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - int32_t od_min_setting, od_max_setting; - - switch (od_feature_bit) { - case PP_OD_FEATURE_GFXCLK_FMIN: - od_min_setting = overdrive_lowerlimits->GfxclkFmin; - od_max_setting = overdrive_upperlimits->GfxclkFmin; - break; - case PP_OD_FEATURE_GFXCLK_FMAX: - od_min_setting = overdrive_lowerlimits->GfxclkFmax; - od_max_setting = overdrive_upperlimits->GfxclkFmax; - break; - case PP_OD_FEATURE_UCLK_FMIN: - od_min_setting = overdrive_lowerlimits->UclkFmin; - od_max_setting = overdrive_upperlimits->UclkFmin; - break; - case PP_OD_FEATURE_UCLK_FMAX: - od_min_setting = overdrive_lowerlimits->UclkFmax; - od_max_setting = overdrive_upperlimits->UclkFmax; - break; - case PP_OD_FEATURE_GFX_VF_CURVE: - od_min_setting = overdrive_lowerlimits->VoltageOffsetPerZoneBoundary[0]; - od_max_setting = overdrive_upperlimits->VoltageOffsetPerZoneBoundary[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_TEMP: - od_min_setting = overdrive_lowerlimits->FanLinearTempPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearTempPoints[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_PWM: - od_min_setting = overdrive_lowerlimits->FanLinearPwmPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearPwmPoints[0]; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT: - od_min_setting = overdrive_lowerlimits->AcousticLimitRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticLimitRpmThreshold; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_TARGET: - od_min_setting = overdrive_lowerlimits->AcousticTargetRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticTargetRpmThreshold; - break; - case PP_OD_FEATURE_FAN_TARGET_TEMPERATURE: - od_min_setting = overdrive_lowerlimits->FanTargetTemperature; - od_max_setting = overdrive_upperlimits->FanTargetTemperature; - break; - case PP_OD_FEATURE_FAN_MINIMUM_PWM: - od_min_setting = overdrive_lowerlimits->FanMinimumPwm; - od_max_setting = overdrive_upperlimits->FanMinimumPwm; - break; - default: - od_min_setting = od_max_setting = INT_MAX; - break; - } - - if (min) - *min = od_min_setting; - if (max) - *max = od_max_setting; -} - -static int smu_v14_0_2_print_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - char *buf) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - struct smu_14_0_dpm_table *single_dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t gen_speed, lane_width; - int i, curr_freq, size = 0; - int32_t min_value, max_value; - int ret = 0; - - smu_cmn_get_sysfs_buf(&buf, &size); - - if (amdgpu_ras_intr_triggered()) { - size += sysfs_emit_at(buf, size, "unavailable\n"); - return size; - } - - switch (clk_type) { - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - case SMU_DCEFCLK: - single_dpm_table = &(dpm_context->dpm_tables.dcef_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_SCLK: - case SMU_MCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - case SMU_DCEFCLK: - ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq); - if (ret) { - dev_err(smu->adev->dev, "Failed to get current clock freq!"); - return ret; - } - - if (single_dpm_table->is_fine_grained) { - /* - * For fine grained dpms, there are only two dpm levels: - * - level 0 -> min clock freq - * - level 1 -> max clock freq - * And the current clock frequency can be any value between them. - * So, if the current clock frequency is not at level 0 or level 1, - * we will fake it as three dpm levels: - * - level 0 -> min clock freq - * - level 1 -> current actual clock freq - * - level 2 -> max clock freq - */ - if ((single_dpm_table->dpm_levels[0].value != curr_freq) && - (single_dpm_table->dpm_levels[1].value != curr_freq)) { - size += sysfs_emit_at(buf, size, "0: %uMhz\n", - single_dpm_table->dpm_levels[0].value); - size += sysfs_emit_at(buf, size, "1: %uMhz *\n", - curr_freq); - size += sysfs_emit_at(buf, size, "2: %uMhz\n", - single_dpm_table->dpm_levels[1].value); - } else { - size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", - single_dpm_table->dpm_levels[0].value, - single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : ""); - size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", - single_dpm_table->dpm_levels[1].value, - single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : ""); - } - } else { - for (i = 0; i < single_dpm_table->count; i++) - size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", - i, single_dpm_table->dpm_levels[i].value, - single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : ""); - } - break; - case SMU_PCIE: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_RATE, - &gen_speed); - if (ret) - return ret; - - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_WIDTH, - &lane_width); - if (ret) - return ret; - - pcie_table = &(dpm_context->dpm_tables.pcie_table); - for (i = 0; i < pcie_table->num_of_link_levels; i++) - size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, - (pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," : - (pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," : - (pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," : - (pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "", - (pcie_table->pcie_lane[i] == 1) ? "x1" : - (pcie_table->pcie_lane[i] == 2) ? "x2" : - (pcie_table->pcie_lane[i] == 3) ? "x4" : - (pcie_table->pcie_lane[i] == 4) ? "x8" : - (pcie_table->pcie_lane[i] == 5) ? "x12" : - (pcie_table->pcie_lane[i] == 6) ? "x16" : "", - pcie_table->clk_freq[i], - (gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) && - (lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ? - "*" : ""); - break; - - case SMU_OD_SCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFXCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", - od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - break; - - case SMU_OD_MCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_UCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMHz\n", - od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); - break; - - case SMU_OD_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_VDDGFX_OFFSET:\n"); - size += sysfs_emit_at(buf, size, "%dmV\n", - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[0]); - break; - - case SMU_OD_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_FAN_CURVE:\n"); - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) - size += sysfs_emit_at(buf, size, "%d: %dC %d%%\n", - i, - (int)od_table->OverDriveTable.FanLinearTempPoints[i], - (int)od_table->OverDriveTable.FanLinearPwmPoints[i]); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(hotspot temp): %uC %uC\n", - min_value, max_value); - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(fan speed): %u%% %u%%\n", - min_value, max_value); - - break; - - case SMU_OD_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_LIMIT:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticLimitRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_LIMIT: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_TARGET:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticTargetRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_TARGET: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_TARGET_TEMPERATURE:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanTargetTemperature); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "TARGET_TEMPERATURE: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_MINIMUM_PWM:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanMinimumPwm); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "MINIMUM_PWM: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_RANGE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "VDDGFX_OFFSET: %7dmv %10dmv\n", - min_value, max_value); - } - break; - - default: - break; - } - - return size; -} - -static int smu_v14_0_2_force_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t mask) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - struct smu_14_0_dpm_table *single_dpm_table; - uint32_t soft_min_level, soft_max_level; - uint32_t min_freq, max_freq; - int ret = 0; - - soft_min_level = mask ? (ffs(mask) - 1) : 0; - soft_max_level = mask ? (fls(mask) - 1) : 0; - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - case SMU_UCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - case SMU_MCLK: - case SMU_UCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - if (single_dpm_table->is_fine_grained) { - /* There is only 2 levels for fine grained DPM */ - soft_max_level = (soft_max_level >= 1 ? 1 : 0); - soft_min_level = (soft_min_level >= 1 ? 1 : 0); - } else { - if ((soft_max_level >= single_dpm_table->count) || - (soft_min_level >= single_dpm_table->count)) - return -EINVAL; - } - - min_freq = single_dpm_table->dpm_levels[soft_min_level].value; - max_freq = single_dpm_table->dpm_levels[soft_max_level].value; - - ret = smu_v14_0_set_soft_freq_limited_range(smu, - clk_type, - min_freq, - max_freq); - break; - case SMU_DCEFCLK: - case SMU_PCIE: - default: - break; - } - - return ret; -} - -static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu, - uint8_t pcie_gen_cap, - uint8_t pcie_width_cap) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_14_0_pcie_table *pcie_table = - &dpm_context->dpm_tables.pcie_table; - uint32_t smu_pcie_arg; - int ret, i; - - for (i = 0; i < pcie_table->num_of_link_levels; i++) { - if (pcie_table->pcie_gen[i] > pcie_gen_cap) - pcie_table->pcie_gen[i] = pcie_gen_cap; - if (pcie_table->pcie_lane[i] > pcie_width_cap) - pcie_table->pcie_lane[i] = pcie_width_cap; - - smu_pcie_arg = i << 16; - smu_pcie_arg |= pcie_table->pcie_gen[i] << 8; - smu_pcie_arg |= pcie_table->pcie_lane[i]; - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_OverridePcieParameters, - smu_pcie_arg, - NULL); - if (ret) - return ret; - } - - return 0; -} - -static const struct smu_temperature_range smu14_thermal_policy[] = { - {-273150, 99000, 99000, -273150, 99000, 99000, -273150, 99000, 99000}, - { 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000}, -}; - -static int smu_v14_0_2_get_thermal_temperature_range(struct smu_context *smu, - struct smu_temperature_range *range) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - PPTable_t *pptable = smu->smu_table.driver_pptable; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!range) - return -EINVAL; - - memcpy(range, &smu14_thermal_policy[0], sizeof(struct smu_temperature_range)); - - range->max = pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->edge_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] + CTF_OFFSET_EDGE) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] + CTF_OFFSET_HOTSPOT) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] + CTF_OFFSET_MEM)* - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->software_shutdown_temp = powerplay_table->software_shutdown_temp; - range->software_shutdown_temp_offset = pptable->CustomSkuTable.FanAbnormalTempLimitOffset; - - return 0; -} - -static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *gfx_table = - &dpm_context->dpm_tables.gfx_table; - struct smu_14_0_dpm_table *mem_table = - &dpm_context->dpm_tables.uclk_table; - struct smu_14_0_dpm_table *soc_table = - &dpm_context->dpm_tables.soc_table; - struct smu_14_0_dpm_table *vclk_table = - &dpm_context->dpm_tables.vclk_table; - struct smu_14_0_dpm_table *dclk_table = - &dpm_context->dpm_tables.dclk_table; - struct smu_14_0_dpm_table *fclk_table = - &dpm_context->dpm_tables.fclk_table; - struct smu_umd_pstate_table *pstate_table = - &smu->pstate_table; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - DriverReportedClocks_t driver_clocks = - pptable->SkuTable.DriverReportedClocks; - - pstate_table->gfxclk_pstate.min = gfx_table->min; - if (driver_clocks.GameClockAc && - (driver_clocks.GameClockAc < gfx_table->max)) - pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc; - else - pstate_table->gfxclk_pstate.peak = gfx_table->max; - - pstate_table->uclk_pstate.min = mem_table->min; - pstate_table->uclk_pstate.peak = mem_table->max; - - pstate_table->socclk_pstate.min = soc_table->min; - pstate_table->socclk_pstate.peak = soc_table->max; - - pstate_table->vclk_pstate.min = vclk_table->min; - pstate_table->vclk_pstate.peak = vclk_table->max; - - pstate_table->dclk_pstate.min = dclk_table->min; - pstate_table->dclk_pstate.peak = dclk_table->max; - - pstate_table->fclk_pstate.min = fclk_table->min; - pstate_table->fclk_pstate.peak = fclk_table->max; - - if (driver_clocks.BaseClockAc && - driver_clocks.BaseClockAc < gfx_table->max) - pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc; - else - pstate_table->gfxclk_pstate.standard = gfx_table->max; - pstate_table->uclk_pstate.standard = mem_table->max; - pstate_table->socclk_pstate.standard = soc_table->min; - pstate_table->vclk_pstate.standard = vclk_table->min; - pstate_table->dclk_pstate.standard = dclk_table->min; - pstate_table->fclk_pstate.standard = fclk_table->min; - - return 0; -} - -static void smu_v14_0_2_get_unique_id(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - struct amdgpu_device *adev = smu->adev; - uint32_t upper32 = 0, lower32 = 0; - int ret; - - ret = smu_cmn_get_metrics_table(smu, NULL, false); - if (ret) - goto out; - - upper32 = metrics->PublicSerialNumberUpper; - lower32 = metrics->PublicSerialNumberLower; - -out: - adev->unique_id = ((uint64_t)upper32 << 32) | lower32; -} - -static int smu_v14_0_2_get_power_limit(struct smu_context *smu, - uint32_t *current_power_limit, - uint32_t *default_power_limit, - uint32_t *max_power_limit, - uint32_t *min_power_limit) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - uint32_t power_limit; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - - if (current_power_limit) - *current_power_limit = power_limit; - if (default_power_limit) - *default_power_limit = power_limit; - - if (max_power_limit) - *max_power_limit = msg_limit; - - if (min_power_limit) - *min_power_limit = 0; - - return 0; -} - -static int smu_v14_0_2_get_power_profile_mode(struct smu_context *smu, - char *buf) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - static const char *title[] = { - "PROFILE_INDEX(NAME)", - "CLOCK_TYPE(NAME)", - "FPS", - "MinActiveFreqType", - "MinActiveFreq", - "BoosterFreqType", - "BoosterFreq", - "PD_Data_limit_c", - "PD_Data_error_coeff", - "PD_Data_error_rate_coeff"}; - int16_t workload_type = 0; - uint32_t i, size = 0; - int result = 0; - - if (!buf) - return -EINVAL; - - size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s\n", - title[0], title[1], title[2], title[3], title[4], title[5], - title[6], title[7], title[8], title[9]); - - for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) { - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - i); - if (workload_type == -ENOTSUPP) - continue; - else if (workload_type < 0) - return -EINVAL; - - result = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - workload_type, - (void *)(&activity_monitor_external), - false); - if (result) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return result; - } - - size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", - i, amdgpu_pp_profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 0, - "GFXCLK", - activity_monitor->Gfx_FPS, - activity_monitor->Gfx_MinActiveFreqType, - activity_monitor->Gfx_MinActiveFreq, - activity_monitor->Gfx_BoosterFreqType, - activity_monitor->Gfx_BoosterFreq, - activity_monitor->Gfx_PD_Data_limit_c, - activity_monitor->Gfx_PD_Data_error_coeff, - activity_monitor->Gfx_PD_Data_error_rate_coeff); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 1, - "FCLK", - activity_monitor->Fclk_FPS, - activity_monitor->Fclk_MinActiveFreqType, - activity_monitor->Fclk_MinActiveFreq, - activity_monitor->Fclk_BoosterFreqType, - activity_monitor->Fclk_BoosterFreq, - activity_monitor->Fclk_PD_Data_limit_c, - activity_monitor->Fclk_PD_Data_error_coeff, - activity_monitor->Fclk_PD_Data_error_rate_coeff); - } - - return size; -} - -static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu, - long *input, - uint32_t size) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - int workload_type, ret = 0; - - smu->power_profile_mode = input[size]; - - if (smu->power_profile_mode >= PP_SMC_POWER_PROFILE_COUNT) { - dev_err(smu->adev->dev, "Invalid power profile mode %d\n", smu->power_profile_mode); - return -EINVAL; - } - - if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) { - if (size != 9) - return -EINVAL; - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - false); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return ret; - } - - switch (input[0]) { - case 0: /* Gfxclk */ - activity_monitor->Gfx_FPS = input[1]; - activity_monitor->Gfx_MinActiveFreqType = input[2]; - activity_monitor->Gfx_MinActiveFreq = input[3]; - activity_monitor->Gfx_BoosterFreqType = input[4]; - activity_monitor->Gfx_BoosterFreq = input[5]; - activity_monitor->Gfx_PD_Data_limit_c = input[6]; - activity_monitor->Gfx_PD_Data_error_coeff = input[7]; - activity_monitor->Gfx_PD_Data_error_rate_coeff = input[8]; - break; - case 1: /* Fclk */ - activity_monitor->Fclk_FPS = input[1]; - activity_monitor->Fclk_MinActiveFreqType = input[2]; - activity_monitor->Fclk_MinActiveFreq = input[3]; - activity_monitor->Fclk_BoosterFreqType = input[4]; - activity_monitor->Fclk_BoosterFreq = input[5]; - activity_monitor->Fclk_PD_Data_limit_c = input[6]; - activity_monitor->Fclk_PD_Data_error_coeff = input[7]; - activity_monitor->Fclk_PD_Data_error_rate_coeff = input[8]; - break; - default: - return -EINVAL; - } - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - true); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to set activity monitor!", __func__); - return ret; - } - } - - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - smu->power_profile_mode); - if (workload_type < 0) - return -EINVAL; - - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetWorkloadMask, - 1 << workload_type, - NULL); -} - -static int smu_v14_0_2_baco_enter(struct smu_context *smu) -{ - struct smu_baco_context *smu_baco = &smu->smu_baco; - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) - return smu_v14_0_baco_set_armd3_sequence(smu, - smu_baco->maco_support ? BACO_SEQ_BAMACO : BACO_SEQ_BACO); - else - return smu_v14_0_baco_enter(smu); -} - -static int smu_v14_0_2_baco_exit(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) { - /* Wait for PMFW handling for the Dstate change */ - usleep_range(10000, 11000); - return smu_v14_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); - } else { - return smu_v14_0_baco_exit(smu); - } -} - -static bool smu_v14_0_2_is_mode1_reset_supported(struct smu_context *smu) -{ - // TODO - - return true; -} - -static int smu_v14_0_2_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msg, int num_msgs) -{ - struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = smu_i2c->adev; - struct smu_context *smu = adev->powerplay.pp_handle; - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *table = &smu_table->driver_table; - SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr; - int i, j, r, c; - u16 dir; - - if (!adev->pm.dpm_enabled) - return -EBUSY; - - req = kzalloc(sizeof(*req), GFP_KERNEL); - if (!req) - return -ENOMEM; - - req->I2CcontrollerPort = smu_i2c->port; - req->I2CSpeed = I2C_SPEED_FAST_400K; - req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */ - dir = msg[0].flags & I2C_M_RD; - - for (c = i = 0; i < num_msgs; i++) { - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &req->SwI2cCmds[c]; - - if (!(msg[i].flags & I2C_M_RD)) { - /* write */ - cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK; - cmd->ReadWriteData = msg[i].buf[j]; - } - - if ((dir ^ msg[i].flags) & I2C_M_RD) { - /* The direction changes. - */ - dir = msg[i].flags & I2C_M_RD; - cmd->CmdConfig |= CMDCONFIG_RESTART_MASK; - } - - req->NumCmds++; - - /* - * Insert STOP if we are at the last byte of either last - * message for the transaction or the client explicitly - * requires a STOP at this particular message. - */ - if ((j == msg[i].len - 1) && - ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) { - cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK; - cmd->CmdConfig |= CMDCONFIG_STOP_MASK; - } - } - } - mutex_lock(&adev->pm.mutex); - r = smu_cmn_update_table(smu, SMU_TABLE_I2C_COMMANDS, 0, req, true); - mutex_unlock(&adev->pm.mutex); - if (r) - goto fail; - - for (c = i = 0; i < num_msgs; i++) { - if (!(msg[i].flags & I2C_M_RD)) { - c += msg[i].len; - continue; - } - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &res->SwI2cCmds[c]; - - msg[i].buf[j] = cmd->ReadWriteData; - } - } - r = num_msgs; -fail: - kfree(req); - return r; -} - -static u32 smu_v14_0_2_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm smu_v14_0_2_i2c_algo = { - .master_xfer = smu_v14_0_2_i2c_xfer, - .functionality = smu_v14_0_2_i2c_func, -}; - -static const struct i2c_adapter_quirks smu_v14_0_2_i2c_control_quirks = { - .flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN, - .max_read_len = MAX_SW_I2C_COMMANDS, - .max_write_len = MAX_SW_I2C_COMMANDS, - .max_comb_1st_msg_len = 2, - .max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2, -}; - -static int smu_v14_0_2_i2c_control_init(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int res, i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - smu_i2c->adev = adev; - smu_i2c->port = i; - mutex_init(&smu_i2c->mutex); - control->owner = THIS_MODULE; - control->dev.parent = &adev->pdev->dev; - control->algo = &smu_v14_0_2_i2c_algo; - snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i); - control->quirks = &smu_v14_0_2_i2c_control_quirks; - i2c_set_adapdata(control, smu_i2c); - - res = i2c_add_adapter(control); - if (res) { - DRM_ERROR("Failed to register hw i2c, err: %d\n", res); - goto Out_err; - } - } - - /* assign the buses used for the FRU EEPROM and RAS EEPROM */ - /* XXX ideally this would be something in a vbios data table */ - adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[1].adapter; - adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter; - - return 0; -Out_err: - for ( ; i >= 0; i--) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - return res; -} - -static void smu_v14_0_2_i2c_control_fini(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - adev->pm.ras_eeprom_i2c_bus = NULL; - adev->pm.fru_eeprom_i2c_bus = NULL; -} - -static int smu_v14_0_2_set_mp1_state(struct smu_context *smu, - enum pp_mp1_state mp1_state) -{ - int ret; - - switch (mp1_state) { - case PP_MP1_STATE_UNLOAD: - ret = smu_cmn_set_mp1_state(smu, mp1_state); - break; - default: - /* Ignore others */ - ret = 0; - } - - return ret; -} - -static int smu_v14_0_2_set_df_cstate(struct smu_context *smu, - enum pp_df_cstate state) -{ - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_DFCstateControl, - state, - NULL); -} - -static int smu_v14_0_2_mode1_reset(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); - if (!ret) { - if (amdgpu_emu_mode == 1) - msleep(50000); - else - msleep(1000); - } - - return ret; -} - -static int smu_v14_0_2_mode2_reset(struct smu_context *smu) -{ - int ret = 0; - - // TODO - - return ret; -} - -static int smu_v14_0_2_enable_gfx_features(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(14, 0, 2)) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnableAllSmuFeatures, - FEATURE_PWR_GFX, NULL); - else - return -EOPNOTSUPP; -} - -static void smu_v14_0_2_set_smu_mailbox_registers(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - smu->param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_82); - smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_66); - smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_90); - - smu->debug_param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_53); - smu->debug_msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_75); - smu->debug_resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_54); -} - -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); -} - -static void smu_v14_0_2_dump_od_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - struct amdgpu_device *adev = smu->adev; - - dev_dbg(adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - dev_dbg(adev->dev, "OD: Uclk: (%d, %d)\n", od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); -} - -static int smu_v14_0_2_upload_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - true); - if (ret) - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - - return ret; -} - -static void smu_v14_0_2_set_supported_od_feature_mask(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - adev->pm.od_feature_mask |= OD_OPS_SUPPORT_FAN_CURVE_RETRIEVE | - OD_OPS_SUPPORT_FAN_CURVE_SET | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_SET | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_SET | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_RETRIEVE | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_SET | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_RETRIEVE | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_SET; -} - -static int smu_v14_0_2_get_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - false); - if (ret) - dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); - - return ret; -} - -static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu) -{ - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - OverDriveTableExternal_t *boot_od_table = - (OverDriveTableExternal_t *)smu->smu_table.boot_overdrive_table; - OverDriveTableExternal_t *user_od_table = - (OverDriveTableExternal_t *)smu->smu_table.user_overdrive_table; - OverDriveTableExternal_t user_od_table_bak; - int ret; - int i; - - ret = smu_v14_0_2_get_overdrive_table(smu, boot_od_table); - if (ret) - return ret; - - smu_v14_0_2_dump_od_table(smu, boot_od_table); - - memcpy(od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - - /* - * For S3/S4/Runpm resume, we need to setup those overdrive tables again, - * but we have to preserve user defined values in "user_od_table". - */ - if (!smu->adev->in_suspend) { - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - smu->user_dpm_profile.user_od = false; - } else if (smu->user_dpm_profile.user_od) { - memcpy(&user_od_table_bak, - user_od_table, - sizeof(OverDriveTableExternal_t)); - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - user_od_table->OverDriveTable.GfxclkFmin = - user_od_table_bak.OverDriveTable.GfxclkFmin; - user_od_table->OverDriveTable.GfxclkFmax = - user_od_table_bak.OverDriveTable.GfxclkFmax; - user_od_table->OverDriveTable.UclkFmin = - user_od_table_bak.OverDriveTable.UclkFmin; - user_od_table->OverDriveTable.UclkFmax = - user_od_table_bak.OverDriveTable.UclkFmax; - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - user_od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = - user_od_table_bak.OverDriveTable.VoltageOffsetPerZoneBoundary[i]; - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) { - user_od_table->OverDriveTable.FanLinearTempPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearTempPoints[i]; - user_od_table->OverDriveTable.FanLinearPwmPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearPwmPoints[i]; - } - user_od_table->OverDriveTable.AcousticLimitRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticLimitRpmThreshold; - user_od_table->OverDriveTable.AcousticTargetRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticTargetRpmThreshold; - user_od_table->OverDriveTable.FanTargetTemperature = - user_od_table_bak.OverDriveTable.FanTargetTemperature; - user_od_table->OverDriveTable.FanMinimumPwm = - user_od_table_bak.OverDriveTable.FanMinimumPwm; - } - - smu_v14_0_2_set_supported_od_feature_mask(smu); - - return 0; -} - -static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = table_context->overdrive_table; - OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; - int res; - - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); - user_od_table->OverDriveTable.FeatureCtrlMask = 0; - if (res == 0) - memcpy(od_table, user_od_table, sizeof(OverDriveTableExternal_t)); - - return res; -} - -static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long input) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *boot_overdrive_table = - (OverDriveTableExternal_t *)table_context->boot_overdrive_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - int i; - - switch (input) { - case PP_OD_EDIT_FAN_CURVE: - for (i = 0; i < NUM_OD_FAN_MAX_POINTS; i++) { - od_table->OverDriveTable.FanLinearTempPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearTempPoints[i]; - od_table->OverDriveTable.FanLinearPwmPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearPwmPoints[i]; - } - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_LIMIT: - od_table->OverDriveTable.AcousticLimitRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticLimitRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_TARGET: - od_table->OverDriveTable.AcousticTargetRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticTargetRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - od_table->OverDriveTable.FanTargetTemperature = - boot_overdrive_table->OverDriveTable.FanTargetTemperature; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_MINIMUM_PWM: - od_table->OverDriveTable.FanMinimumPwm = - boot_overdrive_table->OverDriveTable.FanMinimumPwm; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - default: - dev_info(adev->dev, "Invalid table index: %ld\n", input); - return -EINVAL; - } - - return 0; -} - -static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, - enum PP_OD_DPM_TABLE_COMMAND type, - long input[], - uint32_t size) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - uint32_t offset_of_voltageoffset; - int32_t minimum, maximum; - uint32_t feature_ctrlmask; - int i, ret = 0; - - switch (type) { - case PP_OD_EDIT_SCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - dev_warn(adev->dev, "GFXCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.GfxclkFmin > od_table->OverDriveTable.GfxclkFmax) { - dev_err(adev->dev, - "Invalid setting: GfxclkFmin(%u) is bigger than GfxclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.GfxclkFmin, - (uint32_t)od_table->OverDriveTable.GfxclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_MCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - dev_warn(adev->dev, "UCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.UclkFmin > od_table->OverDriveTable.UclkFmax) { - dev_err(adev->dev, - "Invalid setting: UclkFmin(%u) is bigger than UclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.UclkFmin, - (uint32_t)od_table->OverDriveTable.UclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - dev_warn(adev->dev, "Gfx offset setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "Voltage offset (%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = input[0]; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - if (input[0] >= NUM_OD_FAN_MAX_POINTS - 1 || - input[0] < 0) - return -EINVAL; - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &minimum, - &maximum); - if (input[1] < minimum || - input[1] > maximum) { - dev_info(adev->dev, "Fan curve temp setting(%ld) must be within [%d, %d]!\n", - input[1], minimum, maximum); - return -EINVAL; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &minimum, - &maximum); - if (input[2] < minimum || - input[2] > maximum) { - dev_info(adev->dev, "Fan curve pwm setting(%ld) must be within [%d, %d]!\n", - input[2], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanLinearTempPoints[input[0]] = input[1]; - od_table->OverDriveTable.FanLinearPwmPoints[input[0]] = input[2]; - od_table->OverDriveTable.FanMode = FAN_MODE_MANUAL_LINEAR; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic limit threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticLimitRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic target threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticTargetRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan target temperature setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanTargetTemperature = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan minimum pwm setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanMinimumPwm = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_RESTORE_DEFAULT_TABLE: - if (size == 1) { - ret = smu_v14_0_2_od_restore_table_single(smu, input[0]); - if (ret) - return ret; - } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; - memcpy(od_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; - } - fallthrough; - case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ - offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); - if (memcmp((u8 *)od_table + offset_of_voltageoffset, - table_context->user_overdrive_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset)) { - smu_v14_0_2_dump_od_table(smu, od_table); - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - od_table->OverDriveTable.FeatureCtrlMask = 0; - memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, - (u8 *)od_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); - - if (!memcmp(table_context->user_overdrive_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t))) - smu->user_dpm_profile.user_od = false; - else - smu->user_dpm_profile.user_od = true; - } - break; - - default: - return -ENOSYS; - } - - return ret; -} - -static int smu_v14_0_2_set_power_limit(struct smu_context *smu, - enum smu_ppt_limit_type limit_type, - uint32_t limit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - int ret = 0; - - if (limit_type != SMU_DEFAULT_PPT_LIMIT) - return -EINVAL; - - if (limit <= msg_limit) { - if (smu->current_power_limit > msg_limit) { - od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - } - return smu_v14_0_set_power_limit(smu, limit_type, limit); - } else if (smu->od_enabled) { - ret = smu_v14_0_set_power_limit(smu, limit_type, msg_limit); - if (ret) - return ret; - - od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - smu->current_power_limit = limit; - } else { - return -EINVAL; - } - - return 0; -} - -static const struct pptable_funcs smu_v14_0_2_ppt_funcs = { - .get_allowed_feature_mask = smu_v14_0_2_get_allowed_feature_mask, - .set_default_dpm_table = smu_v14_0_2_set_default_dpm_table, - .i2c_init = smu_v14_0_2_i2c_control_init, - .i2c_fini = smu_v14_0_2_i2c_control_fini, - .is_dpm_running = smu_v14_0_2_is_dpm_running, - .dump_pptable = smu_v14_0_2_dump_pptable, - .init_microcode = smu_v14_0_init_microcode, - .load_microcode = smu_v14_0_load_microcode, - .fini_microcode = smu_v14_0_fini_microcode, - .init_smc_tables = smu_v14_0_2_init_smc_tables, - .fini_smc_tables = smu_v14_0_fini_smc_tables, - .init_power = smu_v14_0_init_power, - .fini_power = smu_v14_0_fini_power, - .check_fw_status = smu_v14_0_check_fw_status, - .setup_pptable = smu_v14_0_2_setup_pptable, - .check_fw_version = smu_v14_0_check_fw_version, - .write_pptable = smu_cmn_write_pptable, - .set_driver_table_location = smu_v14_0_set_driver_table_location, - .system_features_control = smu_v14_0_system_features_control, - .set_allowed_mask = smu_v14_0_set_allowed_mask, - .get_enabled_mask = smu_cmn_get_enabled_mask, - .dpm_set_vcn_enable = smu_v14_0_set_vcn_enable, - .dpm_set_jpeg_enable = smu_v14_0_set_jpeg_enable, - .get_dpm_ultimate_freq = smu_v14_0_2_get_dpm_ultimate_freq, - .get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values, - .read_sensor = smu_v14_0_2_read_sensor, - .feature_is_enabled = smu_cmn_feature_is_enabled, - .print_clk_levels = smu_v14_0_2_print_clk_levels, - .force_clk_levels = smu_v14_0_2_force_clk_levels, - .update_pcie_parameters = smu_v14_0_2_update_pcie_parameters, - .get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range, - .register_irq_handler = smu_v14_0_register_irq_handler, - .enable_thermal_alert = smu_v14_0_enable_thermal_alert, - .disable_thermal_alert = smu_v14_0_disable_thermal_alert, - .notify_memory_pool_location = smu_v14_0_notify_memory_pool_location, - .get_gpu_metrics = smu_v14_0_2_get_gpu_metrics, - .set_soft_freq_limited_range = smu_v14_0_set_soft_freq_limited_range, - .set_default_od_settings = smu_v14_0_2_set_default_od_settings, - .restore_user_od_settings = smu_v14_0_2_restore_user_od_settings, - .od_edit_dpm_table = smu_v14_0_2_od_edit_dpm_table, - .init_pptable_microcode = smu_v14_0_init_pptable_microcode, - .populate_umd_state_clk = smu_v14_0_2_populate_umd_state_clk, - .set_performance_level = smu_v14_0_set_performance_level, - .gfx_off_control = smu_v14_0_gfx_off_control, - .get_unique_id = smu_v14_0_2_get_unique_id, - .get_power_limit = smu_v14_0_2_get_power_limit, - .set_power_limit = smu_v14_0_2_set_power_limit, - .set_power_source = smu_v14_0_set_power_source, - .get_power_profile_mode = smu_v14_0_2_get_power_profile_mode, - .set_power_profile_mode = smu_v14_0_2_set_power_profile_mode, - .run_btc = smu_v14_0_run_btc, - .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, - .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, - .set_tool_table_location = smu_v14_0_set_tool_table_location, - .deep_sleep_control = smu_v14_0_deep_sleep_control, - .gfx_ulv_control = smu_v14_0_gfx_ulv_control, - .get_bamaco_support = smu_v14_0_get_bamaco_support, - .baco_get_state = smu_v14_0_baco_get_state, - .baco_set_state = smu_v14_0_baco_set_state, - .baco_enter = smu_v14_0_2_baco_enter, - .baco_exit = smu_v14_0_2_baco_exit, - .mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported, - .mode1_reset = smu_v14_0_2_mode1_reset, - .mode2_reset = smu_v14_0_2_mode2_reset, - .enable_gfx_features = smu_v14_0_2_enable_gfx_features, - .set_mp1_state = smu_v14_0_2_set_mp1_state, - .set_df_cstate = smu_v14_0_2_set_df_cstate, -#if 0 - .gpo_control = smu_v14_0_gpo_control, -#endif -}; - -void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu) -{ - smu->ppt_funcs = &smu_v14_0_2_ppt_funcs; - smu->message_map = smu_v14_0_2_message_map; - smu->clock_map = smu_v14_0_2_clk_map; - smu->feature_map = smu_v14_0_2_feature_mask_map; - smu->table_map = smu_v14_0_2_table_map; - smu->pwr_src_map = smu_v14_0_2_pwr_src_map; - smu->workload_map = smu_v14_0_2_workload_map; - smu_v14_0_2_set_smu_mailbox_registers(smu); -} diff --git a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage b/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage deleted file mode 100644 index 6878263c313c..000000000000 --- a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage +++ /dev/null @@ -1,2956 +0,0 @@ -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#define SWSMU_CODE_LAYER_L2 - -#include <linux/firmware.h> -#include <linux/pci.h> -#include <linux/i2c.h> -#include "amdgpu.h" -#include "amdgpu_smu.h" -#include "atomfirmware.h" -#include "amdgpu_atomfirmware.h" -#include "amdgpu_atombios.h" -#include "smu_v14_0.h" -#include "smu14_driver_if_v14_0.h" -#include "soc15_common.h" -#include "atom.h" -#include "smu_v14_0_2_ppt.h" -#include "smu_v14_0_2_pptable.h" -#include "smu_v14_0_2_ppsmc.h" -#include "mp/mp_14_0_2_offset.h" -#include "mp/mp_14_0_2_sh_mask.h" - -#include "smu_cmn.h" -#include "amdgpu_ras.h" - -/* - * DO NOT use these for err/warn/info/debug messages. - * Use dev_err, dev_warn, dev_info and dev_dbg instead. - * They are more MGPU friendly. - */ -#undef pr_err -#undef pr_warn -#undef pr_info -#undef pr_debug - -#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) - -#define FEATURE_MASK(feature) (1ULL << feature) -#define SMC_DPM_FEATURE ( \ - FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_LINK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_FCLK_BIT)) - -#define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE 0x4000 -#define DEBUGSMC_MSG_Mode1Reset 2 -#define LINK_SPEED_MAX 3 -<<<<<<< -======= - -#define PP_OD_FEATURE_GFXCLK_FMIN 0 -#define PP_OD_FEATURE_GFXCLK_FMAX 1 -#define PP_OD_FEATURE_UCLK_FMIN 2 -#define PP_OD_FEATURE_UCLK_FMAX 3 -#define PP_OD_FEATURE_GFX_VF_CURVE 4 -#define PP_OD_FEATURE_FAN_CURVE_TEMP 5 -#define PP_OD_FEATURE_FAN_CURVE_PWM 6 -#define PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT 7 -#define PP_OD_FEATURE_FAN_ACOUSTIC_TARGET 8 -#define PP_OD_FEATURE_FAN_TARGET_TEMPERATURE 9 -#define PP_OD_FEATURE_FAN_MINIMUM_PWM 10 ->>>>>>> - -static struct cmn2asic_msg_mapping smu_v14_0_2_message_map[SMU_MSG_MAX_COUNT] = { - MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 1), - MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), - MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 1), - MSG_MAP(SetAllowedFeaturesMaskLow, PPSMC_MSG_SetAllowedFeaturesMaskLow, 0), - MSG_MAP(SetAllowedFeaturesMaskHigh, PPSMC_MSG_SetAllowedFeaturesMaskHigh, 0), - MSG_MAP(EnableAllSmuFeatures, PPSMC_MSG_EnableAllSmuFeatures, 0), - MSG_MAP(DisableAllSmuFeatures, PPSMC_MSG_DisableAllSmuFeatures, 0), - MSG_MAP(EnableSmuFeaturesLow, PPSMC_MSG_EnableSmuFeaturesLow, 1), - MSG_MAP(EnableSmuFeaturesHigh, PPSMC_MSG_EnableSmuFeaturesHigh, 1), - MSG_MAP(DisableSmuFeaturesLow, PPSMC_MSG_DisableSmuFeaturesLow, 1), - MSG_MAP(DisableSmuFeaturesHigh, PPSMC_MSG_DisableSmuFeaturesHigh, 1), - MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetRunningSmuFeaturesLow, 1), - MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetRunningSmuFeaturesHigh, 1), - MSG_MAP(SetWorkloadMask, PPSMC_MSG_SetWorkloadMask, 1), - MSG_MAP(SetPptLimit, PPSMC_MSG_SetPptLimit, 0), - MSG_MAP(SetDriverDramAddrHigh, PPSMC_MSG_SetDriverDramAddrHigh, 1), - MSG_MAP(SetDriverDramAddrLow, PPSMC_MSG_SetDriverDramAddrLow, 1), - MSG_MAP(SetToolsDramAddrHigh, PPSMC_MSG_SetToolsDramAddrHigh, 0), - MSG_MAP(SetToolsDramAddrLow, PPSMC_MSG_SetToolsDramAddrLow, 0), - MSG_MAP(TransferTableSmu2Dram, PPSMC_MSG_TransferTableSmu2Dram, 1), - MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), - MSG_MAP(UseDefaultPPTable, PPSMC_MSG_UseDefaultPPTable, 0), - MSG_MAP(RunDcBtc, PPSMC_MSG_RunDcBtc, 0), - MSG_MAP(EnterBaco, PPSMC_MSG_EnterBaco, 0), - MSG_MAP(ExitBaco, PPSMC_MSG_ExitBaco, 0), - MSG_MAP(SetSoftMinByFreq, PPSMC_MSG_SetSoftMinByFreq, 1), - MSG_MAP(SetSoftMaxByFreq, PPSMC_MSG_SetSoftMaxByFreq, 1), - MSG_MAP(SetHardMinByFreq, PPSMC_MSG_SetHardMinByFreq, 1), - MSG_MAP(SetHardMaxByFreq, PPSMC_MSG_SetHardMaxByFreq, 0), - MSG_MAP(GetMinDpmFreq, PPSMC_MSG_GetMinDpmFreq, 1), - MSG_MAP(GetMaxDpmFreq, PPSMC_MSG_GetMaxDpmFreq, 1), - MSG_MAP(GetDpmFreqByIndex, PPSMC_MSG_GetDpmFreqByIndex, 1), - MSG_MAP(PowerUpVcn, PPSMC_MSG_PowerUpVcn, 0), - MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), - MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), - MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), - MSG_MAP(GetDcModeMaxDpmFreq, PPSMC_MSG_GetDcModeMaxDpmFreq, 1), - MSG_MAP(OverridePcieParameters, PPSMC_MSG_OverridePcieParameters, 0), - MSG_MAP(DramLogSetDramAddrHigh, PPSMC_MSG_DramLogSetDramAddrHigh, 0), - MSG_MAP(DramLogSetDramAddrLow, PPSMC_MSG_DramLogSetDramAddrLow, 0), - MSG_MAP(DramLogSetDramSize, PPSMC_MSG_DramLogSetDramSize, 0), - MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), - MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), - MSG_MAP(GetPptLimit, PPSMC_MSG_GetPptLimit, 0), - MSG_MAP(NotifyPowerSource, PPSMC_MSG_NotifyPowerSource, 0), - MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), - MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), - MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), - MSG_MAP(SetNumBadMemoryPagesRetired, PPSMC_MSG_SetNumBadMemoryPagesRetired, 0), - MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel, - PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel, 0), - MSG_MAP(AllowIHHostInterrupt, PPSMC_MSG_AllowIHHostInterrupt, 0), - MSG_MAP(ReenableAcDcInterrupt, PPSMC_MSG_ReenableAcDcInterrupt, 0), -}; - -static struct cmn2asic_mapping smu_v14_0_2_clk_map[SMU_CLK_COUNT] = { - CLK_MAP(GFXCLK, PPCLK_GFXCLK), - CLK_MAP(SCLK, PPCLK_GFXCLK), - CLK_MAP(SOCCLK, PPCLK_SOCCLK), - CLK_MAP(FCLK, PPCLK_FCLK), - CLK_MAP(UCLK, PPCLK_UCLK), - CLK_MAP(MCLK, PPCLK_UCLK), - CLK_MAP(VCLK, PPCLK_VCLK_0), - CLK_MAP(DCLK, PPCLK_DCLK_0), - CLK_MAP(DCEFCLK, PPCLK_DCFCLK), -}; - -static struct cmn2asic_mapping smu_v14_0_2_feature_mask_map[SMU_FEATURE_COUNT] = { - FEA_MAP(FW_DATA_READ), - FEA_MAP(DPM_GFXCLK), - FEA_MAP(DPM_GFX_POWER_OPTIMIZER), - FEA_MAP(DPM_UCLK), - FEA_MAP(DPM_FCLK), - FEA_MAP(DPM_SOCCLK), - FEA_MAP(DPM_LINK), - FEA_MAP(DPM_DCN), - FEA_MAP(VMEMP_SCALING), - FEA_MAP(VDDIO_MEM_SCALING), - FEA_MAP(DS_GFXCLK), - FEA_MAP(DS_SOCCLK), - FEA_MAP(DS_FCLK), - FEA_MAP(DS_LCLK), - FEA_MAP(DS_DCFCLK), - FEA_MAP(DS_UCLK), - FEA_MAP(GFX_ULV), - FEA_MAP(FW_DSTATE), - FEA_MAP(GFXOFF), - FEA_MAP(BACO), - FEA_MAP(MM_DPM), - FEA_MAP(SOC_MPCLK_DS), - FEA_MAP(BACO_MPCLK_DS), - FEA_MAP(THROTTLERS), - FEA_MAP(SMARTSHIFT), - FEA_MAP(GTHR), - FEA_MAP(ACDC), - FEA_MAP(VR0HOT), - FEA_MAP(FW_CTF), - FEA_MAP(FAN_CONTROL), - FEA_MAP(GFX_DCS), - FEA_MAP(GFX_READ_MARGIN), - FEA_MAP(LED_DISPLAY), - FEA_MAP(GFXCLK_SPREAD_SPECTRUM), - FEA_MAP(OUT_OF_BAND_MONITOR), - FEA_MAP(OPTIMIZED_VMIN), - FEA_MAP(GFX_IMU), - FEA_MAP(BOOT_TIME_CAL), - FEA_MAP(GFX_PCC_DFLL), - FEA_MAP(SOC_CG), - FEA_MAP(DF_CSTATE), - FEA_MAP(GFX_EDC), - FEA_MAP(BOOT_POWER_OPT), - FEA_MAP(CLOCK_POWER_DOWN_BYPASS), - FEA_MAP(DS_VCN), - FEA_MAP(BACO_CG), - FEA_MAP(MEM_TEMP_READ), - FEA_MAP(ATHUB_MMHUB_PG), - FEA_MAP(SOC_PCC), - [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_PPT_BIT] = {1, FEATURE_THROTTLERS_BIT}, -}; - -static struct cmn2asic_mapping smu_v14_0_2_table_map[SMU_TABLE_COUNT] = { - TAB_MAP(PPTABLE), - TAB_MAP(WATERMARKS), - TAB_MAP(AVFS_PSM_DEBUG), - TAB_MAP(PMSTATUSLOG), - TAB_MAP(SMU_METRICS), - TAB_MAP(DRIVER_SMU_CONFIG), - TAB_MAP(ACTIVITY_MONITOR_COEFF), - [SMU_TABLE_COMBO_PPTABLE] = {1, TABLE_COMBO_PPTABLE}, - TAB_MAP(I2C_COMMANDS), - TAB_MAP(ECCINFO), - TAB_MAP(OVERDRIVE), -}; - -static struct cmn2asic_mapping smu_v14_0_2_pwr_src_map[SMU_POWER_SOURCE_COUNT] = { - PWR_MAP(AC), - PWR_MAP(DC), -}; - -static struct cmn2asic_mapping smu_v14_0_2_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT, WORKLOAD_PPLIB_DEFAULT_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_POWERSAVING, WORKLOAD_PPLIB_POWER_SAVING_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_WINDOW3D, WORKLOAD_PPLIB_WINDOW_3D_BIT), -}; - -static const uint8_t smu_v14_0_2_throttler_map[] = { - [THROTTLER_PPT0_BIT] = (SMU_THROTTLER_PPT0_BIT), - [THROTTLER_PPT1_BIT] = (SMU_THROTTLER_PPT1_BIT), - [THROTTLER_PPT2_BIT] = (SMU_THROTTLER_PPT2_BIT), - [THROTTLER_PPT3_BIT] = (SMU_THROTTLER_PPT3_BIT), - [THROTTLER_TDC_GFX_BIT] = (SMU_THROTTLER_TDC_GFX_BIT), - [THROTTLER_TDC_SOC_BIT] = (SMU_THROTTLER_TDC_SOC_BIT), - [THROTTLER_TEMP_EDGE_BIT] = (SMU_THROTTLER_TEMP_EDGE_BIT), - [THROTTLER_TEMP_HOTSPOT_BIT] = (SMU_THROTTLER_TEMP_HOTSPOT_BIT), - [THROTTLER_TEMP_MEM_BIT] = (SMU_THROTTLER_TEMP_MEM_BIT), - [THROTTLER_TEMP_VR_GFX_BIT] = (SMU_THROTTLER_TEMP_VR_GFX_BIT), - [THROTTLER_TEMP_VR_SOC_BIT] = (SMU_THROTTLER_TEMP_VR_SOC_BIT), - [THROTTLER_TEMP_VR_MEM0_BIT] = (SMU_THROTTLER_TEMP_VR_MEM0_BIT), - [THROTTLER_TEMP_VR_MEM1_BIT] = (SMU_THROTTLER_TEMP_VR_MEM1_BIT), - [THROTTLER_TEMP_LIQUID0_BIT] = (SMU_THROTTLER_TEMP_LIQUID0_BIT), - [THROTTLER_TEMP_LIQUID1_BIT] = (SMU_THROTTLER_TEMP_LIQUID1_BIT), - [THROTTLER_GFX_APCC_PLUS_BIT] = (SMU_THROTTLER_APCC_BIT), - [THROTTLER_FIT_BIT] = (SMU_THROTTLER_FIT_BIT), -}; - -static int -smu_v14_0_2_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - /*u32 smu_version;*/ - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0xff, sizeof(uint32_t) * num); - - if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT); - } -#if 0 - if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) || - !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); - - if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); - - /* PMFW 78.58 contains a critical fix for gfxoff feature */ - smu_cmn_get_smc_version(smu, NULL, &smu_version); - if ((smu_version < 0x004e3a00) || - !(adev->pm.pp_feature & PP_GFXOFF_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT); - - if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); - } - - if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - - if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT); - } - - if (!(adev->pm.pp_feature & PP_ULV_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT); -#endif - - return 0; -} - -static int smu_v14_0_2_check_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - struct smu_baco_context *smu_baco = &smu->smu_baco; - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC) - smu->dc_controlled_by_gpio = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_BACO) { - smu_baco->platform_support = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_MACO) - smu_baco->maco_support = true; - } - - if (!overdrive_lowerlimits->FeatureCtrlMask || - !overdrive_upperlimits->FeatureCtrlMask) - smu->od_enabled = false; - - table_context->thermal_controller_type = - powerplay_table->thermal_controller_type; - - /* - * Instead of having its own buffer space and get overdrive_table copied, - * smu->od_settings just points to the actual overdrive_table - */ - smu->od_settings = &powerplay_table->overdrive_table; - - smu->adev->pm.no_fan = - !(pptable->PFE_Settings.FeaturesToRun[0] & (1 << FEATURE_FAN_CONTROL_BIT)); - - return 0; -} - -static int smu_v14_0_2_store_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - - memcpy(table_context->driver_pptable, &powerplay_table->smc_pptable, - sizeof(PPTable_t)); - - return 0; -} - -#ifndef atom_smc_dpm_info_table_14_0_0 -struct atom_smc_dpm_info_table_14_0_0 { - struct atom_common_table_header table_header; - BoardTable_t BoardTable; -}; -#endif - -static int smu_v14_0_2_append_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - struct atom_smc_dpm_info_table_14_0_0 *smc_dpm_table; - BoardTable_t *BoardTable = &smc_pptable->BoardTable; - int index, ret; - - index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, - smc_dpm_info); - - ret = amdgpu_atombios_get_data_table(smu->adev, index, NULL, NULL, NULL, - (uint8_t **)&smc_dpm_table); - if (ret) - return ret; - - memcpy(BoardTable, &smc_dpm_table->BoardTable, sizeof(BoardTable_t)); - - return 0; -} - -#if 0 -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_powerplay_table); - - return 0; -} -#endif - -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_2_powerplay_table); - - return 0; -} - -static int smu_v14_0_2_setup_pptable(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct amdgpu_device *adev = smu->adev; - int ret = 0; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!adev->scpm_enabled) - ret = smu_v14_0_setup_pptable(smu); - else - ret = smu_v14_0_2_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - if (ret) - return ret; - - ret = smu_v14_0_2_store_powerplay_table(smu); - if (ret) - return ret; - - /* - * With SCPM enabled, the operation below will be handled - * by PSP. Driver involvment is unnecessary and useless. - */ - if (!adev->scpm_enabled) { - ret = smu_v14_0_2_append_powerplay_table(smu); - if (ret) - return ret; - } - - ret = smu_v14_0_2_check_powerplay_table(smu); - if (ret) - return ret; - - return ret; -} - -static int smu_v14_0_2_tables_init(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *tables = smu_table->tables; - - SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetricsExternal_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_PMSTATUSLOG, SMU14_TOOL_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ACTIVITY_MONITOR_COEFF, - sizeof(DpmActivityMonitorCoeffIntExternal_t), PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_COMBO_PPTABLE, MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ECCINFO, sizeof(EccInfoTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - - smu_table->metrics_table = kzalloc(sizeof(SmuMetricsExternal_t), GFP_KERNEL); - if (!smu_table->metrics_table) - goto err0_out; - smu_table->metrics_time = 0; - - smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3); - smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); - if (!smu_table->gpu_metrics_table) - goto err1_out; - - smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); - if (!smu_table->watermarks_table) - goto err2_out; - - smu_table->ecc_table = kzalloc(tables[SMU_TABLE_ECCINFO].size, GFP_KERNEL); - if (!smu_table->ecc_table) - goto err3_out; - - return 0; - -err3_out: - kfree(smu_table->watermarks_table); -err2_out: - kfree(smu_table->gpu_metrics_table); -err1_out: - kfree(smu_table->metrics_table); -err0_out: - return -ENOMEM; -} - -static int smu_v14_0_2_allocate_dpm_context(struct smu_context *smu) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - - smu_dpm->dpm_context = kzalloc(sizeof(struct smu_14_0_dpm_context), - GFP_KERNEL); - if (!smu_dpm->dpm_context) - return -ENOMEM; - - smu_dpm->dpm_context_size = sizeof(struct smu_14_0_dpm_context); - - return 0; -} - -static int smu_v14_0_2_init_smc_tables(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_v14_0_2_tables_init(smu); - if (ret) - return ret; - - ret = smu_v14_0_2_allocate_dpm_context(smu); - if (ret) - return ret; - - return smu_v14_0_init_smc_tables(smu); -} - -static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - SkuTable_t *skutable = &pptable->SkuTable; - struct smu_14_0_dpm_table *dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t link_level; - int ret = 0; - - /* socclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.soc_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_SOCCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* gfxclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_GFXCLK, - dpm_table); - if (ret) - return ret; - - /* - * Update the reported maximum shader clock to the value - * which can be guarded to be achieved on all cards. This - * is aligned with Window setting. And considering that value - * might be not the peak frequency the card can achieve, it - * is normal some real-time clock frequency can overtake this - * labelled maximum clock frequency(for example in pp_dpm_sclk - * sysfs output). - */ - if (skutable->DriverReportedClocks.GameClockAc && - (dpm_table->dpm_levels[dpm_table->count - 1].value > - skutable->DriverReportedClocks.GameClockAc)) { - dpm_table->dpm_levels[dpm_table->count - 1].value = - skutable->DriverReportedClocks.GameClockAc; - dpm_table->max = skutable->DriverReportedClocks.GameClockAc; - } - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* uclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_UCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* fclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_FCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* vclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_VCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* dclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* lclk dpm table setup */ - pcie_table = &dpm_context->dpm_tables.pcie_table; - pcie_table->num_of_link_levels = 0; - for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) { - if (!skutable->PcieGenSpeed[link_level] && - !skutable->PcieLaneCount[link_level] && - !skutable->LclkFreq[link_level]) - continue; - - pcie_table->pcie_gen[pcie_table->num_of_link_levels] = - skutable->PcieGenSpeed[link_level]; - pcie_table->pcie_lane[pcie_table->num_of_link_levels] = - skutable->PcieLaneCount[link_level]; - pcie_table->clk_freq[pcie_table->num_of_link_levels] = - skutable->LclkFreq[link_level]; - pcie_table->num_of_link_levels++; - } - - /* dcefclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dcef_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCEFCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - return 0; -} - -static bool smu_v14_0_2_is_dpm_running(struct smu_context *smu) -{ - int ret = 0; - uint64_t feature_enabled; - - ret = smu_cmn_get_enabled_mask(smu, &feature_enabled); - if (ret) - return false; - - return !!(feature_enabled & SMC_DPM_FEATURE); -} - -static void smu_v14_0_2_dump_pptable(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - PFE_Settings_t *PFEsettings = &pptable->PFE_Settings; - - dev_info(smu->adev->dev, "Dumped PPTable:\n"); - - dev_info(smu->adev->dev, "Version = 0x%08x\n", PFEsettings->Version); - dev_info(smu->adev->dev, "FeaturesToRun[0] = 0x%08x\n", PFEsettings->FeaturesToRun[0]); - dev_info(smu->adev->dev, "FeaturesToRun[1] = 0x%08x\n", PFEsettings->FeaturesToRun[1]); -} - -static uint32_t smu_v14_0_2_get_throttler_status(SmuMetrics_t *metrics) -{ - uint32_t throttler_status = 0; - int i; - - for (i = 0; i < THROTTLER_COUNT; i++) - throttler_status |= - (metrics->ThrottlingPercentage[i] ? 1U << i : 0); - - return throttler_status; -} - -#define SMU_14_0_2_BUSY_THRESHOLD 5 -static int smu_v14_0_2_get_smu_metrics_data(struct smu_context *smu, - MetricsMember_t member, - uint32_t *value) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - NULL, - false); - if (ret) - return ret; - - switch (member) { - case METRICS_CURR_GFXCLK: - *value = metrics->CurrClock[PPCLK_GFXCLK]; - break; - case METRICS_CURR_SOCCLK: - *value = metrics->CurrClock[PPCLK_SOCCLK]; - break; - case METRICS_CURR_UCLK: - *value = metrics->CurrClock[PPCLK_UCLK]; - break; - case METRICS_CURR_VCLK: - *value = metrics->CurrClock[PPCLK_VCLK_0]; - break; - case METRICS_CURR_DCLK: - *value = metrics->CurrClock[PPCLK_DCLK_0]; - break; - case METRICS_CURR_FCLK: - *value = metrics->CurrClock[PPCLK_FCLK]; - break; - case METRICS_CURR_DCEFCLK: - *value = metrics->CurrClock[PPCLK_DCFCLK]; - break; - case METRICS_AVERAGE_GFXCLK: - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageGfxclkFrequencyPostDs; - else - *value = metrics->AverageGfxclkFrequencyPreDs; - break; - case METRICS_AVERAGE_FCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageFclkFrequencyPostDs; - else - *value = metrics->AverageFclkFrequencyPreDs; - break; - case METRICS_AVERAGE_UCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageMemclkFrequencyPostDs; - else - *value = metrics->AverageMemclkFrequencyPreDs; - break; - case METRICS_AVERAGE_VCLK: - *value = metrics->AverageVclk0Frequency; - break; - case METRICS_AVERAGE_DCLK: - *value = metrics->AverageDclk0Frequency; - break; - case METRICS_AVERAGE_VCLK1: - *value = metrics->AverageVclk1Frequency; - break; - case METRICS_AVERAGE_DCLK1: - *value = metrics->AverageDclk1Frequency; - break; - case METRICS_AVERAGE_GFXACTIVITY: - *value = metrics->AverageGfxActivity; - break; - case METRICS_AVERAGE_MEMACTIVITY: - *value = metrics->AverageUclkActivity; - break; - case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->AverageSocketPower << 8; - break; - case METRICS_TEMPERATURE_EDGE: - *value = metrics->AvgTemperature[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_HOTSPOT: - *value = metrics->AvgTemperature[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_MEM: - *value = metrics->AvgTemperature[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRGFX: - *value = metrics->AvgTemperature[TEMP_VR_GFX] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRSOC: - *value = metrics->AvgTemperature[TEMP_VR_SOC] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_THROTTLER_STATUS: - *value = smu_v14_0_2_get_throttler_status(metrics); - break; - case METRICS_CURR_FANSPEED: - *value = metrics->AvgFanRpm; - break; - case METRICS_CURR_FANPWM: - *value = metrics->AvgFanPwm; - break; - case METRICS_VOLTAGE_VDDGFX: - *value = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - break; - case METRICS_PCIE_RATE: - *value = metrics->PcieRate; - break; - case METRICS_PCIE_WIDTH: - *value = metrics->PcieWidth; - break; - default: - *value = UINT_MAX; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *min, - uint32_t *max) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *dpm_table; - - switch (clk_type) { - case SMU_MCLK: - case SMU_UCLK: - /* uclk dpm table */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - break; - case SMU_GFXCLK: - case SMU_SCLK: - /* gfxclk dpm table */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - break; - case SMU_SOCCLK: - /* socclk dpm table */ - dpm_table = &dpm_context->dpm_tables.soc_table; - break; - case SMU_FCLK: - /* fclk dpm table */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - break; - case SMU_VCLK: - case SMU_VCLK1: - /* vclk dpm table */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - break; - case SMU_DCLK: - case SMU_DCLK1: - /* dclk dpm table */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - break; - default: - dev_err(smu->adev->dev, "Unsupported clock type!\n"); - return -EINVAL; - } - - if (min) - *min = dpm_table->min; - if (max) - *max = dpm_table->max; - - return 0; -} - -static int smu_v14_0_2_read_sensor(struct smu_context *smu, - enum amd_pp_sensors sensor, - void *data, - uint32_t *size) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - int ret = 0; - - switch (sensor) { - case AMDGPU_PP_SENSOR_MAX_FAN_RPM: - *(uint16_t *)data = smc_pptable->CustomSkuTable.FanMaximumRpm; - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_MEMACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_AVG_POWER: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_SOCKETPOWER, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_HOTSPOT, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_EDGE_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_EDGE, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_MEM, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_MCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_CURR_UCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_VDDGFX: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_VOLTAGE_VDDGFX, - (uint32_t *)data); - *size = 4; - break; - default: - ret = -EOPNOTSUPP; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_current_clk_freq_by_table(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *value) -{ - MetricsMember_t member_type; - int clk_id = 0; - - clk_id = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_CLK, - clk_type); - if (clk_id < 0) - return -EINVAL; - - switch (clk_id) { - case PPCLK_GFXCLK: - member_type = METRICS_AVERAGE_GFXCLK; - break; - case PPCLK_UCLK: - member_type = METRICS_CURR_UCLK; - break; - case PPCLK_FCLK: - member_type = METRICS_CURR_FCLK; - break; - case PPCLK_SOCCLK: - member_type = METRICS_CURR_SOCCLK; - break; - case PPCLK_VCLK_0: - member_type = METRICS_AVERAGE_VCLK; - break; - case PPCLK_DCLK_0: - member_type = METRICS_AVERAGE_DCLK; - break; - case PPCLK_DCFCLK: - member_type = METRICS_CURR_DCEFCLK; - break; - default: - return -EINVAL; - } - - return smu_v14_0_2_get_smu_metrics_data(smu, - member_type, - value); -} - -static bool smu_v14_0_2_is_od_feature_supported(struct smu_context *smu, - int od_feature_bit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); -} - -static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, - int od_feature_bit, - int32_t *min, - int32_t *max) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - int32_t od_min_setting, od_max_setting; - - switch (od_feature_bit) { - case PP_OD_FEATURE_GFXCLK_FMIN: - od_min_setting = overdrive_lowerlimits->GfxclkFmin; - od_max_setting = overdrive_upperlimits->GfxclkFmin; - break; - case PP_OD_FEATURE_GFXCLK_FMAX: - od_min_setting = overdrive_lowerlimits->GfxclkFmax; - od_max_setting = overdrive_upperlimits->GfxclkFmax; - break; - case PP_OD_FEATURE_UCLK_FMIN: - od_min_setting = overdrive_lowerlimits->UclkFmin; - od_max_setting = overdrive_upperlimits->UclkFmin; - break; - case PP_OD_FEATURE_UCLK_FMAX: - od_min_setting = overdrive_lowerlimits->UclkFmax; - od_max_setting = overdrive_upperlimits->UclkFmax; - break; - case PP_OD_FEATURE_GFX_VF_CURVE: - od_min_setting = overdrive_lowerlimits->VoltageOffsetPerZoneBoundary[0]; - od_max_setting = overdrive_upperlimits->VoltageOffsetPerZoneBoundary[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_TEMP: - od_min_setting = overdrive_lowerlimits->FanLinearTempPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearTempPoints[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_PWM: - od_min_setting = overdrive_lowerlimits->FanLinearPwmPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearPwmPoints[0]; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT: - od_min_setting = overdrive_lowerlimits->AcousticLimitRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticLimitRpmThreshold; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_TARGET: - od_min_setting = overdrive_lowerlimits->AcousticTargetRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticTargetRpmThreshold; - break; - case PP_OD_FEATURE_FAN_TARGET_TEMPERATURE: - od_min_setting = overdrive_lowerlimits->FanTargetTemperature; - od_max_setting = overdrive_upperlimits->FanTargetTemperature; - break; - case PP_OD_FEATURE_FAN_MINIMUM_PWM: - od_min_setting = overdrive_lowerlimits->FanMinimumPwm; - od_max_setting = overdrive_upperlimits->FanMinimumPwm; - break; - default: - od_min_setting = od_max_setting = INT_MAX; - break; - } - - if (min) - *min = od_min_setting; - if (max) - *max = od_max_setting; -} - -static int smu_v14_0_2_print_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - char *buf) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - struct smu_14_0_dpm_table *single_dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t gen_speed, lane_width; - int i, curr_freq, size = 0; - int32_t min_value, max_value; - int ret = 0; - - smu_cmn_get_sysfs_buf(&buf, &size); - - if (amdgpu_ras_intr_triggered()) { - size += sysfs_emit_at(buf, size, "unavailable\n"); - return size; - } - - switch (clk_type) { - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - case SMU_DCEFCLK: - single_dpm_table = &(dpm_context->dpm_tables.dcef_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_SCLK: - case SMU_MCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - case SMU_DCEFCLK: - ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq); - if (ret) { - dev_err(smu->adev->dev, "Failed to get current clock freq!"); - return ret; - } - - if (single_dpm_table->is_fine_grained) { - /* - * For fine grained dpms, there are only two dpm levels: - * - level 0 -> min clock freq - * - level 1 -> max clock freq - * And the current clock frequency can be any value between them. - * So, if the current clock frequency is not at level 0 or level 1, - * we will fake it as three dpm levels: - * - level 0 -> min clock freq - * - level 1 -> current actual clock freq - * - level 2 -> max clock freq - */ - if ((single_dpm_table->dpm_levels[0].value != curr_freq) && - (single_dpm_table->dpm_levels[1].value != curr_freq)) { - size += sysfs_emit_at(buf, size, "0: %uMhz\n", - single_dpm_table->dpm_levels[0].value); - size += sysfs_emit_at(buf, size, "1: %uMhz *\n", - curr_freq); - size += sysfs_emit_at(buf, size, "2: %uMhz\n", - single_dpm_table->dpm_levels[1].value); - } else { - size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", - single_dpm_table->dpm_levels[0].value, - single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : ""); - size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", - single_dpm_table->dpm_levels[1].value, - single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : ""); - } - } else { - for (i = 0; i < single_dpm_table->count; i++) - size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", - i, single_dpm_table->dpm_levels[i].value, - single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : ""); - } - break; - case SMU_PCIE: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_RATE, - &gen_speed); - if (ret) - return ret; - - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_WIDTH, - &lane_width); - if (ret) - return ret; - - pcie_table = &(dpm_context->dpm_tables.pcie_table); - for (i = 0; i < pcie_table->num_of_link_levels; i++) - size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, - (pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," : - (pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," : - (pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," : - (pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "", - (pcie_table->pcie_lane[i] == 1) ? "x1" : - (pcie_table->pcie_lane[i] == 2) ? "x2" : - (pcie_table->pcie_lane[i] == 3) ? "x4" : - (pcie_table->pcie_lane[i] == 4) ? "x8" : - (pcie_table->pcie_lane[i] == 5) ? "x12" : - (pcie_table->pcie_lane[i] == 6) ? "x16" : "", - pcie_table->clk_freq[i], - (gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) && - (lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ? - "*" : ""); - break; - - case SMU_OD_SCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFXCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", - od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - break; - - case SMU_OD_MCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_UCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMHz\n", - od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); - break; - - case SMU_OD_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_VDDGFX_OFFSET:\n"); - size += sysfs_emit_at(buf, size, "%dmV\n", - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[0]); - break; - - case SMU_OD_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_FAN_CURVE:\n"); - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) - size += sysfs_emit_at(buf, size, "%d: %dC %d%%\n", - i, - (int)od_table->OverDriveTable.FanLinearTempPoints[i], - (int)od_table->OverDriveTable.FanLinearPwmPoints[i]); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(hotspot temp): %uC %uC\n", - min_value, max_value); - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(fan speed): %u%% %u%%\n", - min_value, max_value); - - break; - - case SMU_OD_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_LIMIT:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticLimitRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_LIMIT: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_TARGET:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticTargetRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_TARGET: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_TARGET_TEMPERATURE:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanTargetTemperature); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "TARGET_TEMPERATURE: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_MINIMUM_PWM:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanMinimumPwm); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "MINIMUM_PWM: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_RANGE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "VDDGFX_OFFSET: %7dmv %10dmv\n", - min_value, max_value); - } - break; - - default: - break; - } - - return size; -} - -static int smu_v14_0_2_force_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t mask) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - struct smu_14_0_dpm_table *single_dpm_table; - uint32_t soft_min_level, soft_max_level; - uint32_t min_freq, max_freq; - int ret = 0; - - soft_min_level = mask ? (ffs(mask) - 1) : 0; - soft_max_level = mask ? (fls(mask) - 1) : 0; - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - case SMU_UCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - case SMU_MCLK: - case SMU_UCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - if (single_dpm_table->is_fine_grained) { - /* There is only 2 levels for fine grained DPM */ - soft_max_level = (soft_max_level >= 1 ? 1 : 0); - soft_min_level = (soft_min_level >= 1 ? 1 : 0); - } else { - if ((soft_max_level >= single_dpm_table->count) || - (soft_min_level >= single_dpm_table->count)) - return -EINVAL; - } - - min_freq = single_dpm_table->dpm_levels[soft_min_level].value; - max_freq = single_dpm_table->dpm_levels[soft_max_level].value; - - ret = smu_v14_0_set_soft_freq_limited_range(smu, - clk_type, - min_freq, - max_freq); - break; - case SMU_DCEFCLK: - case SMU_PCIE: - default: - break; - } - - return ret; -} - -static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu, - uint8_t pcie_gen_cap, - uint8_t pcie_width_cap) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_14_0_pcie_table *pcie_table = - &dpm_context->dpm_tables.pcie_table; - uint32_t smu_pcie_arg; - int ret, i; - - for (i = 0; i < pcie_table->num_of_link_levels; i++) { - if (pcie_table->pcie_gen[i] > pcie_gen_cap) - pcie_table->pcie_gen[i] = pcie_gen_cap; - if (pcie_table->pcie_lane[i] > pcie_width_cap) - pcie_table->pcie_lane[i] = pcie_width_cap; - - smu_pcie_arg = i << 16; - smu_pcie_arg |= pcie_table->pcie_gen[i] << 8; - smu_pcie_arg |= pcie_table->pcie_lane[i]; - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_OverridePcieParameters, - smu_pcie_arg, - NULL); - if (ret) - return ret; - } - - return 0; -} - -static const struct smu_temperature_range smu14_thermal_policy[] = { - {-273150, 99000, 99000, -273150, 99000, 99000, -273150, 99000, 99000}, - { 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000}, -}; - -static int smu_v14_0_2_get_thermal_temperature_range(struct smu_context *smu, - struct smu_temperature_range *range) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - PPTable_t *pptable = smu->smu_table.driver_pptable; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!range) - return -EINVAL; - - memcpy(range, &smu14_thermal_policy[0], sizeof(struct smu_temperature_range)); - - range->max = pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->edge_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] + CTF_OFFSET_EDGE) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] + CTF_OFFSET_HOTSPOT) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] + CTF_OFFSET_MEM)* - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->software_shutdown_temp = powerplay_table->software_shutdown_temp; - range->software_shutdown_temp_offset = pptable->CustomSkuTable.FanAbnormalTempLimitOffset; - - return 0; -} - -static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *gfx_table = - &dpm_context->dpm_tables.gfx_table; - struct smu_14_0_dpm_table *mem_table = - &dpm_context->dpm_tables.uclk_table; - struct smu_14_0_dpm_table *soc_table = - &dpm_context->dpm_tables.soc_table; - struct smu_14_0_dpm_table *vclk_table = - &dpm_context->dpm_tables.vclk_table; - struct smu_14_0_dpm_table *dclk_table = - &dpm_context->dpm_tables.dclk_table; - struct smu_14_0_dpm_table *fclk_table = - &dpm_context->dpm_tables.fclk_table; - struct smu_umd_pstate_table *pstate_table = - &smu->pstate_table; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - DriverReportedClocks_t driver_clocks = - pptable->SkuTable.DriverReportedClocks; - - pstate_table->gfxclk_pstate.min = gfx_table->min; - if (driver_clocks.GameClockAc && - (driver_clocks.GameClockAc < gfx_table->max)) - pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc; - else - pstate_table->gfxclk_pstate.peak = gfx_table->max; - - pstate_table->uclk_pstate.min = mem_table->min; - pstate_table->uclk_pstate.peak = mem_table->max; - - pstate_table->socclk_pstate.min = soc_table->min; - pstate_table->socclk_pstate.peak = soc_table->max; - - pstate_table->vclk_pstate.min = vclk_table->min; - pstate_table->vclk_pstate.peak = vclk_table->max; - - pstate_table->dclk_pstate.min = dclk_table->min; - pstate_table->dclk_pstate.peak = dclk_table->max; - - pstate_table->fclk_pstate.min = fclk_table->min; - pstate_table->fclk_pstate.peak = fclk_table->max; - - if (driver_clocks.BaseClockAc && - driver_clocks.BaseClockAc < gfx_table->max) - pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc; - else - pstate_table->gfxclk_pstate.standard = gfx_table->max; - pstate_table->uclk_pstate.standard = mem_table->max; - pstate_table->socclk_pstate.standard = soc_table->min; - pstate_table->vclk_pstate.standard = vclk_table->min; - pstate_table->dclk_pstate.standard = dclk_table->min; - pstate_table->fclk_pstate.standard = fclk_table->min; - - return 0; -} - -static void smu_v14_0_2_get_unique_id(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - struct amdgpu_device *adev = smu->adev; - uint32_t upper32 = 0, lower32 = 0; - int ret; - - ret = smu_cmn_get_metrics_table(smu, NULL, false); - if (ret) - goto out; - - upper32 = metrics->PublicSerialNumberUpper; - lower32 = metrics->PublicSerialNumberLower; - -out: - adev->unique_id = ((uint64_t)upper32 << 32) | lower32; -} - -static int smu_v14_0_2_get_power_limit(struct smu_context *smu, - uint32_t *current_power_limit, - uint32_t *default_power_limit, - uint32_t *max_power_limit, - uint32_t *min_power_limit) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - uint32_t power_limit; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - - if (current_power_limit) - *current_power_limit = power_limit; - if (default_power_limit) - *default_power_limit = power_limit; - - if (max_power_limit) - *max_power_limit = msg_limit; - - if (min_power_limit) - *min_power_limit = 0; - - return 0; -} - -static int smu_v14_0_2_get_power_profile_mode(struct smu_context *smu, - char *buf) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - static const char *title[] = { - "PROFILE_INDEX(NAME)", - "CLOCK_TYPE(NAME)", - "FPS", - "MinActiveFreqType", - "MinActiveFreq", - "BoosterFreqType", - "BoosterFreq", - "PD_Data_limit_c", - "PD_Data_error_coeff", - "PD_Data_error_rate_coeff"}; - int16_t workload_type = 0; - uint32_t i, size = 0; - int result = 0; - - if (!buf) - return -EINVAL; - - size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s\n", - title[0], title[1], title[2], title[3], title[4], title[5], - title[6], title[7], title[8], title[9]); - - for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) { - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - i); - if (workload_type == -ENOTSUPP) - continue; - else if (workload_type < 0) - return -EINVAL; - - result = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - workload_type, - (void *)(&activity_monitor_external), - false); - if (result) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return result; - } - - size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", - i, amdgpu_pp_profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 0, - "GFXCLK", - activity_monitor->Gfx_FPS, - activity_monitor->Gfx_MinActiveFreqType, - activity_monitor->Gfx_MinActiveFreq, - activity_monitor->Gfx_BoosterFreqType, - activity_monitor->Gfx_BoosterFreq, - activity_monitor->Gfx_PD_Data_limit_c, - activity_monitor->Gfx_PD_Data_error_coeff, - activity_monitor->Gfx_PD_Data_error_rate_coeff); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 1, - "FCLK", - activity_monitor->Fclk_FPS, - activity_monitor->Fclk_MinActiveFreqType, - activity_monitor->Fclk_MinActiveFreq, - activity_monitor->Fclk_BoosterFreqType, - activity_monitor->Fclk_BoosterFreq, - activity_monitor->Fclk_PD_Data_limit_c, - activity_monitor->Fclk_PD_Data_error_coeff, - activity_monitor->Fclk_PD_Data_error_rate_coeff); - } - - return size; -} - -static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu, - long *input, - uint32_t size) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - int workload_type, ret = 0; - - smu->power_profile_mode = input[size]; - - if (smu->power_profile_mode >= PP_SMC_POWER_PROFILE_COUNT) { - dev_err(smu->adev->dev, "Invalid power profile mode %d\n", smu->power_profile_mode); - return -EINVAL; - } - - if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) { - if (size != 9) - return -EINVAL; - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - false); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return ret; - } - - switch (input[0]) { - case 0: /* Gfxclk */ - activity_monitor->Gfx_FPS = input[1]; - activity_monitor->Gfx_MinActiveFreqType = input[2]; - activity_monitor->Gfx_MinActiveFreq = input[3]; - activity_monitor->Gfx_BoosterFreqType = input[4]; - activity_monitor->Gfx_BoosterFreq = input[5]; - activity_monitor->Gfx_PD_Data_limit_c = input[6]; - activity_monitor->Gfx_PD_Data_error_coeff = input[7]; - activity_monitor->Gfx_PD_Data_error_rate_coeff = input[8]; - break; - case 1: /* Fclk */ - activity_monitor->Fclk_FPS = input[1]; - activity_monitor->Fclk_MinActiveFreqType = input[2]; - activity_monitor->Fclk_MinActiveFreq = input[3]; - activity_monitor->Fclk_BoosterFreqType = input[4]; - activity_monitor->Fclk_BoosterFreq = input[5]; - activity_monitor->Fclk_PD_Data_limit_c = input[6]; - activity_monitor->Fclk_PD_Data_error_coeff = input[7]; - activity_monitor->Fclk_PD_Data_error_rate_coeff = input[8]; - break; - default: - return -EINVAL; - } - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - true); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to set activity monitor!", __func__); - return ret; - } - } - - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - smu->power_profile_mode); - if (workload_type < 0) - return -EINVAL; - - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetWorkloadMask, - 1 << workload_type, - NULL); -} - -static int smu_v14_0_2_baco_enter(struct smu_context *smu) -{ - struct smu_baco_context *smu_baco = &smu->smu_baco; - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) - return smu_v14_0_baco_set_armd3_sequence(smu, - smu_baco->maco_support ? BACO_SEQ_BAMACO : BACO_SEQ_BACO); - else - return smu_v14_0_baco_enter(smu); -} - -static int smu_v14_0_2_baco_exit(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) { - /* Wait for PMFW handling for the Dstate change */ - usleep_range(10000, 11000); - return smu_v14_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); - } else { - return smu_v14_0_baco_exit(smu); - } -} - -static bool smu_v14_0_2_is_mode1_reset_supported(struct smu_context *smu) -{ - // TODO - - return true; -} - -static int smu_v14_0_2_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msg, int num_msgs) -{ - struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = smu_i2c->adev; - struct smu_context *smu = adev->powerplay.pp_handle; - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *table = &smu_table->driver_table; - SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr; - int i, j, r, c; - u16 dir; - - if (!adev->pm.dpm_enabled) - return -EBUSY; - - req = kzalloc(sizeof(*req), GFP_KERNEL); - if (!req) - return -ENOMEM; - - req->I2CcontrollerPort = smu_i2c->port; - req->I2CSpeed = I2C_SPEED_FAST_400K; - req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */ - dir = msg[0].flags & I2C_M_RD; - - for (c = i = 0; i < num_msgs; i++) { - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &req->SwI2cCmds[c]; - - if (!(msg[i].flags & I2C_M_RD)) { - /* write */ - cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK; - cmd->ReadWriteData = msg[i].buf[j]; - } - - if ((dir ^ msg[i].flags) & I2C_M_RD) { - /* The direction changes. - */ - dir = msg[i].flags & I2C_M_RD; - cmd->CmdConfig |= CMDCONFIG_RESTART_MASK; - } - - req->NumCmds++; - - /* - * Insert STOP if we are at the last byte of either last - * message for the transaction or the client explicitly - * requires a STOP at this particular message. - */ - if ((j == msg[i].len - 1) && - ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) { - cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK; - cmd->CmdConfig |= CMDCONFIG_STOP_MASK; - } - } - } - mutex_lock(&adev->pm.mutex); - r = smu_cmn_update_table(smu, SMU_TABLE_I2C_COMMANDS, 0, req, true); - mutex_unlock(&adev->pm.mutex); - if (r) - goto fail; - - for (c = i = 0; i < num_msgs; i++) { - if (!(msg[i].flags & I2C_M_RD)) { - c += msg[i].len; - continue; - } - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &res->SwI2cCmds[c]; - - msg[i].buf[j] = cmd->ReadWriteData; - } - } - r = num_msgs; -fail: - kfree(req); - return r; -} - -static u32 smu_v14_0_2_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm smu_v14_0_2_i2c_algo = { - .master_xfer = smu_v14_0_2_i2c_xfer, - .functionality = smu_v14_0_2_i2c_func, -}; - -static const struct i2c_adapter_quirks smu_v14_0_2_i2c_control_quirks = { - .flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN, - .max_read_len = MAX_SW_I2C_COMMANDS, - .max_write_len = MAX_SW_I2C_COMMANDS, - .max_comb_1st_msg_len = 2, - .max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2, -}; - -static int smu_v14_0_2_i2c_control_init(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int res, i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - smu_i2c->adev = adev; - smu_i2c->port = i; - mutex_init(&smu_i2c->mutex); - control->owner = THIS_MODULE; - control->dev.parent = &adev->pdev->dev; - control->algo = &smu_v14_0_2_i2c_algo; - snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i); - control->quirks = &smu_v14_0_2_i2c_control_quirks; - i2c_set_adapdata(control, smu_i2c); - - res = i2c_add_adapter(control); - if (res) { - DRM_ERROR("Failed to register hw i2c, err: %d\n", res); - goto Out_err; - } - } - - /* assign the buses used for the FRU EEPROM and RAS EEPROM */ - /* XXX ideally this would be something in a vbios data table */ - adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[1].adapter; - adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter; - - return 0; -Out_err: - for ( ; i >= 0; i--) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - return res; -} - -static void smu_v14_0_2_i2c_control_fini(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - adev->pm.ras_eeprom_i2c_bus = NULL; - adev->pm.fru_eeprom_i2c_bus = NULL; -} - -static int smu_v14_0_2_set_mp1_state(struct smu_context *smu, - enum pp_mp1_state mp1_state) -{ - int ret; - - switch (mp1_state) { - case PP_MP1_STATE_UNLOAD: - ret = smu_cmn_set_mp1_state(smu, mp1_state); - break; - default: - /* Ignore others */ - ret = 0; - } - - return ret; -} - -static int smu_v14_0_2_set_df_cstate(struct smu_context *smu, - enum pp_df_cstate state) -{ - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_DFCstateControl, - state, - NULL); -} - -static int smu_v14_0_2_mode1_reset(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); - if (!ret) { - if (amdgpu_emu_mode == 1) - msleep(50000); - else - msleep(1000); - } - - return ret; -} - -static int smu_v14_0_2_mode2_reset(struct smu_context *smu) -{ - int ret = 0; - - // TODO - - return ret; -} - -static int smu_v14_0_2_enable_gfx_features(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(14, 0, 2)) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnableAllSmuFeatures, - FEATURE_PWR_GFX, NULL); - else - return -EOPNOTSUPP; -} - -static void smu_v14_0_2_set_smu_mailbox_registers(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - smu->param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_82); - smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_66); - smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_90); - - smu->debug_param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_53); - smu->debug_msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_75); - smu->debug_resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_54); -} - -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); -} - -static void smu_v14_0_2_dump_od_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - struct amdgpu_device *adev = smu->adev; - - dev_dbg(adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - dev_dbg(adev->dev, "OD: Uclk: (%d, %d)\n", od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); -} - -static int smu_v14_0_2_upload_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - true); - if (ret) - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - - return ret; -} - -static void smu_v14_0_2_set_supported_od_feature_mask(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - adev->pm.od_feature_mask |= OD_OPS_SUPPORT_FAN_CURVE_RETRIEVE | - OD_OPS_SUPPORT_FAN_CURVE_SET | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_SET | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_SET | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_RETRIEVE | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_SET | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_RETRIEVE | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_SET; -} - -static int smu_v14_0_2_get_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - false); - if (ret) - dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); - - return ret; -} - -static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu) -{ - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - OverDriveTableExternal_t *boot_od_table = - (OverDriveTableExternal_t *)smu->smu_table.boot_overdrive_table; - OverDriveTableExternal_t *user_od_table = - (OverDriveTableExternal_t *)smu->smu_table.user_overdrive_table; - OverDriveTableExternal_t user_od_table_bak; - int ret; - int i; - - ret = smu_v14_0_2_get_overdrive_table(smu, boot_od_table); - if (ret) - return ret; - - smu_v14_0_2_dump_od_table(smu, boot_od_table); - - memcpy(od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - - /* - * For S3/S4/Runpm resume, we need to setup those overdrive tables again, - * but we have to preserve user defined values in "user_od_table". - */ - if (!smu->adev->in_suspend) { - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - smu->user_dpm_profile.user_od = false; - } else if (smu->user_dpm_profile.user_od) { - memcpy(&user_od_table_bak, - user_od_table, - sizeof(OverDriveTableExternal_t)); - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - user_od_table->OverDriveTable.GfxclkFmin = - user_od_table_bak.OverDriveTable.GfxclkFmin; - user_od_table->OverDriveTable.GfxclkFmax = - user_od_table_bak.OverDriveTable.GfxclkFmax; - user_od_table->OverDriveTable.UclkFmin = - user_od_table_bak.OverDriveTable.UclkFmin; - user_od_table->OverDriveTable.UclkFmax = - user_od_table_bak.OverDriveTable.UclkFmax; - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - user_od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = - user_od_table_bak.OverDriveTable.VoltageOffsetPerZoneBoundary[i]; - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) { - user_od_table->OverDriveTable.FanLinearTempPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearTempPoints[i]; - user_od_table->OverDriveTable.FanLinearPwmPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearPwmPoints[i]; - } - user_od_table->OverDriveTable.AcousticLimitRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticLimitRpmThreshold; - user_od_table->OverDriveTable.AcousticTargetRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticTargetRpmThreshold; - user_od_table->OverDriveTable.FanTargetTemperature = - user_od_table_bak.OverDriveTable.FanTargetTemperature; - user_od_table->OverDriveTable.FanMinimumPwm = - user_od_table_bak.OverDriveTable.FanMinimumPwm; - } - - smu_v14_0_2_set_supported_od_feature_mask(smu); - - return 0; -} - -static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = table_context->overdrive_table; - OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; - int res; - - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); - user_od_table->OverDriveTable.FeatureCtrlMask = 0; - if (res == 0) - memcpy(od_table, user_od_table, sizeof(OverDriveTableExternal_t)); - - return res; -} - -static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long input) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *boot_overdrive_table = - (OverDriveTableExternal_t *)table_context->boot_overdrive_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - int i; - - switch (input) { - case PP_OD_EDIT_FAN_CURVE: - for (i = 0; i < NUM_OD_FAN_MAX_POINTS; i++) { - od_table->OverDriveTable.FanLinearTempPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearTempPoints[i]; - od_table->OverDriveTable.FanLinearPwmPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearPwmPoints[i]; - } - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_LIMIT: - od_table->OverDriveTable.AcousticLimitRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticLimitRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_TARGET: - od_table->OverDriveTable.AcousticTargetRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticTargetRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - od_table->OverDriveTable.FanTargetTemperature = - boot_overdrive_table->OverDriveTable.FanTargetTemperature; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_MINIMUM_PWM: - od_table->OverDriveTable.FanMinimumPwm = - boot_overdrive_table->OverDriveTable.FanMinimumPwm; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - default: - dev_info(adev->dev, "Invalid table index: %ld\n", input); - return -EINVAL; - } - - return 0; -} - -static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, - enum PP_OD_DPM_TABLE_COMMAND type, - long input[], - uint32_t size) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - uint32_t offset_of_voltageoffset; - int32_t minimum, maximum; - uint32_t feature_ctrlmask; - int i, ret = 0; - - switch (type) { - case PP_OD_EDIT_SCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - dev_warn(adev->dev, "GFXCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.GfxclkFmin > od_table->OverDriveTable.GfxclkFmax) { - dev_err(adev->dev, - "Invalid setting: GfxclkFmin(%u) is bigger than GfxclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.GfxclkFmin, - (uint32_t)od_table->OverDriveTable.GfxclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_MCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - dev_warn(adev->dev, "UCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.UclkFmin > od_table->OverDriveTable.UclkFmax) { - dev_err(adev->dev, - "Invalid setting: UclkFmin(%u) is bigger than UclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.UclkFmin, - (uint32_t)od_table->OverDriveTable.UclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - dev_warn(adev->dev, "Gfx offset setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "Voltage offset (%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = input[0]; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - if (input[0] >= NUM_OD_FAN_MAX_POINTS - 1 || - input[0] < 0) - return -EINVAL; - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &minimum, - &maximum); - if (input[1] < minimum || - input[1] > maximum) { - dev_info(adev->dev, "Fan curve temp setting(%ld) must be within [%d, %d]!\n", - input[1], minimum, maximum); - return -EINVAL; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &minimum, - &maximum); - if (input[2] < minimum || - input[2] > maximum) { - dev_info(adev->dev, "Fan curve pwm setting(%ld) must be within [%d, %d]!\n", - input[2], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanLinearTempPoints[input[0]] = input[1]; - od_table->OverDriveTable.FanLinearPwmPoints[input[0]] = input[2]; - od_table->OverDriveTable.FanMode = FAN_MODE_MANUAL_LINEAR; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic limit threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticLimitRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic target threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticTargetRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan target temperature setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanTargetTemperature = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan minimum pwm setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanMinimumPwm = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_RESTORE_DEFAULT_TABLE: - if (size == 1) { - ret = smu_v14_0_2_od_restore_table_single(smu, input[0]); - if (ret) - return ret; - } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; - memcpy(od_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; - } - fallthrough; - case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ - offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); - if (memcmp((u8 *)od_table + offset_of_voltageoffset, - table_context->user_overdrive_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset)) { - smu_v14_0_2_dump_od_table(smu, od_table); - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - od_table->OverDriveTable.FeatureCtrlMask = 0; - memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, - (u8 *)od_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); - - if (!memcmp(table_context->user_overdrive_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t))) - smu->user_dpm_profile.user_od = false; - else - smu->user_dpm_profile.user_od = true; - } - break; - - default: - return -ENOSYS; - } - - return ret; -} - -<<<<<<< -static int smu_v14_0_2_set_power_limit(struct smu_context *smu, - enum smu_ppt_limit_type limit_type, - uint32_t limit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - int ret = 0; - - if (limit_type != SMU_DEFAULT_PPT_LIMIT) - return -EINVAL; - - if (limit <= msg_limit) { - if (smu->current_power_limit > msg_limit) { - od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - } - return smu_v14_0_set_power_limit(smu, limit_type, limit); - } else if (smu->od_enabled) { - ret = smu_v14_0_set_power_limit(smu, limit_type, msg_limit); - if (ret) - return ret; - - od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - smu->current_power_limit = limit; - } else { - return -EINVAL; - } - - return 0; -======= -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); ->>>>>>> -} - -static const struct pptable_funcs smu_v14_0_2_ppt_funcs = { - .get_allowed_feature_mask = smu_v14_0_2_get_allowed_feature_mask, - .set_default_dpm_table = smu_v14_0_2_set_default_dpm_table, - .i2c_init = smu_v14_0_2_i2c_control_init, - .i2c_fini = smu_v14_0_2_i2c_control_fini, - .is_dpm_running = smu_v14_0_2_is_dpm_running, - .dump_pptable = smu_v14_0_2_dump_pptable, - .init_microcode = smu_v14_0_init_microcode, - .load_microcode = smu_v14_0_load_microcode, - .fini_microcode = smu_v14_0_fini_microcode, - .init_smc_tables = smu_v14_0_2_init_smc_tables, - .fini_smc_tables = smu_v14_0_fini_smc_tables, - .init_power = smu_v14_0_init_power, - .fini_power = smu_v14_0_fini_power, - .check_fw_status = smu_v14_0_check_fw_status, - .setup_pptable = smu_v14_0_2_setup_pptable, - .check_fw_version = smu_v14_0_check_fw_version, - .write_pptable = smu_cmn_write_pptable, - .set_driver_table_location = smu_v14_0_set_driver_table_location, - .system_features_control = smu_v14_0_system_features_control, - .set_allowed_mask = smu_v14_0_set_allowed_mask, - .get_enabled_mask = smu_cmn_get_enabled_mask, - .dpm_set_vcn_enable = smu_v14_0_set_vcn_enable, - .dpm_set_jpeg_enable = smu_v14_0_set_jpeg_enable, - .get_dpm_ultimate_freq = smu_v14_0_2_get_dpm_ultimate_freq, - .get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values, - .read_sensor = smu_v14_0_2_read_sensor, - .feature_is_enabled = smu_cmn_feature_is_enabled, - .print_clk_levels = smu_v14_0_2_print_clk_levels, - .force_clk_levels = smu_v14_0_2_force_clk_levels, - .update_pcie_parameters = smu_v14_0_2_update_pcie_parameters, - .get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range, - .register_irq_handler = smu_v14_0_register_irq_handler, - .enable_thermal_alert = smu_v14_0_enable_thermal_alert, - .disable_thermal_alert = smu_v14_0_disable_thermal_alert, - .notify_memory_pool_location = smu_v14_0_notify_memory_pool_location, - .get_gpu_metrics = smu_v14_0_2_get_gpu_metrics, - .set_soft_freq_limited_range = smu_v14_0_set_soft_freq_limited_range, - .set_default_od_settings = smu_v14_0_2_set_default_od_settings, - .restore_user_od_settings = smu_v14_0_2_restore_user_od_settings, - .od_edit_dpm_table = smu_v14_0_2_od_edit_dpm_table, - .init_pptable_microcode = smu_v14_0_init_pptable_microcode, - .populate_umd_state_clk = smu_v14_0_2_populate_umd_state_clk, - .set_performance_level = smu_v14_0_set_performance_level, - .gfx_off_control = smu_v14_0_gfx_off_control, - .get_unique_id = smu_v14_0_2_get_unique_id, - .get_power_limit = smu_v14_0_2_get_power_limit, - .set_power_limit = smu_v14_0_2_set_power_limit, - .set_power_source = smu_v14_0_set_power_source, - .get_power_profile_mode = smu_v14_0_2_get_power_profile_mode, - .set_power_profile_mode = smu_v14_0_2_set_power_profile_mode, - .run_btc = smu_v14_0_run_btc, - .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, - .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, - .set_tool_table_location = smu_v14_0_set_tool_table_location, - .deep_sleep_control = smu_v14_0_deep_sleep_control, - .gfx_ulv_control = smu_v14_0_gfx_ulv_control, - .get_bamaco_support = smu_v14_0_get_bamaco_support, - .baco_get_state = smu_v14_0_baco_get_state, - .baco_set_state = smu_v14_0_baco_set_state, - .baco_enter = smu_v14_0_2_baco_enter, - .baco_exit = smu_v14_0_2_baco_exit, - .mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported, - .mode1_reset = smu_v14_0_2_mode1_reset, - .mode2_reset = smu_v14_0_2_mode2_reset, - .enable_gfx_features = smu_v14_0_2_enable_gfx_features, - .set_mp1_state = smu_v14_0_2_set_mp1_state, - .set_df_cstate = smu_v14_0_2_set_df_cstate, -#if 0 - .gpo_control = smu_v14_0_gpo_control, -#endif -}; - -void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu) -{ - smu->ppt_funcs = &smu_v14_0_2_ppt_funcs; - smu->message_map = smu_v14_0_2_message_map; - smu->clock_map = smu_v14_0_2_clk_map; - smu->feature_map = smu_v14_0_2_feature_mask_map; - smu->table_map = smu_v14_0_2_table_map; - smu->pwr_src_map = smu_v14_0_2_pwr_src_map; - smu->workload_map = smu_v14_0_2_workload_map; - smu_v14_0_2_set_smu_mailbox_registers(smu); -} diff --git a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.1 b/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.1 deleted file mode 100644 index 6878263c313c..000000000000 --- a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.1 +++ /dev/null @@ -1,2956 +0,0 @@ -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#define SWSMU_CODE_LAYER_L2 - -#include <linux/firmware.h> -#include <linux/pci.h> -#include <linux/i2c.h> -#include "amdgpu.h" -#include "amdgpu_smu.h" -#include "atomfirmware.h" -#include "amdgpu_atomfirmware.h" -#include "amdgpu_atombios.h" -#include "smu_v14_0.h" -#include "smu14_driver_if_v14_0.h" -#include "soc15_common.h" -#include "atom.h" -#include "smu_v14_0_2_ppt.h" -#include "smu_v14_0_2_pptable.h" -#include "smu_v14_0_2_ppsmc.h" -#include "mp/mp_14_0_2_offset.h" -#include "mp/mp_14_0_2_sh_mask.h" - -#include "smu_cmn.h" -#include "amdgpu_ras.h" - -/* - * DO NOT use these for err/warn/info/debug messages. - * Use dev_err, dev_warn, dev_info and dev_dbg instead. - * They are more MGPU friendly. - */ -#undef pr_err -#undef pr_warn -#undef pr_info -#undef pr_debug - -#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) - -#define FEATURE_MASK(feature) (1ULL << feature) -#define SMC_DPM_FEATURE ( \ - FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_LINK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_FCLK_BIT)) - -#define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE 0x4000 -#define DEBUGSMC_MSG_Mode1Reset 2 -#define LINK_SPEED_MAX 3 -<<<<<<< -======= - -#define PP_OD_FEATURE_GFXCLK_FMIN 0 -#define PP_OD_FEATURE_GFXCLK_FMAX 1 -#define PP_OD_FEATURE_UCLK_FMIN 2 -#define PP_OD_FEATURE_UCLK_FMAX 3 -#define PP_OD_FEATURE_GFX_VF_CURVE 4 -#define PP_OD_FEATURE_FAN_CURVE_TEMP 5 -#define PP_OD_FEATURE_FAN_CURVE_PWM 6 -#define PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT 7 -#define PP_OD_FEATURE_FAN_ACOUSTIC_TARGET 8 -#define PP_OD_FEATURE_FAN_TARGET_TEMPERATURE 9 -#define PP_OD_FEATURE_FAN_MINIMUM_PWM 10 ->>>>>>> - -static struct cmn2asic_msg_mapping smu_v14_0_2_message_map[SMU_MSG_MAX_COUNT] = { - MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 1), - MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), - MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 1), - MSG_MAP(SetAllowedFeaturesMaskLow, PPSMC_MSG_SetAllowedFeaturesMaskLow, 0), - MSG_MAP(SetAllowedFeaturesMaskHigh, PPSMC_MSG_SetAllowedFeaturesMaskHigh, 0), - MSG_MAP(EnableAllSmuFeatures, PPSMC_MSG_EnableAllSmuFeatures, 0), - MSG_MAP(DisableAllSmuFeatures, PPSMC_MSG_DisableAllSmuFeatures, 0), - MSG_MAP(EnableSmuFeaturesLow, PPSMC_MSG_EnableSmuFeaturesLow, 1), - MSG_MAP(EnableSmuFeaturesHigh, PPSMC_MSG_EnableSmuFeaturesHigh, 1), - MSG_MAP(DisableSmuFeaturesLow, PPSMC_MSG_DisableSmuFeaturesLow, 1), - MSG_MAP(DisableSmuFeaturesHigh, PPSMC_MSG_DisableSmuFeaturesHigh, 1), - MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetRunningSmuFeaturesLow, 1), - MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetRunningSmuFeaturesHigh, 1), - MSG_MAP(SetWorkloadMask, PPSMC_MSG_SetWorkloadMask, 1), - MSG_MAP(SetPptLimit, PPSMC_MSG_SetPptLimit, 0), - MSG_MAP(SetDriverDramAddrHigh, PPSMC_MSG_SetDriverDramAddrHigh, 1), - MSG_MAP(SetDriverDramAddrLow, PPSMC_MSG_SetDriverDramAddrLow, 1), - MSG_MAP(SetToolsDramAddrHigh, PPSMC_MSG_SetToolsDramAddrHigh, 0), - MSG_MAP(SetToolsDramAddrLow, PPSMC_MSG_SetToolsDramAddrLow, 0), - MSG_MAP(TransferTableSmu2Dram, PPSMC_MSG_TransferTableSmu2Dram, 1), - MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), - MSG_MAP(UseDefaultPPTable, PPSMC_MSG_UseDefaultPPTable, 0), - MSG_MAP(RunDcBtc, PPSMC_MSG_RunDcBtc, 0), - MSG_MAP(EnterBaco, PPSMC_MSG_EnterBaco, 0), - MSG_MAP(ExitBaco, PPSMC_MSG_ExitBaco, 0), - MSG_MAP(SetSoftMinByFreq, PPSMC_MSG_SetSoftMinByFreq, 1), - MSG_MAP(SetSoftMaxByFreq, PPSMC_MSG_SetSoftMaxByFreq, 1), - MSG_MAP(SetHardMinByFreq, PPSMC_MSG_SetHardMinByFreq, 1), - MSG_MAP(SetHardMaxByFreq, PPSMC_MSG_SetHardMaxByFreq, 0), - MSG_MAP(GetMinDpmFreq, PPSMC_MSG_GetMinDpmFreq, 1), - MSG_MAP(GetMaxDpmFreq, PPSMC_MSG_GetMaxDpmFreq, 1), - MSG_MAP(GetDpmFreqByIndex, PPSMC_MSG_GetDpmFreqByIndex, 1), - MSG_MAP(PowerUpVcn, PPSMC_MSG_PowerUpVcn, 0), - MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), - MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), - MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), - MSG_MAP(GetDcModeMaxDpmFreq, PPSMC_MSG_GetDcModeMaxDpmFreq, 1), - MSG_MAP(OverridePcieParameters, PPSMC_MSG_OverridePcieParameters, 0), - MSG_MAP(DramLogSetDramAddrHigh, PPSMC_MSG_DramLogSetDramAddrHigh, 0), - MSG_MAP(DramLogSetDramAddrLow, PPSMC_MSG_DramLogSetDramAddrLow, 0), - MSG_MAP(DramLogSetDramSize, PPSMC_MSG_DramLogSetDramSize, 0), - MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), - MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), - MSG_MAP(GetPptLimit, PPSMC_MSG_GetPptLimit, 0), - MSG_MAP(NotifyPowerSource, PPSMC_MSG_NotifyPowerSource, 0), - MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), - MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), - MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), - MSG_MAP(SetNumBadMemoryPagesRetired, PPSMC_MSG_SetNumBadMemoryPagesRetired, 0), - MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel, - PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel, 0), - MSG_MAP(AllowIHHostInterrupt, PPSMC_MSG_AllowIHHostInterrupt, 0), - MSG_MAP(ReenableAcDcInterrupt, PPSMC_MSG_ReenableAcDcInterrupt, 0), -}; - -static struct cmn2asic_mapping smu_v14_0_2_clk_map[SMU_CLK_COUNT] = { - CLK_MAP(GFXCLK, PPCLK_GFXCLK), - CLK_MAP(SCLK, PPCLK_GFXCLK), - CLK_MAP(SOCCLK, PPCLK_SOCCLK), - CLK_MAP(FCLK, PPCLK_FCLK), - CLK_MAP(UCLK, PPCLK_UCLK), - CLK_MAP(MCLK, PPCLK_UCLK), - CLK_MAP(VCLK, PPCLK_VCLK_0), - CLK_MAP(DCLK, PPCLK_DCLK_0), - CLK_MAP(DCEFCLK, PPCLK_DCFCLK), -}; - -static struct cmn2asic_mapping smu_v14_0_2_feature_mask_map[SMU_FEATURE_COUNT] = { - FEA_MAP(FW_DATA_READ), - FEA_MAP(DPM_GFXCLK), - FEA_MAP(DPM_GFX_POWER_OPTIMIZER), - FEA_MAP(DPM_UCLK), - FEA_MAP(DPM_FCLK), - FEA_MAP(DPM_SOCCLK), - FEA_MAP(DPM_LINK), - FEA_MAP(DPM_DCN), - FEA_MAP(VMEMP_SCALING), - FEA_MAP(VDDIO_MEM_SCALING), - FEA_MAP(DS_GFXCLK), - FEA_MAP(DS_SOCCLK), - FEA_MAP(DS_FCLK), - FEA_MAP(DS_LCLK), - FEA_MAP(DS_DCFCLK), - FEA_MAP(DS_UCLK), - FEA_MAP(GFX_ULV), - FEA_MAP(FW_DSTATE), - FEA_MAP(GFXOFF), - FEA_MAP(BACO), - FEA_MAP(MM_DPM), - FEA_MAP(SOC_MPCLK_DS), - FEA_MAP(BACO_MPCLK_DS), - FEA_MAP(THROTTLERS), - FEA_MAP(SMARTSHIFT), - FEA_MAP(GTHR), - FEA_MAP(ACDC), - FEA_MAP(VR0HOT), - FEA_MAP(FW_CTF), - FEA_MAP(FAN_CONTROL), - FEA_MAP(GFX_DCS), - FEA_MAP(GFX_READ_MARGIN), - FEA_MAP(LED_DISPLAY), - FEA_MAP(GFXCLK_SPREAD_SPECTRUM), - FEA_MAP(OUT_OF_BAND_MONITOR), - FEA_MAP(OPTIMIZED_VMIN), - FEA_MAP(GFX_IMU), - FEA_MAP(BOOT_TIME_CAL), - FEA_MAP(GFX_PCC_DFLL), - FEA_MAP(SOC_CG), - FEA_MAP(DF_CSTATE), - FEA_MAP(GFX_EDC), - FEA_MAP(BOOT_POWER_OPT), - FEA_MAP(CLOCK_POWER_DOWN_BYPASS), - FEA_MAP(DS_VCN), - FEA_MAP(BACO_CG), - FEA_MAP(MEM_TEMP_READ), - FEA_MAP(ATHUB_MMHUB_PG), - FEA_MAP(SOC_PCC), - [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_PPT_BIT] = {1, FEATURE_THROTTLERS_BIT}, -}; - -static struct cmn2asic_mapping smu_v14_0_2_table_map[SMU_TABLE_COUNT] = { - TAB_MAP(PPTABLE), - TAB_MAP(WATERMARKS), - TAB_MAP(AVFS_PSM_DEBUG), - TAB_MAP(PMSTATUSLOG), - TAB_MAP(SMU_METRICS), - TAB_MAP(DRIVER_SMU_CONFIG), - TAB_MAP(ACTIVITY_MONITOR_COEFF), - [SMU_TABLE_COMBO_PPTABLE] = {1, TABLE_COMBO_PPTABLE}, - TAB_MAP(I2C_COMMANDS), - TAB_MAP(ECCINFO), - TAB_MAP(OVERDRIVE), -}; - -static struct cmn2asic_mapping smu_v14_0_2_pwr_src_map[SMU_POWER_SOURCE_COUNT] = { - PWR_MAP(AC), - PWR_MAP(DC), -}; - -static struct cmn2asic_mapping smu_v14_0_2_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT, WORKLOAD_PPLIB_DEFAULT_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_POWERSAVING, WORKLOAD_PPLIB_POWER_SAVING_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_WINDOW3D, WORKLOAD_PPLIB_WINDOW_3D_BIT), -}; - -static const uint8_t smu_v14_0_2_throttler_map[] = { - [THROTTLER_PPT0_BIT] = (SMU_THROTTLER_PPT0_BIT), - [THROTTLER_PPT1_BIT] = (SMU_THROTTLER_PPT1_BIT), - [THROTTLER_PPT2_BIT] = (SMU_THROTTLER_PPT2_BIT), - [THROTTLER_PPT3_BIT] = (SMU_THROTTLER_PPT3_BIT), - [THROTTLER_TDC_GFX_BIT] = (SMU_THROTTLER_TDC_GFX_BIT), - [THROTTLER_TDC_SOC_BIT] = (SMU_THROTTLER_TDC_SOC_BIT), - [THROTTLER_TEMP_EDGE_BIT] = (SMU_THROTTLER_TEMP_EDGE_BIT), - [THROTTLER_TEMP_HOTSPOT_BIT] = (SMU_THROTTLER_TEMP_HOTSPOT_BIT), - [THROTTLER_TEMP_MEM_BIT] = (SMU_THROTTLER_TEMP_MEM_BIT), - [THROTTLER_TEMP_VR_GFX_BIT] = (SMU_THROTTLER_TEMP_VR_GFX_BIT), - [THROTTLER_TEMP_VR_SOC_BIT] = (SMU_THROTTLER_TEMP_VR_SOC_BIT), - [THROTTLER_TEMP_VR_MEM0_BIT] = (SMU_THROTTLER_TEMP_VR_MEM0_BIT), - [THROTTLER_TEMP_VR_MEM1_BIT] = (SMU_THROTTLER_TEMP_VR_MEM1_BIT), - [THROTTLER_TEMP_LIQUID0_BIT] = (SMU_THROTTLER_TEMP_LIQUID0_BIT), - [THROTTLER_TEMP_LIQUID1_BIT] = (SMU_THROTTLER_TEMP_LIQUID1_BIT), - [THROTTLER_GFX_APCC_PLUS_BIT] = (SMU_THROTTLER_APCC_BIT), - [THROTTLER_FIT_BIT] = (SMU_THROTTLER_FIT_BIT), -}; - -static int -smu_v14_0_2_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - /*u32 smu_version;*/ - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0xff, sizeof(uint32_t) * num); - - if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT); - } -#if 0 - if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) || - !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); - - if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); - - /* PMFW 78.58 contains a critical fix for gfxoff feature */ - smu_cmn_get_smc_version(smu, NULL, &smu_version); - if ((smu_version < 0x004e3a00) || - !(adev->pm.pp_feature & PP_GFXOFF_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT); - - if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); - } - - if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - - if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT); - } - - if (!(adev->pm.pp_feature & PP_ULV_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT); -#endif - - return 0; -} - -static int smu_v14_0_2_check_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - struct smu_baco_context *smu_baco = &smu->smu_baco; - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC) - smu->dc_controlled_by_gpio = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_BACO) { - smu_baco->platform_support = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_MACO) - smu_baco->maco_support = true; - } - - if (!overdrive_lowerlimits->FeatureCtrlMask || - !overdrive_upperlimits->FeatureCtrlMask) - smu->od_enabled = false; - - table_context->thermal_controller_type = - powerplay_table->thermal_controller_type; - - /* - * Instead of having its own buffer space and get overdrive_table copied, - * smu->od_settings just points to the actual overdrive_table - */ - smu->od_settings = &powerplay_table->overdrive_table; - - smu->adev->pm.no_fan = - !(pptable->PFE_Settings.FeaturesToRun[0] & (1 << FEATURE_FAN_CONTROL_BIT)); - - return 0; -} - -static int smu_v14_0_2_store_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - - memcpy(table_context->driver_pptable, &powerplay_table->smc_pptable, - sizeof(PPTable_t)); - - return 0; -} - -#ifndef atom_smc_dpm_info_table_14_0_0 -struct atom_smc_dpm_info_table_14_0_0 { - struct atom_common_table_header table_header; - BoardTable_t BoardTable; -}; -#endif - -static int smu_v14_0_2_append_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - struct atom_smc_dpm_info_table_14_0_0 *smc_dpm_table; - BoardTable_t *BoardTable = &smc_pptable->BoardTable; - int index, ret; - - index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, - smc_dpm_info); - - ret = amdgpu_atombios_get_data_table(smu->adev, index, NULL, NULL, NULL, - (uint8_t **)&smc_dpm_table); - if (ret) - return ret; - - memcpy(BoardTable, &smc_dpm_table->BoardTable, sizeof(BoardTable_t)); - - return 0; -} - -#if 0 -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_powerplay_table); - - return 0; -} -#endif - -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_2_powerplay_table); - - return 0; -} - -static int smu_v14_0_2_setup_pptable(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct amdgpu_device *adev = smu->adev; - int ret = 0; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!adev->scpm_enabled) - ret = smu_v14_0_setup_pptable(smu); - else - ret = smu_v14_0_2_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - if (ret) - return ret; - - ret = smu_v14_0_2_store_powerplay_table(smu); - if (ret) - return ret; - - /* - * With SCPM enabled, the operation below will be handled - * by PSP. Driver involvment is unnecessary and useless. - */ - if (!adev->scpm_enabled) { - ret = smu_v14_0_2_append_powerplay_table(smu); - if (ret) - return ret; - } - - ret = smu_v14_0_2_check_powerplay_table(smu); - if (ret) - return ret; - - return ret; -} - -static int smu_v14_0_2_tables_init(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *tables = smu_table->tables; - - SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetricsExternal_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_PMSTATUSLOG, SMU14_TOOL_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ACTIVITY_MONITOR_COEFF, - sizeof(DpmActivityMonitorCoeffIntExternal_t), PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_COMBO_PPTABLE, MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ECCINFO, sizeof(EccInfoTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - - smu_table->metrics_table = kzalloc(sizeof(SmuMetricsExternal_t), GFP_KERNEL); - if (!smu_table->metrics_table) - goto err0_out; - smu_table->metrics_time = 0; - - smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3); - smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); - if (!smu_table->gpu_metrics_table) - goto err1_out; - - smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); - if (!smu_table->watermarks_table) - goto err2_out; - - smu_table->ecc_table = kzalloc(tables[SMU_TABLE_ECCINFO].size, GFP_KERNEL); - if (!smu_table->ecc_table) - goto err3_out; - - return 0; - -err3_out: - kfree(smu_table->watermarks_table); -err2_out: - kfree(smu_table->gpu_metrics_table); -err1_out: - kfree(smu_table->metrics_table); -err0_out: - return -ENOMEM; -} - -static int smu_v14_0_2_allocate_dpm_context(struct smu_context *smu) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - - smu_dpm->dpm_context = kzalloc(sizeof(struct smu_14_0_dpm_context), - GFP_KERNEL); - if (!smu_dpm->dpm_context) - return -ENOMEM; - - smu_dpm->dpm_context_size = sizeof(struct smu_14_0_dpm_context); - - return 0; -} - -static int smu_v14_0_2_init_smc_tables(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_v14_0_2_tables_init(smu); - if (ret) - return ret; - - ret = smu_v14_0_2_allocate_dpm_context(smu); - if (ret) - return ret; - - return smu_v14_0_init_smc_tables(smu); -} - -static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - SkuTable_t *skutable = &pptable->SkuTable; - struct smu_14_0_dpm_table *dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t link_level; - int ret = 0; - - /* socclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.soc_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_SOCCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* gfxclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_GFXCLK, - dpm_table); - if (ret) - return ret; - - /* - * Update the reported maximum shader clock to the value - * which can be guarded to be achieved on all cards. This - * is aligned with Window setting. And considering that value - * might be not the peak frequency the card can achieve, it - * is normal some real-time clock frequency can overtake this - * labelled maximum clock frequency(for example in pp_dpm_sclk - * sysfs output). - */ - if (skutable->DriverReportedClocks.GameClockAc && - (dpm_table->dpm_levels[dpm_table->count - 1].value > - skutable->DriverReportedClocks.GameClockAc)) { - dpm_table->dpm_levels[dpm_table->count - 1].value = - skutable->DriverReportedClocks.GameClockAc; - dpm_table->max = skutable->DriverReportedClocks.GameClockAc; - } - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* uclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_UCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* fclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_FCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* vclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_VCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* dclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* lclk dpm table setup */ - pcie_table = &dpm_context->dpm_tables.pcie_table; - pcie_table->num_of_link_levels = 0; - for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) { - if (!skutable->PcieGenSpeed[link_level] && - !skutable->PcieLaneCount[link_level] && - !skutable->LclkFreq[link_level]) - continue; - - pcie_table->pcie_gen[pcie_table->num_of_link_levels] = - skutable->PcieGenSpeed[link_level]; - pcie_table->pcie_lane[pcie_table->num_of_link_levels] = - skutable->PcieLaneCount[link_level]; - pcie_table->clk_freq[pcie_table->num_of_link_levels] = - skutable->LclkFreq[link_level]; - pcie_table->num_of_link_levels++; - } - - /* dcefclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dcef_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCEFCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - return 0; -} - -static bool smu_v14_0_2_is_dpm_running(struct smu_context *smu) -{ - int ret = 0; - uint64_t feature_enabled; - - ret = smu_cmn_get_enabled_mask(smu, &feature_enabled); - if (ret) - return false; - - return !!(feature_enabled & SMC_DPM_FEATURE); -} - -static void smu_v14_0_2_dump_pptable(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - PFE_Settings_t *PFEsettings = &pptable->PFE_Settings; - - dev_info(smu->adev->dev, "Dumped PPTable:\n"); - - dev_info(smu->adev->dev, "Version = 0x%08x\n", PFEsettings->Version); - dev_info(smu->adev->dev, "FeaturesToRun[0] = 0x%08x\n", PFEsettings->FeaturesToRun[0]); - dev_info(smu->adev->dev, "FeaturesToRun[1] = 0x%08x\n", PFEsettings->FeaturesToRun[1]); -} - -static uint32_t smu_v14_0_2_get_throttler_status(SmuMetrics_t *metrics) -{ - uint32_t throttler_status = 0; - int i; - - for (i = 0; i < THROTTLER_COUNT; i++) - throttler_status |= - (metrics->ThrottlingPercentage[i] ? 1U << i : 0); - - return throttler_status; -} - -#define SMU_14_0_2_BUSY_THRESHOLD 5 -static int smu_v14_0_2_get_smu_metrics_data(struct smu_context *smu, - MetricsMember_t member, - uint32_t *value) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - NULL, - false); - if (ret) - return ret; - - switch (member) { - case METRICS_CURR_GFXCLK: - *value = metrics->CurrClock[PPCLK_GFXCLK]; - break; - case METRICS_CURR_SOCCLK: - *value = metrics->CurrClock[PPCLK_SOCCLK]; - break; - case METRICS_CURR_UCLK: - *value = metrics->CurrClock[PPCLK_UCLK]; - break; - case METRICS_CURR_VCLK: - *value = metrics->CurrClock[PPCLK_VCLK_0]; - break; - case METRICS_CURR_DCLK: - *value = metrics->CurrClock[PPCLK_DCLK_0]; - break; - case METRICS_CURR_FCLK: - *value = metrics->CurrClock[PPCLK_FCLK]; - break; - case METRICS_CURR_DCEFCLK: - *value = metrics->CurrClock[PPCLK_DCFCLK]; - break; - case METRICS_AVERAGE_GFXCLK: - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageGfxclkFrequencyPostDs; - else - *value = metrics->AverageGfxclkFrequencyPreDs; - break; - case METRICS_AVERAGE_FCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageFclkFrequencyPostDs; - else - *value = metrics->AverageFclkFrequencyPreDs; - break; - case METRICS_AVERAGE_UCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageMemclkFrequencyPostDs; - else - *value = metrics->AverageMemclkFrequencyPreDs; - break; - case METRICS_AVERAGE_VCLK: - *value = metrics->AverageVclk0Frequency; - break; - case METRICS_AVERAGE_DCLK: - *value = metrics->AverageDclk0Frequency; - break; - case METRICS_AVERAGE_VCLK1: - *value = metrics->AverageVclk1Frequency; - break; - case METRICS_AVERAGE_DCLK1: - *value = metrics->AverageDclk1Frequency; - break; - case METRICS_AVERAGE_GFXACTIVITY: - *value = metrics->AverageGfxActivity; - break; - case METRICS_AVERAGE_MEMACTIVITY: - *value = metrics->AverageUclkActivity; - break; - case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->AverageSocketPower << 8; - break; - case METRICS_TEMPERATURE_EDGE: - *value = metrics->AvgTemperature[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_HOTSPOT: - *value = metrics->AvgTemperature[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_MEM: - *value = metrics->AvgTemperature[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRGFX: - *value = metrics->AvgTemperature[TEMP_VR_GFX] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRSOC: - *value = metrics->AvgTemperature[TEMP_VR_SOC] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_THROTTLER_STATUS: - *value = smu_v14_0_2_get_throttler_status(metrics); - break; - case METRICS_CURR_FANSPEED: - *value = metrics->AvgFanRpm; - break; - case METRICS_CURR_FANPWM: - *value = metrics->AvgFanPwm; - break; - case METRICS_VOLTAGE_VDDGFX: - *value = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - break; - case METRICS_PCIE_RATE: - *value = metrics->PcieRate; - break; - case METRICS_PCIE_WIDTH: - *value = metrics->PcieWidth; - break; - default: - *value = UINT_MAX; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *min, - uint32_t *max) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *dpm_table; - - switch (clk_type) { - case SMU_MCLK: - case SMU_UCLK: - /* uclk dpm table */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - break; - case SMU_GFXCLK: - case SMU_SCLK: - /* gfxclk dpm table */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - break; - case SMU_SOCCLK: - /* socclk dpm table */ - dpm_table = &dpm_context->dpm_tables.soc_table; - break; - case SMU_FCLK: - /* fclk dpm table */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - break; - case SMU_VCLK: - case SMU_VCLK1: - /* vclk dpm table */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - break; - case SMU_DCLK: - case SMU_DCLK1: - /* dclk dpm table */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - break; - default: - dev_err(smu->adev->dev, "Unsupported clock type!\n"); - return -EINVAL; - } - - if (min) - *min = dpm_table->min; - if (max) - *max = dpm_table->max; - - return 0; -} - -static int smu_v14_0_2_read_sensor(struct smu_context *smu, - enum amd_pp_sensors sensor, - void *data, - uint32_t *size) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - int ret = 0; - - switch (sensor) { - case AMDGPU_PP_SENSOR_MAX_FAN_RPM: - *(uint16_t *)data = smc_pptable->CustomSkuTable.FanMaximumRpm; - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_MEMACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_AVG_POWER: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_SOCKETPOWER, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_HOTSPOT, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_EDGE_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_EDGE, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_MEM, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_MCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_CURR_UCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_VDDGFX: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_VOLTAGE_VDDGFX, - (uint32_t *)data); - *size = 4; - break; - default: - ret = -EOPNOTSUPP; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_current_clk_freq_by_table(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *value) -{ - MetricsMember_t member_type; - int clk_id = 0; - - clk_id = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_CLK, - clk_type); - if (clk_id < 0) - return -EINVAL; - - switch (clk_id) { - case PPCLK_GFXCLK: - member_type = METRICS_AVERAGE_GFXCLK; - break; - case PPCLK_UCLK: - member_type = METRICS_CURR_UCLK; - break; - case PPCLK_FCLK: - member_type = METRICS_CURR_FCLK; - break; - case PPCLK_SOCCLK: - member_type = METRICS_CURR_SOCCLK; - break; - case PPCLK_VCLK_0: - member_type = METRICS_AVERAGE_VCLK; - break; - case PPCLK_DCLK_0: - member_type = METRICS_AVERAGE_DCLK; - break; - case PPCLK_DCFCLK: - member_type = METRICS_CURR_DCEFCLK; - break; - default: - return -EINVAL; - } - - return smu_v14_0_2_get_smu_metrics_data(smu, - member_type, - value); -} - -static bool smu_v14_0_2_is_od_feature_supported(struct smu_context *smu, - int od_feature_bit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); -} - -static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, - int od_feature_bit, - int32_t *min, - int32_t *max) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - int32_t od_min_setting, od_max_setting; - - switch (od_feature_bit) { - case PP_OD_FEATURE_GFXCLK_FMIN: - od_min_setting = overdrive_lowerlimits->GfxclkFmin; - od_max_setting = overdrive_upperlimits->GfxclkFmin; - break; - case PP_OD_FEATURE_GFXCLK_FMAX: - od_min_setting = overdrive_lowerlimits->GfxclkFmax; - od_max_setting = overdrive_upperlimits->GfxclkFmax; - break; - case PP_OD_FEATURE_UCLK_FMIN: - od_min_setting = overdrive_lowerlimits->UclkFmin; - od_max_setting = overdrive_upperlimits->UclkFmin; - break; - case PP_OD_FEATURE_UCLK_FMAX: - od_min_setting = overdrive_lowerlimits->UclkFmax; - od_max_setting = overdrive_upperlimits->UclkFmax; - break; - case PP_OD_FEATURE_GFX_VF_CURVE: - od_min_setting = overdrive_lowerlimits->VoltageOffsetPerZoneBoundary[0]; - od_max_setting = overdrive_upperlimits->VoltageOffsetPerZoneBoundary[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_TEMP: - od_min_setting = overdrive_lowerlimits->FanLinearTempPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearTempPoints[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_PWM: - od_min_setting = overdrive_lowerlimits->FanLinearPwmPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearPwmPoints[0]; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT: - od_min_setting = overdrive_lowerlimits->AcousticLimitRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticLimitRpmThreshold; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_TARGET: - od_min_setting = overdrive_lowerlimits->AcousticTargetRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticTargetRpmThreshold; - break; - case PP_OD_FEATURE_FAN_TARGET_TEMPERATURE: - od_min_setting = overdrive_lowerlimits->FanTargetTemperature; - od_max_setting = overdrive_upperlimits->FanTargetTemperature; - break; - case PP_OD_FEATURE_FAN_MINIMUM_PWM: - od_min_setting = overdrive_lowerlimits->FanMinimumPwm; - od_max_setting = overdrive_upperlimits->FanMinimumPwm; - break; - default: - od_min_setting = od_max_setting = INT_MAX; - break; - } - - if (min) - *min = od_min_setting; - if (max) - *max = od_max_setting; -} - -static int smu_v14_0_2_print_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - char *buf) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - struct smu_14_0_dpm_table *single_dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t gen_speed, lane_width; - int i, curr_freq, size = 0; - int32_t min_value, max_value; - int ret = 0; - - smu_cmn_get_sysfs_buf(&buf, &size); - - if (amdgpu_ras_intr_triggered()) { - size += sysfs_emit_at(buf, size, "unavailable\n"); - return size; - } - - switch (clk_type) { - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - case SMU_DCEFCLK: - single_dpm_table = &(dpm_context->dpm_tables.dcef_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_SCLK: - case SMU_MCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - case SMU_DCEFCLK: - ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq); - if (ret) { - dev_err(smu->adev->dev, "Failed to get current clock freq!"); - return ret; - } - - if (single_dpm_table->is_fine_grained) { - /* - * For fine grained dpms, there are only two dpm levels: - * - level 0 -> min clock freq - * - level 1 -> max clock freq - * And the current clock frequency can be any value between them. - * So, if the current clock frequency is not at level 0 or level 1, - * we will fake it as three dpm levels: - * - level 0 -> min clock freq - * - level 1 -> current actual clock freq - * - level 2 -> max clock freq - */ - if ((single_dpm_table->dpm_levels[0].value != curr_freq) && - (single_dpm_table->dpm_levels[1].value != curr_freq)) { - size += sysfs_emit_at(buf, size, "0: %uMhz\n", - single_dpm_table->dpm_levels[0].value); - size += sysfs_emit_at(buf, size, "1: %uMhz *\n", - curr_freq); - size += sysfs_emit_at(buf, size, "2: %uMhz\n", - single_dpm_table->dpm_levels[1].value); - } else { - size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", - single_dpm_table->dpm_levels[0].value, - single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : ""); - size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", - single_dpm_table->dpm_levels[1].value, - single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : ""); - } - } else { - for (i = 0; i < single_dpm_table->count; i++) - size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", - i, single_dpm_table->dpm_levels[i].value, - single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : ""); - } - break; - case SMU_PCIE: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_RATE, - &gen_speed); - if (ret) - return ret; - - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_WIDTH, - &lane_width); - if (ret) - return ret; - - pcie_table = &(dpm_context->dpm_tables.pcie_table); - for (i = 0; i < pcie_table->num_of_link_levels; i++) - size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, - (pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," : - (pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," : - (pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," : - (pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "", - (pcie_table->pcie_lane[i] == 1) ? "x1" : - (pcie_table->pcie_lane[i] == 2) ? "x2" : - (pcie_table->pcie_lane[i] == 3) ? "x4" : - (pcie_table->pcie_lane[i] == 4) ? "x8" : - (pcie_table->pcie_lane[i] == 5) ? "x12" : - (pcie_table->pcie_lane[i] == 6) ? "x16" : "", - pcie_table->clk_freq[i], - (gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) && - (lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ? - "*" : ""); - break; - - case SMU_OD_SCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFXCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", - od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - break; - - case SMU_OD_MCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_UCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMHz\n", - od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); - break; - - case SMU_OD_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_VDDGFX_OFFSET:\n"); - size += sysfs_emit_at(buf, size, "%dmV\n", - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[0]); - break; - - case SMU_OD_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_FAN_CURVE:\n"); - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) - size += sysfs_emit_at(buf, size, "%d: %dC %d%%\n", - i, - (int)od_table->OverDriveTable.FanLinearTempPoints[i], - (int)od_table->OverDriveTable.FanLinearPwmPoints[i]); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(hotspot temp): %uC %uC\n", - min_value, max_value); - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(fan speed): %u%% %u%%\n", - min_value, max_value); - - break; - - case SMU_OD_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_LIMIT:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticLimitRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_LIMIT: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_TARGET:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticTargetRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_TARGET: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_TARGET_TEMPERATURE:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanTargetTemperature); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "TARGET_TEMPERATURE: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_MINIMUM_PWM:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanMinimumPwm); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "MINIMUM_PWM: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_RANGE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "VDDGFX_OFFSET: %7dmv %10dmv\n", - min_value, max_value); - } - break; - - default: - break; - } - - return size; -} - -static int smu_v14_0_2_force_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t mask) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - struct smu_14_0_dpm_table *single_dpm_table; - uint32_t soft_min_level, soft_max_level; - uint32_t min_freq, max_freq; - int ret = 0; - - soft_min_level = mask ? (ffs(mask) - 1) : 0; - soft_max_level = mask ? (fls(mask) - 1) : 0; - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - case SMU_UCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - case SMU_MCLK: - case SMU_UCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - if (single_dpm_table->is_fine_grained) { - /* There is only 2 levels for fine grained DPM */ - soft_max_level = (soft_max_level >= 1 ? 1 : 0); - soft_min_level = (soft_min_level >= 1 ? 1 : 0); - } else { - if ((soft_max_level >= single_dpm_table->count) || - (soft_min_level >= single_dpm_table->count)) - return -EINVAL; - } - - min_freq = single_dpm_table->dpm_levels[soft_min_level].value; - max_freq = single_dpm_table->dpm_levels[soft_max_level].value; - - ret = smu_v14_0_set_soft_freq_limited_range(smu, - clk_type, - min_freq, - max_freq); - break; - case SMU_DCEFCLK: - case SMU_PCIE: - default: - break; - } - - return ret; -} - -static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu, - uint8_t pcie_gen_cap, - uint8_t pcie_width_cap) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_14_0_pcie_table *pcie_table = - &dpm_context->dpm_tables.pcie_table; - uint32_t smu_pcie_arg; - int ret, i; - - for (i = 0; i < pcie_table->num_of_link_levels; i++) { - if (pcie_table->pcie_gen[i] > pcie_gen_cap) - pcie_table->pcie_gen[i] = pcie_gen_cap; - if (pcie_table->pcie_lane[i] > pcie_width_cap) - pcie_table->pcie_lane[i] = pcie_width_cap; - - smu_pcie_arg = i << 16; - smu_pcie_arg |= pcie_table->pcie_gen[i] << 8; - smu_pcie_arg |= pcie_table->pcie_lane[i]; - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_OverridePcieParameters, - smu_pcie_arg, - NULL); - if (ret) - return ret; - } - - return 0; -} - -static const struct smu_temperature_range smu14_thermal_policy[] = { - {-273150, 99000, 99000, -273150, 99000, 99000, -273150, 99000, 99000}, - { 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000}, -}; - -static int smu_v14_0_2_get_thermal_temperature_range(struct smu_context *smu, - struct smu_temperature_range *range) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - PPTable_t *pptable = smu->smu_table.driver_pptable; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!range) - return -EINVAL; - - memcpy(range, &smu14_thermal_policy[0], sizeof(struct smu_temperature_range)); - - range->max = pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->edge_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] + CTF_OFFSET_EDGE) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] + CTF_OFFSET_HOTSPOT) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] + CTF_OFFSET_MEM)* - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->software_shutdown_temp = powerplay_table->software_shutdown_temp; - range->software_shutdown_temp_offset = pptable->CustomSkuTable.FanAbnormalTempLimitOffset; - - return 0; -} - -static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *gfx_table = - &dpm_context->dpm_tables.gfx_table; - struct smu_14_0_dpm_table *mem_table = - &dpm_context->dpm_tables.uclk_table; - struct smu_14_0_dpm_table *soc_table = - &dpm_context->dpm_tables.soc_table; - struct smu_14_0_dpm_table *vclk_table = - &dpm_context->dpm_tables.vclk_table; - struct smu_14_0_dpm_table *dclk_table = - &dpm_context->dpm_tables.dclk_table; - struct smu_14_0_dpm_table *fclk_table = - &dpm_context->dpm_tables.fclk_table; - struct smu_umd_pstate_table *pstate_table = - &smu->pstate_table; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - DriverReportedClocks_t driver_clocks = - pptable->SkuTable.DriverReportedClocks; - - pstate_table->gfxclk_pstate.min = gfx_table->min; - if (driver_clocks.GameClockAc && - (driver_clocks.GameClockAc < gfx_table->max)) - pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc; - else - pstate_table->gfxclk_pstate.peak = gfx_table->max; - - pstate_table->uclk_pstate.min = mem_table->min; - pstate_table->uclk_pstate.peak = mem_table->max; - - pstate_table->socclk_pstate.min = soc_table->min; - pstate_table->socclk_pstate.peak = soc_table->max; - - pstate_table->vclk_pstate.min = vclk_table->min; - pstate_table->vclk_pstate.peak = vclk_table->max; - - pstate_table->dclk_pstate.min = dclk_table->min; - pstate_table->dclk_pstate.peak = dclk_table->max; - - pstate_table->fclk_pstate.min = fclk_table->min; - pstate_table->fclk_pstate.peak = fclk_table->max; - - if (driver_clocks.BaseClockAc && - driver_clocks.BaseClockAc < gfx_table->max) - pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc; - else - pstate_table->gfxclk_pstate.standard = gfx_table->max; - pstate_table->uclk_pstate.standard = mem_table->max; - pstate_table->socclk_pstate.standard = soc_table->min; - pstate_table->vclk_pstate.standard = vclk_table->min; - pstate_table->dclk_pstate.standard = dclk_table->min; - pstate_table->fclk_pstate.standard = fclk_table->min; - - return 0; -} - -static void smu_v14_0_2_get_unique_id(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - struct amdgpu_device *adev = smu->adev; - uint32_t upper32 = 0, lower32 = 0; - int ret; - - ret = smu_cmn_get_metrics_table(smu, NULL, false); - if (ret) - goto out; - - upper32 = metrics->PublicSerialNumberUpper; - lower32 = metrics->PublicSerialNumberLower; - -out: - adev->unique_id = ((uint64_t)upper32 << 32) | lower32; -} - -static int smu_v14_0_2_get_power_limit(struct smu_context *smu, - uint32_t *current_power_limit, - uint32_t *default_power_limit, - uint32_t *max_power_limit, - uint32_t *min_power_limit) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - uint32_t power_limit; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - - if (current_power_limit) - *current_power_limit = power_limit; - if (default_power_limit) - *default_power_limit = power_limit; - - if (max_power_limit) - *max_power_limit = msg_limit; - - if (min_power_limit) - *min_power_limit = 0; - - return 0; -} - -static int smu_v14_0_2_get_power_profile_mode(struct smu_context *smu, - char *buf) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - static const char *title[] = { - "PROFILE_INDEX(NAME)", - "CLOCK_TYPE(NAME)", - "FPS", - "MinActiveFreqType", - "MinActiveFreq", - "BoosterFreqType", - "BoosterFreq", - "PD_Data_limit_c", - "PD_Data_error_coeff", - "PD_Data_error_rate_coeff"}; - int16_t workload_type = 0; - uint32_t i, size = 0; - int result = 0; - - if (!buf) - return -EINVAL; - - size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s\n", - title[0], title[1], title[2], title[3], title[4], title[5], - title[6], title[7], title[8], title[9]); - - for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) { - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - i); - if (workload_type == -ENOTSUPP) - continue; - else if (workload_type < 0) - return -EINVAL; - - result = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - workload_type, - (void *)(&activity_monitor_external), - false); - if (result) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return result; - } - - size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", - i, amdgpu_pp_profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 0, - "GFXCLK", - activity_monitor->Gfx_FPS, - activity_monitor->Gfx_MinActiveFreqType, - activity_monitor->Gfx_MinActiveFreq, - activity_monitor->Gfx_BoosterFreqType, - activity_monitor->Gfx_BoosterFreq, - activity_monitor->Gfx_PD_Data_limit_c, - activity_monitor->Gfx_PD_Data_error_coeff, - activity_monitor->Gfx_PD_Data_error_rate_coeff); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 1, - "FCLK", - activity_monitor->Fclk_FPS, - activity_monitor->Fclk_MinActiveFreqType, - activity_monitor->Fclk_MinActiveFreq, - activity_monitor->Fclk_BoosterFreqType, - activity_monitor->Fclk_BoosterFreq, - activity_monitor->Fclk_PD_Data_limit_c, - activity_monitor->Fclk_PD_Data_error_coeff, - activity_monitor->Fclk_PD_Data_error_rate_coeff); - } - - return size; -} - -static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu, - long *input, - uint32_t size) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - int workload_type, ret = 0; - - smu->power_profile_mode = input[size]; - - if (smu->power_profile_mode >= PP_SMC_POWER_PROFILE_COUNT) { - dev_err(smu->adev->dev, "Invalid power profile mode %d\n", smu->power_profile_mode); - return -EINVAL; - } - - if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) { - if (size != 9) - return -EINVAL; - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - false); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return ret; - } - - switch (input[0]) { - case 0: /* Gfxclk */ - activity_monitor->Gfx_FPS = input[1]; - activity_monitor->Gfx_MinActiveFreqType = input[2]; - activity_monitor->Gfx_MinActiveFreq = input[3]; - activity_monitor->Gfx_BoosterFreqType = input[4]; - activity_monitor->Gfx_BoosterFreq = input[5]; - activity_monitor->Gfx_PD_Data_limit_c = input[6]; - activity_monitor->Gfx_PD_Data_error_coeff = input[7]; - activity_monitor->Gfx_PD_Data_error_rate_coeff = input[8]; - break; - case 1: /* Fclk */ - activity_monitor->Fclk_FPS = input[1]; - activity_monitor->Fclk_MinActiveFreqType = input[2]; - activity_monitor->Fclk_MinActiveFreq = input[3]; - activity_monitor->Fclk_BoosterFreqType = input[4]; - activity_monitor->Fclk_BoosterFreq = input[5]; - activity_monitor->Fclk_PD_Data_limit_c = input[6]; - activity_monitor->Fclk_PD_Data_error_coeff = input[7]; - activity_monitor->Fclk_PD_Data_error_rate_coeff = input[8]; - break; - default: - return -EINVAL; - } - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - true); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to set activity monitor!", __func__); - return ret; - } - } - - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - smu->power_profile_mode); - if (workload_type < 0) - return -EINVAL; - - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetWorkloadMask, - 1 << workload_type, - NULL); -} - -static int smu_v14_0_2_baco_enter(struct smu_context *smu) -{ - struct smu_baco_context *smu_baco = &smu->smu_baco; - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) - return smu_v14_0_baco_set_armd3_sequence(smu, - smu_baco->maco_support ? BACO_SEQ_BAMACO : BACO_SEQ_BACO); - else - return smu_v14_0_baco_enter(smu); -} - -static int smu_v14_0_2_baco_exit(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) { - /* Wait for PMFW handling for the Dstate change */ - usleep_range(10000, 11000); - return smu_v14_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); - } else { - return smu_v14_0_baco_exit(smu); - } -} - -static bool smu_v14_0_2_is_mode1_reset_supported(struct smu_context *smu) -{ - // TODO - - return true; -} - -static int smu_v14_0_2_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msg, int num_msgs) -{ - struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = smu_i2c->adev; - struct smu_context *smu = adev->powerplay.pp_handle; - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *table = &smu_table->driver_table; - SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr; - int i, j, r, c; - u16 dir; - - if (!adev->pm.dpm_enabled) - return -EBUSY; - - req = kzalloc(sizeof(*req), GFP_KERNEL); - if (!req) - return -ENOMEM; - - req->I2CcontrollerPort = smu_i2c->port; - req->I2CSpeed = I2C_SPEED_FAST_400K; - req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */ - dir = msg[0].flags & I2C_M_RD; - - for (c = i = 0; i < num_msgs; i++) { - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &req->SwI2cCmds[c]; - - if (!(msg[i].flags & I2C_M_RD)) { - /* write */ - cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK; - cmd->ReadWriteData = msg[i].buf[j]; - } - - if ((dir ^ msg[i].flags) & I2C_M_RD) { - /* The direction changes. - */ - dir = msg[i].flags & I2C_M_RD; - cmd->CmdConfig |= CMDCONFIG_RESTART_MASK; - } - - req->NumCmds++; - - /* - * Insert STOP if we are at the last byte of either last - * message for the transaction or the client explicitly - * requires a STOP at this particular message. - */ - if ((j == msg[i].len - 1) && - ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) { - cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK; - cmd->CmdConfig |= CMDCONFIG_STOP_MASK; - } - } - } - mutex_lock(&adev->pm.mutex); - r = smu_cmn_update_table(smu, SMU_TABLE_I2C_COMMANDS, 0, req, true); - mutex_unlock(&adev->pm.mutex); - if (r) - goto fail; - - for (c = i = 0; i < num_msgs; i++) { - if (!(msg[i].flags & I2C_M_RD)) { - c += msg[i].len; - continue; - } - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &res->SwI2cCmds[c]; - - msg[i].buf[j] = cmd->ReadWriteData; - } - } - r = num_msgs; -fail: - kfree(req); - return r; -} - -static u32 smu_v14_0_2_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm smu_v14_0_2_i2c_algo = { - .master_xfer = smu_v14_0_2_i2c_xfer, - .functionality = smu_v14_0_2_i2c_func, -}; - -static const struct i2c_adapter_quirks smu_v14_0_2_i2c_control_quirks = { - .flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN, - .max_read_len = MAX_SW_I2C_COMMANDS, - .max_write_len = MAX_SW_I2C_COMMANDS, - .max_comb_1st_msg_len = 2, - .max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2, -}; - -static int smu_v14_0_2_i2c_control_init(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int res, i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - smu_i2c->adev = adev; - smu_i2c->port = i; - mutex_init(&smu_i2c->mutex); - control->owner = THIS_MODULE; - control->dev.parent = &adev->pdev->dev; - control->algo = &smu_v14_0_2_i2c_algo; - snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i); - control->quirks = &smu_v14_0_2_i2c_control_quirks; - i2c_set_adapdata(control, smu_i2c); - - res = i2c_add_adapter(control); - if (res) { - DRM_ERROR("Failed to register hw i2c, err: %d\n", res); - goto Out_err; - } - } - - /* assign the buses used for the FRU EEPROM and RAS EEPROM */ - /* XXX ideally this would be something in a vbios data table */ - adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[1].adapter; - adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter; - - return 0; -Out_err: - for ( ; i >= 0; i--) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - return res; -} - -static void smu_v14_0_2_i2c_control_fini(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - adev->pm.ras_eeprom_i2c_bus = NULL; - adev->pm.fru_eeprom_i2c_bus = NULL; -} - -static int smu_v14_0_2_set_mp1_state(struct smu_context *smu, - enum pp_mp1_state mp1_state) -{ - int ret; - - switch (mp1_state) { - case PP_MP1_STATE_UNLOAD: - ret = smu_cmn_set_mp1_state(smu, mp1_state); - break; - default: - /* Ignore others */ - ret = 0; - } - - return ret; -} - -static int smu_v14_0_2_set_df_cstate(struct smu_context *smu, - enum pp_df_cstate state) -{ - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_DFCstateControl, - state, - NULL); -} - -static int smu_v14_0_2_mode1_reset(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); - if (!ret) { - if (amdgpu_emu_mode == 1) - msleep(50000); - else - msleep(1000); - } - - return ret; -} - -static int smu_v14_0_2_mode2_reset(struct smu_context *smu) -{ - int ret = 0; - - // TODO - - return ret; -} - -static int smu_v14_0_2_enable_gfx_features(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(14, 0, 2)) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnableAllSmuFeatures, - FEATURE_PWR_GFX, NULL); - else - return -EOPNOTSUPP; -} - -static void smu_v14_0_2_set_smu_mailbox_registers(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - smu->param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_82); - smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_66); - smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_90); - - smu->debug_param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_53); - smu->debug_msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_75); - smu->debug_resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_54); -} - -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); -} - -static void smu_v14_0_2_dump_od_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - struct amdgpu_device *adev = smu->adev; - - dev_dbg(adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - dev_dbg(adev->dev, "OD: Uclk: (%d, %d)\n", od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); -} - -static int smu_v14_0_2_upload_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - true); - if (ret) - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - - return ret; -} - -static void smu_v14_0_2_set_supported_od_feature_mask(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - adev->pm.od_feature_mask |= OD_OPS_SUPPORT_FAN_CURVE_RETRIEVE | - OD_OPS_SUPPORT_FAN_CURVE_SET | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_SET | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_SET | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_RETRIEVE | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_SET | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_RETRIEVE | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_SET; -} - -static int smu_v14_0_2_get_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - false); - if (ret) - dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); - - return ret; -} - -static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu) -{ - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - OverDriveTableExternal_t *boot_od_table = - (OverDriveTableExternal_t *)smu->smu_table.boot_overdrive_table; - OverDriveTableExternal_t *user_od_table = - (OverDriveTableExternal_t *)smu->smu_table.user_overdrive_table; - OverDriveTableExternal_t user_od_table_bak; - int ret; - int i; - - ret = smu_v14_0_2_get_overdrive_table(smu, boot_od_table); - if (ret) - return ret; - - smu_v14_0_2_dump_od_table(smu, boot_od_table); - - memcpy(od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - - /* - * For S3/S4/Runpm resume, we need to setup those overdrive tables again, - * but we have to preserve user defined values in "user_od_table". - */ - if (!smu->adev->in_suspend) { - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - smu->user_dpm_profile.user_od = false; - } else if (smu->user_dpm_profile.user_od) { - memcpy(&user_od_table_bak, - user_od_table, - sizeof(OverDriveTableExternal_t)); - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - user_od_table->OverDriveTable.GfxclkFmin = - user_od_table_bak.OverDriveTable.GfxclkFmin; - user_od_table->OverDriveTable.GfxclkFmax = - user_od_table_bak.OverDriveTable.GfxclkFmax; - user_od_table->OverDriveTable.UclkFmin = - user_od_table_bak.OverDriveTable.UclkFmin; - user_od_table->OverDriveTable.UclkFmax = - user_od_table_bak.OverDriveTable.UclkFmax; - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - user_od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = - user_od_table_bak.OverDriveTable.VoltageOffsetPerZoneBoundary[i]; - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) { - user_od_table->OverDriveTable.FanLinearTempPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearTempPoints[i]; - user_od_table->OverDriveTable.FanLinearPwmPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearPwmPoints[i]; - } - user_od_table->OverDriveTable.AcousticLimitRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticLimitRpmThreshold; - user_od_table->OverDriveTable.AcousticTargetRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticTargetRpmThreshold; - user_od_table->OverDriveTable.FanTargetTemperature = - user_od_table_bak.OverDriveTable.FanTargetTemperature; - user_od_table->OverDriveTable.FanMinimumPwm = - user_od_table_bak.OverDriveTable.FanMinimumPwm; - } - - smu_v14_0_2_set_supported_od_feature_mask(smu); - - return 0; -} - -static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = table_context->overdrive_table; - OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; - int res; - - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); - user_od_table->OverDriveTable.FeatureCtrlMask = 0; - if (res == 0) - memcpy(od_table, user_od_table, sizeof(OverDriveTableExternal_t)); - - return res; -} - -static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long input) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *boot_overdrive_table = - (OverDriveTableExternal_t *)table_context->boot_overdrive_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - int i; - - switch (input) { - case PP_OD_EDIT_FAN_CURVE: - for (i = 0; i < NUM_OD_FAN_MAX_POINTS; i++) { - od_table->OverDriveTable.FanLinearTempPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearTempPoints[i]; - od_table->OverDriveTable.FanLinearPwmPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearPwmPoints[i]; - } - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_LIMIT: - od_table->OverDriveTable.AcousticLimitRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticLimitRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_TARGET: - od_table->OverDriveTable.AcousticTargetRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticTargetRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - od_table->OverDriveTable.FanTargetTemperature = - boot_overdrive_table->OverDriveTable.FanTargetTemperature; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_MINIMUM_PWM: - od_table->OverDriveTable.FanMinimumPwm = - boot_overdrive_table->OverDriveTable.FanMinimumPwm; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - default: - dev_info(adev->dev, "Invalid table index: %ld\n", input); - return -EINVAL; - } - - return 0; -} - -static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, - enum PP_OD_DPM_TABLE_COMMAND type, - long input[], - uint32_t size) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - uint32_t offset_of_voltageoffset; - int32_t minimum, maximum; - uint32_t feature_ctrlmask; - int i, ret = 0; - - switch (type) { - case PP_OD_EDIT_SCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - dev_warn(adev->dev, "GFXCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.GfxclkFmin > od_table->OverDriveTable.GfxclkFmax) { - dev_err(adev->dev, - "Invalid setting: GfxclkFmin(%u) is bigger than GfxclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.GfxclkFmin, - (uint32_t)od_table->OverDriveTable.GfxclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_MCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - dev_warn(adev->dev, "UCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.UclkFmin > od_table->OverDriveTable.UclkFmax) { - dev_err(adev->dev, - "Invalid setting: UclkFmin(%u) is bigger than UclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.UclkFmin, - (uint32_t)od_table->OverDriveTable.UclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - dev_warn(adev->dev, "Gfx offset setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "Voltage offset (%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = input[0]; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - if (input[0] >= NUM_OD_FAN_MAX_POINTS - 1 || - input[0] < 0) - return -EINVAL; - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &minimum, - &maximum); - if (input[1] < minimum || - input[1] > maximum) { - dev_info(adev->dev, "Fan curve temp setting(%ld) must be within [%d, %d]!\n", - input[1], minimum, maximum); - return -EINVAL; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &minimum, - &maximum); - if (input[2] < minimum || - input[2] > maximum) { - dev_info(adev->dev, "Fan curve pwm setting(%ld) must be within [%d, %d]!\n", - input[2], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanLinearTempPoints[input[0]] = input[1]; - od_table->OverDriveTable.FanLinearPwmPoints[input[0]] = input[2]; - od_table->OverDriveTable.FanMode = FAN_MODE_MANUAL_LINEAR; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic limit threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticLimitRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic target threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticTargetRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan target temperature setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanTargetTemperature = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan minimum pwm setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanMinimumPwm = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_RESTORE_DEFAULT_TABLE: - if (size == 1) { - ret = smu_v14_0_2_od_restore_table_single(smu, input[0]); - if (ret) - return ret; - } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; - memcpy(od_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; - } - fallthrough; - case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ - offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); - if (memcmp((u8 *)od_table + offset_of_voltageoffset, - table_context->user_overdrive_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset)) { - smu_v14_0_2_dump_od_table(smu, od_table); - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - od_table->OverDriveTable.FeatureCtrlMask = 0; - memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, - (u8 *)od_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); - - if (!memcmp(table_context->user_overdrive_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t))) - smu->user_dpm_profile.user_od = false; - else - smu->user_dpm_profile.user_od = true; - } - break; - - default: - return -ENOSYS; - } - - return ret; -} - -<<<<<<< -static int smu_v14_0_2_set_power_limit(struct smu_context *smu, - enum smu_ppt_limit_type limit_type, - uint32_t limit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - int ret = 0; - - if (limit_type != SMU_DEFAULT_PPT_LIMIT) - return -EINVAL; - - if (limit <= msg_limit) { - if (smu->current_power_limit > msg_limit) { - od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - } - return smu_v14_0_set_power_limit(smu, limit_type, limit); - } else if (smu->od_enabled) { - ret = smu_v14_0_set_power_limit(smu, limit_type, msg_limit); - if (ret) - return ret; - - od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - smu->current_power_limit = limit; - } else { - return -EINVAL; - } - - return 0; -======= -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); ->>>>>>> -} - -static const struct pptable_funcs smu_v14_0_2_ppt_funcs = { - .get_allowed_feature_mask = smu_v14_0_2_get_allowed_feature_mask, - .set_default_dpm_table = smu_v14_0_2_set_default_dpm_table, - .i2c_init = smu_v14_0_2_i2c_control_init, - .i2c_fini = smu_v14_0_2_i2c_control_fini, - .is_dpm_running = smu_v14_0_2_is_dpm_running, - .dump_pptable = smu_v14_0_2_dump_pptable, - .init_microcode = smu_v14_0_init_microcode, - .load_microcode = smu_v14_0_load_microcode, - .fini_microcode = smu_v14_0_fini_microcode, - .init_smc_tables = smu_v14_0_2_init_smc_tables, - .fini_smc_tables = smu_v14_0_fini_smc_tables, - .init_power = smu_v14_0_init_power, - .fini_power = smu_v14_0_fini_power, - .check_fw_status = smu_v14_0_check_fw_status, - .setup_pptable = smu_v14_0_2_setup_pptable, - .check_fw_version = smu_v14_0_check_fw_version, - .write_pptable = smu_cmn_write_pptable, - .set_driver_table_location = smu_v14_0_set_driver_table_location, - .system_features_control = smu_v14_0_system_features_control, - .set_allowed_mask = smu_v14_0_set_allowed_mask, - .get_enabled_mask = smu_cmn_get_enabled_mask, - .dpm_set_vcn_enable = smu_v14_0_set_vcn_enable, - .dpm_set_jpeg_enable = smu_v14_0_set_jpeg_enable, - .get_dpm_ultimate_freq = smu_v14_0_2_get_dpm_ultimate_freq, - .get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values, - .read_sensor = smu_v14_0_2_read_sensor, - .feature_is_enabled = smu_cmn_feature_is_enabled, - .print_clk_levels = smu_v14_0_2_print_clk_levels, - .force_clk_levels = smu_v14_0_2_force_clk_levels, - .update_pcie_parameters = smu_v14_0_2_update_pcie_parameters, - .get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range, - .register_irq_handler = smu_v14_0_register_irq_handler, - .enable_thermal_alert = smu_v14_0_enable_thermal_alert, - .disable_thermal_alert = smu_v14_0_disable_thermal_alert, - .notify_memory_pool_location = smu_v14_0_notify_memory_pool_location, - .get_gpu_metrics = smu_v14_0_2_get_gpu_metrics, - .set_soft_freq_limited_range = smu_v14_0_set_soft_freq_limited_range, - .set_default_od_settings = smu_v14_0_2_set_default_od_settings, - .restore_user_od_settings = smu_v14_0_2_restore_user_od_settings, - .od_edit_dpm_table = smu_v14_0_2_od_edit_dpm_table, - .init_pptable_microcode = smu_v14_0_init_pptable_microcode, - .populate_umd_state_clk = smu_v14_0_2_populate_umd_state_clk, - .set_performance_level = smu_v14_0_set_performance_level, - .gfx_off_control = smu_v14_0_gfx_off_control, - .get_unique_id = smu_v14_0_2_get_unique_id, - .get_power_limit = smu_v14_0_2_get_power_limit, - .set_power_limit = smu_v14_0_2_set_power_limit, - .set_power_source = smu_v14_0_set_power_source, - .get_power_profile_mode = smu_v14_0_2_get_power_profile_mode, - .set_power_profile_mode = smu_v14_0_2_set_power_profile_mode, - .run_btc = smu_v14_0_run_btc, - .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, - .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, - .set_tool_table_location = smu_v14_0_set_tool_table_location, - .deep_sleep_control = smu_v14_0_deep_sleep_control, - .gfx_ulv_control = smu_v14_0_gfx_ulv_control, - .get_bamaco_support = smu_v14_0_get_bamaco_support, - .baco_get_state = smu_v14_0_baco_get_state, - .baco_set_state = smu_v14_0_baco_set_state, - .baco_enter = smu_v14_0_2_baco_enter, - .baco_exit = smu_v14_0_2_baco_exit, - .mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported, - .mode1_reset = smu_v14_0_2_mode1_reset, - .mode2_reset = smu_v14_0_2_mode2_reset, - .enable_gfx_features = smu_v14_0_2_enable_gfx_features, - .set_mp1_state = smu_v14_0_2_set_mp1_state, - .set_df_cstate = smu_v14_0_2_set_df_cstate, -#if 0 - .gpo_control = smu_v14_0_gpo_control, -#endif -}; - -void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu) -{ - smu->ppt_funcs = &smu_v14_0_2_ppt_funcs; - smu->message_map = smu_v14_0_2_message_map; - smu->clock_map = smu_v14_0_2_clk_map; - smu->feature_map = smu_v14_0_2_feature_mask_map; - smu->table_map = smu_v14_0_2_table_map; - smu->pwr_src_map = smu_v14_0_2_pwr_src_map; - smu->workload_map = smu_v14_0_2_workload_map; - smu_v14_0_2_set_smu_mailbox_registers(smu); -} diff --git a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.2 b/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.2 deleted file mode 100644 index 6878263c313c..000000000000 --- a/rr-cache/02c9ed6a88e6cfc33649cb70da4aaaf2dc874129/preimage.2 +++ /dev/null @@ -1,2956 +0,0 @@ -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#define SWSMU_CODE_LAYER_L2 - -#include <linux/firmware.h> -#include <linux/pci.h> -#include <linux/i2c.h> -#include "amdgpu.h" -#include "amdgpu_smu.h" -#include "atomfirmware.h" -#include "amdgpu_atomfirmware.h" -#include "amdgpu_atombios.h" -#include "smu_v14_0.h" -#include "smu14_driver_if_v14_0.h" -#include "soc15_common.h" -#include "atom.h" -#include "smu_v14_0_2_ppt.h" -#include "smu_v14_0_2_pptable.h" -#include "smu_v14_0_2_ppsmc.h" -#include "mp/mp_14_0_2_offset.h" -#include "mp/mp_14_0_2_sh_mask.h" - -#include "smu_cmn.h" -#include "amdgpu_ras.h" - -/* - * DO NOT use these for err/warn/info/debug messages. - * Use dev_err, dev_warn, dev_info and dev_dbg instead. - * They are more MGPU friendly. - */ -#undef pr_err -#undef pr_warn -#undef pr_info -#undef pr_debug - -#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) - -#define FEATURE_MASK(feature) (1ULL << feature) -#define SMC_DPM_FEATURE ( \ - FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_LINK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_FCLK_BIT)) - -#define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE 0x4000 -#define DEBUGSMC_MSG_Mode1Reset 2 -#define LINK_SPEED_MAX 3 -<<<<<<< -======= - -#define PP_OD_FEATURE_GFXCLK_FMIN 0 -#define PP_OD_FEATURE_GFXCLK_FMAX 1 -#define PP_OD_FEATURE_UCLK_FMIN 2 -#define PP_OD_FEATURE_UCLK_FMAX 3 -#define PP_OD_FEATURE_GFX_VF_CURVE 4 -#define PP_OD_FEATURE_FAN_CURVE_TEMP 5 -#define PP_OD_FEATURE_FAN_CURVE_PWM 6 -#define PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT 7 -#define PP_OD_FEATURE_FAN_ACOUSTIC_TARGET 8 -#define PP_OD_FEATURE_FAN_TARGET_TEMPERATURE 9 -#define PP_OD_FEATURE_FAN_MINIMUM_PWM 10 ->>>>>>> - -static struct cmn2asic_msg_mapping smu_v14_0_2_message_map[SMU_MSG_MAX_COUNT] = { - MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 1), - MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), - MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 1), - MSG_MAP(SetAllowedFeaturesMaskLow, PPSMC_MSG_SetAllowedFeaturesMaskLow, 0), - MSG_MAP(SetAllowedFeaturesMaskHigh, PPSMC_MSG_SetAllowedFeaturesMaskHigh, 0), - MSG_MAP(EnableAllSmuFeatures, PPSMC_MSG_EnableAllSmuFeatures, 0), - MSG_MAP(DisableAllSmuFeatures, PPSMC_MSG_DisableAllSmuFeatures, 0), - MSG_MAP(EnableSmuFeaturesLow, PPSMC_MSG_EnableSmuFeaturesLow, 1), - MSG_MAP(EnableSmuFeaturesHigh, PPSMC_MSG_EnableSmuFeaturesHigh, 1), - MSG_MAP(DisableSmuFeaturesLow, PPSMC_MSG_DisableSmuFeaturesLow, 1), - MSG_MAP(DisableSmuFeaturesHigh, PPSMC_MSG_DisableSmuFeaturesHigh, 1), - MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetRunningSmuFeaturesLow, 1), - MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetRunningSmuFeaturesHigh, 1), - MSG_MAP(SetWorkloadMask, PPSMC_MSG_SetWorkloadMask, 1), - MSG_MAP(SetPptLimit, PPSMC_MSG_SetPptLimit, 0), - MSG_MAP(SetDriverDramAddrHigh, PPSMC_MSG_SetDriverDramAddrHigh, 1), - MSG_MAP(SetDriverDramAddrLow, PPSMC_MSG_SetDriverDramAddrLow, 1), - MSG_MAP(SetToolsDramAddrHigh, PPSMC_MSG_SetToolsDramAddrHigh, 0), - MSG_MAP(SetToolsDramAddrLow, PPSMC_MSG_SetToolsDramAddrLow, 0), - MSG_MAP(TransferTableSmu2Dram, PPSMC_MSG_TransferTableSmu2Dram, 1), - MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), - MSG_MAP(UseDefaultPPTable, PPSMC_MSG_UseDefaultPPTable, 0), - MSG_MAP(RunDcBtc, PPSMC_MSG_RunDcBtc, 0), - MSG_MAP(EnterBaco, PPSMC_MSG_EnterBaco, 0), - MSG_MAP(ExitBaco, PPSMC_MSG_ExitBaco, 0), - MSG_MAP(SetSoftMinByFreq, PPSMC_MSG_SetSoftMinByFreq, 1), - MSG_MAP(SetSoftMaxByFreq, PPSMC_MSG_SetSoftMaxByFreq, 1), - MSG_MAP(SetHardMinByFreq, PPSMC_MSG_SetHardMinByFreq, 1), - MSG_MAP(SetHardMaxByFreq, PPSMC_MSG_SetHardMaxByFreq, 0), - MSG_MAP(GetMinDpmFreq, PPSMC_MSG_GetMinDpmFreq, 1), - MSG_MAP(GetMaxDpmFreq, PPSMC_MSG_GetMaxDpmFreq, 1), - MSG_MAP(GetDpmFreqByIndex, PPSMC_MSG_GetDpmFreqByIndex, 1), - MSG_MAP(PowerUpVcn, PPSMC_MSG_PowerUpVcn, 0), - MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), - MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), - MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), - MSG_MAP(GetDcModeMaxDpmFreq, PPSMC_MSG_GetDcModeMaxDpmFreq, 1), - MSG_MAP(OverridePcieParameters, PPSMC_MSG_OverridePcieParameters, 0), - MSG_MAP(DramLogSetDramAddrHigh, PPSMC_MSG_DramLogSetDramAddrHigh, 0), - MSG_MAP(DramLogSetDramAddrLow, PPSMC_MSG_DramLogSetDramAddrLow, 0), - MSG_MAP(DramLogSetDramSize, PPSMC_MSG_DramLogSetDramSize, 0), - MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), - MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), - MSG_MAP(GetPptLimit, PPSMC_MSG_GetPptLimit, 0), - MSG_MAP(NotifyPowerSource, PPSMC_MSG_NotifyPowerSource, 0), - MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), - MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), - MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), - MSG_MAP(SetNumBadMemoryPagesRetired, PPSMC_MSG_SetNumBadMemoryPagesRetired, 0), - MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel, - PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel, 0), - MSG_MAP(AllowIHHostInterrupt, PPSMC_MSG_AllowIHHostInterrupt, 0), - MSG_MAP(ReenableAcDcInterrupt, PPSMC_MSG_ReenableAcDcInterrupt, 0), -}; - -static struct cmn2asic_mapping smu_v14_0_2_clk_map[SMU_CLK_COUNT] = { - CLK_MAP(GFXCLK, PPCLK_GFXCLK), - CLK_MAP(SCLK, PPCLK_GFXCLK), - CLK_MAP(SOCCLK, PPCLK_SOCCLK), - CLK_MAP(FCLK, PPCLK_FCLK), - CLK_MAP(UCLK, PPCLK_UCLK), - CLK_MAP(MCLK, PPCLK_UCLK), - CLK_MAP(VCLK, PPCLK_VCLK_0), - CLK_MAP(DCLK, PPCLK_DCLK_0), - CLK_MAP(DCEFCLK, PPCLK_DCFCLK), -}; - -static struct cmn2asic_mapping smu_v14_0_2_feature_mask_map[SMU_FEATURE_COUNT] = { - FEA_MAP(FW_DATA_READ), - FEA_MAP(DPM_GFXCLK), - FEA_MAP(DPM_GFX_POWER_OPTIMIZER), - FEA_MAP(DPM_UCLK), - FEA_MAP(DPM_FCLK), - FEA_MAP(DPM_SOCCLK), - FEA_MAP(DPM_LINK), - FEA_MAP(DPM_DCN), - FEA_MAP(VMEMP_SCALING), - FEA_MAP(VDDIO_MEM_SCALING), - FEA_MAP(DS_GFXCLK), - FEA_MAP(DS_SOCCLK), - FEA_MAP(DS_FCLK), - FEA_MAP(DS_LCLK), - FEA_MAP(DS_DCFCLK), - FEA_MAP(DS_UCLK), - FEA_MAP(GFX_ULV), - FEA_MAP(FW_DSTATE), - FEA_MAP(GFXOFF), - FEA_MAP(BACO), - FEA_MAP(MM_DPM), - FEA_MAP(SOC_MPCLK_DS), - FEA_MAP(BACO_MPCLK_DS), - FEA_MAP(THROTTLERS), - FEA_MAP(SMARTSHIFT), - FEA_MAP(GTHR), - FEA_MAP(ACDC), - FEA_MAP(VR0HOT), - FEA_MAP(FW_CTF), - FEA_MAP(FAN_CONTROL), - FEA_MAP(GFX_DCS), - FEA_MAP(GFX_READ_MARGIN), - FEA_MAP(LED_DISPLAY), - FEA_MAP(GFXCLK_SPREAD_SPECTRUM), - FEA_MAP(OUT_OF_BAND_MONITOR), - FEA_MAP(OPTIMIZED_VMIN), - FEA_MAP(GFX_IMU), - FEA_MAP(BOOT_TIME_CAL), - FEA_MAP(GFX_PCC_DFLL), - FEA_MAP(SOC_CG), - FEA_MAP(DF_CSTATE), - FEA_MAP(GFX_EDC), - FEA_MAP(BOOT_POWER_OPT), - FEA_MAP(CLOCK_POWER_DOWN_BYPASS), - FEA_MAP(DS_VCN), - FEA_MAP(BACO_CG), - FEA_MAP(MEM_TEMP_READ), - FEA_MAP(ATHUB_MMHUB_PG), - FEA_MAP(SOC_PCC), - [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, - [SMU_FEATURE_PPT_BIT] = {1, FEATURE_THROTTLERS_BIT}, -}; - -static struct cmn2asic_mapping smu_v14_0_2_table_map[SMU_TABLE_COUNT] = { - TAB_MAP(PPTABLE), - TAB_MAP(WATERMARKS), - TAB_MAP(AVFS_PSM_DEBUG), - TAB_MAP(PMSTATUSLOG), - TAB_MAP(SMU_METRICS), - TAB_MAP(DRIVER_SMU_CONFIG), - TAB_MAP(ACTIVITY_MONITOR_COEFF), - [SMU_TABLE_COMBO_PPTABLE] = {1, TABLE_COMBO_PPTABLE}, - TAB_MAP(I2C_COMMANDS), - TAB_MAP(ECCINFO), - TAB_MAP(OVERDRIVE), -}; - -static struct cmn2asic_mapping smu_v14_0_2_pwr_src_map[SMU_POWER_SOURCE_COUNT] = { - PWR_MAP(AC), - PWR_MAP(DC), -}; - -static struct cmn2asic_mapping smu_v14_0_2_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT, WORKLOAD_PPLIB_DEFAULT_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_POWERSAVING, WORKLOAD_PPLIB_POWER_SAVING_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), - WORKLOAD_MAP(PP_SMC_POWER_PROFILE_WINDOW3D, WORKLOAD_PPLIB_WINDOW_3D_BIT), -}; - -static const uint8_t smu_v14_0_2_throttler_map[] = { - [THROTTLER_PPT0_BIT] = (SMU_THROTTLER_PPT0_BIT), - [THROTTLER_PPT1_BIT] = (SMU_THROTTLER_PPT1_BIT), - [THROTTLER_PPT2_BIT] = (SMU_THROTTLER_PPT2_BIT), - [THROTTLER_PPT3_BIT] = (SMU_THROTTLER_PPT3_BIT), - [THROTTLER_TDC_GFX_BIT] = (SMU_THROTTLER_TDC_GFX_BIT), - [THROTTLER_TDC_SOC_BIT] = (SMU_THROTTLER_TDC_SOC_BIT), - [THROTTLER_TEMP_EDGE_BIT] = (SMU_THROTTLER_TEMP_EDGE_BIT), - [THROTTLER_TEMP_HOTSPOT_BIT] = (SMU_THROTTLER_TEMP_HOTSPOT_BIT), - [THROTTLER_TEMP_MEM_BIT] = (SMU_THROTTLER_TEMP_MEM_BIT), - [THROTTLER_TEMP_VR_GFX_BIT] = (SMU_THROTTLER_TEMP_VR_GFX_BIT), - [THROTTLER_TEMP_VR_SOC_BIT] = (SMU_THROTTLER_TEMP_VR_SOC_BIT), - [THROTTLER_TEMP_VR_MEM0_BIT] = (SMU_THROTTLER_TEMP_VR_MEM0_BIT), - [THROTTLER_TEMP_VR_MEM1_BIT] = (SMU_THROTTLER_TEMP_VR_MEM1_BIT), - [THROTTLER_TEMP_LIQUID0_BIT] = (SMU_THROTTLER_TEMP_LIQUID0_BIT), - [THROTTLER_TEMP_LIQUID1_BIT] = (SMU_THROTTLER_TEMP_LIQUID1_BIT), - [THROTTLER_GFX_APCC_PLUS_BIT] = (SMU_THROTTLER_APCC_BIT), - [THROTTLER_FIT_BIT] = (SMU_THROTTLER_FIT_BIT), -}; - -static int -smu_v14_0_2_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - /*u32 smu_version;*/ - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0xff, sizeof(uint32_t) * num); - - if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT); - } -#if 0 - if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) || - !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); - - if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); - - /* PMFW 78.58 contains a critical fix for gfxoff feature */ - smu_cmn_get_smc_version(smu, NULL, &smu_version); - if ((smu_version < 0x004e3a00) || - !(adev->pm.pp_feature & PP_GFXOFF_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT); - - if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); - } - - if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - - if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) { - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT); - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT); - } - - if (!(adev->pm.pp_feature & PP_ULV_MASK)) - *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT); -#endif - - return 0; -} - -static int smu_v14_0_2_check_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - struct smu_baco_context *smu_baco = &smu->smu_baco; - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC) - smu->dc_controlled_by_gpio = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_BACO) { - smu_baco->platform_support = true; - - if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_MACO) - smu_baco->maco_support = true; - } - - if (!overdrive_lowerlimits->FeatureCtrlMask || - !overdrive_upperlimits->FeatureCtrlMask) - smu->od_enabled = false; - - table_context->thermal_controller_type = - powerplay_table->thermal_controller_type; - - /* - * Instead of having its own buffer space and get overdrive_table copied, - * smu->od_settings just points to the actual overdrive_table - */ - smu->od_settings = &powerplay_table->overdrive_table; - - smu->adev->pm.no_fan = - !(pptable->PFE_Settings.FeaturesToRun[0] & (1 << FEATURE_FAN_CONTROL_BIT)); - - return 0; -} - -static int smu_v14_0_2_store_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - - memcpy(table_context->driver_pptable, &powerplay_table->smc_pptable, - sizeof(PPTable_t)); - - return 0; -} - -#ifndef atom_smc_dpm_info_table_14_0_0 -struct atom_smc_dpm_info_table_14_0_0 { - struct atom_common_table_header table_header; - BoardTable_t BoardTable; -}; -#endif - -static int smu_v14_0_2_append_powerplay_table(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - struct atom_smc_dpm_info_table_14_0_0 *smc_dpm_table; - BoardTable_t *BoardTable = &smc_pptable->BoardTable; - int index, ret; - - index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, - smc_dpm_info); - - ret = amdgpu_atombios_get_data_table(smu->adev, index, NULL, NULL, NULL, - (uint8_t **)&smc_dpm_table); - if (ret) - return ret; - - memcpy(BoardTable, &smc_dpm_table->BoardTable, sizeof(BoardTable_t)); - - return 0; -} - -#if 0 -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_powerplay_table); - - return 0; -} -#endif - -static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu, - void **table, - uint32_t *size) -{ - struct smu_table_context *smu_table = &smu->smu_table; - void *combo_pptable = smu_table->combo_pptable; - int ret = 0; - - ret = smu_cmn_get_combo_pptable(smu); - if (ret) - return ret; - - *table = combo_pptable; - *size = sizeof(struct smu_14_0_2_powerplay_table); - - return 0; -} - -static int smu_v14_0_2_setup_pptable(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct amdgpu_device *adev = smu->adev; - int ret = 0; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!adev->scpm_enabled) - ret = smu_v14_0_setup_pptable(smu); - else - ret = smu_v14_0_2_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - if (ret) - return ret; - - ret = smu_v14_0_2_store_powerplay_table(smu); - if (ret) - return ret; - - /* - * With SCPM enabled, the operation below will be handled - * by PSP. Driver involvment is unnecessary and useless. - */ - if (!adev->scpm_enabled) { - ret = smu_v14_0_2_append_powerplay_table(smu); - if (ret) - return ret; - } - - ret = smu_v14_0_2_check_powerplay_table(smu); - if (ret) - return ret; - - return ret; -} - -static int smu_v14_0_2_tables_init(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *tables = smu_table->tables; - - SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetricsExternal_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_PMSTATUSLOG, SMU14_TOOL_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ACTIVITY_MONITOR_COEFF, - sizeof(DpmActivityMonitorCoeffIntExternal_t), PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_COMBO_PPTABLE, MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_ECCINFO, sizeof(EccInfoTable_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - - smu_table->metrics_table = kzalloc(sizeof(SmuMetricsExternal_t), GFP_KERNEL); - if (!smu_table->metrics_table) - goto err0_out; - smu_table->metrics_time = 0; - - smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3); - smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); - if (!smu_table->gpu_metrics_table) - goto err1_out; - - smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); - if (!smu_table->watermarks_table) - goto err2_out; - - smu_table->ecc_table = kzalloc(tables[SMU_TABLE_ECCINFO].size, GFP_KERNEL); - if (!smu_table->ecc_table) - goto err3_out; - - return 0; - -err3_out: - kfree(smu_table->watermarks_table); -err2_out: - kfree(smu_table->gpu_metrics_table); -err1_out: - kfree(smu_table->metrics_table); -err0_out: - return -ENOMEM; -} - -static int smu_v14_0_2_allocate_dpm_context(struct smu_context *smu) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - - smu_dpm->dpm_context = kzalloc(sizeof(struct smu_14_0_dpm_context), - GFP_KERNEL); - if (!smu_dpm->dpm_context) - return -ENOMEM; - - smu_dpm->dpm_context_size = sizeof(struct smu_14_0_dpm_context); - - return 0; -} - -static int smu_v14_0_2_init_smc_tables(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_v14_0_2_tables_init(smu); - if (ret) - return ret; - - ret = smu_v14_0_2_allocate_dpm_context(smu); - if (ret) - return ret; - - return smu_v14_0_init_smc_tables(smu); -} - -static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - SkuTable_t *skutable = &pptable->SkuTable; - struct smu_14_0_dpm_table *dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t link_level; - int ret = 0; - - /* socclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.soc_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_SOCCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* gfxclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_GFXCLK, - dpm_table); - if (ret) - return ret; - - /* - * Update the reported maximum shader clock to the value - * which can be guarded to be achieved on all cards. This - * is aligned with Window setting. And considering that value - * might be not the peak frequency the card can achieve, it - * is normal some real-time clock frequency can overtake this - * labelled maximum clock frequency(for example in pp_dpm_sclk - * sysfs output). - */ - if (skutable->DriverReportedClocks.GameClockAc && - (dpm_table->dpm_levels[dpm_table->count - 1].value > - skutable->DriverReportedClocks.GameClockAc)) { - dpm_table->dpm_levels[dpm_table->count - 1].value = - skutable->DriverReportedClocks.GameClockAc; - dpm_table->max = skutable->DriverReportedClocks.GameClockAc; - } - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* uclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_UCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* fclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_FCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* vclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_VCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* dclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - /* lclk dpm table setup */ - pcie_table = &dpm_context->dpm_tables.pcie_table; - pcie_table->num_of_link_levels = 0; - for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) { - if (!skutable->PcieGenSpeed[link_level] && - !skutable->PcieLaneCount[link_level] && - !skutable->LclkFreq[link_level]) - continue; - - pcie_table->pcie_gen[pcie_table->num_of_link_levels] = - skutable->PcieGenSpeed[link_level]; - pcie_table->pcie_lane[pcie_table->num_of_link_levels] = - skutable->PcieLaneCount[link_level]; - pcie_table->clk_freq[pcie_table->num_of_link_levels] = - skutable->LclkFreq[link_level]; - pcie_table->num_of_link_levels++; - } - - /* dcefclk dpm table setup */ - dpm_table = &dpm_context->dpm_tables.dcef_table; - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) { - ret = smu_v14_0_set_single_dpm_table(smu, - SMU_DCEFCLK, - dpm_table); - if (ret) - return ret; - } else { - dpm_table->count = 1; - dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100; - dpm_table->dpm_levels[0].enabled = true; - dpm_table->min = dpm_table->dpm_levels[0].value; - dpm_table->max = dpm_table->dpm_levels[0].value; - } - - return 0; -} - -static bool smu_v14_0_2_is_dpm_running(struct smu_context *smu) -{ - int ret = 0; - uint64_t feature_enabled; - - ret = smu_cmn_get_enabled_mask(smu, &feature_enabled); - if (ret) - return false; - - return !!(feature_enabled & SMC_DPM_FEATURE); -} - -static void smu_v14_0_2_dump_pptable(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - PFE_Settings_t *PFEsettings = &pptable->PFE_Settings; - - dev_info(smu->adev->dev, "Dumped PPTable:\n"); - - dev_info(smu->adev->dev, "Version = 0x%08x\n", PFEsettings->Version); - dev_info(smu->adev->dev, "FeaturesToRun[0] = 0x%08x\n", PFEsettings->FeaturesToRun[0]); - dev_info(smu->adev->dev, "FeaturesToRun[1] = 0x%08x\n", PFEsettings->FeaturesToRun[1]); -} - -static uint32_t smu_v14_0_2_get_throttler_status(SmuMetrics_t *metrics) -{ - uint32_t throttler_status = 0; - int i; - - for (i = 0; i < THROTTLER_COUNT; i++) - throttler_status |= - (metrics->ThrottlingPercentage[i] ? 1U << i : 0); - - return throttler_status; -} - -#define SMU_14_0_2_BUSY_THRESHOLD 5 -static int smu_v14_0_2_get_smu_metrics_data(struct smu_context *smu, - MetricsMember_t member, - uint32_t *value) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - NULL, - false); - if (ret) - return ret; - - switch (member) { - case METRICS_CURR_GFXCLK: - *value = metrics->CurrClock[PPCLK_GFXCLK]; - break; - case METRICS_CURR_SOCCLK: - *value = metrics->CurrClock[PPCLK_SOCCLK]; - break; - case METRICS_CURR_UCLK: - *value = metrics->CurrClock[PPCLK_UCLK]; - break; - case METRICS_CURR_VCLK: - *value = metrics->CurrClock[PPCLK_VCLK_0]; - break; - case METRICS_CURR_DCLK: - *value = metrics->CurrClock[PPCLK_DCLK_0]; - break; - case METRICS_CURR_FCLK: - *value = metrics->CurrClock[PPCLK_FCLK]; - break; - case METRICS_CURR_DCEFCLK: - *value = metrics->CurrClock[PPCLK_DCFCLK]; - break; - case METRICS_AVERAGE_GFXCLK: - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageGfxclkFrequencyPostDs; - else - *value = metrics->AverageGfxclkFrequencyPreDs; - break; - case METRICS_AVERAGE_FCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageFclkFrequencyPostDs; - else - *value = metrics->AverageFclkFrequencyPreDs; - break; - case METRICS_AVERAGE_UCLK: - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - *value = metrics->AverageMemclkFrequencyPostDs; - else - *value = metrics->AverageMemclkFrequencyPreDs; - break; - case METRICS_AVERAGE_VCLK: - *value = metrics->AverageVclk0Frequency; - break; - case METRICS_AVERAGE_DCLK: - *value = metrics->AverageDclk0Frequency; - break; - case METRICS_AVERAGE_VCLK1: - *value = metrics->AverageVclk1Frequency; - break; - case METRICS_AVERAGE_DCLK1: - *value = metrics->AverageDclk1Frequency; - break; - case METRICS_AVERAGE_GFXACTIVITY: - *value = metrics->AverageGfxActivity; - break; - case METRICS_AVERAGE_MEMACTIVITY: - *value = metrics->AverageUclkActivity; - break; - case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->AverageSocketPower << 8; - break; - case METRICS_TEMPERATURE_EDGE: - *value = metrics->AvgTemperature[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_HOTSPOT: - *value = metrics->AvgTemperature[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_MEM: - *value = metrics->AvgTemperature[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRGFX: - *value = metrics->AvgTemperature[TEMP_VR_GFX] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_TEMPERATURE_VRSOC: - *value = metrics->AvgTemperature[TEMP_VR_SOC] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - break; - case METRICS_THROTTLER_STATUS: - *value = smu_v14_0_2_get_throttler_status(metrics); - break; - case METRICS_CURR_FANSPEED: - *value = metrics->AvgFanRpm; - break; - case METRICS_CURR_FANPWM: - *value = metrics->AvgFanPwm; - break; - case METRICS_VOLTAGE_VDDGFX: - *value = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - break; - case METRICS_PCIE_RATE: - *value = metrics->PcieRate; - break; - case METRICS_PCIE_WIDTH: - *value = metrics->PcieWidth; - break; - default: - *value = UINT_MAX; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *min, - uint32_t *max) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *dpm_table; - - switch (clk_type) { - case SMU_MCLK: - case SMU_UCLK: - /* uclk dpm table */ - dpm_table = &dpm_context->dpm_tables.uclk_table; - break; - case SMU_GFXCLK: - case SMU_SCLK: - /* gfxclk dpm table */ - dpm_table = &dpm_context->dpm_tables.gfx_table; - break; - case SMU_SOCCLK: - /* socclk dpm table */ - dpm_table = &dpm_context->dpm_tables.soc_table; - break; - case SMU_FCLK: - /* fclk dpm table */ - dpm_table = &dpm_context->dpm_tables.fclk_table; - break; - case SMU_VCLK: - case SMU_VCLK1: - /* vclk dpm table */ - dpm_table = &dpm_context->dpm_tables.vclk_table; - break; - case SMU_DCLK: - case SMU_DCLK1: - /* dclk dpm table */ - dpm_table = &dpm_context->dpm_tables.dclk_table; - break; - default: - dev_err(smu->adev->dev, "Unsupported clock type!\n"); - return -EINVAL; - } - - if (min) - *min = dpm_table->min; - if (max) - *max = dpm_table->max; - - return 0; -} - -static int smu_v14_0_2_read_sensor(struct smu_context *smu, - enum amd_pp_sensors sensor, - void *data, - uint32_t *size) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *smc_pptable = table_context->driver_pptable; - int ret = 0; - - switch (sensor) { - case AMDGPU_PP_SENSOR_MAX_FAN_RPM: - *(uint16_t *)data = smc_pptable->CustomSkuTable.FanMaximumRpm; - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_MEMACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_LOAD: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXACTIVITY, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GPU_AVG_POWER: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_SOCKETPOWER, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_HOTSPOT, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_EDGE_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_EDGE, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_TEMPERATURE_MEM, - (uint32_t *)data); - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_MCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_CURR_UCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_AVERAGE_GFXCLK, - (uint32_t *)data); - *(uint32_t *)data *= 100; - *size = 4; - break; - case AMDGPU_PP_SENSOR_VDDGFX: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_VOLTAGE_VDDGFX, - (uint32_t *)data); - *size = 4; - break; - default: - ret = -EOPNOTSUPP; - break; - } - - return ret; -} - -static int smu_v14_0_2_get_current_clk_freq_by_table(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t *value) -{ - MetricsMember_t member_type; - int clk_id = 0; - - clk_id = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_CLK, - clk_type); - if (clk_id < 0) - return -EINVAL; - - switch (clk_id) { - case PPCLK_GFXCLK: - member_type = METRICS_AVERAGE_GFXCLK; - break; - case PPCLK_UCLK: - member_type = METRICS_CURR_UCLK; - break; - case PPCLK_FCLK: - member_type = METRICS_CURR_FCLK; - break; - case PPCLK_SOCCLK: - member_type = METRICS_CURR_SOCCLK; - break; - case PPCLK_VCLK_0: - member_type = METRICS_AVERAGE_VCLK; - break; - case PPCLK_DCLK_0: - member_type = METRICS_AVERAGE_DCLK; - break; - case PPCLK_DCFCLK: - member_type = METRICS_CURR_DCEFCLK; - break; - default: - return -EINVAL; - } - - return smu_v14_0_2_get_smu_metrics_data(smu, - member_type, - value); -} - -static bool smu_v14_0_2_is_od_feature_supported(struct smu_context *smu, - int od_feature_bit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - - return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); -} - -static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, - int od_feature_bit, - int32_t *min, - int32_t *max) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - const OverDriveLimits_t * const overdrive_upperlimits = - &pptable->SkuTable.OverDriveLimitsBasicMax; - const OverDriveLimits_t * const overdrive_lowerlimits = - &pptable->SkuTable.OverDriveLimitsBasicMin; - int32_t od_min_setting, od_max_setting; - - switch (od_feature_bit) { - case PP_OD_FEATURE_GFXCLK_FMIN: - od_min_setting = overdrive_lowerlimits->GfxclkFmin; - od_max_setting = overdrive_upperlimits->GfxclkFmin; - break; - case PP_OD_FEATURE_GFXCLK_FMAX: - od_min_setting = overdrive_lowerlimits->GfxclkFmax; - od_max_setting = overdrive_upperlimits->GfxclkFmax; - break; - case PP_OD_FEATURE_UCLK_FMIN: - od_min_setting = overdrive_lowerlimits->UclkFmin; - od_max_setting = overdrive_upperlimits->UclkFmin; - break; - case PP_OD_FEATURE_UCLK_FMAX: - od_min_setting = overdrive_lowerlimits->UclkFmax; - od_max_setting = overdrive_upperlimits->UclkFmax; - break; - case PP_OD_FEATURE_GFX_VF_CURVE: - od_min_setting = overdrive_lowerlimits->VoltageOffsetPerZoneBoundary[0]; - od_max_setting = overdrive_upperlimits->VoltageOffsetPerZoneBoundary[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_TEMP: - od_min_setting = overdrive_lowerlimits->FanLinearTempPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearTempPoints[0]; - break; - case PP_OD_FEATURE_FAN_CURVE_PWM: - od_min_setting = overdrive_lowerlimits->FanLinearPwmPoints[0]; - od_max_setting = overdrive_upperlimits->FanLinearPwmPoints[0]; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT: - od_min_setting = overdrive_lowerlimits->AcousticLimitRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticLimitRpmThreshold; - break; - case PP_OD_FEATURE_FAN_ACOUSTIC_TARGET: - od_min_setting = overdrive_lowerlimits->AcousticTargetRpmThreshold; - od_max_setting = overdrive_upperlimits->AcousticTargetRpmThreshold; - break; - case PP_OD_FEATURE_FAN_TARGET_TEMPERATURE: - od_min_setting = overdrive_lowerlimits->FanTargetTemperature; - od_max_setting = overdrive_upperlimits->FanTargetTemperature; - break; - case PP_OD_FEATURE_FAN_MINIMUM_PWM: - od_min_setting = overdrive_lowerlimits->FanMinimumPwm; - od_max_setting = overdrive_upperlimits->FanMinimumPwm; - break; - default: - od_min_setting = od_max_setting = INT_MAX; - break; - } - - if (min) - *min = od_min_setting; - if (max) - *max = od_max_setting; -} - -static int smu_v14_0_2_print_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - char *buf) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - struct smu_14_0_dpm_table *single_dpm_table; - struct smu_14_0_pcie_table *pcie_table; - uint32_t gen_speed, lane_width; - int i, curr_freq, size = 0; - int32_t min_value, max_value; - int ret = 0; - - smu_cmn_get_sysfs_buf(&buf, &size); - - if (amdgpu_ras_intr_triggered()) { - size += sysfs_emit_at(buf, size, "unavailable\n"); - return size; - } - - switch (clk_type) { - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - case SMU_DCEFCLK: - single_dpm_table = &(dpm_context->dpm_tables.dcef_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_SCLK: - case SMU_MCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - case SMU_DCEFCLK: - ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq); - if (ret) { - dev_err(smu->adev->dev, "Failed to get current clock freq!"); - return ret; - } - - if (single_dpm_table->is_fine_grained) { - /* - * For fine grained dpms, there are only two dpm levels: - * - level 0 -> min clock freq - * - level 1 -> max clock freq - * And the current clock frequency can be any value between them. - * So, if the current clock frequency is not at level 0 or level 1, - * we will fake it as three dpm levels: - * - level 0 -> min clock freq - * - level 1 -> current actual clock freq - * - level 2 -> max clock freq - */ - if ((single_dpm_table->dpm_levels[0].value != curr_freq) && - (single_dpm_table->dpm_levels[1].value != curr_freq)) { - size += sysfs_emit_at(buf, size, "0: %uMhz\n", - single_dpm_table->dpm_levels[0].value); - size += sysfs_emit_at(buf, size, "1: %uMhz *\n", - curr_freq); - size += sysfs_emit_at(buf, size, "2: %uMhz\n", - single_dpm_table->dpm_levels[1].value); - } else { - size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", - single_dpm_table->dpm_levels[0].value, - single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : ""); - size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", - single_dpm_table->dpm_levels[1].value, - single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : ""); - } - } else { - for (i = 0; i < single_dpm_table->count; i++) - size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", - i, single_dpm_table->dpm_levels[i].value, - single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : ""); - } - break; - case SMU_PCIE: - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_RATE, - &gen_speed); - if (ret) - return ret; - - ret = smu_v14_0_2_get_smu_metrics_data(smu, - METRICS_PCIE_WIDTH, - &lane_width); - if (ret) - return ret; - - pcie_table = &(dpm_context->dpm_tables.pcie_table); - for (i = 0; i < pcie_table->num_of_link_levels; i++) - size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, - (pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," : - (pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," : - (pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," : - (pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "", - (pcie_table->pcie_lane[i] == 1) ? "x1" : - (pcie_table->pcie_lane[i] == 2) ? "x2" : - (pcie_table->pcie_lane[i] == 3) ? "x4" : - (pcie_table->pcie_lane[i] == 4) ? "x8" : - (pcie_table->pcie_lane[i] == 5) ? "x12" : - (pcie_table->pcie_lane[i] == 6) ? "x16" : "", - pcie_table->clk_freq[i], - (gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) && - (lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ? - "*" : ""); - break; - - case SMU_OD_SCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFXCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", - od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - break; - - case SMU_OD_MCLK: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_UCLK_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); - size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMHz\n", - od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); - break; - - case SMU_OD_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_VDDGFX_OFFSET:\n"); - size += sysfs_emit_at(buf, size, "%dmV\n", - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[0]); - break; - - case SMU_OD_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_FAN_CURVE:\n"); - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) - size += sysfs_emit_at(buf, size, "%d: %dC %d%%\n", - i, - (int)od_table->OverDriveTable.FanLinearTempPoints[i], - (int)od_table->OverDriveTable.FanLinearPwmPoints[i]); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(hotspot temp): %uC %uC\n", - min_value, max_value); - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "FAN_CURVE(fan speed): %u%% %u%%\n", - min_value, max_value); - - break; - - case SMU_OD_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_LIMIT:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticLimitRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_LIMIT: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "OD_ACOUSTIC_TARGET:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.AcousticTargetRpmThreshold); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "ACOUSTIC_TARGET: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_TARGET_TEMPERATURE:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanTargetTemperature); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "TARGET_TEMPERATURE: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "FAN_MINIMUM_PWM:\n"); - size += sysfs_emit_at(buf, size, "%d\n", - (int)od_table->OverDriveTable.FanMinimumPwm); - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "MINIMUM_PWM: %u %u\n", - min_value, max_value); - break; - - case SMU_OD_RANGE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT) && - !smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) - break; - - size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &min_value, - NULL); - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - NULL, - &max_value); - size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", - min_value, max_value); - } - - if (smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &min_value, - &max_value); - size += sysfs_emit_at(buf, size, "VDDGFX_OFFSET: %7dmv %10dmv\n", - min_value, max_value); - } - break; - - default: - break; - } - - return size; -} - -static int smu_v14_0_2_force_clk_levels(struct smu_context *smu, - enum smu_clk_type clk_type, - uint32_t mask) -{ - struct smu_dpm_context *smu_dpm = &smu->smu_dpm; - struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context; - struct smu_14_0_dpm_table *single_dpm_table; - uint32_t soft_min_level, soft_max_level; - uint32_t min_freq, max_freq; - int ret = 0; - - soft_min_level = mask ? (ffs(mask) - 1) : 0; - soft_max_level = mask ? (fls(mask) - 1) : 0; - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - single_dpm_table = &(dpm_context->dpm_tables.gfx_table); - break; - case SMU_MCLK: - case SMU_UCLK: - single_dpm_table = &(dpm_context->dpm_tables.uclk_table); - break; - case SMU_SOCCLK: - single_dpm_table = &(dpm_context->dpm_tables.soc_table); - break; - case SMU_FCLK: - single_dpm_table = &(dpm_context->dpm_tables.fclk_table); - break; - case SMU_VCLK: - case SMU_VCLK1: - single_dpm_table = &(dpm_context->dpm_tables.vclk_table); - break; - case SMU_DCLK: - case SMU_DCLK1: - single_dpm_table = &(dpm_context->dpm_tables.dclk_table); - break; - default: - break; - } - - switch (clk_type) { - case SMU_GFXCLK: - case SMU_SCLK: - case SMU_MCLK: - case SMU_UCLK: - case SMU_SOCCLK: - case SMU_FCLK: - case SMU_VCLK: - case SMU_VCLK1: - case SMU_DCLK: - case SMU_DCLK1: - if (single_dpm_table->is_fine_grained) { - /* There is only 2 levels for fine grained DPM */ - soft_max_level = (soft_max_level >= 1 ? 1 : 0); - soft_min_level = (soft_min_level >= 1 ? 1 : 0); - } else { - if ((soft_max_level >= single_dpm_table->count) || - (soft_min_level >= single_dpm_table->count)) - return -EINVAL; - } - - min_freq = single_dpm_table->dpm_levels[soft_min_level].value; - max_freq = single_dpm_table->dpm_levels[soft_max_level].value; - - ret = smu_v14_0_set_soft_freq_limited_range(smu, - clk_type, - min_freq, - max_freq); - break; - case SMU_DCEFCLK: - case SMU_PCIE: - default: - break; - } - - return ret; -} - -static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu, - uint8_t pcie_gen_cap, - uint8_t pcie_width_cap) -{ - struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; - struct smu_14_0_pcie_table *pcie_table = - &dpm_context->dpm_tables.pcie_table; - uint32_t smu_pcie_arg; - int ret, i; - - for (i = 0; i < pcie_table->num_of_link_levels; i++) { - if (pcie_table->pcie_gen[i] > pcie_gen_cap) - pcie_table->pcie_gen[i] = pcie_gen_cap; - if (pcie_table->pcie_lane[i] > pcie_width_cap) - pcie_table->pcie_lane[i] = pcie_width_cap; - - smu_pcie_arg = i << 16; - smu_pcie_arg |= pcie_table->pcie_gen[i] << 8; - smu_pcie_arg |= pcie_table->pcie_lane[i]; - - ret = smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_OverridePcieParameters, - smu_pcie_arg, - NULL); - if (ret) - return ret; - } - - return 0; -} - -static const struct smu_temperature_range smu14_thermal_policy[] = { - {-273150, 99000, 99000, -273150, 99000, 99000, -273150, 99000, 99000}, - { 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000, 120000}, -}; - -static int smu_v14_0_2_get_thermal_temperature_range(struct smu_context *smu, - struct smu_temperature_range *range) -{ - struct smu_table_context *table_context = &smu->smu_table; - struct smu_14_0_2_powerplay_table *powerplay_table = - table_context->power_play_table; - PPTable_t *pptable = smu->smu_table.driver_pptable; - - if (amdgpu_sriov_vf(smu->adev)) - return 0; - - if (!range) - return -EINVAL; - - memcpy(range, &smu14_thermal_policy[0], sizeof(struct smu_temperature_range)); - - range->max = pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->edge_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_EDGE] + CTF_OFFSET_EDGE) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->hotspot_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_HOTSPOT] + CTF_OFFSET_HOTSPOT) * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_crit_max = pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] * - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->mem_emergency_max = (pptable->CustomSkuTable.TemperatureLimit[TEMP_MEM] + CTF_OFFSET_MEM)* - SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; - range->software_shutdown_temp = powerplay_table->software_shutdown_temp; - range->software_shutdown_temp_offset = pptable->CustomSkuTable.FanAbnormalTempLimitOffset; - - return 0; -} - -static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu) -{ - struct smu_14_0_dpm_context *dpm_context = - smu->smu_dpm.dpm_context; - struct smu_14_0_dpm_table *gfx_table = - &dpm_context->dpm_tables.gfx_table; - struct smu_14_0_dpm_table *mem_table = - &dpm_context->dpm_tables.uclk_table; - struct smu_14_0_dpm_table *soc_table = - &dpm_context->dpm_tables.soc_table; - struct smu_14_0_dpm_table *vclk_table = - &dpm_context->dpm_tables.vclk_table; - struct smu_14_0_dpm_table *dclk_table = - &dpm_context->dpm_tables.dclk_table; - struct smu_14_0_dpm_table *fclk_table = - &dpm_context->dpm_tables.fclk_table; - struct smu_umd_pstate_table *pstate_table = - &smu->pstate_table; - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - DriverReportedClocks_t driver_clocks = - pptable->SkuTable.DriverReportedClocks; - - pstate_table->gfxclk_pstate.min = gfx_table->min; - if (driver_clocks.GameClockAc && - (driver_clocks.GameClockAc < gfx_table->max)) - pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc; - else - pstate_table->gfxclk_pstate.peak = gfx_table->max; - - pstate_table->uclk_pstate.min = mem_table->min; - pstate_table->uclk_pstate.peak = mem_table->max; - - pstate_table->socclk_pstate.min = soc_table->min; - pstate_table->socclk_pstate.peak = soc_table->max; - - pstate_table->vclk_pstate.min = vclk_table->min; - pstate_table->vclk_pstate.peak = vclk_table->max; - - pstate_table->dclk_pstate.min = dclk_table->min; - pstate_table->dclk_pstate.peak = dclk_table->max; - - pstate_table->fclk_pstate.min = fclk_table->min; - pstate_table->fclk_pstate.peak = fclk_table->max; - - if (driver_clocks.BaseClockAc && - driver_clocks.BaseClockAc < gfx_table->max) - pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc; - else - pstate_table->gfxclk_pstate.standard = gfx_table->max; - pstate_table->uclk_pstate.standard = mem_table->max; - pstate_table->socclk_pstate.standard = soc_table->min; - pstate_table->vclk_pstate.standard = vclk_table->min; - pstate_table->dclk_pstate.standard = dclk_table->min; - pstate_table->fclk_pstate.standard = fclk_table->min; - - return 0; -} - -static void smu_v14_0_2_get_unique_id(struct smu_context *smu) -{ - struct smu_table_context *smu_table = &smu->smu_table; - SmuMetrics_t *metrics = - &(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics); - struct amdgpu_device *adev = smu->adev; - uint32_t upper32 = 0, lower32 = 0; - int ret; - - ret = smu_cmn_get_metrics_table(smu, NULL, false); - if (ret) - goto out; - - upper32 = metrics->PublicSerialNumberUpper; - lower32 = metrics->PublicSerialNumberLower; - -out: - adev->unique_id = ((uint64_t)upper32 << 32) | lower32; -} - -static int smu_v14_0_2_get_power_limit(struct smu_context *smu, - uint32_t *current_power_limit, - uint32_t *default_power_limit, - uint32_t *max_power_limit, - uint32_t *min_power_limit) -{ - struct smu_table_context *table_context = &smu->smu_table; - PPTable_t *pptable = table_context->driver_pptable; - CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - uint32_t power_limit; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - - if (current_power_limit) - *current_power_limit = power_limit; - if (default_power_limit) - *default_power_limit = power_limit; - - if (max_power_limit) - *max_power_limit = msg_limit; - - if (min_power_limit) - *min_power_limit = 0; - - return 0; -} - -static int smu_v14_0_2_get_power_profile_mode(struct smu_context *smu, - char *buf) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - static const char *title[] = { - "PROFILE_INDEX(NAME)", - "CLOCK_TYPE(NAME)", - "FPS", - "MinActiveFreqType", - "MinActiveFreq", - "BoosterFreqType", - "BoosterFreq", - "PD_Data_limit_c", - "PD_Data_error_coeff", - "PD_Data_error_rate_coeff"}; - int16_t workload_type = 0; - uint32_t i, size = 0; - int result = 0; - - if (!buf) - return -EINVAL; - - size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s\n", - title[0], title[1], title[2], title[3], title[4], title[5], - title[6], title[7], title[8], title[9]); - - for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) { - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - i); - if (workload_type == -ENOTSUPP) - continue; - else if (workload_type < 0) - return -EINVAL; - - result = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - workload_type, - (void *)(&activity_monitor_external), - false); - if (result) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return result; - } - - size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", - i, amdgpu_pp_profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 0, - "GFXCLK", - activity_monitor->Gfx_FPS, - activity_monitor->Gfx_MinActiveFreqType, - activity_monitor->Gfx_MinActiveFreq, - activity_monitor->Gfx_BoosterFreqType, - activity_monitor->Gfx_BoosterFreq, - activity_monitor->Gfx_PD_Data_limit_c, - activity_monitor->Gfx_PD_Data_error_coeff, - activity_monitor->Gfx_PD_Data_error_rate_coeff); - - size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n", - " ", - 1, - "FCLK", - activity_monitor->Fclk_FPS, - activity_monitor->Fclk_MinActiveFreqType, - activity_monitor->Fclk_MinActiveFreq, - activity_monitor->Fclk_BoosterFreqType, - activity_monitor->Fclk_BoosterFreq, - activity_monitor->Fclk_PD_Data_limit_c, - activity_monitor->Fclk_PD_Data_error_coeff, - activity_monitor->Fclk_PD_Data_error_rate_coeff); - } - - return size; -} - -static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu, - long *input, - uint32_t size) -{ - DpmActivityMonitorCoeffIntExternal_t activity_monitor_external; - DpmActivityMonitorCoeffInt_t *activity_monitor = - &(activity_monitor_external.DpmActivityMonitorCoeffInt); - int workload_type, ret = 0; - - smu->power_profile_mode = input[size]; - - if (smu->power_profile_mode >= PP_SMC_POWER_PROFILE_COUNT) { - dev_err(smu->adev->dev, "Invalid power profile mode %d\n", smu->power_profile_mode); - return -EINVAL; - } - - if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) { - if (size != 9) - return -EINVAL; - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - false); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__); - return ret; - } - - switch (input[0]) { - case 0: /* Gfxclk */ - activity_monitor->Gfx_FPS = input[1]; - activity_monitor->Gfx_MinActiveFreqType = input[2]; - activity_monitor->Gfx_MinActiveFreq = input[3]; - activity_monitor->Gfx_BoosterFreqType = input[4]; - activity_monitor->Gfx_BoosterFreq = input[5]; - activity_monitor->Gfx_PD_Data_limit_c = input[6]; - activity_monitor->Gfx_PD_Data_error_coeff = input[7]; - activity_monitor->Gfx_PD_Data_error_rate_coeff = input[8]; - break; - case 1: /* Fclk */ - activity_monitor->Fclk_FPS = input[1]; - activity_monitor->Fclk_MinActiveFreqType = input[2]; - activity_monitor->Fclk_MinActiveFreq = input[3]; - activity_monitor->Fclk_BoosterFreqType = input[4]; - activity_monitor->Fclk_BoosterFreq = input[5]; - activity_monitor->Fclk_PD_Data_limit_c = input[6]; - activity_monitor->Fclk_PD_Data_error_coeff = input[7]; - activity_monitor->Fclk_PD_Data_error_rate_coeff = input[8]; - break; - default: - return -EINVAL; - } - - ret = smu_cmn_update_table(smu, - SMU_TABLE_ACTIVITY_MONITOR_COEFF, - WORKLOAD_PPLIB_CUSTOM_BIT, - (void *)(&activity_monitor_external), - true); - if (ret) { - dev_err(smu->adev->dev, "[%s] Failed to set activity monitor!", __func__); - return ret; - } - } - - /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ - workload_type = smu_cmn_to_asic_specific_index(smu, - CMN2ASIC_MAPPING_WORKLOAD, - smu->power_profile_mode); - if (workload_type < 0) - return -EINVAL; - - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_SetWorkloadMask, - 1 << workload_type, - NULL); -} - -static int smu_v14_0_2_baco_enter(struct smu_context *smu) -{ - struct smu_baco_context *smu_baco = &smu->smu_baco; - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) - return smu_v14_0_baco_set_armd3_sequence(smu, - smu_baco->maco_support ? BACO_SEQ_BAMACO : BACO_SEQ_BACO); - else - return smu_v14_0_baco_enter(smu); -} - -static int smu_v14_0_2_baco_exit(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) { - /* Wait for PMFW handling for the Dstate change */ - usleep_range(10000, 11000); - return smu_v14_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); - } else { - return smu_v14_0_baco_exit(smu); - } -} - -static bool smu_v14_0_2_is_mode1_reset_supported(struct smu_context *smu) -{ - // TODO - - return true; -} - -static int smu_v14_0_2_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msg, int num_msgs) -{ - struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = smu_i2c->adev; - struct smu_context *smu = adev->powerplay.pp_handle; - struct smu_table_context *smu_table = &smu->smu_table; - struct smu_table *table = &smu_table->driver_table; - SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr; - int i, j, r, c; - u16 dir; - - if (!adev->pm.dpm_enabled) - return -EBUSY; - - req = kzalloc(sizeof(*req), GFP_KERNEL); - if (!req) - return -ENOMEM; - - req->I2CcontrollerPort = smu_i2c->port; - req->I2CSpeed = I2C_SPEED_FAST_400K; - req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */ - dir = msg[0].flags & I2C_M_RD; - - for (c = i = 0; i < num_msgs; i++) { - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &req->SwI2cCmds[c]; - - if (!(msg[i].flags & I2C_M_RD)) { - /* write */ - cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK; - cmd->ReadWriteData = msg[i].buf[j]; - } - - if ((dir ^ msg[i].flags) & I2C_M_RD) { - /* The direction changes. - */ - dir = msg[i].flags & I2C_M_RD; - cmd->CmdConfig |= CMDCONFIG_RESTART_MASK; - } - - req->NumCmds++; - - /* - * Insert STOP if we are at the last byte of either last - * message for the transaction or the client explicitly - * requires a STOP at this particular message. - */ - if ((j == msg[i].len - 1) && - ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) { - cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK; - cmd->CmdConfig |= CMDCONFIG_STOP_MASK; - } - } - } - mutex_lock(&adev->pm.mutex); - r = smu_cmn_update_table(smu, SMU_TABLE_I2C_COMMANDS, 0, req, true); - mutex_unlock(&adev->pm.mutex); - if (r) - goto fail; - - for (c = i = 0; i < num_msgs; i++) { - if (!(msg[i].flags & I2C_M_RD)) { - c += msg[i].len; - continue; - } - for (j = 0; j < msg[i].len; j++, c++) { - SwI2cCmd_t *cmd = &res->SwI2cCmds[c]; - - msg[i].buf[j] = cmd->ReadWriteData; - } - } - r = num_msgs; -fail: - kfree(req); - return r; -} - -static u32 smu_v14_0_2_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm smu_v14_0_2_i2c_algo = { - .master_xfer = smu_v14_0_2_i2c_xfer, - .functionality = smu_v14_0_2_i2c_func, -}; - -static const struct i2c_adapter_quirks smu_v14_0_2_i2c_control_quirks = { - .flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN, - .max_read_len = MAX_SW_I2C_COMMANDS, - .max_write_len = MAX_SW_I2C_COMMANDS, - .max_comb_1st_msg_len = 2, - .max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2, -}; - -static int smu_v14_0_2_i2c_control_init(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int res, i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - smu_i2c->adev = adev; - smu_i2c->port = i; - mutex_init(&smu_i2c->mutex); - control->owner = THIS_MODULE; - control->dev.parent = &adev->pdev->dev; - control->algo = &smu_v14_0_2_i2c_algo; - snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i); - control->quirks = &smu_v14_0_2_i2c_control_quirks; - i2c_set_adapdata(control, smu_i2c); - - res = i2c_add_adapter(control); - if (res) { - DRM_ERROR("Failed to register hw i2c, err: %d\n", res); - goto Out_err; - } - } - - /* assign the buses used for the FRU EEPROM and RAS EEPROM */ - /* XXX ideally this would be something in a vbios data table */ - adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[1].adapter; - adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter; - - return 0; -Out_err: - for ( ; i >= 0; i--) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - return res; -} - -static void smu_v14_0_2_i2c_control_fini(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - int i; - - for (i = 0; i < MAX_SMU_I2C_BUSES; i++) { - struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i]; - struct i2c_adapter *control = &smu_i2c->adapter; - - i2c_del_adapter(control); - } - adev->pm.ras_eeprom_i2c_bus = NULL; - adev->pm.fru_eeprom_i2c_bus = NULL; -} - -static int smu_v14_0_2_set_mp1_state(struct smu_context *smu, - enum pp_mp1_state mp1_state) -{ - int ret; - - switch (mp1_state) { - case PP_MP1_STATE_UNLOAD: - ret = smu_cmn_set_mp1_state(smu, mp1_state); - break; - default: - /* Ignore others */ - ret = 0; - } - - return ret; -} - -static int smu_v14_0_2_set_df_cstate(struct smu_context *smu, - enum pp_df_cstate state) -{ - return smu_cmn_send_smc_msg_with_param(smu, - SMU_MSG_DFCstateControl, - state, - NULL); -} - -static int smu_v14_0_2_mode1_reset(struct smu_context *smu) -{ - int ret = 0; - - ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); - if (!ret) { - if (amdgpu_emu_mode == 1) - msleep(50000); - else - msleep(1000); - } - - return ret; -} - -static int smu_v14_0_2_mode2_reset(struct smu_context *smu) -{ - int ret = 0; - - // TODO - - return ret; -} - -static int smu_v14_0_2_enable_gfx_features(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(14, 0, 2)) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnableAllSmuFeatures, - FEATURE_PWR_GFX, NULL); - else - return -EOPNOTSUPP; -} - -static void smu_v14_0_2_set_smu_mailbox_registers(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - smu->param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_82); - smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_66); - smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_90); - - smu->debug_param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_53); - smu->debug_msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_75); - smu->debug_resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_54); -} - -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); -} - -static void smu_v14_0_2_dump_od_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - struct amdgpu_device *adev = smu->adev; - - dev_dbg(adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->OverDriveTable.GfxclkFmin, - od_table->OverDriveTable.GfxclkFmax); - dev_dbg(adev->dev, "OD: Uclk: (%d, %d)\n", od_table->OverDriveTable.UclkFmin, - od_table->OverDriveTable.UclkFmax); -} - -static int smu_v14_0_2_upload_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - true); - if (ret) - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - - return ret; -} - -static void smu_v14_0_2_set_supported_od_feature_mask(struct smu_context *smu) -{ - struct amdgpu_device *adev = smu->adev; - - if (smu_v14_0_2_is_od_feature_supported(smu, - PP_OD_FEATURE_FAN_CURVE_BIT)) - adev->pm.od_feature_mask |= OD_OPS_SUPPORT_FAN_CURVE_RETRIEVE | - OD_OPS_SUPPORT_FAN_CURVE_SET | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_LIMIT_THRESHOLD_SET | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_RETRIEVE | - OD_OPS_SUPPORT_ACOUSTIC_TARGET_THRESHOLD_SET | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_RETRIEVE | - OD_OPS_SUPPORT_FAN_TARGET_TEMPERATURE_SET | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_RETRIEVE | - OD_OPS_SUPPORT_FAN_MINIMUM_PWM_SET; -} - -static int smu_v14_0_2_get_overdrive_table(struct smu_context *smu, - OverDriveTableExternal_t *od_table) -{ - int ret; - ret = smu_cmn_update_table(smu, - SMU_TABLE_OVERDRIVE, - 0, - (void *)od_table, - false); - if (ret) - dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); - - return ret; -} - -static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu) -{ - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)smu->smu_table.overdrive_table; - OverDriveTableExternal_t *boot_od_table = - (OverDriveTableExternal_t *)smu->smu_table.boot_overdrive_table; - OverDriveTableExternal_t *user_od_table = - (OverDriveTableExternal_t *)smu->smu_table.user_overdrive_table; - OverDriveTableExternal_t user_od_table_bak; - int ret; - int i; - - ret = smu_v14_0_2_get_overdrive_table(smu, boot_od_table); - if (ret) - return ret; - - smu_v14_0_2_dump_od_table(smu, boot_od_table); - - memcpy(od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - - /* - * For S3/S4/Runpm resume, we need to setup those overdrive tables again, - * but we have to preserve user defined values in "user_od_table". - */ - if (!smu->adev->in_suspend) { - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - smu->user_dpm_profile.user_od = false; - } else if (smu->user_dpm_profile.user_od) { - memcpy(&user_od_table_bak, - user_od_table, - sizeof(OverDriveTableExternal_t)); - memcpy(user_od_table, - boot_od_table, - sizeof(OverDriveTableExternal_t)); - user_od_table->OverDriveTable.GfxclkFmin = - user_od_table_bak.OverDriveTable.GfxclkFmin; - user_od_table->OverDriveTable.GfxclkFmax = - user_od_table_bak.OverDriveTable.GfxclkFmax; - user_od_table->OverDriveTable.UclkFmin = - user_od_table_bak.OverDriveTable.UclkFmin; - user_od_table->OverDriveTable.UclkFmax = - user_od_table_bak.OverDriveTable.UclkFmax; - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - user_od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = - user_od_table_bak.OverDriveTable.VoltageOffsetPerZoneBoundary[i]; - for (i = 0; i < NUM_OD_FAN_MAX_POINTS - 1; i++) { - user_od_table->OverDriveTable.FanLinearTempPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearTempPoints[i]; - user_od_table->OverDriveTable.FanLinearPwmPoints[i] = - user_od_table_bak.OverDriveTable.FanLinearPwmPoints[i]; - } - user_od_table->OverDriveTable.AcousticLimitRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticLimitRpmThreshold; - user_od_table->OverDriveTable.AcousticTargetRpmThreshold = - user_od_table_bak.OverDriveTable.AcousticTargetRpmThreshold; - user_od_table->OverDriveTable.FanTargetTemperature = - user_od_table_bak.OverDriveTable.FanTargetTemperature; - user_od_table->OverDriveTable.FanMinimumPwm = - user_od_table_bak.OverDriveTable.FanMinimumPwm; - } - - smu_v14_0_2_set_supported_od_feature_mask(smu); - - return 0; -} - -static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = table_context->overdrive_table; - OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; - int res; - - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); - user_od_table->OverDriveTable.FeatureCtrlMask = 0; - if (res == 0) - memcpy(od_table, user_od_table, sizeof(OverDriveTableExternal_t)); - - return res; -} - -static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long input) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *boot_overdrive_table = - (OverDriveTableExternal_t *)table_context->boot_overdrive_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - int i; - - switch (input) { - case PP_OD_EDIT_FAN_CURVE: - for (i = 0; i < NUM_OD_FAN_MAX_POINTS; i++) { - od_table->OverDriveTable.FanLinearTempPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearTempPoints[i]; - od_table->OverDriveTable.FanLinearPwmPoints[i] = - boot_overdrive_table->OverDriveTable.FanLinearPwmPoints[i]; - } - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_LIMIT: - od_table->OverDriveTable.AcousticLimitRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticLimitRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_ACOUSTIC_TARGET: - od_table->OverDriveTable.AcousticTargetRpmThreshold = - boot_overdrive_table->OverDriveTable.AcousticTargetRpmThreshold; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - od_table->OverDriveTable.FanTargetTemperature = - boot_overdrive_table->OverDriveTable.FanTargetTemperature; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - case PP_OD_EDIT_FAN_MINIMUM_PWM: - od_table->OverDriveTable.FanMinimumPwm = - boot_overdrive_table->OverDriveTable.FanMinimumPwm; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - default: - dev_info(adev->dev, "Invalid table index: %ld\n", input); - return -EINVAL; - } - - return 0; -} - -static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, - enum PP_OD_DPM_TABLE_COMMAND type, - long input[], - uint32_t size) -{ - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - struct amdgpu_device *adev = smu->adev; - uint32_t offset_of_voltageoffset; - int32_t minimum, maximum; - uint32_t feature_ctrlmask; - int i, ret = 0; - - switch (type) { - case PP_OD_EDIT_SCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFXCLK_BIT)) { - dev_warn(adev->dev, "GFXCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFXCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "GfxclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.GfxclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_GFXCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.GfxclkFmin > od_table->OverDriveTable.GfxclkFmax) { - dev_err(adev->dev, - "Invalid setting: GfxclkFmin(%u) is bigger than GfxclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.GfxclkFmin, - (uint32_t)od_table->OverDriveTable.GfxclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_MCLK_VDDC_TABLE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_UCLK_BIT)) { - dev_warn(adev->dev, "UCLK_LIMITS setting not supported!\n"); - return -ENOTSUPP; - } - - for (i = 0; i < size; i += 2) { - if (i + 2 > size) { - dev_info(adev->dev, "invalid number of input parameters %d\n", size); - return -EINVAL; - } - - switch (input[i]) { - case 0: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMIN, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmin (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmin = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - case 1: - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_UCLK_FMAX, - &minimum, - &maximum); - if (input[i + 1] < minimum || - input[i + 1] > maximum) { - dev_info(adev->dev, "UclkFmax (%ld) must be within [%u, %u]!\n", - input[i + 1], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.UclkFmax = input[i + 1]; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_UCLK_BIT; - break; - - default: - dev_info(adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); - dev_info(adev->dev, "Supported indices: [0:min,1:max]\n"); - return -EINVAL; - } - } - - if (od_table->OverDriveTable.UclkFmin > od_table->OverDriveTable.UclkFmax) { - dev_err(adev->dev, - "Invalid setting: UclkFmin(%u) is bigger than UclkFmax(%u)\n", - (uint32_t)od_table->OverDriveTable.UclkFmin, - (uint32_t)od_table->OverDriveTable.UclkFmax); - return -EINVAL; - } - break; - - case PP_OD_EDIT_VDDGFX_OFFSET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_GFX_VF_CURVE_BIT)) { - dev_warn(adev->dev, "Gfx offset setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_GFX_VF_CURVE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "Voltage offset (%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - for (i = 0; i < PP_NUM_OD_VF_CURVE_POINTS; i++) - od_table->OverDriveTable.VoltageOffsetPerZoneBoundary[i] = input[0]; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_CURVE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - if (input[0] >= NUM_OD_FAN_MAX_POINTS - 1 || - input[0] < 0) - return -EINVAL; - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_TEMP, - &minimum, - &maximum); - if (input[1] < minimum || - input[1] > maximum) { - dev_info(adev->dev, "Fan curve temp setting(%ld) must be within [%d, %d]!\n", - input[1], minimum, maximum); - return -EINVAL; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_CURVE_PWM, - &minimum, - &maximum); - if (input[2] < minimum || - input[2] > maximum) { - dev_info(adev->dev, "Fan curve pwm setting(%ld) must be within [%d, %d]!\n", - input[2], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanLinearTempPoints[input[0]] = input[1]; - od_table->OverDriveTable.FanLinearPwmPoints[input[0]] = input[2]; - od_table->OverDriveTable.FanMode = FAN_MODE_MANUAL_LINEAR; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_LIMIT: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_LIMIT, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic limit threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticLimitRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_ACOUSTIC_TARGET: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_ACOUSTIC_TARGET, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "acoustic target threshold setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.AcousticTargetRpmThreshold = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_TARGET_TEMPERATURE: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_TARGET_TEMPERATURE, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan target temperature setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanTargetTemperature = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_EDIT_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { - dev_warn(adev->dev, "Fan curve setting not supported!\n"); - return -ENOTSUPP; - } - - smu_v14_0_2_get_od_setting_limits(smu, - PP_OD_FEATURE_FAN_MINIMUM_PWM, - &minimum, - &maximum); - if (input[0] < minimum || - input[0] > maximum) { - dev_info(adev->dev, "fan minimum pwm setting(%ld) must be within [%d, %d]!\n", - input[0], minimum, maximum); - return -EINVAL; - } - - od_table->OverDriveTable.FanMinimumPwm = input[0]; - od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); - break; - - case PP_OD_RESTORE_DEFAULT_TABLE: - if (size == 1) { - ret = smu_v14_0_2_od_restore_table_single(smu, input[0]); - if (ret) - return ret; - } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; - memcpy(od_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; - } - fallthrough; - case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ - offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); - if (memcmp((u8 *)od_table + offset_of_voltageoffset, - table_context->user_overdrive_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset)) { - smu_v14_0_2_dump_od_table(smu, od_table); - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - od_table->OverDriveTable.FeatureCtrlMask = 0; - memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, - (u8 *)od_table + offset_of_voltageoffset, - sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); - - if (!memcmp(table_context->user_overdrive_table, - table_context->boot_overdrive_table, - sizeof(OverDriveTableExternal_t))) - smu->user_dpm_profile.user_od = false; - else - smu->user_dpm_profile.user_od = true; - } - break; - - default: - return -ENOSYS; - } - - return ret; -} - -<<<<<<< -static int smu_v14_0_2_set_power_limit(struct smu_context *smu, - enum smu_ppt_limit_type limit_type, - uint32_t limit) -{ - PPTable_t *pptable = smu->smu_table.driver_pptable; - uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - struct smu_table_context *table_context = &smu->smu_table; - OverDriveTableExternal_t *od_table = - (OverDriveTableExternal_t *)table_context->overdrive_table; - int ret = 0; - - if (limit_type != SMU_DEFAULT_PPT_LIMIT) - return -EINVAL; - - if (limit <= msg_limit) { - if (smu->current_power_limit > msg_limit) { - od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - } - return smu_v14_0_set_power_limit(smu, limit_type, limit); - } else if (smu->od_enabled) { - ret = smu_v14_0_set_power_limit(smu, limit_type, msg_limit); - if (ret) - return ret; - - od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - - ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); - if (ret) { - dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); - return ret; - } - - smu->current_power_limit = limit; - } else { - return -EINVAL; - } - - return 0; -======= -static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu, - void **table) -{ - struct smu_table_context *smu_table = &smu->smu_table; - struct gpu_metrics_v1_3 *gpu_metrics = - (struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table; - SmuMetricsExternal_t metrics_ext; - SmuMetrics_t *metrics = &metrics_ext.SmuMetrics; - int ret = 0; - - ret = smu_cmn_get_metrics_table(smu, - &metrics_ext, - true); - if (ret) - return ret; - - smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3); - - gpu_metrics->temperature_edge = metrics->AvgTemperature[TEMP_EDGE]; - gpu_metrics->temperature_hotspot = metrics->AvgTemperature[TEMP_HOTSPOT]; - gpu_metrics->temperature_mem = metrics->AvgTemperature[TEMP_MEM]; - gpu_metrics->temperature_vrgfx = metrics->AvgTemperature[TEMP_VR_GFX]; - gpu_metrics->temperature_vrsoc = metrics->AvgTemperature[TEMP_VR_SOC]; - gpu_metrics->temperature_vrmem = max(metrics->AvgTemperature[TEMP_VR_MEM0], - metrics->AvgTemperature[TEMP_VR_MEM1]); - - gpu_metrics->average_gfx_activity = metrics->AverageGfxActivity; - gpu_metrics->average_umc_activity = metrics->AverageUclkActivity; - gpu_metrics->average_mm_activity = max(metrics->Vcn0ActivityPercentage, - metrics->Vcn1ActivityPercentage); - - gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - - if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; - else - gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPreDs; - - if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD) - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPostDs; - else - gpu_metrics->average_uclk_frequency = metrics->AverageMemclkFrequencyPreDs; - - gpu_metrics->average_vclk0_frequency = metrics->AverageVclk0Frequency; - gpu_metrics->average_dclk0_frequency = metrics->AverageDclk0Frequency; - gpu_metrics->average_vclk1_frequency = metrics->AverageVclk1Frequency; - gpu_metrics->average_dclk1_frequency = metrics->AverageDclk1Frequency; - - gpu_metrics->current_gfxclk = gpu_metrics->average_gfxclk_frequency; - gpu_metrics->current_socclk = metrics->CurrClock[PPCLK_SOCCLK]; - gpu_metrics->current_uclk = metrics->CurrClock[PPCLK_UCLK]; - gpu_metrics->current_vclk0 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk0 = metrics->CurrClock[PPCLK_DCLK_0]; - gpu_metrics->current_vclk1 = metrics->CurrClock[PPCLK_VCLK_0]; - gpu_metrics->current_dclk1 = metrics->CurrClock[PPCLK_DCLK_0]; - - gpu_metrics->throttle_status = - smu_v14_0_2_get_throttler_status(metrics); - gpu_metrics->indep_throttle_status = - smu_cmn_get_indep_throttler_status(gpu_metrics->throttle_status, - smu_v14_0_2_throttler_map); - - gpu_metrics->current_fan_speed = metrics->AvgFanRpm; - - gpu_metrics->pcie_link_width = metrics->PcieWidth; - if ((metrics->PcieRate - 1) > LINK_SPEED_MAX) - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(1); - else - gpu_metrics->pcie_link_speed = pcie_gen_to_speed(metrics->PcieRate); - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); - - gpu_metrics->voltage_gfx = metrics->AvgVoltage[SVI_PLANE_VDD_GFX]; - gpu_metrics->voltage_soc = metrics->AvgVoltage[SVI_PLANE_VDD_SOC]; - gpu_metrics->voltage_mem = metrics->AvgVoltage[SVI_PLANE_VDDIO_MEM]; - - *table = (void *)gpu_metrics; - - return sizeof(struct gpu_metrics_v1_3); ->>>>>>> -} - -static const struct pptable_funcs smu_v14_0_2_ppt_funcs = { - .get_allowed_feature_mask = smu_v14_0_2_get_allowed_feature_mask, - .set_default_dpm_table = smu_v14_0_2_set_default_dpm_table, - .i2c_init = smu_v14_0_2_i2c_control_init, - .i2c_fini = smu_v14_0_2_i2c_control_fini, - .is_dpm_running = smu_v14_0_2_is_dpm_running, - .dump_pptable = smu_v14_0_2_dump_pptable, - .init_microcode = smu_v14_0_init_microcode, - .load_microcode = smu_v14_0_load_microcode, - .fini_microcode = smu_v14_0_fini_microcode, - .init_smc_tables = smu_v14_0_2_init_smc_tables, - .fini_smc_tables = smu_v14_0_fini_smc_tables, - .init_power = smu_v14_0_init_power, - .fini_power = smu_v14_0_fini_power, - .check_fw_status = smu_v14_0_check_fw_status, - .setup_pptable = smu_v14_0_2_setup_pptable, - .check_fw_version = smu_v14_0_check_fw_version, - .write_pptable = smu_cmn_write_pptable, - .set_driver_table_location = smu_v14_0_set_driver_table_location, - .system_features_control = smu_v14_0_system_features_control, - .set_allowed_mask = smu_v14_0_set_allowed_mask, - .get_enabled_mask = smu_cmn_get_enabled_mask, - .dpm_set_vcn_enable = smu_v14_0_set_vcn_enable, - .dpm_set_jpeg_enable = smu_v14_0_set_jpeg_enable, - .get_dpm_ultimate_freq = smu_v14_0_2_get_dpm_ultimate_freq, - .get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values, - .read_sensor = smu_v14_0_2_read_sensor, - .feature_is_enabled = smu_cmn_feature_is_enabled, - .print_clk_levels = smu_v14_0_2_print_clk_levels, - .force_clk_levels = smu_v14_0_2_force_clk_levels, - .update_pcie_parameters = smu_v14_0_2_update_pcie_parameters, - .get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range, - .register_irq_handler = smu_v14_0_register_irq_handler, - .enable_thermal_alert = smu_v14_0_enable_thermal_alert, - .disable_thermal_alert = smu_v14_0_disable_thermal_alert, - .notify_memory_pool_location = smu_v14_0_notify_memory_pool_location, - .get_gpu_metrics = smu_v14_0_2_get_gpu_metrics, - .set_soft_freq_limited_range = smu_v14_0_set_soft_freq_limited_range, - .set_default_od_settings = smu_v14_0_2_set_default_od_settings, - .restore_user_od_settings = smu_v14_0_2_restore_user_od_settings, - .od_edit_dpm_table = smu_v14_0_2_od_edit_dpm_table, - .init_pptable_microcode = smu_v14_0_init_pptable_microcode, - .populate_umd_state_clk = smu_v14_0_2_populate_umd_state_clk, - .set_performance_level = smu_v14_0_set_performance_level, - .gfx_off_control = smu_v14_0_gfx_off_control, - .get_unique_id = smu_v14_0_2_get_unique_id, - .get_power_limit = smu_v14_0_2_get_power_limit, - .set_power_limit = smu_v14_0_2_set_power_limit, - .set_power_source = smu_v14_0_set_power_source, - .get_power_profile_mode = smu_v14_0_2_get_power_profile_mode, - .set_power_profile_mode = smu_v14_0_2_set_power_profile_mode, - .run_btc = smu_v14_0_run_btc, - .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, - .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, - .set_tool_table_location = smu_v14_0_set_tool_table_location, - .deep_sleep_control = smu_v14_0_deep_sleep_control, - .gfx_ulv_control = smu_v14_0_gfx_ulv_control, - .get_bamaco_support = smu_v14_0_get_bamaco_support, - .baco_get_state = smu_v14_0_baco_get_state, - .baco_set_state = smu_v14_0_baco_set_state, - .baco_enter = smu_v14_0_2_baco_enter, - .baco_exit = smu_v14_0_2_baco_exit, - .mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported, - .mode1_reset = smu_v14_0_2_mode1_reset, - .mode2_reset = smu_v14_0_2_mode2_reset, - .enable_gfx_features = smu_v14_0_2_enable_gfx_features, - .set_mp1_state = smu_v14_0_2_set_mp1_state, - .set_df_cstate = smu_v14_0_2_set_df_cstate, -#if 0 - .gpo_control = smu_v14_0_gpo_control, -#endif -}; - -void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu) -{ - smu->ppt_funcs = &smu_v14_0_2_ppt_funcs; - smu->message_map = smu_v14_0_2_message_map; - smu->clock_map = smu_v14_0_2_clk_map; - smu->feature_map = smu_v14_0_2_feature_mask_map; - smu->table_map = smu_v14_0_2_table_map; - smu->pwr_src_map = smu_v14_0_2_pwr_src_map; - smu->workload_map = smu_v14_0_2_workload_map; - smu_v14_0_2_set_smu_mailbox_registers(smu); -} diff --git a/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/postimage b/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/postimage deleted file mode 100644 index 15541932b809..000000000000 --- a/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/postimage +++ /dev/null @@ -1,2104 +0,0 @@ -/* - * Copyright © 2008 Intel Corporation - * 2014 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_edid.h> -#include <drm/drm_fixed.h> -#include <drm/drm_probe_helper.h> - -#include "i915_drv.h" -#include "i915_reg.h" -#include "intel_atomic.h" -#include "intel_audio.h" -#include "intel_connector.h" -#include "intel_crtc.h" -#include "intel_ddi.h" -#include "intel_de.h" -#include "intel_display_driver.h" -#include "intel_display_types.h" -#include "intel_dp.h" -#include "intel_dp_hdcp.h" -#include "intel_dp_mst.h" -#include "intel_dp_tunnel.h" -#include "intel_dp_link_training.h" -#include "intel_dpio_phy.h" -#include "intel_hdcp.h" -#include "intel_hotplug.h" -#include "intel_link_bw.h" -#include "intel_psr.h" -#include "intel_vdsc.h" -#include "skl_scaler.h" - -static int intel_dp_mst_max_dpt_bpp(const struct intel_crtc_state *crtc_state, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - if (!intel_dp_is_uhbr(crtc_state) || DISPLAY_VER(i915) >= 20 || !dsc) - return INT_MAX; - - /* - * DSC->DPT interface width: - * ICL-MTL: 72 bits (each branch has 72 bits, only left branch is used) - * LNL+: 144 bits (not a bottleneck in any config) - * - * Bspec/49259 suggests that the FEC overhead needs to be - * applied here, though HW people claim that neither this FEC - * or any other overhead is applicable here (that is the actual - * available_bw is just symbol_clock * 72). However based on - * testing on MTL-P the - * - DELL U3224KBA display - * - Unigraf UCD-500 CTS test sink - * devices the - * - 5120x2880/995.59Mhz - * - 6016x3384/1357.23Mhz - * - 6144x3456/1413.39Mhz - * modes (all the ones having a DPT limit on the above devices), - * both the channel coding efficiency and an additional 3% - * overhead needs to be accounted for. - */ - return div64_u64(mul_u32_u32(intel_dp_link_symbol_clock(crtc_state->port_clock) * 72, - drm_dp_bw_channel_coding_efficiency(true)), - mul_u32_u32(adjusted_mode->crtc_clock, 1030000)); -} - -static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - bool ssc, bool dsc, int bpp_x16) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - unsigned long flags = DRM_DP_BW_OVERHEAD_MST; - int dsc_slice_count = 0; - int overhead; - - flags |= intel_dp_is_uhbr(crtc_state) ? DRM_DP_BW_OVERHEAD_UHBR : 0; - flags |= ssc ? DRM_DP_BW_OVERHEAD_SSC_REF_CLK : 0; - flags |= crtc_state->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0; - - if (dsc) { - flags |= DRM_DP_BW_OVERHEAD_DSC; - dsc_slice_count = intel_dp_dsc_get_slice_count(connector, - adjusted_mode->clock, - adjusted_mode->hdisplay, - crtc_state->joiner_pipes); - } - - overhead = drm_dp_bw_overhead(crtc_state->lane_count, - adjusted_mode->hdisplay, - dsc_slice_count, - bpp_x16, - flags); - - /* - * TODO: clarify whether a minimum required by the fixed FEC overhead - * in the bspec audio programming sequence is required here. - */ - return max(overhead, intel_dp_bw_fec_overhead(crtc_state->fec_enable)); -} - -static void intel_dp_mst_compute_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - int overhead, - int bpp_x16, - struct intel_link_m_n *m_n) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - /* TODO: Check WA 14013163432 to set data M/N for full BW utilization. */ - intel_link_compute_m_n(bpp_x16, crtc_state->lane_count, - adjusted_mode->crtc_clock, - crtc_state->port_clock, - overhead, - m_n); - - m_n->tu = DIV_ROUND_UP_ULL(mul_u32_u32(m_n->data_m, 64), m_n->data_n); -} - -static int intel_dp_mst_calc_pbn(int pixel_clock, int bpp_x16, int bw_overhead) -{ - int effective_data_rate = - intel_dp_effective_data_rate(pixel_clock, bpp_x16, bw_overhead); - - /* - * TODO: Use drm_dp_calc_pbn_mode() instead, once it's converted - * to calculate PBN with the BW overhead passed to it. - */ - return DIV_ROUND_UP(effective_data_rate * 64, 54 * 1000); -} - -static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - int max_bpp, - int min_bpp, - struct link_config_limits *limits, - struct drm_connector_state *conn_state, - int step, - bool dsc) -{ - struct drm_atomic_state *state = crtc_state->uapi.state; - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_state *mst_state; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - int bpp, slots = -EINVAL; - int max_dpt_bpp; - int ret = 0; - - mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - crtc_state->lane_count = limits->max_lane_count; - crtc_state->port_clock = limits->max_rate; - - if (dsc) { - if (!intel_dp_supports_fec(intel_dp, connector, crtc_state)) - return -EINVAL; - - crtc_state->fec_enable = !intel_dp_is_uhbr(crtc_state); - } - - mst_state->pbn_div = drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr, - crtc_state->port_clock, - crtc_state->lane_count); - - max_dpt_bpp = intel_dp_mst_max_dpt_bpp(crtc_state, dsc); - if (max_bpp > max_dpt_bpp) { - drm_dbg_kms(&i915->drm, "Limiting bpp to max DPT bpp (%d -> %d)\n", - max_bpp, max_dpt_bpp); - max_bpp = max_dpt_bpp; - } - - drm_dbg_kms(&i915->drm, "Looking for slots in range min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - for (bpp = max_bpp; bpp >= min_bpp; bpp -= step) { - int local_bw_overhead; - int remote_bw_overhead; - int link_bpp_x16; - int remote_tu; - fixed20_12 pbn; - - drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp); - - link_bpp_x16 = fxp_q4_from_int(dsc ? bpp : - intel_dp_output_bpp(crtc_state->output_format, bpp)); - - local_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - false, dsc, link_bpp_x16); - remote_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - true, dsc, link_bpp_x16); - - intel_dp_mst_compute_m_n(crtc_state, connector, - local_bw_overhead, - link_bpp_x16, - &crtc_state->dp_m_n); - - /* - * The TU size programmed to the HW determines which slots in - * an MTP frame are used for this stream, which needs to match - * the payload size programmed to the first downstream branch - * device's payload table. - * - * Note that atm the payload's PBN value DRM core sends via - * the ALLOCATE_PAYLOAD side-band message matches the payload - * size (which it calculates from the PBN value) it programs - * to the first branch device's payload table. The allocation - * in the payload table could be reduced though (to - * crtc_state->dp_m_n.tu), provided that the driver doesn't - * enable SSC on the corresponding link. - */ - pbn.full = dfixed_const(intel_dp_mst_calc_pbn(adjusted_mode->crtc_clock, - link_bpp_x16, - remote_bw_overhead)); - remote_tu = DIV_ROUND_UP(pbn.full, mst_state->pbn_div.full); - - /* - * Aligning the TUs ensures that symbols consisting of multiple - * (4) symbol cycles don't get split between two consecutive - * MTPs, as required by Bspec. - * TODO: remove the alignment restriction for 128b/132b links - * on some platforms, where Bspec allows this. - */ - remote_tu = ALIGN(remote_tu, 4 / crtc_state->lane_count); - - /* - * Also align PBNs accordingly, since MST core will derive its - * own copy of TU from the PBN in drm_dp_atomic_find_time_slots(). - * The above comment about the difference between the PBN - * allocated for the whole path and the TUs allocated for the - * first branch device's link also applies here. - */ - pbn.full = remote_tu * mst_state->pbn_div.full; - crtc_state->pbn = dfixed_trunc(pbn); - - drm_WARN_ON(&i915->drm, remote_tu < crtc_state->dp_m_n.tu); - crtc_state->dp_m_n.tu = remote_tu; - - slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr, - connector->port, - crtc_state->pbn); - if (slots == -EDEADLK) - return slots; - - if (slots >= 0) { - drm_WARN_ON(&i915->drm, slots != crtc_state->dp_m_n.tu); - - break; - } - } - - /* We failed to find a proper bpp/timeslots, return error */ - if (ret) - slots = ret; - - if (slots < 0) { - drm_dbg_kms(&i915->drm, "failed finding vcpi slots:%d\n", - slots); - } else { - if (!dsc) - crtc_state->pipe_bpp = bpp; - else - crtc_state->dsc.compressed_bpp_x16 = fxp_q4_from_int(bpp); - drm_dbg_kms(&i915->drm, "Got %d slots for pipe bpp %d dsc %d\n", slots, bpp, dsc); - } - - return slots; -} - -static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - int slots = -EINVAL; - - /* - * FIXME: allocate the BW according to link_bpp, which in the case of - * YUV420 is only half of the pipe bpp value. - */ - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, - fxp_q4_to_int(limits->link.max_bpp_x16), - fxp_q4_to_int(limits->link.min_bpp_x16), - limits, - conn_state, 2 * 3, false); - - if (slots < 0) - return slots; - - return 0; -} - -static int intel_dp_dsc_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - int slots = -EINVAL; - int i, num_bpc; - u8 dsc_bpc[3] = {}; - int min_bpp, max_bpp, sink_min_bpp, sink_max_bpp; - u8 dsc_max_bpc; - int min_compressed_bpp, max_compressed_bpp; - - /* Max DSC Input BPC for ICL is 10 and for TGL+ is 12 */ - if (DISPLAY_VER(i915) >= 12) - dsc_max_bpc = min_t(u8, 12, conn_state->max_requested_bpc); - else - dsc_max_bpc = min_t(u8, 10, conn_state->max_requested_bpc); - - max_bpp = min_t(u8, dsc_max_bpc * 3, limits->pipe.max_bpp); - min_bpp = limits->pipe.min_bpp; - - num_bpc = drm_dp_dsc_sink_supported_input_bpcs(connector->dp.dsc_dpcd, - dsc_bpc); - - drm_dbg_kms(&i915->drm, "DSC Source supported min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - sink_max_bpp = dsc_bpc[0] * 3; - sink_min_bpp = sink_max_bpp; - - for (i = 1; i < num_bpc; i++) { - if (sink_min_bpp > dsc_bpc[i] * 3) - sink_min_bpp = dsc_bpc[i] * 3; - if (sink_max_bpp < dsc_bpc[i] * 3) - sink_max_bpp = dsc_bpc[i] * 3; - } - - drm_dbg_kms(&i915->drm, "DSC Sink supported min bpp %d max bpp %d\n", - sink_min_bpp, sink_max_bpp); - - if (min_bpp < sink_min_bpp) - min_bpp = sink_min_bpp; - - if (max_bpp > sink_max_bpp) - max_bpp = sink_max_bpp; - - crtc_state->pipe_bpp = max_bpp; - - max_compressed_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector, - crtc_state, - max_bpp / 3); - max_compressed_bpp = min(max_compressed_bpp, - fxp_q4_to_int(limits->link.max_bpp_x16)); - - min_compressed_bpp = intel_dp_dsc_sink_min_compressed_bpp(crtc_state); - min_compressed_bpp = max(min_compressed_bpp, - fxp_q4_to_int_roundup(limits->link.min_bpp_x16)); - - drm_dbg_kms(&i915->drm, "DSC Sink supported compressed min bpp %d compressed max bpp %d\n", - min_compressed_bpp, max_compressed_bpp); - - /* Align compressed bpps according to our own constraints */ - max_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, max_compressed_bpp, - crtc_state->pipe_bpp); - min_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, min_compressed_bpp, - crtc_state->pipe_bpp); - - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, max_compressed_bpp, - min_compressed_bpp, limits, - conn_state, 1, true); - - if (slots < 0) - return slots; - - return 0; -} -static int intel_dp_mst_update_slots(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_topology_state *topology_state; - u8 link_coding_cap = intel_dp_is_uhbr(crtc_state) ? - DP_CAP_ANSI_128B132B : DP_CAP_ANSI_8B10B; - - topology_state = drm_atomic_get_mst_topology_state(conn_state->state, mgr); - if (IS_ERR(topology_state)) { - drm_dbg_kms(&i915->drm, "slot update failed\n"); - return PTR_ERR(topology_state); - } - - drm_dp_mst_update_slots(topology_state, link_coding_cap); - - return 0; -} - -static int mode_hblank_period_ns(const struct drm_display_mode *mode) -{ - return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(mode->htotal - mode->hdisplay, - NSEC_PER_SEC / 1000), - mode->crtc_clock); -} - -static bool -hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - const struct link_config_limits *limits) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - bool is_uhbr_sink = connector->mst_port && - drm_dp_128b132b_supported(connector->mst_port->dpcd); - int hblank_limit = is_uhbr_sink ? 500 : 300; - - if (!connector->dp.dsc_hblank_expansion_quirk) - return false; - - if (is_uhbr_sink && !drm_dp_is_uhbr_rate(limits->max_rate)) - return false; - - if (mode_hblank_period_ns(adjusted_mode) > hblank_limit) - return false; - - return true; -} - -static bool -adjust_limits_for_dsc_hblank_expansion_quirk(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - struct link_config_limits *limits, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - int min_bpp_x16 = limits->link.min_bpp_x16; - - if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state, limits)) - return true; - - if (!dsc) { - if (intel_dp_supports_dsc(connector, crtc_state)) { - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] DSC needed by hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - return false; - } - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to 24 due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - - if (limits->link.max_bpp_x16 < fxp_q4_from_int(24)) - return false; - - limits->link.min_bpp_x16 = fxp_q4_from_int(24); - - return true; - } - - drm_WARN_ON(&i915->drm, limits->min_rate != limits->max_rate); - - if (limits->max_rate < 540000) - min_bpp_x16 = fxp_q4_from_int(13); - else if (limits->max_rate < 810000) - min_bpp_x16 = fxp_q4_from_int(10); - - if (limits->link.min_bpp_x16 >= min_bpp_x16) - return true; - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to " FXP_Q4_FMT " in DSC mode due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name, - FXP_Q4_ARGS(min_bpp_x16)); - - if (limits->link.max_bpp_x16 < min_bpp_x16) - return false; - - limits->link.min_bpp_x16 = min_bpp_x16; - - return true; -} - -static bool -intel_dp_mst_compute_config_limits(struct intel_dp *intel_dp, - const struct intel_connector *connector, - struct intel_crtc_state *crtc_state, - bool dsc, - struct link_config_limits *limits) -{ - /* - * for MST we always configure max link bw - the spec doesn't - * seem to suggest we should do otherwise. - */ - limits->min_rate = limits->max_rate = - intel_dp_max_link_rate(intel_dp); - - limits->min_lane_count = limits->max_lane_count = - intel_dp_max_lane_count(intel_dp); - - limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); - /* - * FIXME: If all the streams can't fit into the link with - * their current pipe_bpp we should reduce pipe_bpp across - * the board until things start to fit. Until then we - * limit to <= 8bpc since that's what was hardcoded for all - * MST streams previously. This hack should be removed once - * we have the proper retry logic in place. - */ - limits->pipe.max_bpp = min(crtc_state->pipe_bpp, 24); - - intel_dp_adjust_compliance_config(intel_dp, crtc_state, limits); - - if (!intel_dp_compute_config_link_bpp_limits(intel_dp, - crtc_state, - dsc, - limits)) - return false; - - return adjust_limits_for_dsc_hblank_expansion_quirk(connector, - crtc_state, - limits, - dsc); -} - -static int intel_dp_mst_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - struct link_config_limits limits; - bool dsc_needed, joiner_needs_dsc; - int ret = 0; - - if (pipe_config->fec_enable && - !intel_dp_supports_fec(intel_dp, connector, pipe_config)) - return -EINVAL; - - if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return -EINVAL; - - if (intel_dp_need_joiner(intel_dp, connector, - adjusted_mode->crtc_hdisplay, - adjusted_mode->crtc_clock)) - pipe_config->joiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe); - - pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->has_pch_encoder = false; - - joiner_needs_dsc = intel_dp_joiner_needs_dsc(dev_priv, pipe_config->joiner_pipes); - - dsc_needed = joiner_needs_dsc || intel_dp->force_dsc_en || - !intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - false, - &limits); - - if (!dsc_needed) { - ret = intel_dp_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - - if (ret == -EDEADLK) - return ret; - - if (ret) - dsc_needed = true; - } - - /* enable compression if the mode doesn't fit available BW */ - if (dsc_needed) { - drm_dbg_kms(&dev_priv->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n", - str_yes_no(ret), str_yes_no(joiner_needs_dsc), - str_yes_no(intel_dp->force_dsc_en)); - - if (!intel_dp_supports_dsc(connector, pipe_config)) - return -EINVAL; - - if (!intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - true, - &limits)) - return -EINVAL; - - /* - * FIXME: As bpc is hardcoded to 8, as mentioned above, - * WARN and ignore the debug flag force_dsc_bpc for now. - */ - drm_WARN(&dev_priv->drm, intel_dp->force_dsc_bpc, "Cannot Force BPC for MST\n"); - /* - * Try to get at least some timeslots and then see, if - * we can fit there with DSC. - */ - drm_dbg_kms(&dev_priv->drm, "Trying to find VCPI slots in DSC mode\n"); - - ret = intel_dp_dsc_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - if (ret < 0) - return ret; - - ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, - conn_state, &limits, - pipe_config->dp_m_n.tu, false); - } - - if (ret) - return ret; - - ret = intel_dp_mst_update_slots(encoder, pipe_config, conn_state); - if (ret) - return ret; - - pipe_config->limited_color_range = - intel_dp_limited_color_range(pipe_config, conn_state); - - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - pipe_config->lane_lat_optim_mask = - bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); - - intel_dp_audio_compute_config(encoder, pipe_config, conn_state); - - intel_ddi_compute_min_voltage_level(pipe_config); - - intel_psr_compute_config(intel_dp, pipe_config, conn_state); - - return intel_dp_tunnel_atomic_compute_stream_bw(state, intel_dp, connector, - pipe_config); -} - -/* - * Iterate over all connectors and return a mask of - * all CPU transcoders streaming over the same DP link. - */ -static unsigned int -intel_dp_mst_transcoder_mask(struct intel_atomic_state *state, - struct intel_dp *mst_port) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 transcoders = 0; - int i; - - if (DISPLAY_VER(dev_priv) < 12) - return 0; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - const struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector->mst_port != mst_port || !conn_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_state->base.crtc); - crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - - if (!crtc_state->hw.active) - continue; - - transcoders |= BIT(crtc_state->cpu_transcoder); - } - - return transcoders; -} - -static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_port *parent_port) -{ - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 mask = 0; - int i; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - if (!conn_state->base.crtc) - continue; - - if (&connector->mst_port->mst_mgr != mst_mgr) - continue; - - if (connector->port != parent_port && - !drm_dp_mst_port_downstream_of_parent(mst_mgr, - connector->port, - parent_port)) - continue; - - mask |= BIT(to_intel_crtc(conn_state->base.crtc)->pipe); - } - - return mask; -} - -static int intel_dp_mst_check_fec_change(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct intel_link_bw_limits *limits) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - struct intel_crtc *crtc; - u8 mst_pipe_mask; - u8 fec_pipe_mask = 0; - int ret; - - mst_pipe_mask = get_pipes_downstream_of_mst_port(state, mst_mgr, NULL); - - for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, mst_pipe_mask) { - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - /* Atomic connector check should've added all the MST CRTCs. */ - if (drm_WARN_ON(&i915->drm, !crtc_state)) - return -EINVAL; - - if (crtc_state->fec_enable) - fec_pipe_mask |= BIT(crtc->pipe); - } - - if (!fec_pipe_mask || mst_pipe_mask == fec_pipe_mask) - return 0; - - limits->force_fec_pipes |= mst_pipe_mask; - - ret = intel_modeset_pipes_in_mask_early(state, "MST FEC", - mst_pipe_mask); - - return ret ? : -EAGAIN; -} - -static int intel_dp_mst_check_bw(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_topology_state *mst_state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_port *mst_port; - u8 mst_port_pipes; - int ret; - - ret = drm_dp_mst_atomic_check_mgr(&state->base, mst_mgr, mst_state, &mst_port); - if (ret != -ENOSPC) - return ret; - - mst_port_pipes = get_pipes_downstream_of_mst_port(state, mst_mgr, mst_port); - - ret = intel_link_bw_reduce_bpp(state, limits, - mst_port_pipes, "MST link BW"); - - return ret ? : -EAGAIN; -} - -/** - * intel_dp_mst_atomic_check_link - check all modeset MST link configuration - * @state: intel atomic state - * @limits: link BW limits - * - * Check the link configuration for all modeset MST outputs. If the - * configuration is invalid @limits will be updated if possible to - * reduce the total BW, after which the configuration for all CRTCs in - * @state must be recomputed with the updated @limits. - * - * Returns: - * - 0 if the confugration is valid - * - %-EAGAIN, if the configuration is invalid and @limits got updated - * with fallback values with which the configuration of all CRTCs in - * @state must be recomputed - * - Other negative error, if the configuration is invalid without a - * fallback possibility, or the check failed for another reason - */ -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_topology_mgr *mgr; - struct drm_dp_mst_topology_state *mst_state; - int ret; - int i; - - for_each_new_mst_mgr_in_state(&state->base, mgr, mst_state, i) { - ret = intel_dp_mst_check_fec_change(state, mgr, limits); - if (ret) - return ret; - - ret = intel_dp_mst_check_bw(state, mgr, mst_state, - limits); - if (ret) - return ret; - } - - return 0; -} - -static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - /* lowest numbered transcoder will be designated master */ - crtc_state->mst_master_transcoder = - ffs(intel_dp_mst_transcoder_mask(state, intel_dp)) - 1; - - return 0; -} - -/* - * If one of the connectors in a MST stream needs a modeset, mark all CRTCs - * that shares the same MST stream as mode changed, - * intel_modeset_pipe_config()+intel_crtc_check_fastset() will take care to do - * a fastset when possible. - * - * On TGL+ this is required since each stream go through a master transcoder, - * so if the master transcoder needs modeset, all other streams in the - * topology need a modeset. All platforms need to add the atomic state - * for all streams in the topology, since a modeset on one may require - * changing the MST link BW usage of the others, which in turn needs a - * recomputation of the corresponding CRTC states. - */ -static int -intel_dp_mst_atomic_topology_check(struct intel_connector *connector, - struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct drm_connector_list_iter connector_list_iter; - struct intel_connector *connector_iter; - int ret = 0; - - if (!intel_connector_needs_modeset(state, &connector->base)) - return 0; - - drm_connector_list_iter_begin(&dev_priv->drm, &connector_list_iter); - for_each_intel_connector_iter(connector_iter, &connector_list_iter) { - struct intel_digital_connector_state *conn_iter_state; - struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector_iter->mst_port != connector->mst_port || - connector_iter == connector) - continue; - - conn_iter_state = intel_atomic_get_digital_connector_state(state, - connector_iter); - if (IS_ERR(conn_iter_state)) { - ret = PTR_ERR(conn_iter_state); - break; - } - - if (!conn_iter_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_iter_state->base.crtc); - crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - break; - } - - ret = drm_atomic_add_affected_planes(&state->base, &crtc->base); - if (ret) - break; - crtc_state->uapi.mode_changed = true; - } - drm_connector_list_iter_end(&connector_list_iter); - - return ret; -} - -static int -intel_dp_mst_atomic_check(struct drm_connector *connector, - struct drm_atomic_state *_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(_state); - struct intel_connector *intel_connector = - to_intel_connector(connector); - int ret; - - ret = intel_digital_connector_atomic_check(connector, &state->base); - if (ret) - return ret; - - ret = intel_dp_mst_atomic_topology_check(intel_connector, state); - if (ret) - return ret; - - if (intel_connector_needs_modeset(state, connector)) { - ret = intel_dp_tunnel_atomic_check_state(state, - intel_connector->mst_port, - intel_connector); - if (ret) - return ret; - } - - return drm_dp_atomic_release_time_slots(&state->base, - &intel_connector->mst_port->mst_mgr, - intel_connector->port); -} - -static void clear_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - - intel_de_write(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT); -} - -static void wait_for_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - if (intel_de_wait_for_set(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT, 1)) - drm_err(&i915->drm, "Timed out waiting for ACT sent\n"); - - drm_dp_check_act_status(&intel_dp->mst_mgr); -} - -static void intel_mst_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - - drm_dbg_kms(&i915->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (intel_dp->active_mst_links == 1) - intel_dp->link_trained = false; - - intel_hdcp_disable(intel_mst->connector); - - intel_dp_sink_disable_decompression(state, connector, old_crtc_state); -} - -static void intel_mst_post_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_dp_mst_topology_state *old_mst_state = - drm_atomic_get_old_mst_topology_state(&state->base, &intel_dp->mst_mgr); - struct drm_dp_mst_topology_state *new_mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - const struct drm_dp_mst_atomic_payload *old_payload = - drm_atomic_get_mst_payload_state(old_mst_state, connector->port); - struct drm_dp_mst_atomic_payload *new_payload = - drm_atomic_get_mst_payload_state(new_mst_state, connector->port); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_crtc *pipe_crtc; - bool last_mst_stream; - - intel_dp->active_mst_links--; - last_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && last_mst_stream && - !intel_dp_mst_is_master_trans(old_crtc_state)); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_off(old_pipe_crtc_state); - } - - intel_disable_transcoder(old_crtc_state); - - drm_dp_remove_payload_part1(&intel_dp->mst_mgr, new_mst_state, new_payload); - - clear_act_sent(encoder, old_crtc_state); - - intel_de_rmw(dev_priv, - TRANS_DDI_FUNC_CTL(dev_priv, old_crtc_state->cpu_transcoder), - TRANS_DDI_DP_VC_PAYLOAD_ALLOC, 0); - - wait_for_act_sent(encoder, old_crtc_state); - - drm_dp_remove_payload_part2(&intel_dp->mst_mgr, new_mst_state, - old_payload, new_payload); - - intel_ddi_disable_transcoder_func(old_crtc_state); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_dsc_disable(old_pipe_crtc_state); - - if (DISPLAY_VER(dev_priv) >= 9) - skl_scaler_disable(old_pipe_crtc_state); - else - ilk_pfit_disable(old_pipe_crtc_state); - } - - /* - * Power down mst path before disabling the port, otherwise we end - * up getting interrupts from the sink upon detecting link loss. - */ - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, - false); - - /* - * BSpec 4287: disable DIP after the transcoder is disabled and before - * the transcoder clock select is set to none. - */ - intel_dp_set_infoframes(&dig_port->base, false, - old_crtc_state, NULL); - /* - * From TGL spec: "If multi-stream slave transcoder: Configure - * Transcoder Clock Select to direct no clock to the transcoder" - * - * From older GENs spec: "Configure Transcoder Clock Select to direct - * no clock to the transcoder" - */ - if (DISPLAY_VER(dev_priv) < 12 || !last_mst_stream) - intel_ddi_disable_transcoder_clock(old_crtc_state); - - - intel_mst->connector = NULL; - if (last_mst_stream) - dig_port->base.post_disable(state, &dig_port->base, - old_crtc_state, NULL); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); -} - -static void intel_mst_post_pll_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0 && - dig_port->base.post_pll_disable) - dig_port->base.post_pll_disable(state, encoder, old_crtc_state, old_conn_state); -} - -static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0) - dig_port->base.pre_pll_enable(state, &dig_port->base, - pipe_config, NULL); - else - /* - * The port PLL state needs to get updated for secondary - * streams as for the primary stream. - */ - intel_ddi_update_active_dpll(state, &dig_port->base, - to_intel_crtc(pipe_config->uapi.crtc)); -} - -static bool intel_mst_probed_link_params_valid(struct intel_dp *intel_dp, - int link_rate, int lane_count) -{ - return intel_dp->link.mst_probed_rate == link_rate && - intel_dp->link.mst_probed_lane_count == lane_count; -} - -static void intel_mst_set_probed_link_params(struct intel_dp *intel_dp, - int link_rate, int lane_count) -{ - intel_dp->link.mst_probed_rate = link_rate; - intel_dp->link.mst_probed_lane_count = lane_count; -} - -static void intel_mst_reprobe_topology(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - if (intel_mst_probed_link_params_valid(intel_dp, - crtc_state->port_clock, crtc_state->lane_count)) - return; - - drm_dp_mst_topology_queue_probe(&intel_dp->mst_mgr); - - intel_mst_set_probed_link_params(intel_dp, - crtc_state->port_clock, crtc_state->lane_count); -} - -static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - int ret; - bool first_mst_stream; - - /* MST encoders are bound to a crtc, not to a connector, - * force the mapping here for get_hw_state. - */ - connector->encoder = encoder; - intel_mst->connector = connector; - first_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && first_mst_stream && - !intel_dp_mst_is_master_trans(pipe_config)); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (first_mst_stream) - intel_dp_set_power(intel_dp, DP_SET_POWER_D0); - - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true); - - intel_dp_sink_enable_decompression(state, connector, pipe_config); - - if (first_mst_stream) { - dig_port->base.pre_enable(state, &dig_port->base, - pipe_config, NULL); - - intel_mst_reprobe_topology(intel_dp, pipe_config); - } - - intel_dp->active_mst_links++; - - ret = drm_dp_add_payload_part1(&intel_dp->mst_mgr, mst_state, - drm_atomic_get_mst_payload_state(mst_state, connector->port)); - if (ret < 0) - intel_dp_queue_modeset_retry_for_link(state, &dig_port->base, pipe_config); - - /* - * Before Gen 12 this is not done as part of - * dig_port->base.pre_enable() and should be done here. For - * Gen 12+ the step in which this should be done is different for the - * first MST stream, so it's done on the DDI for the first stream and - * here for the following ones. - */ - if (DISPLAY_VER(dev_priv) < 12 || !first_mst_stream) - intel_ddi_enable_transcoder_clock(encoder, pipe_config); - - intel_dsc_dp_pps_write(&dig_port->base, pipe_config); - intel_ddi_set_dp_msa(pipe_config, conn_state); -} - -static void enable_bs_jitter_was(const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - u32 clear = 0; - u32 set = 0; - - if (!IS_ALDERLAKE_P(i915)) - return; - - if (!IS_DISPLAY_STEP(i915, STEP_D0, STEP_FOREVER)) - return; - - /* Wa_14013163432:adlp */ - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_FEC_BS_JITTER_WA(crtc_state->cpu_transcoder); - - /* Wa_14014143976:adlp */ - if (IS_DISPLAY_STEP(i915, STEP_E0, STEP_FOREVER)) { - if (intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - else if (crtc_state->fec_enable) - clear |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_DPT_DPTP_ALIGN_WA(crtc_state->cpu_transcoder); - } - - if (!clear && !set) - return; - - intel_de_rmw(i915, CHICKEN_MISC_3, clear, set); -} - -static void intel_mst_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - enum transcoder trans = pipe_config->cpu_transcoder; - bool first_mst_stream = intel_dp->active_mst_links == 1; - struct intel_crtc *pipe_crtc; - int ret; - - drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder); - - if (intel_dp_is_uhbr(pipe_config)) { - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - u64 crtc_clock_hz = KHz(adjusted_mode->crtc_clock); - - intel_de_write(dev_priv, TRANS_DP2_VFREQHIGH(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz >> 24)); - intel_de_write(dev_priv, TRANS_DP2_VFREQLOW(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz & 0xffffff)); - } - - enable_bs_jitter_was(pipe_config); - - intel_ddi_enable_transcoder_func(encoder, pipe_config); - - clear_act_sent(encoder, pipe_config); - - intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(dev_priv, trans), 0, - TRANS_DDI_DP_VC_PAYLOAD_ALLOC); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - wait_for_act_sent(encoder, pipe_config); - - if (first_mst_stream) - intel_ddi_wait_for_fec_status(encoder, pipe_config, true); - - ret = drm_dp_add_payload_part2(&intel_dp->mst_mgr, - drm_atomic_get_mst_payload_state(mst_state, - connector->port)); - if (ret < 0) - intel_dp_queue_modeset_retry_for_link(state, &dig_port->base, pipe_config); - - if (DISPLAY_VER(dev_priv) >= 12) - intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, trans), - FECSTALL_DIS_DPTSTREAM_DPTTG, - pipe_config->fec_enable ? FECSTALL_DIS_DPTSTREAM_DPTTG : 0); - - intel_audio_sdp_split_update(pipe_config); - - intel_enable_transcoder(pipe_config); - - for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(pipe_config)) { - const struct intel_crtc_state *pipe_crtc_state = - intel_atomic_get_new_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_on(pipe_crtc_state); - } - - intel_hdcp_enable(state, encoder, pipe_config, conn_state); -} - -static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder, - enum pipe *pipe) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - *pipe = intel_mst->pipe; - if (intel_mst->connector) - return true; - return false; -} - -static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - dig_port->base.get_config(&dig_port->base, pipe_config); -} - -static bool intel_dp_mst_initial_fastset_check(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - return intel_dp_initial_fastset_check(&dig_port->base, crtc_state); -} - -static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - struct drm_i915_private *i915 = to_i915(intel_connector->base.dev); - struct intel_dp *intel_dp = intel_connector->mst_port; - const struct drm_edid *drm_edid; - int ret; - - if (drm_connector_is_unregistered(connector)) - return intel_connector_update_modes(connector, NULL); - - if (!intel_display_driver_check_access(i915)) - return drm_edid_connector_add_modes(connector); - - drm_edid = drm_dp_mst_edid_read(connector, &intel_dp->mst_mgr, intel_connector->port); - - ret = intel_connector_update_modes(connector, drm_edid); - - drm_edid_free(drm_edid); - - return ret; -} - -static int -intel_dp_mst_connector_late_register(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - int ret; - - ret = drm_dp_mst_connector_late_register(connector, - intel_connector->port); - if (ret < 0) - return ret; - - ret = intel_connector_register(connector); - if (ret < 0) - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); - - return ret; -} - -static void -intel_dp_mst_connector_early_unregister(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - intel_connector_unregister(connector); - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); -} - -static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { - .fill_modes = drm_helper_probe_single_connector_modes, - .atomic_get_property = intel_digital_connector_atomic_get_property, - .atomic_set_property = intel_digital_connector_atomic_set_property, - .late_register = intel_dp_mst_connector_late_register, - .early_unregister = intel_dp_mst_connector_early_unregister, - .destroy = intel_connector_destroy, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_duplicate_state = intel_digital_connector_duplicate_state, -}; - -static int intel_dp_mst_get_modes(struct drm_connector *connector) -{ - return intel_dp_mst_get_ddc_modes(connector); -} - -static int -intel_dp_mst_mode_valid_ctx(struct drm_connector *connector, - struct drm_display_mode *mode, - struct drm_modeset_acquire_ctx *ctx, - enum drm_mode_status *status) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_port *port = intel_connector->port; - const int min_bpp = 18; - int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq; - int max_rate, mode_rate, max_lanes, max_link_clock; - int ret; - bool dsc = false, joiner = false; - u16 dsc_max_compressed_bpp = 0; - u8 dsc_slice_count = 0; - int target_clock = mode->clock; - - if (drm_connector_is_unregistered(connector)) { - *status = MODE_ERROR; - return 0; - } - - *status = intel_cpu_transcoder_mode_valid(dev_priv, mode); - if (*status != MODE_OK) - return 0; - - if (mode->flags & DRM_MODE_FLAG_DBLCLK) { - *status = MODE_H_ILLEGAL; - return 0; - } - - if (mode->clock < 10000) { - *status = MODE_CLOCK_LOW; - return 0; - } - - max_link_clock = intel_dp_max_link_rate(intel_dp); - max_lanes = intel_dp_max_lane_count(intel_dp); - - max_rate = intel_dp_max_link_data_rate(intel_dp, - max_link_clock, max_lanes); - mode_rate = intel_dp_link_required(mode->clock, min_bpp); - - /* - * TODO: - * - Also check if compression would allow for the mode - * - Calculate the overhead using drm_dp_bw_overhead() / - * drm_dp_bw_channel_coding_efficiency(), similarly to the - * compute config code, as drm_dp_calc_pbn_mode() doesn't - * account with all the overheads. - * - Check here and during compute config the BW reported by - * DFP_Link_Available_Payload_Bandwidth_Number (or the - * corresponding link capabilities of the sink) in case the - * stream is uncompressed for it by the last branch device. - */ - if (intel_dp_need_joiner(intel_dp, intel_connector, - mode->hdisplay, target_clock)) { - joiner = true; - max_dotclk *= 2; - } - - ret = drm_modeset_lock(&mgr->base.lock, ctx); - if (ret) - return ret; - - if (mode_rate > max_rate || mode->clock > max_dotclk || - drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (intel_dp_has_dsc(intel_connector)) { - /* - * TBD pass the connector BPC, - * for now U8_MAX so that max BPC on that platform would be picked - */ - int pipe_bpp = intel_dp_dsc_compute_max_bpp(intel_connector, U8_MAX); - - if (drm_dp_sink_supports_fec(intel_connector->dp.fec_capability)) { - dsc_max_compressed_bpp = - intel_dp_dsc_get_max_compressed_bpp(dev_priv, - max_link_clock, - max_lanes, - target_clock, - mode->hdisplay, - joiner, - INTEL_OUTPUT_FORMAT_RGB, - pipe_bpp, 64); - dsc_slice_count = - intel_dp_dsc_get_slice_count(intel_connector, - target_clock, - mode->hdisplay, - joiner); - } - - dsc = dsc_max_compressed_bpp && dsc_slice_count; - } - - if (intel_dp_joiner_needs_dsc(dev_priv, joiner) && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (mode_rate > max_rate && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - *status = intel_mode_valid_max_plane_size(dev_priv, mode, joiner); - return 0; -} - -static struct drm_encoder *intel_mst_atomic_best_encoder(struct drm_connector *connector, - struct drm_atomic_state *state) -{ - struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state, - connector); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct intel_crtc *crtc = to_intel_crtc(connector_state->crtc); - - return &intel_dp->mst_encoders[crtc->pipe]->base.base; -} - -static int -intel_dp_mst_detect(struct drm_connector *connector, - struct drm_modeset_acquire_ctx *ctx, bool force) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - - if (!intel_display_device_enabled(i915)) - return connector_status_disconnected; - - if (drm_connector_is_unregistered(connector)) - return connector_status_disconnected; - - if (!intel_display_driver_check_access(i915)) - return connector->status; - - return drm_dp_mst_detect_port(connector, ctx, &intel_dp->mst_mgr, - intel_connector->port); -} - -static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = { - .get_modes = intel_dp_mst_get_modes, - .mode_valid_ctx = intel_dp_mst_mode_valid_ctx, - .atomic_best_encoder = intel_mst_atomic_best_encoder, - .atomic_check = intel_dp_mst_atomic_check, - .detect_ctx = intel_dp_mst_detect, -}; - -static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(to_intel_encoder(encoder)); - - drm_encoder_cleanup(encoder); - kfree(intel_mst); -} - -static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = { - .destroy = intel_dp_mst_encoder_destroy, -}; - -static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) -{ - if (intel_attached_encoder(connector) && connector->base.state->crtc) { - enum pipe pipe; - if (!intel_attached_encoder(connector)->get_hw_state(intel_attached_encoder(connector), &pipe)) - return false; - return true; - } - return false; -} - -static int intel_dp_mst_add_properties(struct intel_dp *intel_dp, - struct drm_connector *connector, - const char *pathprop) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - - drm_object_attach_property(&connector->base, - i915->drm.mode_config.path_property, 0); - drm_object_attach_property(&connector->base, - i915->drm.mode_config.tile_property, 0); - - intel_attach_force_audio_property(connector); - intel_attach_broadcast_rgb_property(connector); - - /* - * Reuse the prop from the SST connector because we're - * not allowed to create new props after device registration. - */ - connector->max_bpc_property = - intel_dp->attached_connector->base.max_bpc_property; - if (connector->max_bpc_property) - drm_connector_attach_max_bpc_property(connector, 6, 12); - - return drm_connector_set_path_property(connector, pathprop); -} - -static void -intel_dp_mst_read_decompression_port_dsc_caps(struct intel_dp *intel_dp, - struct intel_connector *connector) -{ - u8 dpcd_caps[DP_RECEIVER_CAP_SIZE]; - - if (!connector->dp.dsc_decompression_aux) - return; - - if (drm_dp_read_dpcd_caps(connector->dp.dsc_decompression_aux, dpcd_caps) < 0) - return; - - intel_dp_get_dsc_sink_cap(dpcd_caps[DP_DPCD_REV], connector); -} - -static bool detect_dsc_hblank_expansion_quirk(const struct intel_connector *connector) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct drm_dp_aux *aux = connector->dp.dsc_decompression_aux; - struct drm_dp_desc desc; - u8 dpcd[DP_RECEIVER_CAP_SIZE]; - - if (!aux) - return false; - - /* - * A logical port's OUI (at least for affected sinks) is all 0, so - * instead of that the parent port's OUI is used for identification. - */ - if (drm_dp_mst_port_is_logical(connector->port)) { - aux = drm_dp_mst_aux_for_parent(connector->port); - if (!aux) - aux = &connector->mst_port->aux; - } - - if (drm_dp_read_dpcd_caps(aux, dpcd) < 0) - return false; - - if (drm_dp_read_desc(aux, &desc, drm_dp_is_branch(dpcd)) < 0) - return false; - - if (!drm_dp_has_quirk(&desc, - DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC)) - return false; - - /* - * UHBR (MST sink) devices requiring this quirk don't advertise the - * HBLANK expansion support. Presuming that they perform HBLANK - * expansion internally, or are affected by this issue on modes with a - * short HBLANK for other reasons. - */ - if (!drm_dp_128b132b_supported(dpcd) && - !(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE)) - return false; - - drm_dbg_kms(&i915->drm, - "[CONNECTOR:%d:%s] DSC HBLANK expansion quirk detected\n", - connector->base.base.id, connector->base.name); - - return true; -} - -static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, - struct drm_dp_mst_port *port, - const char *pathprop) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_connector *intel_connector; - struct drm_connector *connector; - enum pipe pipe; - int ret; - - intel_connector = intel_connector_alloc(); - if (!intel_connector) - return NULL; - - intel_connector->get_hw_state = intel_dp_mst_get_hw_state; - intel_connector->sync_state = intel_dp_connector_sync_state; - intel_connector->mst_port = intel_dp; - intel_connector->port = port; - drm_dp_mst_get_port_malloc(port); - - intel_dp_init_modeset_retry_work(intel_connector); - - intel_connector->dp.dsc_decompression_aux = drm_dp_mst_dsc_aux_for_port(port); - intel_dp_mst_read_decompression_port_dsc_caps(intel_dp, intel_connector); - intel_connector->dp.dsc_hblank_expansion_quirk = - detect_dsc_hblank_expansion_quirk(intel_connector); - - connector = &intel_connector->base; - ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, - DRM_MODE_CONNECTOR_DisplayPort); - if (ret) { - drm_dp_mst_put_port_malloc(port); - intel_connector_free(intel_connector); - return NULL; - } - - drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs); - - for_each_pipe(dev_priv, pipe) { - struct drm_encoder *enc = - &intel_dp->mst_encoders[pipe]->base.base; - - ret = drm_connector_attach_encoder(&intel_connector->base, enc); - if (ret) - goto err; - } - - ret = intel_dp_mst_add_properties(intel_dp, connector, pathprop); - if (ret) - goto err; - - ret = intel_dp_hdcp_init(dig_port, intel_connector); - if (ret) - drm_dbg_kms(&dev_priv->drm, "[%s:%d] HDCP MST init failed, skipping.\n", - connector->name, connector->base.id); - - return connector; - -err: - drm_connector_cleanup(connector); - return NULL; -} - -static void -intel_dp_mst_poll_hpd_irq(struct drm_dp_mst_topology_mgr *mgr) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - - intel_hpd_trigger_irq(dp_to_dig_port(intel_dp)); -} - -static const struct drm_dp_mst_topology_cbs mst_cbs = { - .add_connector = intel_dp_add_mst_connector, - .poll_hpd_irq = intel_dp_mst_poll_hpd_irq, -}; - -static struct intel_dp_mst_encoder * -intel_dp_create_fake_mst_encoder(struct intel_digital_port *dig_port, enum pipe pipe) -{ - struct intel_dp_mst_encoder *intel_mst; - struct intel_encoder *intel_encoder; - struct drm_device *dev = dig_port->base.base.dev; - - intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL); - - if (!intel_mst) - return NULL; - - intel_mst->pipe = pipe; - intel_encoder = &intel_mst->base; - intel_mst->primary = dig_port; - - drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, - DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe)); - - intel_encoder->type = INTEL_OUTPUT_DP_MST; - intel_encoder->power_domain = dig_port->base.power_domain; - intel_encoder->port = dig_port->base.port; - intel_encoder->cloneable = 0; - /* - * This is wrong, but broken userspace uses the intersection - * of possible_crtcs of all the encoders of a given connector - * to figure out which crtcs can drive said connector. What - * should be used instead is the union of possible_crtcs. - * To keep such userspace functioning we must misconfigure - * this to make sure the intersection is not empty :( - */ - intel_encoder->pipe_mask = ~0; - - intel_encoder->compute_config = intel_dp_mst_compute_config; - intel_encoder->compute_config_late = intel_dp_mst_compute_config_late; - intel_encoder->disable = intel_mst_disable_dp; - intel_encoder->post_disable = intel_mst_post_disable_dp; - intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp; - intel_encoder->update_pipe = intel_ddi_update_pipe; - intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; - intel_encoder->pre_enable = intel_mst_pre_enable_dp; - intel_encoder->enable = intel_mst_enable_dp; - intel_encoder->audio_enable = intel_audio_codec_enable; - intel_encoder->audio_disable = intel_audio_codec_disable; - intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state; - intel_encoder->get_config = intel_dp_mst_enc_get_config; - intel_encoder->initial_fastset_check = intel_dp_mst_initial_fastset_check; - - return intel_mst; - -} - -static bool -intel_dp_create_fake_mst_encoders(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum pipe pipe; - - for_each_pipe(dev_priv, pipe) - intel_dp->mst_encoders[pipe] = intel_dp_create_fake_mst_encoder(dig_port, pipe); - return true; -} - -int -intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port) -{ - return dig_port->dp.active_mst_links; -} - -int -intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id) -{ - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - struct intel_dp *intel_dp = &dig_port->dp; - enum port port = dig_port->base.port; - int ret; - - if (!HAS_DP_MST(i915) || intel_dp_is_edp(intel_dp)) - return 0; - - if (DISPLAY_VER(i915) < 12 && port == PORT_A) - return 0; - - if (DISPLAY_VER(i915) < 11 && port == PORT_E) - return 0; - - intel_dp->mst_mgr.cbs = &mst_cbs; - - /* create encoders */ - intel_dp_create_fake_mst_encoders(dig_port); - ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, &i915->drm, - &intel_dp->aux, 16, 3, conn_base_id); - if (ret) { - intel_dp->mst_mgr.cbs = NULL; - return ret; - } - - return 0; -} - -bool intel_dp_mst_source_support(struct intel_dp *intel_dp) -{ - return intel_dp->mst_mgr.cbs; -} - -void -intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - - if (!intel_dp_mst_source_support(intel_dp)) - return; - - drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr); - /* encoders will get killed by normal cleanup */ - - intel_dp->mst_mgr.cbs = NULL; -} - -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder == crtc_state->cpu_transcoder; -} - -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder != INVALID_TRANSCODER && - crtc_state->mst_master_transcoder != crtc_state->cpu_transcoder; -} - -/** - * intel_dp_mst_add_topology_state_for_connector - add MST topology state for a connector - * @state: atomic state - * @connector: connector to add the state for - * @crtc: the CRTC @connector is attached to - * - * Add the MST topology state for @connector to @state. - * - * Returns 0 on success, negative error code on failure. - */ -static int -intel_dp_mst_add_topology_state_for_connector(struct intel_atomic_state *state, - struct intel_connector *connector, - struct intel_crtc *crtc) -{ - struct drm_dp_mst_topology_state *mst_state; - - if (!connector->mst_port) - return 0; - - mst_state = drm_atomic_get_mst_topology_state(&state->base, - &connector->mst_port->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - mst_state->pending_crtc_mask |= drm_crtc_mask(&crtc->base); - - return 0; -} - -/** - * intel_dp_mst_add_topology_state_for_crtc - add MST topology state for a CRTC - * @state: atomic state - * @crtc: CRTC to add the state for - * - * Add the MST topology state for @crtc to @state. - * - * Returns 0 on success, negative error code on failure. - */ -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_connector *_connector; - struct drm_connector_state *conn_state; - int i; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - struct intel_connector *connector = to_intel_connector(_connector); - int ret; - - if (conn_state->crtc != &crtc->base) - continue; - - ret = intel_dp_mst_add_topology_state_for_connector(state, connector, crtc); - if (ret) - return ret; - } - - return 0; -} - -static struct intel_connector * -get_connector_in_state_for_crtc(struct intel_atomic_state *state, - const struct intel_crtc *crtc) -{ - struct drm_connector_state *old_conn_state; - struct drm_connector_state *new_conn_state; - struct drm_connector *_connector; - int i; - - for_each_oldnew_connector_in_state(&state->base, _connector, - old_conn_state, new_conn_state, i) { - struct intel_connector *connector = - to_intel_connector(_connector); - - if (old_conn_state->crtc == &crtc->base || - new_conn_state->crtc == &crtc->base) - return connector; - } - - return NULL; -} - -/** - * intel_dp_mst_crtc_needs_modeset - check if changes in topology need to modeset the given CRTC - * @state: atomic state - * @crtc: CRTC for which to check the modeset requirement - * - * Check if any change in a MST topology requires a forced modeset on @crtc in - * this topology. One such change is enabling/disabling the DSC decompression - * state in the first branch device's UFP DPCD as required by one CRTC, while - * the other @crtc in the same topology is still active, requiring a full modeset - * on @crtc. - */ -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - const struct intel_connector *crtc_connector; - const struct drm_connector_state *conn_state; - const struct drm_connector *_connector; - int i; - - if (!intel_crtc_has_type(intel_atomic_get_new_crtc_state(state, crtc), - INTEL_OUTPUT_DP_MST)) - return false; - - crtc_connector = get_connector_in_state_for_crtc(state, crtc); - - if (!crtc_connector) - /* None of the connectors in the topology needs modeset */ - return false; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - const struct intel_connector *connector = - to_intel_connector(_connector); - const struct intel_crtc_state *new_crtc_state; - const struct intel_crtc_state *old_crtc_state; - struct intel_crtc *crtc_iter; - - if (connector->mst_port != crtc_connector->mst_port || - !conn_state->crtc) - continue; - - crtc_iter = to_intel_crtc(conn_state->crtc); - - new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc_iter); - old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc_iter); - - if (!intel_crtc_needs_modeset(new_crtc_state)) - continue; - - if (old_crtc_state->dsc.compression_enable == - new_crtc_state->dsc.compression_enable) - continue; - /* - * Toggling the decompression flag because of this stream in - * the first downstream branch device's UFP DPCD may reset the - * whole branch device. To avoid the reset while other streams - * are also active modeset the whole MST topology in this - * case. - */ - if (connector->dp.dsc_decompression_aux == - &connector->mst_port->aux) - return true; - } - - return false; -} - -/** - * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing - * @intel_dp: DP port object - * - * Prepare an MST link for topology probing, programming the target - * link parameters to DPCD. This step is a requirement of the enumaration - * of path resources during probing. - */ -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp) -{ - int link_rate = intel_dp_max_link_rate(intel_dp); - int lane_count = intel_dp_max_lane_count(intel_dp); - u8 rate_select; - u8 link_bw; - - if (intel_dp->link_trained) - return; - - if (intel_mst_probed_link_params_valid(intel_dp, link_rate, lane_count)) - return; - - intel_dp_compute_rate(intel_dp, link_rate, &link_bw, &rate_select); - - intel_dp_link_training_set_mode(intel_dp, link_rate, false); - intel_dp_link_training_set_bw(intel_dp, link_bw, rate_select, lane_count, - drm_dp_enhanced_frame_cap(intel_dp->dpcd)); - - intel_mst_set_probed_link_params(intel_dp, link_rate, lane_count); -} - -/* - * intel_dp_mst_verify_dpcd_state - verify the MST SW enabled state wrt. the DPCD - * @intel_dp: DP port object - * - * Verify if @intel_dp's MST enabled SW state matches the corresponding DPCD - * state. A long HPD pulse - not long enough to be detected as a disconnected - * state - could've reset the DPCD state, which requires tearing - * down/recreating the MST topology. - * - * Returns %true if the SW MST enabled and DPCD states match, %false - * otherwise. - */ -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp) -{ - struct intel_display *display = to_intel_display(intel_dp); - struct intel_connector *connector = intel_dp->attached_connector; - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *encoder = &dig_port->base; - int ret; - u8 val; - - if (!intel_dp->is_mst) - return true; - - ret = drm_dp_dpcd_readb(intel_dp->mst_mgr.aux, DP_MSTM_CTRL, &val); - - /* Adjust the expected register value for SST + SideBand. */ - if (ret < 0 || val != (DP_MST_EN | DP_UP_REQ_EN | DP_UPSTREAM_IS_SRC)) { - drm_dbg_kms(display->drm, - "[CONNECTOR:%d:%s][ENCODER:%d:%s] MST mode got reset, removing topology (ret=%d, ctrl=0x%02x)\n", - connector->base.base.id, connector->base.name, - encoder->base.base.id, encoder->base.name, - ret, val); - - return false; - } - - return true; -} diff --git a/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/preimage b/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/preimage deleted file mode 100644 index 7583b460ecc8..000000000000 --- a/rr-cache/0c1c391f6e92e878a32995cabf64a0f916149ff2/preimage +++ /dev/null @@ -1,2105 +0,0 @@ -/* - * Copyright © 2008 Intel Corporation - * 2014 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_edid.h> -#include <drm/drm_fixed.h> -#include <drm/drm_probe_helper.h> - -#include "i915_drv.h" -#include "i915_reg.h" -#include "intel_atomic.h" -#include "intel_audio.h" -#include "intel_connector.h" -#include "intel_crtc.h" -#include "intel_ddi.h" -#include "intel_de.h" -#include "intel_display_driver.h" -#include "intel_display_types.h" -#include "intel_dp.h" -#include "intel_dp_hdcp.h" -#include "intel_dp_mst.h" -#include "intel_dp_tunnel.h" -#include "intel_dp_link_training.h" -#include "intel_dpio_phy.h" -#include "intel_hdcp.h" -#include "intel_hotplug.h" -#include "intel_link_bw.h" -#include "intel_psr.h" -#include "intel_vdsc.h" -#include "skl_scaler.h" - -static int intel_dp_mst_max_dpt_bpp(const struct intel_crtc_state *crtc_state, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - if (!intel_dp_is_uhbr(crtc_state) || DISPLAY_VER(i915) >= 20 || !dsc) - return INT_MAX; - - /* - * DSC->DPT interface width: - * ICL-MTL: 72 bits (each branch has 72 bits, only left branch is used) - * LNL+: 144 bits (not a bottleneck in any config) - * - * Bspec/49259 suggests that the FEC overhead needs to be - * applied here, though HW people claim that neither this FEC - * or any other overhead is applicable here (that is the actual - * available_bw is just symbol_clock * 72). However based on - * testing on MTL-P the - * - DELL U3224KBA display - * - Unigraf UCD-500 CTS test sink - * devices the - * - 5120x2880/995.59Mhz - * - 6016x3384/1357.23Mhz - * - 6144x3456/1413.39Mhz - * modes (all the ones having a DPT limit on the above devices), - * both the channel coding efficiency and an additional 3% - * overhead needs to be accounted for. - */ - return div64_u64(mul_u32_u32(intel_dp_link_symbol_clock(crtc_state->port_clock) * 72, - drm_dp_bw_channel_coding_efficiency(true)), - mul_u32_u32(adjusted_mode->crtc_clock, 1030000)); -} - -static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - bool ssc, bool dsc, int bpp_x16) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - unsigned long flags = DRM_DP_BW_OVERHEAD_MST; - int dsc_slice_count = 0; - int overhead; - - flags |= intel_dp_is_uhbr(crtc_state) ? DRM_DP_BW_OVERHEAD_UHBR : 0; - flags |= ssc ? DRM_DP_BW_OVERHEAD_SSC_REF_CLK : 0; - flags |= crtc_state->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0; - - if (dsc) { - flags |= DRM_DP_BW_OVERHEAD_DSC; - dsc_slice_count = intel_dp_dsc_get_slice_count(connector, - adjusted_mode->clock, - adjusted_mode->hdisplay, - crtc_state->joiner_pipes); - } - - overhead = drm_dp_bw_overhead(crtc_state->lane_count, - adjusted_mode->hdisplay, - dsc_slice_count, - bpp_x16, - flags); - - /* - * TODO: clarify whether a minimum required by the fixed FEC overhead - * in the bspec audio programming sequence is required here. - */ - return max(overhead, intel_dp_bw_fec_overhead(crtc_state->fec_enable)); -} - -static void intel_dp_mst_compute_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - int overhead, - int bpp_x16, - struct intel_link_m_n *m_n) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - /* TODO: Check WA 14013163432 to set data M/N for full BW utilization. */ - intel_link_compute_m_n(bpp_x16, crtc_state->lane_count, - adjusted_mode->crtc_clock, - crtc_state->port_clock, - overhead, - m_n); - - m_n->tu = DIV_ROUND_UP_ULL(mul_u32_u32(m_n->data_m, 64), m_n->data_n); -} - -static int intel_dp_mst_calc_pbn(int pixel_clock, int bpp_x16, int bw_overhead) -{ - int effective_data_rate = - intel_dp_effective_data_rate(pixel_clock, bpp_x16, bw_overhead); - - /* - * TODO: Use drm_dp_calc_pbn_mode() instead, once it's converted - * to calculate PBN with the BW overhead passed to it. - */ - return DIV_ROUND_UP(effective_data_rate * 64, 54 * 1000); -} - -static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - int max_bpp, - int min_bpp, - struct link_config_limits *limits, - struct drm_connector_state *conn_state, - int step, - bool dsc) -{ - struct drm_atomic_state *state = crtc_state->uapi.state; - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_state *mst_state; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - int bpp, slots = -EINVAL; - int max_dpt_bpp; - int ret = 0; - - mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - crtc_state->lane_count = limits->max_lane_count; - crtc_state->port_clock = limits->max_rate; - - if (dsc) { - if (!intel_dp_supports_fec(intel_dp, connector, crtc_state)) - return -EINVAL; - - crtc_state->fec_enable = !intel_dp_is_uhbr(crtc_state); - } - - mst_state->pbn_div = drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr, - crtc_state->port_clock, - crtc_state->lane_count); - - max_dpt_bpp = intel_dp_mst_max_dpt_bpp(crtc_state, dsc); - if (max_bpp > max_dpt_bpp) { - drm_dbg_kms(&i915->drm, "Limiting bpp to max DPT bpp (%d -> %d)\n", - max_bpp, max_dpt_bpp); - max_bpp = max_dpt_bpp; - } - - drm_dbg_kms(&i915->drm, "Looking for slots in range min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - for (bpp = max_bpp; bpp >= min_bpp; bpp -= step) { - int local_bw_overhead; - int remote_bw_overhead; - int link_bpp_x16; - int remote_tu; - fixed20_12 pbn; - - drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp); - - link_bpp_x16 = fxp_q4_from_int(dsc ? bpp : - intel_dp_output_bpp(crtc_state->output_format, bpp)); - - local_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - false, dsc, link_bpp_x16); - remote_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - true, dsc, link_bpp_x16); - - intel_dp_mst_compute_m_n(crtc_state, connector, - local_bw_overhead, - link_bpp_x16, - &crtc_state->dp_m_n); - - /* - * The TU size programmed to the HW determines which slots in - * an MTP frame are used for this stream, which needs to match - * the payload size programmed to the first downstream branch - * device's payload table. - * - * Note that atm the payload's PBN value DRM core sends via - * the ALLOCATE_PAYLOAD side-band message matches the payload - * size (which it calculates from the PBN value) it programs - * to the first branch device's payload table. The allocation - * in the payload table could be reduced though (to - * crtc_state->dp_m_n.tu), provided that the driver doesn't - * enable SSC on the corresponding link. - */ - pbn.full = dfixed_const(intel_dp_mst_calc_pbn(adjusted_mode->crtc_clock, - link_bpp_x16, - remote_bw_overhead)); - remote_tu = DIV_ROUND_UP(pbn.full, mst_state->pbn_div.full); - - /* - * Aligning the TUs ensures that symbols consisting of multiple - * (4) symbol cycles don't get split between two consecutive - * MTPs, as required by Bspec. - * TODO: remove the alignment restriction for 128b/132b links - * on some platforms, where Bspec allows this. - */ - remote_tu = ALIGN(remote_tu, 4 / crtc_state->lane_count); - - /* - * Also align PBNs accordingly, since MST core will derive its - * own copy of TU from the PBN in drm_dp_atomic_find_time_slots(). - * The above comment about the difference between the PBN - * allocated for the whole path and the TUs allocated for the - * first branch device's link also applies here. - */ - pbn.full = remote_tu * mst_state->pbn_div.full; - crtc_state->pbn = dfixed_trunc(pbn); - - drm_WARN_ON(&i915->drm, remote_tu < crtc_state->dp_m_n.tu); - crtc_state->dp_m_n.tu = remote_tu; - - slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr, - connector->port, - crtc_state->pbn); - if (slots == -EDEADLK) - return slots; - - if (slots >= 0) { - drm_WARN_ON(&i915->drm, slots != crtc_state->dp_m_n.tu); - - break; - } - } - - /* We failed to find a proper bpp/timeslots, return error */ - if (ret) - slots = ret; - - if (slots < 0) { - drm_dbg_kms(&i915->drm, "failed finding vcpi slots:%d\n", - slots); - } else { - if (!dsc) - crtc_state->pipe_bpp = bpp; - else - crtc_state->dsc.compressed_bpp_x16 = fxp_q4_from_int(bpp); - drm_dbg_kms(&i915->drm, "Got %d slots for pipe bpp %d dsc %d\n", slots, bpp, dsc); - } - - return slots; -} - -static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - int slots = -EINVAL; - - /* - * FIXME: allocate the BW according to link_bpp, which in the case of - * YUV420 is only half of the pipe bpp value. - */ - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, - fxp_q4_to_int(limits->link.max_bpp_x16), - fxp_q4_to_int(limits->link.min_bpp_x16), - limits, - conn_state, 2 * 3, false); - - if (slots < 0) - return slots; - - return 0; -} - -static int intel_dp_dsc_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - int slots = -EINVAL; - int i, num_bpc; - u8 dsc_bpc[3] = {}; - int min_bpp, max_bpp, sink_min_bpp, sink_max_bpp; - u8 dsc_max_bpc; - int min_compressed_bpp, max_compressed_bpp; - - /* Max DSC Input BPC for ICL is 10 and for TGL+ is 12 */ - if (DISPLAY_VER(i915) >= 12) - dsc_max_bpc = min_t(u8, 12, conn_state->max_requested_bpc); - else - dsc_max_bpc = min_t(u8, 10, conn_state->max_requested_bpc); - - max_bpp = min_t(u8, dsc_max_bpc * 3, limits->pipe.max_bpp); - min_bpp = limits->pipe.min_bpp; - - num_bpc = drm_dp_dsc_sink_supported_input_bpcs(connector->dp.dsc_dpcd, - dsc_bpc); - - drm_dbg_kms(&i915->drm, "DSC Source supported min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - sink_max_bpp = dsc_bpc[0] * 3; - sink_min_bpp = sink_max_bpp; - - for (i = 1; i < num_bpc; i++) { - if (sink_min_bpp > dsc_bpc[i] * 3) - sink_min_bpp = dsc_bpc[i] * 3; - if (sink_max_bpp < dsc_bpc[i] * 3) - sink_max_bpp = dsc_bpc[i] * 3; - } - - drm_dbg_kms(&i915->drm, "DSC Sink supported min bpp %d max bpp %d\n", - sink_min_bpp, sink_max_bpp); - - if (min_bpp < sink_min_bpp) - min_bpp = sink_min_bpp; - - if (max_bpp > sink_max_bpp) - max_bpp = sink_max_bpp; - - crtc_state->pipe_bpp = max_bpp; - - max_compressed_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector, - crtc_state, - max_bpp / 3); - max_compressed_bpp = min(max_compressed_bpp, - fxp_q4_to_int(limits->link.max_bpp_x16)); - - min_compressed_bpp = intel_dp_dsc_sink_min_compressed_bpp(crtc_state); - min_compressed_bpp = max(min_compressed_bpp, - fxp_q4_to_int_roundup(limits->link.min_bpp_x16)); - - drm_dbg_kms(&i915->drm, "DSC Sink supported compressed min bpp %d compressed max bpp %d\n", - min_compressed_bpp, max_compressed_bpp); - - /* Align compressed bpps according to our own constraints */ - max_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, max_compressed_bpp, - crtc_state->pipe_bpp); - min_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, min_compressed_bpp, - crtc_state->pipe_bpp); - - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, max_compressed_bpp, - min_compressed_bpp, limits, - conn_state, 1, true); - - if (slots < 0) - return slots; - - return 0; -} -static int intel_dp_mst_update_slots(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_topology_state *topology_state; - u8 link_coding_cap = intel_dp_is_uhbr(crtc_state) ? - DP_CAP_ANSI_128B132B : DP_CAP_ANSI_8B10B; - - topology_state = drm_atomic_get_mst_topology_state(conn_state->state, mgr); - if (IS_ERR(topology_state)) { - drm_dbg_kms(&i915->drm, "slot update failed\n"); - return PTR_ERR(topology_state); - } - - drm_dp_mst_update_slots(topology_state, link_coding_cap); - - return 0; -} - -static int mode_hblank_period_ns(const struct drm_display_mode *mode) -{ - return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(mode->htotal - mode->hdisplay, - NSEC_PER_SEC / 1000), - mode->crtc_clock); -} - -static bool -hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - const struct link_config_limits *limits) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - bool is_uhbr_sink = connector->mst_port && - drm_dp_128b132b_supported(connector->mst_port->dpcd); - int hblank_limit = is_uhbr_sink ? 500 : 300; - - if (!connector->dp.dsc_hblank_expansion_quirk) - return false; - - if (is_uhbr_sink && !drm_dp_is_uhbr_rate(limits->max_rate)) - return false; - - if (mode_hblank_period_ns(adjusted_mode) > hblank_limit) - return false; - - return true; -} - -static bool -adjust_limits_for_dsc_hblank_expansion_quirk(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - struct link_config_limits *limits, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - int min_bpp_x16 = limits->link.min_bpp_x16; - - if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state, limits)) - return true; - - if (!dsc) { - if (intel_dp_supports_dsc(connector, crtc_state)) { - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] DSC needed by hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - return false; - } - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to 24 due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - - if (limits->link.max_bpp_x16 < fxp_q4_from_int(24)) - return false; - - limits->link.min_bpp_x16 = fxp_q4_from_int(24); - - return true; - } - - drm_WARN_ON(&i915->drm, limits->min_rate != limits->max_rate); - - if (limits->max_rate < 540000) - min_bpp_x16 = fxp_q4_from_int(13); - else if (limits->max_rate < 810000) - min_bpp_x16 = fxp_q4_from_int(10); - - if (limits->link.min_bpp_x16 >= min_bpp_x16) - return true; - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to " FXP_Q4_FMT " in DSC mode due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name, - FXP_Q4_ARGS(min_bpp_x16)); - - if (limits->link.max_bpp_x16 < min_bpp_x16) - return false; - - limits->link.min_bpp_x16 = min_bpp_x16; - - return true; -} - -static bool -intel_dp_mst_compute_config_limits(struct intel_dp *intel_dp, - const struct intel_connector *connector, - struct intel_crtc_state *crtc_state, - bool dsc, - struct link_config_limits *limits) -{ - /* - * for MST we always configure max link bw - the spec doesn't - * seem to suggest we should do otherwise. - */ - limits->min_rate = limits->max_rate = - intel_dp_max_link_rate(intel_dp); - - limits->min_lane_count = limits->max_lane_count = - intel_dp_max_lane_count(intel_dp); - - limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); - /* - * FIXME: If all the streams can't fit into the link with - * their current pipe_bpp we should reduce pipe_bpp across - * the board until things start to fit. Until then we - * limit to <= 8bpc since that's what was hardcoded for all - * MST streams previously. This hack should be removed once - * we have the proper retry logic in place. - */ - limits->pipe.max_bpp = min(crtc_state->pipe_bpp, 24); - - intel_dp_adjust_compliance_config(intel_dp, crtc_state, limits); - - if (!intel_dp_compute_config_link_bpp_limits(intel_dp, - crtc_state, - dsc, - limits)) - return false; - - return adjust_limits_for_dsc_hblank_expansion_quirk(connector, - crtc_state, - limits, - dsc); -} - -static int intel_dp_mst_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - struct link_config_limits limits; - bool dsc_needed, joiner_needs_dsc; - int ret = 0; - - if (pipe_config->fec_enable && - !intel_dp_supports_fec(intel_dp, connector, pipe_config)) - return -EINVAL; - - if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return -EINVAL; - - if (intel_dp_need_joiner(intel_dp, connector, - adjusted_mode->crtc_hdisplay, - adjusted_mode->crtc_clock)) - pipe_config->joiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe); - - pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->has_pch_encoder = false; - - joiner_needs_dsc = intel_dp_joiner_needs_dsc(dev_priv, pipe_config->joiner_pipes); - - dsc_needed = joiner_needs_dsc || intel_dp->force_dsc_en || - !intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - false, - &limits); - - if (!dsc_needed) { - ret = intel_dp_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - - if (ret == -EDEADLK) - return ret; - - if (ret) - dsc_needed = true; - } - - /* enable compression if the mode doesn't fit available BW */ - if (dsc_needed) { - drm_dbg_kms(&dev_priv->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n", - str_yes_no(ret), str_yes_no(joiner_needs_dsc), - str_yes_no(intel_dp->force_dsc_en)); - - if (!intel_dp_supports_dsc(connector, pipe_config)) - return -EINVAL; - - if (!intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - true, - &limits)) - return -EINVAL; - - /* - * FIXME: As bpc is hardcoded to 8, as mentioned above, - * WARN and ignore the debug flag force_dsc_bpc for now. - */ - drm_WARN(&dev_priv->drm, intel_dp->force_dsc_bpc, "Cannot Force BPC for MST\n"); - /* - * Try to get at least some timeslots and then see, if - * we can fit there with DSC. - */ - drm_dbg_kms(&dev_priv->drm, "Trying to find VCPI slots in DSC mode\n"); - - ret = intel_dp_dsc_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - if (ret < 0) - return ret; - - ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, - conn_state, &limits, - pipe_config->dp_m_n.tu, false); - } - - if (ret) - return ret; - - ret = intel_dp_mst_update_slots(encoder, pipe_config, conn_state); - if (ret) - return ret; - - pipe_config->limited_color_range = - intel_dp_limited_color_range(pipe_config, conn_state); - - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - pipe_config->lane_lat_optim_mask = - bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); - - intel_dp_audio_compute_config(encoder, pipe_config, conn_state); - - intel_ddi_compute_min_voltage_level(pipe_config); - - intel_psr_compute_config(intel_dp, pipe_config, conn_state); - - return intel_dp_tunnel_atomic_compute_stream_bw(state, intel_dp, connector, - pipe_config); -} - -/* - * Iterate over all connectors and return a mask of - * all CPU transcoders streaming over the same DP link. - */ -static unsigned int -intel_dp_mst_transcoder_mask(struct intel_atomic_state *state, - struct intel_dp *mst_port) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 transcoders = 0; - int i; - - if (DISPLAY_VER(dev_priv) < 12) - return 0; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - const struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector->mst_port != mst_port || !conn_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_state->base.crtc); - crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - - if (!crtc_state->hw.active) - continue; - - transcoders |= BIT(crtc_state->cpu_transcoder); - } - - return transcoders; -} - -static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_port *parent_port) -{ - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 mask = 0; - int i; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - if (!conn_state->base.crtc) - continue; - - if (&connector->mst_port->mst_mgr != mst_mgr) - continue; - - if (connector->port != parent_port && - !drm_dp_mst_port_downstream_of_parent(mst_mgr, - connector->port, - parent_port)) - continue; - - mask |= BIT(to_intel_crtc(conn_state->base.crtc)->pipe); - } - - return mask; -} - -static int intel_dp_mst_check_fec_change(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct intel_link_bw_limits *limits) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - struct intel_crtc *crtc; - u8 mst_pipe_mask; - u8 fec_pipe_mask = 0; - int ret; - - mst_pipe_mask = get_pipes_downstream_of_mst_port(state, mst_mgr, NULL); - - for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, mst_pipe_mask) { - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - /* Atomic connector check should've added all the MST CRTCs. */ - if (drm_WARN_ON(&i915->drm, !crtc_state)) - return -EINVAL; - - if (crtc_state->fec_enable) - fec_pipe_mask |= BIT(crtc->pipe); - } - - if (!fec_pipe_mask || mst_pipe_mask == fec_pipe_mask) - return 0; - - limits->force_fec_pipes |= mst_pipe_mask; - - ret = intel_modeset_pipes_in_mask_early(state, "MST FEC", - mst_pipe_mask); - - return ret ? : -EAGAIN; -} - -static int intel_dp_mst_check_bw(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_topology_state *mst_state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_port *mst_port; - u8 mst_port_pipes; - int ret; - - ret = drm_dp_mst_atomic_check_mgr(&state->base, mst_mgr, mst_state, &mst_port); - if (ret != -ENOSPC) - return ret; - - mst_port_pipes = get_pipes_downstream_of_mst_port(state, mst_mgr, mst_port); - - ret = intel_link_bw_reduce_bpp(state, limits, - mst_port_pipes, "MST link BW"); - - return ret ? : -EAGAIN; -} - -/** - * intel_dp_mst_atomic_check_link - check all modeset MST link configuration - * @state: intel atomic state - * @limits: link BW limits - * - * Check the link configuration for all modeset MST outputs. If the - * configuration is invalid @limits will be updated if possible to - * reduce the total BW, after which the configuration for all CRTCs in - * @state must be recomputed with the updated @limits. - * - * Returns: - * - 0 if the confugration is valid - * - %-EAGAIN, if the configuration is invalid and @limits got updated - * with fallback values with which the configuration of all CRTCs in - * @state must be recomputed - * - Other negative error, if the configuration is invalid without a - * fallback possibility, or the check failed for another reason - */ -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_topology_mgr *mgr; - struct drm_dp_mst_topology_state *mst_state; - int ret; - int i; - - for_each_new_mst_mgr_in_state(&state->base, mgr, mst_state, i) { - ret = intel_dp_mst_check_fec_change(state, mgr, limits); - if (ret) - return ret; - - ret = intel_dp_mst_check_bw(state, mgr, mst_state, - limits); - if (ret) - return ret; - } - - return 0; -} - -static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - /* lowest numbered transcoder will be designated master */ - crtc_state->mst_master_transcoder = - ffs(intel_dp_mst_transcoder_mask(state, intel_dp)) - 1; - - return 0; -} - -/* - * If one of the connectors in a MST stream needs a modeset, mark all CRTCs - * that shares the same MST stream as mode changed, - * intel_modeset_pipe_config()+intel_crtc_check_fastset() will take care to do - * a fastset when possible. - * - * On TGL+ this is required since each stream go through a master transcoder, - * so if the master transcoder needs modeset, all other streams in the - * topology need a modeset. All platforms need to add the atomic state - * for all streams in the topology, since a modeset on one may require - * changing the MST link BW usage of the others, which in turn needs a - * recomputation of the corresponding CRTC states. - */ -static int -intel_dp_mst_atomic_topology_check(struct intel_connector *connector, - struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct drm_connector_list_iter connector_list_iter; - struct intel_connector *connector_iter; - int ret = 0; - - if (!intel_connector_needs_modeset(state, &connector->base)) - return 0; - - drm_connector_list_iter_begin(&dev_priv->drm, &connector_list_iter); - for_each_intel_connector_iter(connector_iter, &connector_list_iter) { - struct intel_digital_connector_state *conn_iter_state; - struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector_iter->mst_port != connector->mst_port || - connector_iter == connector) - continue; - - conn_iter_state = intel_atomic_get_digital_connector_state(state, - connector_iter); - if (IS_ERR(conn_iter_state)) { - ret = PTR_ERR(conn_iter_state); - break; - } - - if (!conn_iter_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_iter_state->base.crtc); - crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - break; - } - - ret = drm_atomic_add_affected_planes(&state->base, &crtc->base); - if (ret) - break; - crtc_state->uapi.mode_changed = true; - } - drm_connector_list_iter_end(&connector_list_iter); - - return ret; -} - -static int -intel_dp_mst_atomic_check(struct drm_connector *connector, - struct drm_atomic_state *_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(_state); - struct intel_connector *intel_connector = - to_intel_connector(connector); - int ret; - - ret = intel_digital_connector_atomic_check(connector, &state->base); - if (ret) - return ret; - - ret = intel_dp_mst_atomic_topology_check(intel_connector, state); - if (ret) - return ret; - - if (intel_connector_needs_modeset(state, connector)) { - ret = intel_dp_tunnel_atomic_check_state(state, - intel_connector->mst_port, - intel_connector); - if (ret) - return ret; - } - - return drm_dp_atomic_release_time_slots(&state->base, - &intel_connector->mst_port->mst_mgr, - intel_connector->port); -} - -static void clear_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - - intel_de_write(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT); -} - -static void wait_for_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - if (intel_de_wait_for_set(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT, 1)) - drm_err(&i915->drm, "Timed out waiting for ACT sent\n"); - - drm_dp_check_act_status(&intel_dp->mst_mgr); -} - -static void intel_mst_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - - drm_dbg_kms(&i915->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (intel_dp->active_mst_links == 1) - intel_dp->link_trained = false; - - intel_hdcp_disable(intel_mst->connector); - - intel_dp_sink_disable_decompression(state, connector, old_crtc_state); -} - -static void intel_mst_post_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_dp_mst_topology_state *old_mst_state = - drm_atomic_get_old_mst_topology_state(&state->base, &intel_dp->mst_mgr); - struct drm_dp_mst_topology_state *new_mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - const struct drm_dp_mst_atomic_payload *old_payload = - drm_atomic_get_mst_payload_state(old_mst_state, connector->port); - struct drm_dp_mst_atomic_payload *new_payload = - drm_atomic_get_mst_payload_state(new_mst_state, connector->port); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_crtc *pipe_crtc; - bool last_mst_stream; - - intel_dp->active_mst_links--; - last_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && last_mst_stream && - !intel_dp_mst_is_master_trans(old_crtc_state)); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_off(old_pipe_crtc_state); - } - - intel_disable_transcoder(old_crtc_state); - - drm_dp_remove_payload_part1(&intel_dp->mst_mgr, new_mst_state, new_payload); - - clear_act_sent(encoder, old_crtc_state); - - intel_de_rmw(dev_priv, - TRANS_DDI_FUNC_CTL(dev_priv, old_crtc_state->cpu_transcoder), - TRANS_DDI_DP_VC_PAYLOAD_ALLOC, 0); - - wait_for_act_sent(encoder, old_crtc_state); - - drm_dp_remove_payload_part2(&intel_dp->mst_mgr, new_mst_state, - old_payload, new_payload); - - intel_ddi_disable_transcoder_func(old_crtc_state); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_dsc_disable(old_pipe_crtc_state); - - if (DISPLAY_VER(dev_priv) >= 9) - skl_scaler_disable(old_pipe_crtc_state); - else - ilk_pfit_disable(old_pipe_crtc_state); - } - - /* - * Power down mst path before disabling the port, otherwise we end - * up getting interrupts from the sink upon detecting link loss. - */ - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, - false); - - /* - * BSpec 4287: disable DIP after the transcoder is disabled and before - * the transcoder clock select is set to none. - */ - intel_dp_set_infoframes(&dig_port->base, false, - old_crtc_state, NULL); - /* - * From TGL spec: "If multi-stream slave transcoder: Configure - * Transcoder Clock Select to direct no clock to the transcoder" - * - * From older GENs spec: "Configure Transcoder Clock Select to direct - * no clock to the transcoder" - */ - if (DISPLAY_VER(dev_priv) < 12 || !last_mst_stream) - intel_ddi_disable_transcoder_clock(old_crtc_state); - - - intel_mst->connector = NULL; - if (last_mst_stream) - dig_port->base.post_disable(state, &dig_port->base, - old_crtc_state, NULL); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); -} - -static void intel_mst_post_pll_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0 && - dig_port->base.post_pll_disable) - dig_port->base.post_pll_disable(state, encoder, old_crtc_state, old_conn_state); -} - -static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0) - dig_port->base.pre_pll_enable(state, &dig_port->base, - pipe_config, NULL); - else - /* - * The port PLL state needs to get updated for secondary - * streams as for the primary stream. - */ - intel_ddi_update_active_dpll(state, &dig_port->base, - to_intel_crtc(pipe_config->uapi.crtc)); -} - -static bool intel_mst_probed_link_params_valid(struct intel_dp *intel_dp, - int link_rate, int lane_count) -{ - return intel_dp->link.mst_probed_rate == link_rate && - intel_dp->link.mst_probed_lane_count == lane_count; -} - -static void intel_mst_set_probed_link_params(struct intel_dp *intel_dp, - int link_rate, int lane_count) -{ - intel_dp->link.mst_probed_rate = link_rate; - intel_dp->link.mst_probed_lane_count = lane_count; -} - -static void intel_mst_reprobe_topology(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - if (intel_mst_probed_link_params_valid(intel_dp, - crtc_state->port_clock, crtc_state->lane_count)) - return; - - drm_dp_mst_topology_queue_probe(&intel_dp->mst_mgr); - - intel_mst_set_probed_link_params(intel_dp, - crtc_state->port_clock, crtc_state->lane_count); -} - -static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - int ret; - bool first_mst_stream; - - /* MST encoders are bound to a crtc, not to a connector, - * force the mapping here for get_hw_state. - */ - connector->encoder = encoder; - intel_mst->connector = connector; - first_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && first_mst_stream && - !intel_dp_mst_is_master_trans(pipe_config)); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (first_mst_stream) - intel_dp_set_power(intel_dp, DP_SET_POWER_D0); - - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true); - - intel_dp_sink_enable_decompression(state, connector, pipe_config); - - if (first_mst_stream) { - dig_port->base.pre_enable(state, &dig_port->base, - pipe_config, NULL); - - intel_mst_reprobe_topology(intel_dp, pipe_config); - } - - intel_dp->active_mst_links++; - - ret = drm_dp_add_payload_part1(&intel_dp->mst_mgr, mst_state, - drm_atomic_get_mst_payload_state(mst_state, connector->port)); - if (ret < 0) - intel_dp_queue_modeset_retry_for_link(state, &dig_port->base, pipe_config); - - /* - * Before Gen 12 this is not done as part of - * dig_port->base.pre_enable() and should be done here. For - * Gen 12+ the step in which this should be done is different for the - * first MST stream, so it's done on the DDI for the first stream and - * here for the following ones. - */ - if (DISPLAY_VER(dev_priv) < 12 || !first_mst_stream) - intel_ddi_enable_transcoder_clock(encoder, pipe_config); - - intel_dsc_dp_pps_write(&dig_port->base, pipe_config); - intel_ddi_set_dp_msa(pipe_config, conn_state); -} - -static void enable_bs_jitter_was(const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - u32 clear = 0; - u32 set = 0; - - if (!IS_ALDERLAKE_P(i915)) - return; - - if (!IS_DISPLAY_STEP(i915, STEP_D0, STEP_FOREVER)) - return; - - /* Wa_14013163432:adlp */ - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_FEC_BS_JITTER_WA(crtc_state->cpu_transcoder); - - /* Wa_14014143976:adlp */ - if (IS_DISPLAY_STEP(i915, STEP_E0, STEP_FOREVER)) { - if (intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - else if (crtc_state->fec_enable) - clear |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_DPT_DPTP_ALIGN_WA(crtc_state->cpu_transcoder); - } - - if (!clear && !set) - return; - - intel_de_rmw(i915, CHICKEN_MISC_3, clear, set); -} - -static void intel_mst_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - enum transcoder trans = pipe_config->cpu_transcoder; - bool first_mst_stream = intel_dp->active_mst_links == 1; - struct intel_crtc *pipe_crtc; - int ret; - - drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder); - - if (intel_dp_is_uhbr(pipe_config)) { - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - u64 crtc_clock_hz = KHz(adjusted_mode->crtc_clock); - - intel_de_write(dev_priv, TRANS_DP2_VFREQHIGH(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz >> 24)); - intel_de_write(dev_priv, TRANS_DP2_VFREQLOW(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz & 0xffffff)); - } - - enable_bs_jitter_was(pipe_config); - - intel_ddi_enable_transcoder_func(encoder, pipe_config); - - clear_act_sent(encoder, pipe_config); - - intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(dev_priv, trans), 0, - TRANS_DDI_DP_VC_PAYLOAD_ALLOC); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - wait_for_act_sent(encoder, pipe_config); - - if (first_mst_stream) - intel_ddi_wait_for_fec_status(encoder, pipe_config, true); - - ret = drm_dp_add_payload_part2(&intel_dp->mst_mgr, - drm_atomic_get_mst_payload_state(mst_state, - connector->port)); - if (ret < 0) - intel_dp_queue_modeset_retry_for_link(state, &dig_port->base, pipe_config); - - if (DISPLAY_VER(dev_priv) >= 12) - intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, trans), - FECSTALL_DIS_DPTSTREAM_DPTTG, - pipe_config->fec_enable ? FECSTALL_DIS_DPTSTREAM_DPTTG : 0); - - intel_audio_sdp_split_update(pipe_config); - - intel_enable_transcoder(pipe_config); - - for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(pipe_config)) { - const struct intel_crtc_state *pipe_crtc_state = - intel_atomic_get_new_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_on(pipe_crtc_state); - } - - intel_hdcp_enable(state, encoder, pipe_config, conn_state); -} - -static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder, - enum pipe *pipe) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - *pipe = intel_mst->pipe; - if (intel_mst->connector) - return true; - return false; -} - -static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - dig_port->base.get_config(&dig_port->base, pipe_config); -} - -static bool intel_dp_mst_initial_fastset_check(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - return intel_dp_initial_fastset_check(&dig_port->base, crtc_state); -} - -static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - struct drm_i915_private *i915 = to_i915(intel_connector->base.dev); - struct intel_dp *intel_dp = intel_connector->mst_port; - const struct drm_edid *drm_edid; - int ret; - - if (drm_connector_is_unregistered(connector)) - return intel_connector_update_modes(connector, NULL); - - if (!intel_display_driver_check_access(i915)) - return drm_edid_connector_add_modes(connector); - - drm_edid = drm_dp_mst_edid_read(connector, &intel_dp->mst_mgr, intel_connector->port); - - ret = intel_connector_update_modes(connector, drm_edid); - - drm_edid_free(drm_edid); - - return ret; -} - -static int -intel_dp_mst_connector_late_register(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - int ret; - - ret = drm_dp_mst_connector_late_register(connector, - intel_connector->port); - if (ret < 0) - return ret; - - ret = intel_connector_register(connector); - if (ret < 0) - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); - - return ret; -} - -static void -intel_dp_mst_connector_early_unregister(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - intel_connector_unregister(connector); - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); -} - -static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { - .fill_modes = drm_helper_probe_single_connector_modes, - .atomic_get_property = intel_digital_connector_atomic_get_property, - .atomic_set_property = intel_digital_connector_atomic_set_property, - .late_register = intel_dp_mst_connector_late_register, - .early_unregister = intel_dp_mst_connector_early_unregister, - .destroy = intel_connector_destroy, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_duplicate_state = intel_digital_connector_duplicate_state, -}; - -static int intel_dp_mst_get_modes(struct drm_connector *connector) -{ - return intel_dp_mst_get_ddc_modes(connector); -} - -static int -intel_dp_mst_mode_valid_ctx(struct drm_connector *connector, - struct drm_display_mode *mode, - struct drm_modeset_acquire_ctx *ctx, - enum drm_mode_status *status) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_port *port = intel_connector->port; - const int min_bpp = 18; - int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq; - int max_rate, mode_rate, max_lanes, max_link_clock; - int ret; - bool dsc = false, joiner = false; - u16 dsc_max_compressed_bpp = 0; - u8 dsc_slice_count = 0; - int target_clock = mode->clock; - - if (drm_connector_is_unregistered(connector)) { - *status = MODE_ERROR; - return 0; - } - - *status = intel_cpu_transcoder_mode_valid(dev_priv, mode); - if (*status != MODE_OK) - return 0; - - if (mode->flags & DRM_MODE_FLAG_DBLCLK) { - *status = MODE_H_ILLEGAL; - return 0; - } - - if (mode->clock < 10000) { - *status = MODE_CLOCK_LOW; - return 0; - } - - max_link_clock = intel_dp_max_link_rate(intel_dp); - max_lanes = intel_dp_max_lane_count(intel_dp); - - max_rate = intel_dp_max_link_data_rate(intel_dp, - max_link_clock, max_lanes); - mode_rate = intel_dp_link_required(mode->clock, min_bpp); - - /* - * TODO: - * - Also check if compression would allow for the mode - * - Calculate the overhead using drm_dp_bw_overhead() / - * drm_dp_bw_channel_coding_efficiency(), similarly to the - * compute config code, as drm_dp_calc_pbn_mode() doesn't - * account with all the overheads. - * - Check here and during compute config the BW reported by - * DFP_Link_Available_Payload_Bandwidth_Number (or the - * corresponding link capabilities of the sink) in case the - * stream is uncompressed for it by the last branch device. - */ - if (intel_dp_need_joiner(intel_dp, intel_connector, - mode->hdisplay, target_clock)) { - joiner = true; - max_dotclk *= 2; - } - - ret = drm_modeset_lock(&mgr->base.lock, ctx); - if (ret) - return ret; - - if (mode_rate > max_rate || mode->clock > max_dotclk || - drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (intel_dp_has_dsc(intel_connector)) { - /* - * TBD pass the connector BPC, - * for now U8_MAX so that max BPC on that platform would be picked - */ - int pipe_bpp = intel_dp_dsc_compute_max_bpp(intel_connector, U8_MAX); - - if (drm_dp_sink_supports_fec(intel_connector->dp.fec_capability)) { - dsc_max_compressed_bpp = - intel_dp_dsc_get_max_compressed_bpp(dev_priv, - max_link_clock, - max_lanes, - target_clock, - mode->hdisplay, - joiner, - INTEL_OUTPUT_FORMAT_RGB, - pipe_bpp, 64); - dsc_slice_count = - intel_dp_dsc_get_slice_count(intel_connector, - target_clock, - mode->hdisplay, - joiner); - } - - dsc = dsc_max_compressed_bpp && dsc_slice_count; - } - - if (intel_dp_joiner_needs_dsc(dev_priv, joiner) && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (mode_rate > max_rate && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - *status = intel_mode_valid_max_plane_size(dev_priv, mode, joiner); - return 0; -} - -static struct drm_encoder *intel_mst_atomic_best_encoder(struct drm_connector *connector, - struct drm_atomic_state *state) -{ - struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state, - connector); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct intel_crtc *crtc = to_intel_crtc(connector_state->crtc); - - return &intel_dp->mst_encoders[crtc->pipe]->base.base; -} - -static int -intel_dp_mst_detect(struct drm_connector *connector, - struct drm_modeset_acquire_ctx *ctx, bool force) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - - if (!intel_display_device_enabled(i915)) - return connector_status_disconnected; - - if (drm_connector_is_unregistered(connector)) - return connector_status_disconnected; - - if (!intel_display_driver_check_access(i915)) - return connector->status; - - return drm_dp_mst_detect_port(connector, ctx, &intel_dp->mst_mgr, - intel_connector->port); -} - -static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = { - .get_modes = intel_dp_mst_get_modes, - .mode_valid_ctx = intel_dp_mst_mode_valid_ctx, - .atomic_best_encoder = intel_mst_atomic_best_encoder, - .atomic_check = intel_dp_mst_atomic_check, - .detect_ctx = intel_dp_mst_detect, -}; - -static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(to_intel_encoder(encoder)); - - drm_encoder_cleanup(encoder); - kfree(intel_mst); -} - -static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = { - .destroy = intel_dp_mst_encoder_destroy, -}; - -static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) -{ - if (intel_attached_encoder(connector) && connector->base.state->crtc) { - enum pipe pipe; - if (!intel_attached_encoder(connector)->get_hw_state(intel_attached_encoder(connector), &pipe)) - return false; - return true; - } - return false; -} - -static int intel_dp_mst_add_properties(struct intel_dp *intel_dp, - struct drm_connector *connector, - const char *pathprop) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - - drm_object_attach_property(&connector->base, - i915->drm.mode_config.path_property, 0); - drm_object_attach_property(&connector->base, - i915->drm.mode_config.tile_property, 0); - - intel_attach_force_audio_property(connector); - intel_attach_broadcast_rgb_property(connector); - - /* - * Reuse the prop from the SST connector because we're - * not allowed to create new props after device registration. - */ - connector->max_bpc_property = - intel_dp->attached_connector->base.max_bpc_property; - if (connector->max_bpc_property) - drm_connector_attach_max_bpc_property(connector, 6, 12); - - return drm_connector_set_path_property(connector, pathprop); -} - -static void -intel_dp_mst_read_decompression_port_dsc_caps(struct intel_dp *intel_dp, - struct intel_connector *connector) -{ - u8 dpcd_caps[DP_RECEIVER_CAP_SIZE]; - - if (!connector->dp.dsc_decompression_aux) - return; - - if (drm_dp_read_dpcd_caps(connector->dp.dsc_decompression_aux, dpcd_caps) < 0) - return; - - intel_dp_get_dsc_sink_cap(dpcd_caps[DP_DPCD_REV], connector); -} - -static bool detect_dsc_hblank_expansion_quirk(const struct intel_connector *connector) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct drm_dp_aux *aux = connector->dp.dsc_decompression_aux; - struct drm_dp_desc desc; - u8 dpcd[DP_RECEIVER_CAP_SIZE]; - - if (!aux) - return false; - - /* - * A logical port's OUI (at least for affected sinks) is all 0, so - * instead of that the parent port's OUI is used for identification. - */ - if (drm_dp_mst_port_is_logical(connector->port)) { - aux = drm_dp_mst_aux_for_parent(connector->port); - if (!aux) - aux = &connector->mst_port->aux; - } - - if (drm_dp_read_dpcd_caps(aux, dpcd) < 0) - return false; - - if (drm_dp_read_desc(aux, &desc, drm_dp_is_branch(dpcd)) < 0) - return false; - - if (!drm_dp_has_quirk(&desc, - DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC)) - return false; - - /* - * UHBR (MST sink) devices requiring this quirk don't advertise the - * HBLANK expansion support. Presuming that they perform HBLANK - * expansion internally, or are affected by this issue on modes with a - * short HBLANK for other reasons. - */ - if (!drm_dp_128b132b_supported(dpcd) && - !(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE)) - return false; - - drm_dbg_kms(&i915->drm, - "[CONNECTOR:%d:%s] DSC HBLANK expansion quirk detected\n", - connector->base.base.id, connector->base.name); - - return true; -} - -static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, - struct drm_dp_mst_port *port, - const char *pathprop) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_connector *intel_connector; - struct drm_connector *connector; - enum pipe pipe; - int ret; - - intel_connector = intel_connector_alloc(); - if (!intel_connector) - return NULL; - - intel_connector->get_hw_state = intel_dp_mst_get_hw_state; - intel_connector->sync_state = intel_dp_connector_sync_state; - intel_connector->mst_port = intel_dp; - intel_connector->port = port; - drm_dp_mst_get_port_malloc(port); - - intel_dp_init_modeset_retry_work(intel_connector); - - intel_connector->dp.dsc_decompression_aux = drm_dp_mst_dsc_aux_for_port(port); - intel_dp_mst_read_decompression_port_dsc_caps(intel_dp, intel_connector); - intel_connector->dp.dsc_hblank_expansion_quirk = - detect_dsc_hblank_expansion_quirk(intel_connector); - - connector = &intel_connector->base; - ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, - DRM_MODE_CONNECTOR_DisplayPort); - if (ret) { - drm_dp_mst_put_port_malloc(port); - intel_connector_free(intel_connector); - return NULL; - } - - drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs); - - for_each_pipe(dev_priv, pipe) { - struct drm_encoder *enc = - &intel_dp->mst_encoders[pipe]->base.base; - - ret = drm_connector_attach_encoder(&intel_connector->base, enc); - if (ret) - goto err; - } - - ret = intel_dp_mst_add_properties(intel_dp, connector, pathprop); - if (ret) - goto err; - - ret = intel_dp_hdcp_init(dig_port, intel_connector); - if (ret) - drm_dbg_kms(&dev_priv->drm, "[%s:%d] HDCP MST init failed, skipping.\n", - connector->name, connector->base.id); - - return connector; - -err: - drm_connector_cleanup(connector); - return NULL; -} - -static void -intel_dp_mst_poll_hpd_irq(struct drm_dp_mst_topology_mgr *mgr) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - - intel_hpd_trigger_irq(dp_to_dig_port(intel_dp)); -} - -static const struct drm_dp_mst_topology_cbs mst_cbs = { - .add_connector = intel_dp_add_mst_connector, - .poll_hpd_irq = intel_dp_mst_poll_hpd_irq, -}; - -static struct intel_dp_mst_encoder * -intel_dp_create_fake_mst_encoder(struct intel_digital_port *dig_port, enum pipe pipe) -{ - struct intel_dp_mst_encoder *intel_mst; - struct intel_encoder *intel_encoder; - struct drm_device *dev = dig_port->base.base.dev; - - intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL); - - if (!intel_mst) - return NULL; - - intel_mst->pipe = pipe; - intel_encoder = &intel_mst->base; - intel_mst->primary = dig_port; - - drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, - DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe)); - - intel_encoder->type = INTEL_OUTPUT_DP_MST; - intel_encoder->power_domain = dig_port->base.power_domain; - intel_encoder->port = dig_port->base.port; - intel_encoder->cloneable = 0; - /* - * This is wrong, but broken userspace uses the intersection - * of possible_crtcs of all the encoders of a given connector - * to figure out which crtcs can drive said connector. What - * should be used instead is the union of possible_crtcs. - * To keep such userspace functioning we must misconfigure - * this to make sure the intersection is not empty :( - */ - intel_encoder->pipe_mask = ~0; - - intel_encoder->compute_config = intel_dp_mst_compute_config; - intel_encoder->compute_config_late = intel_dp_mst_compute_config_late; - intel_encoder->disable = intel_mst_disable_dp; - intel_encoder->post_disable = intel_mst_post_disable_dp; - intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp; - intel_encoder->update_pipe = intel_ddi_update_pipe; - intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; - intel_encoder->pre_enable = intel_mst_pre_enable_dp; - intel_encoder->enable = intel_mst_enable_dp; - intel_encoder->audio_enable = intel_audio_codec_enable; - intel_encoder->audio_disable = intel_audio_codec_disable; - intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state; - intel_encoder->get_config = intel_dp_mst_enc_get_config; - intel_encoder->initial_fastset_check = intel_dp_mst_initial_fastset_check; - - return intel_mst; - -} - -static bool -intel_dp_create_fake_mst_encoders(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum pipe pipe; - - for_each_pipe(dev_priv, pipe) - intel_dp->mst_encoders[pipe] = intel_dp_create_fake_mst_encoder(dig_port, pipe); - return true; -} - -int -intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port) -{ - return dig_port->dp.active_mst_links; -} - -int -intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id) -{ - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - struct intel_dp *intel_dp = &dig_port->dp; - enum port port = dig_port->base.port; - int ret; - - if (!HAS_DP_MST(i915) || intel_dp_is_edp(intel_dp)) - return 0; - - if (DISPLAY_VER(i915) < 12 && port == PORT_A) - return 0; - - if (DISPLAY_VER(i915) < 11 && port == PORT_E) - return 0; - - intel_dp->mst_mgr.cbs = &mst_cbs; - - /* create encoders */ - intel_dp_create_fake_mst_encoders(dig_port); - ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, &i915->drm, - &intel_dp->aux, 16, 3, conn_base_id); - if (ret) { - intel_dp->mst_mgr.cbs = NULL; - return ret; - } - - return 0; -} - -bool intel_dp_mst_source_support(struct intel_dp *intel_dp) -{ - return intel_dp->mst_mgr.cbs; -} - -void -intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - - if (!intel_dp_mst_source_support(intel_dp)) - return; - - drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr); - /* encoders will get killed by normal cleanup */ - - intel_dp->mst_mgr.cbs = NULL; -} - -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder == crtc_state->cpu_transcoder; -} - -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder != INVALID_TRANSCODER && - crtc_state->mst_master_transcoder != crtc_state->cpu_transcoder; -} - -/** - * intel_dp_mst_add_topology_state_for_connector - add MST topology state for a connector - * @state: atomic state - * @connector: connector to add the state for - * @crtc: the CRTC @connector is attached to - * - * Add the MST topology state for @connector to @state. - * - * Returns 0 on success, negative error code on failure. - */ -static int -intel_dp_mst_add_topology_state_for_connector(struct intel_atomic_state *state, - struct intel_connector *connector, - struct intel_crtc *crtc) -{ - struct drm_dp_mst_topology_state *mst_state; - - if (!connector->mst_port) - return 0; - - mst_state = drm_atomic_get_mst_topology_state(&state->base, - &connector->mst_port->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - mst_state->pending_crtc_mask |= drm_crtc_mask(&crtc->base); - - return 0; -} - -/** - * intel_dp_mst_add_topology_state_for_crtc - add MST topology state for a CRTC - * @state: atomic state - * @crtc: CRTC to add the state for - * - * Add the MST topology state for @crtc to @state. - * - * Returns 0 on success, negative error code on failure. - */ -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_connector *_connector; - struct drm_connector_state *conn_state; - int i; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - struct intel_connector *connector = to_intel_connector(_connector); - int ret; - - if (conn_state->crtc != &crtc->base) - continue; - - ret = intel_dp_mst_add_topology_state_for_connector(state, connector, crtc); - if (ret) - return ret; - } - - return 0; -} - -static struct intel_connector * -get_connector_in_state_for_crtc(struct intel_atomic_state *state, - const struct intel_crtc *crtc) -{ - struct drm_connector_state *old_conn_state; - struct drm_connector_state *new_conn_state; - struct drm_connector *_connector; - int i; - - for_each_oldnew_connector_in_state(&state->base, _connector, - old_conn_state, new_conn_state, i) { - struct intel_connector *connector = - to_intel_connector(_connector); - - if (old_conn_state->crtc == &crtc->base || - new_conn_state->crtc == &crtc->base) - return connector; - } - - return NULL; -} - -/** - * intel_dp_mst_crtc_needs_modeset - check if changes in topology need to modeset the given CRTC - * @state: atomic state - * @crtc: CRTC for which to check the modeset requirement - * - * Check if any change in a MST topology requires a forced modeset on @crtc in - * this topology. One such change is enabling/disabling the DSC decompression - * state in the first branch device's UFP DPCD as required by one CRTC, while - * the other @crtc in the same topology is still active, requiring a full modeset - * on @crtc. - */ -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - const struct intel_connector *crtc_connector; - const struct drm_connector_state *conn_state; - const struct drm_connector *_connector; - int i; - - if (!intel_crtc_has_type(intel_atomic_get_new_crtc_state(state, crtc), - INTEL_OUTPUT_DP_MST)) - return false; - - crtc_connector = get_connector_in_state_for_crtc(state, crtc); - - if (!crtc_connector) - /* None of the connectors in the topology needs modeset */ - return false; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - const struct intel_connector *connector = - to_intel_connector(_connector); - const struct intel_crtc_state *new_crtc_state; - const struct intel_crtc_state *old_crtc_state; - struct intel_crtc *crtc_iter; - - if (connector->mst_port != crtc_connector->mst_port || - !conn_state->crtc) - continue; - - crtc_iter = to_intel_crtc(conn_state->crtc); - - new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc_iter); - old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc_iter); - - if (!intel_crtc_needs_modeset(new_crtc_state)) - continue; - - if (old_crtc_state->dsc.compression_enable == - new_crtc_state->dsc.compression_enable) - continue; - /* - * Toggling the decompression flag because of this stream in - * the first downstream branch device's UFP DPCD may reset the - * whole branch device. To avoid the reset while other streams - * are also active modeset the whole MST topology in this - * case. - */ - if (connector->dp.dsc_decompression_aux == - &connector->mst_port->aux) - return true; - } - - return false; -} - -<<<<<<< -/* - * intel_dp_mst_verify_dpcd_state - verify the MST SW enabled state wrt. the DPCD - * @intel_dp: DP port object - * - * Verify if @intel_dp's MST enabled SW state matches the corresponding DPCD - * state. A long HPD pulse - not long enough to be detected as a disconnected - * state - could've reset the DPCD state, which requires tearing - * down/recreating the MST topology. - * - * Returns %true if the SW MST enabled and DPCD states match, %false - * otherwise. - */ -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp) -{ - struct intel_display *display = to_intel_display(intel_dp); - struct intel_connector *connector = intel_dp->attached_connector; - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *encoder = &dig_port->base; - int ret; - u8 val; - - if (!intel_dp->is_mst) - return true; - - ret = drm_dp_dpcd_readb(intel_dp->mst_mgr.aux, DP_MSTM_CTRL, &val); - - /* Adjust the expected register value for SST + SideBand. */ - if (ret < 0 || val != (DP_MST_EN | DP_UP_REQ_EN | DP_UPSTREAM_IS_SRC)) { - drm_dbg_kms(display->drm, - "[CONNECTOR:%d:%s][ENCODER:%d:%s] MST mode got reset, removing topology (ret=%d, ctrl=0x%02x)\n", - connector->base.base.id, connector->base.name, - encoder->base.base.id, encoder->base.name, - ret, val); - - return false; - } - - return true; -======= -/** - * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing - * @intel_dp: DP port object - * - * Prepare an MST link for topology probing, programming the target - * link parameters to DPCD. This step is a requirement of the enumaration - * of path resources during probing. - */ -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp) -{ - int link_rate = intel_dp_max_link_rate(intel_dp); - int lane_count = intel_dp_max_lane_count(intel_dp); - u8 rate_select; - u8 link_bw; - - if (intel_dp->link_trained) - return; - - if (intel_mst_probed_link_params_valid(intel_dp, link_rate, lane_count)) - return; - - intel_dp_compute_rate(intel_dp, link_rate, &link_bw, &rate_select); - - intel_dp_link_training_set_mode(intel_dp, link_rate, false); - intel_dp_link_training_set_bw(intel_dp, link_bw, rate_select, lane_count, - drm_dp_enhanced_frame_cap(intel_dp->dpcd)); - - intel_mst_set_probed_link_params(intel_dp, link_rate, lane_count); ->>>>>>> -} diff --git a/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/postimage b/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/postimage deleted file mode 100644 index 4b433e16206d..000000000000 --- a/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/postimage +++ /dev/null @@ -1,12343 +0,0 @@ -/* - * Copyright 2015 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: AMD - * - */ - -/* The caprices of the preprocessor require that this be declared right here */ -#define CREATE_TRACE_POINTS - -#include "dm_services_types.h" -#include "dc.h" -#include "link_enc_cfg.h" -#include "dc/inc/core_types.h" -#include "dal_asic_id.h" -#include "dmub/dmub_srv.h" -#include "dc/inc/hw/dmcu.h" -#include "dc/inc/hw/abm.h" -#include "dc/dc_dmub_srv.h" -#include "dc/dc_edid_parser.h" -#include "dc/dc_stat.h" -#include "dc/dc_state.h" -#include "amdgpu_dm_trace.h" -#include "dpcd_defs.h" -#include "link/protocols/link_dpcd.h" -#include "link_service_types.h" -#include "link/protocols/link_dp_capability.h" -#include "link/protocols/link_ddc.h" - -#include "vid.h" -#include "amdgpu.h" -#include "amdgpu_display.h" -#include "amdgpu_ucode.h" -#include "atom.h" -#include "amdgpu_dm.h" -#include "amdgpu_dm_plane.h" -#include "amdgpu_dm_crtc.h" -#include "amdgpu_dm_hdcp.h" -#include <drm/display/drm_hdcp_helper.h> -#include "amdgpu_dm_wb.h" -#include "amdgpu_pm.h" -#include "amdgpu_atombios.h" - -#include "amd_shared.h" -#include "amdgpu_dm_irq.h" -#include "dm_helpers.h" -#include "amdgpu_dm_mst_types.h" -#if defined(CONFIG_DEBUG_FS) -#include "amdgpu_dm_debugfs.h" -#endif -#include "amdgpu_dm_psr.h" -#include "amdgpu_dm_replay.h" - -#include "ivsrcid/ivsrcid_vislands30.h" - -#include <linux/backlight.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/types.h> -#include <linux/pm_runtime.h> -#include <linux/pci.h> -#include <linux/power_supply.h> -#include <linux/firmware.h> -#include <linux/component.h> -#include <linux/dmi.h> -#include <linux/sort.h> - -#include <drm/display/drm_dp_mst_helper.h> -#include <drm/display/drm_hdmi_helper.h> -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_uapi.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_blend.h> -#include <drm/drm_fixed.h> -#include <drm/drm_fourcc.h> -#include <drm/drm_edid.h> -#include <drm/drm_eld.h> -#include <drm/drm_vblank.h> -#include <drm/drm_audio_component.h> -#include <drm/drm_gem_atomic_helper.h> - -#include <acpi/video.h> - -#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h" - -#include "dcn/dcn_1_0_offset.h" -#include "dcn/dcn_1_0_sh_mask.h" -#include "soc15_hw_ip.h" -#include "soc15_common.h" -#include "vega10_ip_offset.h" - -#include "gc/gc_11_0_0_offset.h" -#include "gc/gc_11_0_0_sh_mask.h" - -#include "modules/inc/mod_freesync.h" -#include "modules/power/power_helpers.h" - -#define FIRMWARE_RENOIR_DMUB "amdgpu/renoir_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_RENOIR_DMUB); -#define FIRMWARE_SIENNA_CICHLID_DMUB "amdgpu/sienna_cichlid_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID_DMUB); -#define FIRMWARE_NAVY_FLOUNDER_DMUB "amdgpu/navy_flounder_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER_DMUB); -#define FIRMWARE_GREEN_SARDINE_DMUB "amdgpu/green_sardine_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE_DMUB); -#define FIRMWARE_VANGOGH_DMUB "amdgpu/vangogh_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_VANGOGH_DMUB); -#define FIRMWARE_DIMGREY_CAVEFISH_DMUB "amdgpu/dimgrey_cavefish_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH_DMUB); -#define FIRMWARE_BEIGE_GOBY_DMUB "amdgpu/beige_goby_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY_DMUB); -#define FIRMWARE_YELLOW_CARP_DMUB "amdgpu/yellow_carp_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP_DMUB); -#define FIRMWARE_DCN_314_DMUB "amdgpu/dcn_3_1_4_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_314_DMUB); -#define FIRMWARE_DCN_315_DMUB "amdgpu/dcn_3_1_5_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_315_DMUB); -#define FIRMWARE_DCN316_DMUB "amdgpu/dcn_3_1_6_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN316_DMUB); - -#define FIRMWARE_DCN_V3_2_0_DMCUB "amdgpu/dcn_3_2_0_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_0_DMCUB); -#define FIRMWARE_DCN_V3_2_1_DMCUB "amdgpu/dcn_3_2_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_1_DMCUB); - -#define FIRMWARE_RAVEN_DMCU "amdgpu/raven_dmcu.bin" -MODULE_FIRMWARE(FIRMWARE_RAVEN_DMCU); - -#define FIRMWARE_NAVI12_DMCU "amdgpu/navi12_dmcu.bin" -MODULE_FIRMWARE(FIRMWARE_NAVI12_DMCU); - -#define FIRMWARE_DCN_35_DMUB "amdgpu/dcn_3_5_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_35_DMUB); - -#define FIRMWARE_DCN_351_DMUB "amdgpu/dcn_3_5_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_351_DMUB); - -#define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB); - -/* Number of bytes in PSP header for firmware. */ -#define PSP_HEADER_BYTES 0x100 - -/* Number of bytes in PSP footer for firmware. */ -#define PSP_FOOTER_BYTES 0x100 - -/** - * DOC: overview - * - * The AMDgpu display manager, **amdgpu_dm** (or even simpler, - * **dm**) sits between DRM and DC. It acts as a liaison, converting DRM - * requests into DC requests, and DC responses into DRM responses. - * - * The root control structure is &struct amdgpu_display_manager. - */ - -/* basic init/fini API */ -static int amdgpu_dm_init(struct amdgpu_device *adev); -static void amdgpu_dm_fini(struct amdgpu_device *adev); -static bool is_freesync_video_mode(const struct drm_display_mode *mode, struct amdgpu_dm_connector *aconnector); -static void reset_freesync_config_for_crtc(struct dm_crtc_state *new_crtc_state); - -static enum drm_mode_subconnector get_subconnector_type(struct dc_link *link) -{ - switch (link->dpcd_caps.dongle_type) { - case DISPLAY_DONGLE_NONE: - return DRM_MODE_SUBCONNECTOR_Native; - case DISPLAY_DONGLE_DP_VGA_CONVERTER: - return DRM_MODE_SUBCONNECTOR_VGA; - case DISPLAY_DONGLE_DP_DVI_CONVERTER: - case DISPLAY_DONGLE_DP_DVI_DONGLE: - return DRM_MODE_SUBCONNECTOR_DVID; - case DISPLAY_DONGLE_DP_HDMI_CONVERTER: - case DISPLAY_DONGLE_DP_HDMI_DONGLE: - return DRM_MODE_SUBCONNECTOR_HDMIA; - case DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE: - default: - return DRM_MODE_SUBCONNECTOR_Unknown; - } -} - -static void update_subconnector_property(struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = aconnector->dc_link; - struct drm_connector *connector = &aconnector->base; - enum drm_mode_subconnector subconnector = DRM_MODE_SUBCONNECTOR_Unknown; - - if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) - return; - - if (aconnector->dc_sink) - subconnector = get_subconnector_type(link); - - drm_object_property_set_value(&connector->base, - connector->dev->mode_config.dp_subconnector_property, - subconnector); -} - -/* - * initializes drm_device display related structures, based on the information - * provided by DAL. The drm strcutures are: drm_crtc, drm_connector, - * drm_encoder, drm_mode_config - * - * Returns 0 on success - */ -static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev); -/* removes and deallocates the drm structures, created by the above function */ -static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm); - -static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *amdgpu_dm_connector, - u32 link_index, - struct amdgpu_encoder *amdgpu_encoder); -static int amdgpu_dm_encoder_init(struct drm_device *dev, - struct amdgpu_encoder *aencoder, - uint32_t link_index); - -static int amdgpu_dm_connector_get_modes(struct drm_connector *connector); - -static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state); - -static int amdgpu_dm_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state); - -static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector); -static void handle_hpd_rx_irq(void *param); - -static bool -is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state); -/* - * dm_vblank_get_counter - * - * @brief - * Get counter for number of vertical blanks - * - * @param - * struct amdgpu_device *adev - [in] desired amdgpu device - * int disp_idx - [in] which CRTC to get the counter from - * - * @return - * Counter for vertical blanks - */ -static u32 dm_vblank_get_counter(struct amdgpu_device *adev, int crtc) -{ - struct amdgpu_crtc *acrtc = NULL; - - if (crtc >= adev->mode_info.num_crtc) - return 0; - - acrtc = adev->mode_info.crtcs[crtc]; - - if (!acrtc->dm_irq_params.stream) { - DRM_ERROR("dc_stream_state is NULL for crtc '%d'!\n", - crtc); - return 0; - } - - return dc_stream_get_vblank_counter(acrtc->dm_irq_params.stream); -} - -static int dm_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc, - u32 *vbl, u32 *position) -{ - u32 v_blank_start = 0, v_blank_end = 0, h_position = 0, v_position = 0; - struct amdgpu_crtc *acrtc = NULL; - struct dc *dc = adev->dm.dc; - - if ((crtc < 0) || (crtc >= adev->mode_info.num_crtc)) - return -EINVAL; - - acrtc = adev->mode_info.crtcs[crtc]; - - if (!acrtc->dm_irq_params.stream) { - DRM_ERROR("dc_stream_state is NULL for crtc '%d'!\n", - crtc); - return 0; - } - - if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed) - dc_allow_idle_optimizations(dc, false); - - /* - * TODO rework base driver to use values directly. - * for now parse it back into reg-format - */ - dc_stream_get_scanoutpos(acrtc->dm_irq_params.stream, - &v_blank_start, - &v_blank_end, - &h_position, - &v_position); - - *position = v_position | (h_position << 16); - *vbl = v_blank_start | (v_blank_end << 16); - - return 0; -} - -static bool dm_is_idle(void *handle) -{ - /* XXX todo */ - return true; -} - -static int dm_wait_for_idle(void *handle) -{ - /* XXX todo */ - return 0; -} - -static bool dm_check_soft_reset(void *handle) -{ - return false; -} - -static int dm_soft_reset(void *handle) -{ - /* XXX todo */ - return 0; -} - -static struct amdgpu_crtc * -get_crtc_by_otg_inst(struct amdgpu_device *adev, - int otg_inst) -{ - struct drm_device *dev = adev_to_drm(adev); - struct drm_crtc *crtc; - struct amdgpu_crtc *amdgpu_crtc; - - if (WARN_ON(otg_inst == -1)) - return adev->mode_info.crtcs[0]; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - amdgpu_crtc = to_amdgpu_crtc(crtc); - - if (amdgpu_crtc->otg_inst == otg_inst) - return amdgpu_crtc; - } - - return NULL; -} - -static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state, - struct dm_crtc_state *new_state) -{ - if (new_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED) - return true; - else if (amdgpu_dm_crtc_vrr_active(old_state) != amdgpu_dm_crtc_vrr_active(new_state)) - return true; - else - return false; -} - -/* - * DC will program planes with their z-order determined by their ordering - * in the dc_surface_updates array. This comparator is used to sort them - * by descending zpos. - */ -static int dm_plane_layer_index_cmp(const void *a, const void *b) -{ - const struct dc_surface_update *sa = (struct dc_surface_update *)a; - const struct dc_surface_update *sb = (struct dc_surface_update *)b; - - /* Sort by descending dc_plane layer_index (i.e. normalized_zpos) */ - return sb->surface->layer_index - sa->surface->layer_index; -} - -/** - * update_planes_and_stream_adapter() - Send planes to be updated in DC - * - * DC has a generic way to update planes and stream via - * dc_update_planes_and_stream function; however, DM might need some - * adjustments and preparation before calling it. This function is a wrapper - * for the dc_update_planes_and_stream that does any required configuration - * before passing control to DC. - * - * @dc: Display Core control structure - * @update_type: specify whether it is FULL/MEDIUM/FAST update - * @planes_count: planes count to update - * @stream: stream state - * @stream_update: stream update - * @array_of_surface_update: dc surface update pointer - * - */ -static inline bool update_planes_and_stream_adapter(struct dc *dc, - int update_type, - int planes_count, - struct dc_stream_state *stream, - struct dc_stream_update *stream_update, - struct dc_surface_update *array_of_surface_update) -{ - sort(array_of_surface_update, planes_count, - sizeof(*array_of_surface_update), dm_plane_layer_index_cmp, NULL); - - /* - * Previous frame finished and HW is ready for optimization. - */ - if (update_type == UPDATE_TYPE_FAST) - dc_post_update_surfaces_to_stream(dc); - - return dc_update_planes_and_stream(dc, - array_of_surface_update, - planes_count, - stream, - stream_update); -} - -/** - * dm_pflip_high_irq() - Handle pageflip interrupt - * @interrupt_params: ignored - * - * Handles the pageflip interrupt by notifying all interested parties - * that the pageflip has been completed. - */ -static void dm_pflip_high_irq(void *interrupt_params) -{ - struct amdgpu_crtc *amdgpu_crtc; - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct drm_device *dev = adev_to_drm(adev); - unsigned long flags; - struct drm_pending_vblank_event *e; - u32 vpos, hpos, v_blank_start, v_blank_end; - bool vrr_active; - - amdgpu_crtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP); - - /* IRQ could occur when in initial stage */ - /* TODO work and BO cleanup */ - if (amdgpu_crtc == NULL) { - drm_dbg_state(dev, "CRTC is null, returning.\n"); - return; - } - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - - if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { - drm_dbg_state(dev, - "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n", - amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED, - amdgpu_crtc->crtc_id, amdgpu_crtc); - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - return; - } - - /* page flip completed. */ - e = amdgpu_crtc->event; - amdgpu_crtc->event = NULL; - - WARN_ON(!e); - - vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc); - - /* Fixed refresh rate, or VRR scanout position outside front-porch? */ - if (!vrr_active || - !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start, - &v_blank_end, &hpos, &vpos) || - (vpos < v_blank_start)) { - /* Update to correct count and vblank timestamp if racing with - * vblank irq. This also updates to the correct vblank timestamp - * even in VRR mode, as scanout is past the front-porch atm. - */ - drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); - - /* Wake up userspace by sending the pageflip event with proper - * count and timestamp of vblank of flip completion. - */ - if (e) { - drm_crtc_send_vblank_event(&amdgpu_crtc->base, e); - - /* Event sent, so done with vblank for this flip */ - drm_crtc_vblank_put(&amdgpu_crtc->base); - } - } else if (e) { - /* VRR active and inside front-porch: vblank count and - * timestamp for pageflip event will only be up to date after - * drm_crtc_handle_vblank() has been executed from late vblank - * irq handler after start of back-porch (vline 0). We queue the - * pageflip event for send-out by drm_crtc_handle_vblank() with - * updated timestamp and count, once it runs after us. - * - * We need to open-code this instead of using the helper - * drm_crtc_arm_vblank_event(), as that helper would - * call drm_crtc_accurate_vblank_count(), which we must - * not call in VRR mode while we are in front-porch! - */ - - /* sequence will be replaced by real count during send-out. */ - e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); - e->pipe = amdgpu_crtc->crtc_id; - - list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); - e = NULL; - } - - /* Keep track of vblank of this flip for flip throttling. We use the - * cooked hw counter, as that one incremented at start of this vblank - * of pageflip completion, so last_flip_vblank is the forbidden count - * for queueing new pageflips if vsync + VRR is enabled. - */ - amdgpu_crtc->dm_irq_params.last_flip_vblank = - amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); - - amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - - drm_dbg_state(dev, - "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", - amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e); -} - -static void dm_vupdate_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - struct drm_device *drm_dev; - struct drm_vblank_crtc *vblank; - ktime_t frame_duration_ns, previous_timestamp; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); - - if (acrtc) { - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - drm_dev = acrtc->base.dev; - vblank = drm_crtc_vblank_crtc(&acrtc->base); - previous_timestamp = atomic64_read(&irq_params->previous_timestamp); - frame_duration_ns = vblank->time - previous_timestamp; - - if (frame_duration_ns > 0) { - trace_amdgpu_refresh_rate_track(acrtc->base.index, - frame_duration_ns, - ktime_divns(NSEC_PER_SEC, frame_duration_ns)); - atomic64_set(&irq_params->previous_timestamp, vblank->time); - } - - drm_dbg_vbl(drm_dev, - "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, - vrr_active); - - /* Core vblank handling is done here after end of front-porch in - * vrr mode, as vblank timestamping will give valid results - * while now done after front-porch. This will also deliver - * page-flip completion events that have been queued to us - * if a pageflip happened inside front-porch. - */ - if (vrr_active) { - amdgpu_dm_crtc_handle_vblank(acrtc); - - /* BTR processing for pre-DCE12 ASICs */ - if (acrtc->dm_irq_params.stream && - adev->family < AMDGPU_FAMILY_AI) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - mod_freesync_handle_v_update( - adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - dc_stream_adjust_vmin_vmax( - adev->dm.dc, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } - } - } -} - -/** - * dm_crtc_high_irq() - Handles CRTC interrupt - * @interrupt_params: used for determining the CRTC instance - * - * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK - * event handler. - */ -static void dm_crtc_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct drm_writeback_job *job; - struct amdgpu_crtc *acrtc; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); - if (!acrtc) - return; - - if (acrtc->wb_conn) { - spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags); - - if (acrtc->wb_pending) { - job = list_first_entry_or_null(&acrtc->wb_conn->job_queue, - struct drm_writeback_job, - list_entry); - acrtc->wb_pending = false; - spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); - - if (job) { - unsigned int v_total, refresh_hz; - struct dc_stream_state *stream = acrtc->dm_irq_params.stream; - - v_total = stream->adjust.v_total_max ? - stream->adjust.v_total_max : stream->timing.v_total; - refresh_hz = div_u64((uint64_t) stream->timing.pix_clk_100hz * - 100LL, (v_total * stream->timing.h_total)); - mdelay(1000 / refresh_hz); - - drm_writeback_signal_completion(acrtc->wb_conn, 0); - dc_stream_fc_disable_writeback(adev->dm.dc, - acrtc->dm_irq_params.stream, 0); - } - } else - spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); - } - - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - - drm_dbg_vbl(adev_to_drm(adev), - "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id, - vrr_active, acrtc->dm_irq_params.active_planes); - - /** - * Core vblank handling at start of front-porch is only possible - * in non-vrr mode, as only there vblank timestamping will give - * valid results while done in front-porch. Otherwise defer it - * to dm_vupdate_high_irq after end of front-porch. - */ - if (!vrr_active) - amdgpu_dm_crtc_handle_vblank(acrtc); - - /** - * Following stuff must happen at start of vblank, for crc - * computation and below-the-range btr support in vrr mode. - */ - amdgpu_dm_crtc_handle_crc_irq(&acrtc->base); - - /* BTR updates need to happen before VUPDATE on Vega and above. */ - if (adev->family < AMDGPU_FAMILY_AI) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - - if (acrtc->dm_irq_params.stream && - acrtc->dm_irq_params.vrr_params.supported && - acrtc->dm_irq_params.freesync_config.state == - VRR_STATE_ACTIVE_VARIABLE) { - mod_freesync_handle_v_update(adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - dc_stream_adjust_vmin_vmax(adev->dm.dc, acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - } - - /* - * If there aren't any active_planes then DCH HUBP may be clock-gated. - * In that case, pageflip completion interrupts won't fire and pageflip - * completion events won't get delivered. Prevent this by sending - * pending pageflip events from here if a flip is still pending. - * - * If any planes are enabled, use dm_pflip_high_irq() instead, to - * avoid race conditions between flip programming and completion, - * which could cause too early flip completion events. - */ - if (adev->family >= AMDGPU_FAMILY_RV && - acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && - acrtc->dm_irq_params.active_planes == 0) { - if (acrtc->event) { - drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); - acrtc->event = NULL; - drm_crtc_vblank_put(&acrtc->base); - } - acrtc->pflip_status = AMDGPU_FLIP_NONE; - } - - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -/** - * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for - * DCN generation ASICs - * @interrupt_params: interrupt parameters - * - * Used to set crc window/read out crc value at vertical line 0 position - */ -static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0); - - if (!acrtc) - return; - - amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base); -} -#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ - -/** - * dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command. - * @adev: amdgpu_device pointer - * @notify: dmub notification structure - * - * Dmub AUX or SET_CONFIG command completion processing callback - * Copies dmub notification to DM which is to be read by AUX command. - * issuing thread and also signals the event to wake up the thread. - */ -static void dmub_aux_setconfig_callback(struct amdgpu_device *adev, - struct dmub_notification *notify) -{ - if (adev->dm.dmub_notify) - memcpy(adev->dm.dmub_notify, notify, sizeof(struct dmub_notification)); - if (notify->type == DMUB_NOTIFICATION_AUX_REPLY) - complete(&adev->dm.dmub_aux_transfer_done); -} - -/** - * dmub_hpd_callback - DMUB HPD interrupt processing callback. - * @adev: amdgpu_device pointer - * @notify: dmub notification structure - * - * Dmub Hpd interrupt processing callback. Gets displayindex through the - * ink index and calls helper to do the processing. - */ -static void dmub_hpd_callback(struct amdgpu_device *adev, - struct dmub_notification *notify) -{ - struct amdgpu_dm_connector *aconnector; - struct amdgpu_dm_connector *hpd_aconnector = NULL; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct dc_link *link; - u8 link_index = 0; - struct drm_device *dev; - - if (adev == NULL) - return; - - if (notify == NULL) { - DRM_ERROR("DMUB HPD callback notification was NULL"); - return; - } - - if (notify->link_index > adev->dm.dc->link_count) { - DRM_ERROR("DMUB HPD index (%u)is abnormal", notify->link_index); - return; - } - - link_index = notify->link_index; - link = adev->dm.dc->links[link_index]; - dev = adev->dm.ddev; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (link && aconnector->dc_link == link) { - if (notify->type == DMUB_NOTIFICATION_HPD) - DRM_INFO("DMUB HPD IRQ callback: link_index=%u\n", link_index); - else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) - DRM_INFO("DMUB HPD RX IRQ callback: link_index=%u\n", link_index); - else - DRM_WARN("DMUB Unknown HPD callback type %d, link_index=%u\n", - notify->type, link_index); - - hpd_aconnector = aconnector; - break; - } - } - drm_connector_list_iter_end(&iter); - - if (hpd_aconnector) { - if (notify->type == DMUB_NOTIFICATION_HPD) { - if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG)) - DRM_WARN("DMUB reported hpd status unchanged. link_index=%u\n", link_index); - handle_hpd_irq_helper(hpd_aconnector); - } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) { - handle_hpd_rx_irq(hpd_aconnector); - } - } -} - -/** - * register_dmub_notify_callback - Sets callback for DMUB notify - * @adev: amdgpu_device pointer - * @type: Type of dmub notification - * @callback: Dmub interrupt callback function - * @dmub_int_thread_offload: offload indicator - * - * API to register a dmub callback handler for a dmub notification - * Also sets indicator whether callback processing to be offloaded. - * to dmub interrupt handling thread - * Return: true if successfully registered, false if there is existing registration - */ -static bool register_dmub_notify_callback(struct amdgpu_device *adev, - enum dmub_notification_type type, - dmub_notify_interrupt_callback_t callback, - bool dmub_int_thread_offload) -{ - if (callback != NULL && type < ARRAY_SIZE(adev->dm.dmub_thread_offload)) { - adev->dm.dmub_callback[type] = callback; - adev->dm.dmub_thread_offload[type] = dmub_int_thread_offload; - } else - return false; - - return true; -} - -static void dm_handle_hpd_work(struct work_struct *work) -{ - struct dmub_hpd_work *dmub_hpd_wrk; - - dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work); - - if (!dmub_hpd_wrk->dmub_notify) { - DRM_ERROR("dmub_hpd_wrk dmub_notify is NULL"); - return; - } - - if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) { - dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev, - dmub_hpd_wrk->dmub_notify); - } - - kfree(dmub_hpd_wrk->dmub_notify); - kfree(dmub_hpd_wrk); - -} - -#define DMUB_TRACE_MAX_READ 64 -/** - * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt - * @interrupt_params: used for determining the Outbox instance - * - * Handles the Outbox Interrupt - * event handler. - */ -static void dm_dmub_outbox1_low_irq(void *interrupt_params) -{ - struct dmub_notification notify = {0}; - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_display_manager *dm = &adev->dm; - struct dmcub_trace_buf_entry entry = { 0 }; - u32 count = 0; - struct dmub_hpd_work *dmub_hpd_wrk; - static const char *const event_type[] = { - "NO_DATA", - "AUX_REPLY", - "HPD", - "HPD_IRQ", - "SET_CONFIGC_REPLY", - "DPIA_NOTIFICATION", - "HPD_SENSE_NOTIFY", - }; - - do { - if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) { - trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count, - entry.param0, entry.param1); - - DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n", - entry.trace_code, entry.tick_count, entry.param0, entry.param1); - } else - break; - - count++; - - } while (count <= DMUB_TRACE_MAX_READ); - - if (count > DMUB_TRACE_MAX_READ) - DRM_DEBUG_DRIVER("Warning : count > DMUB_TRACE_MAX_READ"); - - if (dc_enable_dmub_notifications(adev->dm.dc) && - irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) { - - do { - dc_stat_get_dmub_notification(adev->dm.dc, ¬ify); - if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) { - DRM_ERROR("DM: notify type %d invalid!", notify.type); - continue; - } - if (!dm->dmub_callback[notify.type]) { - DRM_WARN("DMUB notification skipped due to no handler: type=%s\n", - event_type[notify.type]); - continue; - } - if (dm->dmub_thread_offload[notify.type] == true) { - dmub_hpd_wrk = kzalloc(sizeof(*dmub_hpd_wrk), GFP_ATOMIC); - if (!dmub_hpd_wrk) { - DRM_ERROR("Failed to allocate dmub_hpd_wrk"); - return; - } - dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification), - GFP_ATOMIC); - if (!dmub_hpd_wrk->dmub_notify) { - kfree(dmub_hpd_wrk); - DRM_ERROR("Failed to allocate dmub_hpd_wrk->dmub_notify"); - return; - } - INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work); - dmub_hpd_wrk->adev = adev; - queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work); - } else { - dm->dmub_callback[notify.type](adev, ¬ify); - } - } while (notify.pending_notification); - } -} - -static int dm_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - return 0; -} - -static int dm_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -/* Prototypes of private functions */ -static int dm_early_init(void *handle); - -/* Allocate memory for FBC compressed data */ -static void amdgpu_dm_fbc_init(struct drm_connector *connector) -{ - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct dm_compressor_info *compressor = &adev->dm.compressor; - struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector); - struct drm_display_mode *mode; - unsigned long max_size = 0; - - if (adev->dm.dc->fbc_compressor == NULL) - return; - - if (aconn->dc_link->connector_signal != SIGNAL_TYPE_EDP) - return; - - if (compressor->bo_ptr) - return; - - - list_for_each_entry(mode, &connector->modes, head) { - if (max_size < (unsigned long) mode->htotal * mode->vtotal) - max_size = (unsigned long) mode->htotal * mode->vtotal; - } - - if (max_size) { - int r = amdgpu_bo_create_kernel(adev, max_size * 4, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_GTT, &compressor->bo_ptr, - &compressor->gpu_addr, &compressor->cpu_addr); - - if (r) - DRM_ERROR("DM: Failed to initialize FBC\n"); - else { - adev->dm.dc->ctx->fbc_gpu_addr = compressor->gpu_addr; - DRM_INFO("DM: FBC alloc %lu\n", max_size*4); - } - - } - -} - -static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port, - int pipe, bool *enabled, - unsigned char *buf, int max_bytes) -{ - struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = drm_to_adev(dev); - struct drm_connector *connector; - struct drm_connector_list_iter conn_iter; - struct amdgpu_dm_connector *aconnector; - int ret = 0; - - *enabled = false; - - mutex_lock(&adev->dm.audio_lock); - - drm_connector_list_iter_begin(dev, &conn_iter); - drm_for_each_connector_iter(connector, &conn_iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->audio_inst != port) - continue; - - *enabled = true; - ret = drm_eld_size(connector->eld); - memcpy(buf, connector->eld, min(max_bytes, ret)); - - break; - } - drm_connector_list_iter_end(&conn_iter); - - mutex_unlock(&adev->dm.audio_lock); - - DRM_DEBUG_KMS("Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled); - - return ret; -} - -static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = { - .get_eld = amdgpu_dm_audio_component_get_eld, -}; - -static int amdgpu_dm_audio_component_bind(struct device *kdev, - struct device *hda_kdev, void *data) -{ - struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = drm_to_adev(dev); - struct drm_audio_component *acomp = data; - - acomp->ops = &amdgpu_dm_audio_component_ops; - acomp->dev = kdev; - adev->dm.audio_component = acomp; - - return 0; -} - -static void amdgpu_dm_audio_component_unbind(struct device *kdev, - struct device *hda_kdev, void *data) -{ - struct amdgpu_device *adev = drm_to_adev(dev_get_drvdata(kdev)); - struct drm_audio_component *acomp = data; - - acomp->ops = NULL; - acomp->dev = NULL; - adev->dm.audio_component = NULL; -} - -static const struct component_ops amdgpu_dm_audio_component_bind_ops = { - .bind = amdgpu_dm_audio_component_bind, - .unbind = amdgpu_dm_audio_component_unbind, -}; - -static int amdgpu_dm_audio_init(struct amdgpu_device *adev) -{ - int i, ret; - - if (!amdgpu_audio) - return 0; - - adev->mode_info.audio.enabled = true; - - adev->mode_info.audio.num_pins = adev->dm.dc->res_pool->audio_count; - - for (i = 0; i < adev->mode_info.audio.num_pins; i++) { - adev->mode_info.audio.pin[i].channels = -1; - adev->mode_info.audio.pin[i].rate = -1; - adev->mode_info.audio.pin[i].bits_per_sample = -1; - adev->mode_info.audio.pin[i].status_bits = 0; - adev->mode_info.audio.pin[i].category_code = 0; - adev->mode_info.audio.pin[i].connected = false; - adev->mode_info.audio.pin[i].id = - adev->dm.dc->res_pool->audios[i]->inst; - adev->mode_info.audio.pin[i].offset = 0; - } - - ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops); - if (ret < 0) - return ret; - - adev->dm.audio_registered = true; - - return 0; -} - -static void amdgpu_dm_audio_fini(struct amdgpu_device *adev) -{ - if (!amdgpu_audio) - return; - - if (!adev->mode_info.audio.enabled) - return; - - if (adev->dm.audio_registered) { - component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops); - adev->dm.audio_registered = false; - } - - /* TODO: Disable audio? */ - - adev->mode_info.audio.enabled = false; -} - -static void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin) -{ - struct drm_audio_component *acomp = adev->dm.audio_component; - - if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) { - DRM_DEBUG_KMS("Notify ELD: %d\n", pin); - - acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, - pin, -1); - } -} - -static int dm_dmub_hw_init(struct amdgpu_device *adev) -{ - const struct dmcub_firmware_header_v1_0 *hdr; - struct dmub_srv *dmub_srv = adev->dm.dmub_srv; - struct dmub_srv_fb_info *fb_info = adev->dm.dmub_fb_info; - const struct firmware *dmub_fw = adev->dm.dmub_fw; - struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu; - struct abm *abm = adev->dm.dc->res_pool->abm; - struct dc_context *ctx = adev->dm.dc->ctx; - struct dmub_srv_hw_params hw_params; - enum dmub_status status; - const unsigned char *fw_inst_const, *fw_bss_data; - u32 i, fw_inst_const_size, fw_bss_data_size; - bool has_hw_support; - - if (!dmub_srv) - /* DMUB isn't supported on the ASIC. */ - return 0; - - if (!fb_info) { - DRM_ERROR("No framebuffer info for DMUB service.\n"); - return -EINVAL; - } - - if (!dmub_fw) { - /* Firmware required for DMUB support. */ - DRM_ERROR("No firmware provided for DMUB.\n"); - return -EINVAL; - } - - /* initialize register offsets for ASICs with runtime initialization available */ - if (dmub_srv->hw_funcs.init_reg_offsets) - dmub_srv->hw_funcs.init_reg_offsets(dmub_srv, ctx); - - status = dmub_srv_has_hw_support(dmub_srv, &has_hw_support); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error checking HW support for DMUB: %d\n", status); - return -EINVAL; - } - - if (!has_hw_support) { - DRM_INFO("DMUB unsupported on ASIC\n"); - return 0; - } - - /* Reset DMCUB if it was previously running - before we overwrite its memory. */ - status = dmub_srv_hw_reset(dmub_srv); - if (status != DMUB_STATUS_OK) - DRM_WARN("Error resetting DMUB HW: %d\n", status); - - hdr = (const struct dmcub_firmware_header_v1_0 *)dmub_fw->data; - - fw_inst_const = dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - PSP_HEADER_BYTES; - - fw_bss_data = dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - le32_to_cpu(hdr->inst_const_bytes); - - /* Copy firmware and bios info into FB memory. */ - fw_inst_const_size = le32_to_cpu(hdr->inst_const_bytes) - - PSP_HEADER_BYTES - PSP_FOOTER_BYTES; - - fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes); - - /* if adev->firmware.load_type == AMDGPU_FW_LOAD_PSP, - * amdgpu_ucode_init_single_fw will load dmub firmware - * fw_inst_const part to cw0; otherwise, the firmware back door load - * will be done by dm_dmub_hw_init - */ - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { - memcpy(fb_info->fb[DMUB_WINDOW_0_INST_CONST].cpu_addr, fw_inst_const, - fw_inst_const_size); - } - - if (fw_bss_data_size) - memcpy(fb_info->fb[DMUB_WINDOW_2_BSS_DATA].cpu_addr, - fw_bss_data, fw_bss_data_size); - - /* Copy firmware bios info into FB memory. */ - memcpy(fb_info->fb[DMUB_WINDOW_3_VBIOS].cpu_addr, adev->bios, - adev->bios_size); - - /* Reset regions that need to be reset. */ - memset(fb_info->fb[DMUB_WINDOW_4_MAILBOX].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_4_MAILBOX].size); - - memset(fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size); - - memset(fb_info->fb[DMUB_WINDOW_6_FW_STATE].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_6_FW_STATE].size); - - memset(fb_info->fb[DMUB_WINDOW_SHARED_STATE].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_SHARED_STATE].size); - - /* Initialize hardware. */ - memset(&hw_params, 0, sizeof(hw_params)); - hw_params.fb_base = adev->gmc.fb_start; - hw_params.fb_offset = adev->vm_manager.vram_base_offset; - - /* backdoor load firmware and trigger dmub running */ - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - hw_params.load_inst_const = true; - - if (dmcu) - hw_params.psp_version = dmcu->psp_version; - - for (i = 0; i < fb_info->num_fb; ++i) - hw_params.fb[i] = &fb_info->fb[i]; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - hw_params.dpia_supported = true; - hw_params.disable_dpia = adev->dm.dc->debug.dpia_debug.bits.disable_dpia; - break; - default: - break; - } - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - hw_params.ips_sequential_ono = adev->external_rev_id > 0x10; - break; - default: - break; - } - - status = dmub_srv_hw_init(dmub_srv, &hw_params); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error initializing DMUB HW: %d\n", status); - return -EINVAL; - } - - /* Wait for firmware load to finish. */ - status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); - if (status != DMUB_STATUS_OK) - DRM_WARN("Wait for DMUB auto-load failed: %d\n", status); - - /* Init DMCU and ABM if available. */ - if (dmcu && abm) { - dmcu->funcs->dmcu_init(dmcu); - abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu); - } - - if (!adev->dm.dc->ctx->dmub_srv) - adev->dm.dc->ctx->dmub_srv = dc_dmub_srv_create(adev->dm.dc, dmub_srv); - if (!adev->dm.dc->ctx->dmub_srv) { - DRM_ERROR("Couldn't allocate DC DMUB server!\n"); - return -ENOMEM; - } - - DRM_INFO("DMUB hardware initialized: version=0x%08X\n", - adev->dm.dmcub_fw_version); - - return 0; -} - -static void dm_dmub_hw_resume(struct amdgpu_device *adev) -{ - struct dmub_srv *dmub_srv = adev->dm.dmub_srv; - enum dmub_status status; - bool init; - int r; - - if (!dmub_srv) { - /* DMUB isn't supported on the ASIC. */ - return; - } - - status = dmub_srv_is_hw_init(dmub_srv, &init); - if (status != DMUB_STATUS_OK) - DRM_WARN("DMUB hardware init check failed: %d\n", status); - - if (status == DMUB_STATUS_OK && init) { - /* Wait for firmware load to finish. */ - status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); - if (status != DMUB_STATUS_OK) - DRM_WARN("Wait for DMUB auto-load failed: %d\n", status); - } else { - /* Perform the full hardware initialization. */ - r = dm_dmub_hw_init(adev); - if (r) - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - } -} - -static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config) -{ - u64 pt_base; - u32 logical_addr_low; - u32 logical_addr_high; - u32 agp_base, agp_bot, agp_top; - PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base; - - memset(pa_config, 0, sizeof(*pa_config)); - - agp_base = 0; - agp_bot = adev->gmc.agp_start >> 24; - agp_top = adev->gmc.agp_end >> 24; - - /* AGP aperture is disabled */ - if (agp_bot > agp_top) { - logical_addr_low = adev->gmc.fb_start >> 18; - if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | - AMD_APU_IS_RENOIR | - AMD_APU_IS_GREEN_SARDINE)) - /* - * Raven2 has a HW issue that it is unable to use the vram which - * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the - * workaround that increase system aperture high address (add 1) - * to get rid of the VM fault and hardware hang. - */ - logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1; - else - logical_addr_high = adev->gmc.fb_end >> 18; - } else { - logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18; - if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | - AMD_APU_IS_RENOIR | - AMD_APU_IS_GREEN_SARDINE)) - /* - * Raven2 has a HW issue that it is unable to use the vram which - * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the - * workaround that increase system aperture high address (add 1) - * to get rid of the VM fault and hardware hang. - */ - logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18); - else - logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18; - } - - pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); - - page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_base.high_part = upper_32_bits(pt_base); - page_table_base.low_part = lower_32_bits(pt_base); - - pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18; - pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18; - - pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24; - pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24; - pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24; - - pa_config->system_aperture.fb_base = adev->gmc.fb_start; - pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset; - pa_config->system_aperture.fb_top = adev->gmc.fb_end; - - pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12; - pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12; - pa_config->gart_config.page_table_base_addr = page_table_base.quad_part; - - pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support; - -} - -static void force_connector_state( - struct amdgpu_dm_connector *aconnector, - enum drm_connector_force force_state) -{ - struct drm_connector *connector = &aconnector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - aconnector->base.force = force_state; - mutex_unlock(&connector->dev->mode_config.mutex); - - mutex_lock(&aconnector->hpd_lock); - drm_kms_helper_connector_hotplug_event(connector); - mutex_unlock(&aconnector->hpd_lock); -} - -static void dm_handle_hpd_rx_offload_work(struct work_struct *work) -{ - struct hpd_rx_irq_offload_work *offload_work; - struct amdgpu_dm_connector *aconnector; - struct dc_link *dc_link; - struct amdgpu_device *adev; - enum dc_connection_type new_connection_type = dc_connection_none; - unsigned long flags; - union test_response test_response; - - memset(&test_response, 0, sizeof(test_response)); - - offload_work = container_of(work, struct hpd_rx_irq_offload_work, work); - aconnector = offload_work->offload_wq->aconnector; - - if (!aconnector) { - DRM_ERROR("Can't retrieve aconnector in hpd_rx_irq_offload_work"); - goto skip; - } - - adev = drm_to_adev(aconnector->base.dev); - dc_link = aconnector->dc_link; - - mutex_lock(&aconnector->hpd_lock); - if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - mutex_unlock(&aconnector->hpd_lock); - - if (new_connection_type == dc_connection_none) - goto skip; - - if (amdgpu_in_reset(adev)) - goto skip; - - if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || - offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { - dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT); - spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); - offload_work->offload_wq->is_handling_mst_msg_rdy_event = false; - spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); - goto skip; - } - - mutex_lock(&adev->dm.dc_lock); - if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) { - dc_link_dp_handle_automated_test(dc_link); - - if (aconnector->timing_changed) { - /* force connector disconnect and reconnect */ - force_connector_state(aconnector, DRM_FORCE_OFF); - msleep(100); - force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED); - } - - test_response.bits.ACK = 1; - - core_link_write_dpcd( - dc_link, - DP_TEST_RESPONSE, - &test_response.raw, - sizeof(test_response)); - } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) && - dc_link_check_link_loss_status(dc_link, &offload_work->data) && - dc_link_dp_allow_hpd_rx_irq(dc_link)) { - /* offload_work->data is from handle_hpd_rx_irq-> - * schedule_hpd_rx_offload_work.this is defer handle - * for hpd short pulse. upon here, link status may be - * changed, need get latest link status from dpcd - * registers. if link status is good, skip run link - * training again. - */ - union hpd_irq_data irq_data; - - memset(&irq_data, 0, sizeof(irq_data)); - - /* before dc_link_dp_handle_link_loss, allow new link lost handle - * request be added to work queue if link lost at end of dc_link_ - * dp_handle_link_loss - */ - spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); - offload_work->offload_wq->is_handling_link_loss = false; - spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); - - if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) && - dc_link_check_link_loss_status(dc_link, &irq_data)) - dc_link_dp_handle_link_loss(dc_link); - } - mutex_unlock(&adev->dm.dc_lock); - -skip: - kfree(offload_work); - -} - -static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct dc *dc) -{ - int max_caps = dc->caps.max_links; - int i = 0; - struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL; - - hpd_rx_offload_wq = kcalloc(max_caps, sizeof(*hpd_rx_offload_wq), GFP_KERNEL); - - if (!hpd_rx_offload_wq) - return NULL; - - - for (i = 0; i < max_caps; i++) { - hpd_rx_offload_wq[i].wq = - create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq"); - - if (hpd_rx_offload_wq[i].wq == NULL) { - DRM_ERROR("create amdgpu_dm_hpd_rx_offload_wq fail!"); - goto out_err; - } - - spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); - } - - return hpd_rx_offload_wq; - -out_err: - for (i = 0; i < max_caps; i++) { - if (hpd_rx_offload_wq[i].wq) - destroy_workqueue(hpd_rx_offload_wq[i].wq); - } - kfree(hpd_rx_offload_wq); - return NULL; -} - -struct amdgpu_stutter_quirk { - u16 chip_vendor; - u16 chip_device; - u16 subsys_vendor; - u16 subsys_device; - u8 revision; -}; - -static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = { - /* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */ - { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, - { 0, 0, 0, 0, 0 }, -}; - -static bool dm_should_disable_stutter(struct pci_dev *pdev) -{ - const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list; - - while (p && p->chip_device != 0) { - if (pdev->vendor == p->chip_vendor && - pdev->device == p->chip_device && - pdev->subsystem_vendor == p->subsys_vendor && - pdev->subsystem_device == p->subsys_device && - pdev->revision == p->revision) { - return true; - } - ++p; - } - return false; -} - -static const struct dmi_system_id hpd_disconnect_quirk_table[] = { - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3660"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3260"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3460"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Tower Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Tower 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex SFF Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex SFF 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Micro Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Micro 7010"), - }, - }, - {} - /* TODO: refactor this from a fixed table to a dynamic option */ -}; - -static void retrieve_dmi_info(struct amdgpu_display_manager *dm) -{ - const struct dmi_system_id *dmi_id; - - dm->aux_hpd_discon_quirk = false; - - dmi_id = dmi_first_match(hpd_disconnect_quirk_table); - if (dmi_id) { - dm->aux_hpd_discon_quirk = true; - DRM_INFO("aux_hpd_discon_quirk attached\n"); - } -} - -void* -dm_allocate_gpu_mem( - struct amdgpu_device *adev, - enum dc_gpu_mem_alloc_type type, - size_t size, - long long *addr) -{ - struct dal_allocation *da; - u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ? - AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM; - int ret; - - da = kzalloc(sizeof(struct dal_allocation), GFP_KERNEL); - if (!da) - return NULL; - - ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, - domain, &da->bo, - &da->gpu_addr, &da->cpu_ptr); - - *addr = da->gpu_addr; - - if (ret) { - kfree(da); - return NULL; - } - - /* add da to list in dm */ - list_add(&da->list, &adev->dm.da_list); - - return da->cpu_ptr; -} - -static enum dmub_status -dm_dmub_send_vbios_gpint_command(struct amdgpu_device *adev, - enum dmub_gpint_command command_code, - uint16_t param, - uint32_t timeout_us) -{ - union dmub_gpint_data_register reg, test; - uint32_t i; - - /* Assume that VBIOS DMUB is ready to take commands */ - - reg.bits.status = 1; - reg.bits.command_code = command_code; - reg.bits.param = param; - - cgs_write_register(adev->dm.cgs_device, 0x34c0 + 0x01f8, reg.all); - - for (i = 0; i < timeout_us; ++i) { - udelay(1); - - /* Check if our GPINT got acked */ - reg.bits.status = 0; - test = (union dmub_gpint_data_register) - cgs_read_register(adev->dm.cgs_device, 0x34c0 + 0x01f8); - - if (test.all == reg.all) - return DMUB_STATUS_OK; - } - - return DMUB_STATUS_TIMEOUT; -} - -static struct dml2_soc_bb *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev) -{ - struct dml2_soc_bb *bb; - long long addr; - int i = 0; - uint16_t chunk; - enum dmub_gpint_command send_addrs[] = { - DMUB_GPINT__SET_BB_ADDR_WORD0, - DMUB_GPINT__SET_BB_ADDR_WORD1, - DMUB_GPINT__SET_BB_ADDR_WORD2, - DMUB_GPINT__SET_BB_ADDR_WORD3, - }; - enum dmub_status ret; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(4, 0, 1): - break; - default: - return NULL; - } - - bb = dm_allocate_gpu_mem(adev, - DC_MEM_ALLOC_TYPE_GART, - sizeof(struct dml2_soc_bb), - &addr); - if (!bb) - return NULL; - - for (i = 0; i < 4; i++) { - /* Extract 16-bit chunk */ - chunk = ((uint64_t) addr >> (i * 16)) & 0xFFFF; - /* Send the chunk */ - ret = dm_dmub_send_vbios_gpint_command(adev, send_addrs[i], chunk, 30000); - if (ret != DMUB_STATUS_OK) - /* No need to free bb here since it shall be done in dm_sw_fini() */ - return NULL; - } - - /* Now ask DMUB to copy the bb */ - ret = dm_dmub_send_vbios_gpint_command(adev, DMUB_GPINT__BB_COPY, 1, 200000); - if (ret != DMUB_STATUS_OK) - return NULL; - - return bb; -} - -static int amdgpu_dm_init(struct amdgpu_device *adev) -{ - struct dc_init_data init_data; - struct dc_callback_init init_params; - int r; - - adev->dm.ddev = adev_to_drm(adev); - adev->dm.adev = adev; - - /* Zero all the fields */ - memset(&init_data, 0, sizeof(init_data)); - memset(&init_params, 0, sizeof(init_params)); - - mutex_init(&adev->dm.dpia_aux_lock); - mutex_init(&adev->dm.dc_lock); - mutex_init(&adev->dm.audio_lock); - - if (amdgpu_dm_irq_init(adev)) { - DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); - goto error; - } - - init_data.asic_id.chip_family = adev->family; - - init_data.asic_id.pci_revision_id = adev->pdev->revision; - init_data.asic_id.hw_internal_rev = adev->external_rev_id; - init_data.asic_id.chip_id = adev->pdev->device; - - init_data.asic_id.vram_width = adev->gmc.vram_width; - /* TODO: initialize init_data.asic_id.vram_type here!!!! */ - init_data.asic_id.atombios_base_address = - adev->mode_info.atom_context->bios; - - init_data.driver = adev; - - /* cgs_device was created in dm_sw_init() */ - init_data.cgs_device = adev->dm.cgs_device; - - init_data.dce_environment = DCE_ENV_PRODUCTION_DRV; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - switch (adev->dm.dmcub_fw_version) { - case 0: /* development */ - case 0x1: /* linux-firmware.git hash 6d9f399 */ - case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */ - init_data.flags.disable_dmcu = false; - break; - default: - init_data.flags.disable_dmcu = true; - } - break; - case IP_VERSION(2, 0, 3): - init_data.flags.disable_dmcu = true; - break; - default: - break; - } - - /* APU support S/G display by default except: - * ASICs before Carrizo, - * RAVEN1 (Users reported stability issue) - */ - - if (adev->asic_type < CHIP_CARRIZO) { - init_data.flags.gpu_vm_support = false; - } else if (adev->asic_type == CHIP_RAVEN) { - if (adev->apu_flags & AMD_APU_IS_RAVEN) - init_data.flags.gpu_vm_support = false; - else - init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0); - } else { - init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU); - } - - adev->mode_info.gpu_vm_support = init_data.flags.gpu_vm_support; - - if (amdgpu_dc_feature_mask & DC_FBC_MASK) - init_data.flags.fbc_support = true; - - if (amdgpu_dc_feature_mask & DC_MULTI_MON_PP_MCLK_SWITCH_MASK) - init_data.flags.multi_mon_pp_mclk_switch = true; - - if (amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK) - init_data.flags.disable_fractional_pwm = true; - - if (amdgpu_dc_feature_mask & DC_EDP_NO_POWER_SEQUENCING) - init_data.flags.edp_no_power_sequencing = true; - - if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP1_4A) - init_data.flags.allow_lttpr_non_transparent_mode.bits.DP1_4A = true; - if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP2_0) - init_data.flags.allow_lttpr_non_transparent_mode.bits.DP2_0 = true; - - init_data.flags.seamless_boot_edp_requested = false; - - if (amdgpu_device_seamless_boot_supported(adev)) { - init_data.flags.seamless_boot_edp_requested = true; - init_data.flags.allow_seamless_boot_optimization = true; - DRM_INFO("Seamless boot condition check passed\n"); - } - - init_data.flags.enable_mipi_converter_optimization = true; - - init_data.dcn_reg_offsets = adev->reg_offset[DCE_HWIP][0]; - init_data.nbio_reg_offsets = adev->reg_offset[NBIO_HWIP][0]; - init_data.clk_reg_offsets = adev->reg_offset[CLK_HWIP][0]; - - if (amdgpu_dc_debug_mask & DC_DISABLE_IPS) - init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL; - else - init_data.flags.disable_ips = DMUB_IPS_ENABLE; - - init_data.flags.disable_ips_in_vpb = 0; - - /* Enable DWB for tested platforms only */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0)) - init_data.num_virtual_links = 1; - - retrieve_dmi_info(&adev->dm); - - if (adev->dm.bb_from_dmub) - init_data.bb_from_dmub = adev->dm.bb_from_dmub; - else - init_data.bb_from_dmub = NULL; - - /* Display Core create. */ - adev->dm.dc = dc_create(&init_data); - - if (adev->dm.dc) { - DRM_INFO("Display Core v%s initialized on %s\n", DC_VER, - dce_version_to_string(adev->dm.dc->ctx->dce_version)); - } else { - DRM_INFO("Display Core failed to initialize with v%s!\n", DC_VER); - goto error; - } - - if (amdgpu_dc_debug_mask & DC_DISABLE_PIPE_SPLIT) { - adev->dm.dc->debug.force_single_disp_pipe_split = false; - adev->dm.dc->debug.pipe_split_policy = MPC_SPLIT_AVOID; - } - - if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY) - adev->dm.dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true; - if (dm_should_disable_stutter(adev->pdev)) - adev->dm.dc->debug.disable_stutter = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER) - adev->dm.dc->debug.disable_stutter = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) - adev->dm.dc->debug.disable_dsc = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING) - adev->dm.dc->debug.disable_clock_gate = true; - - if (amdgpu_dc_debug_mask & DC_FORCE_SUBVP_MCLK_SWITCH) - adev->dm.dc->debug.force_subvp_mclk_switch = true; - - if (amdgpu_dc_debug_mask & DC_ENABLE_DML2) { - adev->dm.dc->debug.using_dml2 = true; - adev->dm.dc->debug.using_dml21 = true; - } - - adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; - - /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ - adev->dm.dc->debug.ignore_cable_id = true; - - if (adev->dm.dc->caps.dp_hdmi21_pcon_support) - DRM_INFO("DP-HDMI FRL PCON supported\n"); - - r = dm_dmub_hw_init(adev); - if (r) { - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - goto error; - } - - dc_hardware_init(adev->dm.dc); - - adev->dm.hpd_rx_offload_wq = hpd_rx_irq_create_workqueue(adev->dm.dc); - if (!adev->dm.hpd_rx_offload_wq) { - DRM_ERROR("amdgpu: failed to create hpd rx offload workqueue.\n"); - goto error; - } - - if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) { - struct dc_phy_addr_space_config pa_config; - - mmhub_read_system_context(adev, &pa_config); - - // Call the DC init_memory func - dc_setup_system_context(adev->dm.dc, &pa_config); - } - - adev->dm.freesync_module = mod_freesync_create(adev->dm.dc); - if (!adev->dm.freesync_module) { - DRM_ERROR( - "amdgpu: failed to initialize freesync_module.\n"); - } else - DRM_DEBUG_DRIVER("amdgpu: freesync_module init done %p.\n", - adev->dm.freesync_module); - - amdgpu_dm_init_color_mod(); - - if (adev->dm.dc->caps.max_links > 0) { - adev->dm.vblank_control_workqueue = - create_singlethread_workqueue("dm_vblank_control_workqueue"); - if (!adev->dm.vblank_control_workqueue) - DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); - } - - if (adev->dm.dc->caps.ips_support && adev->dm.dc->config.disable_ips == DMUB_IPS_ENABLE) - adev->dm.idle_workqueue = idle_create_workqueue(adev); - - if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) { - adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); - - if (!adev->dm.hdcp_workqueue) - DRM_ERROR("amdgpu: failed to initialize hdcp_workqueue.\n"); - else - DRM_DEBUG_DRIVER("amdgpu: hdcp_workqueue init done %p.\n", adev->dm.hdcp_workqueue); - - dc_init_callbacks(adev->dm.dc, &init_params); - } - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - init_completion(&adev->dm.dmub_aux_transfer_done); - adev->dm.dmub_notify = kzalloc(sizeof(struct dmub_notification), GFP_KERNEL); - if (!adev->dm.dmub_notify) { - DRM_INFO("amdgpu: fail to allocate adev->dm.dmub_notify"); - goto error; - } - - adev->dm.delayed_hpd_wq = create_singlethread_workqueue("amdgpu_dm_hpd_wq"); - if (!adev->dm.delayed_hpd_wq) { - DRM_ERROR("amdgpu: failed to create hpd offload workqueue.\n"); - goto error; - } - - amdgpu_dm_outbox_init(adev); - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_AUX_REPLY, - dmub_aux_setconfig_callback, false)) { - DRM_ERROR("amdgpu: fail to register dmub aux callback"); - goto error; - } - /* Enable outbox notification only after IRQ handlers are registered and DMUB is alive. - * It is expected that DMUB will resend any pending notifications at this point. Note - * that hpd and hpd_irq handler registration are deferred to register_hpd_handlers() to - * align legacy interface initialization sequence. Connection status will be proactivly - * detected once in the amdgpu_dm_initialize_drm_device. - */ - dc_enable_dmub_outbox(adev->dm.dc); - - /* DPIA trace goes to dmesg logs only if outbox is enabled */ - if (amdgpu_dc_debug_mask & DC_ENABLE_DPIA_TRACE) - dc_dmub_srv_enable_dpia_trace(adev->dm.dc); - } - - if (amdgpu_dm_initialize_drm_device(adev)) { - DRM_ERROR( - "amdgpu: failed to initialize sw for display support.\n"); - goto error; - } - - /* create fake encoders for MST */ - dm_dp_create_fake_mst_encoders(adev); - - /* TODO: Add_display_info? */ - - /* TODO use dynamic cursor width */ - adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size; - adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size; - - if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) { - DRM_ERROR( - "amdgpu: failed to initialize sw for display support.\n"); - goto error; - } - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - adev->dm.secure_display_ctxs = amdgpu_dm_crtc_secure_display_create_contexts(adev); - if (!adev->dm.secure_display_ctxs) - DRM_ERROR("amdgpu: failed to initialize secure display contexts.\n"); -#endif - - DRM_DEBUG_DRIVER("KMS initialized.\n"); - - return 0; -error: - amdgpu_dm_fini(adev); - - return -EINVAL; -} - -static int amdgpu_dm_early_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_dm_audio_fini(adev); - - return 0; -} - -static void amdgpu_dm_fini(struct amdgpu_device *adev) -{ - int i; - - if (adev->dm.vblank_control_workqueue) { - destroy_workqueue(adev->dm.vblank_control_workqueue); - adev->dm.vblank_control_workqueue = NULL; - } - - if (adev->dm.idle_workqueue) { - if (adev->dm.idle_workqueue->running) { - adev->dm.idle_workqueue->enable = false; - flush_work(&adev->dm.idle_workqueue->work); - } - - kfree(adev->dm.idle_workqueue); - adev->dm.idle_workqueue = NULL; - } - - amdgpu_dm_destroy_drm_device(&adev->dm); - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - if (adev->dm.secure_display_ctxs) { - for (i = 0; i < adev->mode_info.num_crtc; i++) { - if (adev->dm.secure_display_ctxs[i].crtc) { - flush_work(&adev->dm.secure_display_ctxs[i].notify_ta_work); - flush_work(&adev->dm.secure_display_ctxs[i].forward_roi_work); - } - } - kfree(adev->dm.secure_display_ctxs); - adev->dm.secure_display_ctxs = NULL; - } -#endif - if (adev->dm.hdcp_workqueue) { - hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue); - adev->dm.hdcp_workqueue = NULL; - } - - if (adev->dm.dc) { - dc_deinit_callbacks(adev->dm.dc); - dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv); - if (dc_enable_dmub_notifications(adev->dm.dc)) { - kfree(adev->dm.dmub_notify); - adev->dm.dmub_notify = NULL; - destroy_workqueue(adev->dm.delayed_hpd_wq); - adev->dm.delayed_hpd_wq = NULL; - } - } - - if (adev->dm.dmub_bo) - amdgpu_bo_free_kernel(&adev->dm.dmub_bo, - &adev->dm.dmub_bo_gpu_addr, - &adev->dm.dmub_bo_cpu_addr); - - if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) { - for (i = 0; i < adev->dm.dc->caps.max_links; i++) { - if (adev->dm.hpd_rx_offload_wq[i].wq) { - destroy_workqueue(adev->dm.hpd_rx_offload_wq[i].wq); - adev->dm.hpd_rx_offload_wq[i].wq = NULL; - } - } - - kfree(adev->dm.hpd_rx_offload_wq); - adev->dm.hpd_rx_offload_wq = NULL; - } - - /* DC Destroy TODO: Replace destroy DAL */ - if (adev->dm.dc) - dc_destroy(&adev->dm.dc); - /* - * TODO: pageflip, vlank interrupt - * - * amdgpu_dm_irq_fini(adev); - */ - - if (adev->dm.cgs_device) { - amdgpu_cgs_destroy_device(adev->dm.cgs_device); - adev->dm.cgs_device = NULL; - } - if (adev->dm.freesync_module) { - mod_freesync_destroy(adev->dm.freesync_module); - adev->dm.freesync_module = NULL; - } - - mutex_destroy(&adev->dm.audio_lock); - mutex_destroy(&adev->dm.dc_lock); - mutex_destroy(&adev->dm.dpia_aux_lock); -} - -static int load_dmcu_fw(struct amdgpu_device *adev) -{ - const char *fw_name_dmcu = NULL; - int r; - const struct dmcu_firmware_header_v1_0 *hdr; - - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - case CHIP_OLAND: -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - case CHIP_TONGA: - case CHIP_FIJI: - case CHIP_CARRIZO: - case CHIP_STONEY: - case CHIP_POLARIS11: - case CHIP_POLARIS10: - case CHIP_POLARIS12: - case CHIP_VEGAM: - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - return 0; - case CHIP_NAVI12: - fw_name_dmcu = FIRMWARE_NAVI12_DMCU; - break; - case CHIP_RAVEN: - if (ASICREV_IS_PICASSO(adev->external_rev_id)) - fw_name_dmcu = FIRMWARE_RAVEN_DMCU; - else if (ASICREV_IS_RAVEN2(adev->external_rev_id)) - fw_name_dmcu = FIRMWARE_RAVEN_DMCU; - else - return 0; - break; - default: - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 3): - case IP_VERSION(2, 0, 0): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 0, 2): - case IP_VERSION(3, 0, 3): - case IP_VERSION(3, 0, 1): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - return 0; - default: - break; - } - DRM_ERROR("Unsupported ASIC type: 0x%X\n", adev->asic_type); - return -EINVAL; - } - - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { - DRM_DEBUG_KMS("dm: DMCU firmware not supported on direct or SMU loading\n"); - return 0; - } - - r = amdgpu_ucode_request(adev, &adev->dm.fw_dmcu, "%s", fw_name_dmcu); - if (r == -ENODEV) { - /* DMCU firmware is not necessary, so don't raise a fuss if it's missing */ - DRM_DEBUG_KMS("dm: DMCU firmware not found\n"); - adev->dm.fw_dmcu = NULL; - return 0; - } - if (r) { - dev_err(adev->dev, "amdgpu_dm: Can't validate firmware \"%s\"\n", - fw_name_dmcu); - amdgpu_ucode_release(&adev->dm.fw_dmcu); - return r; - } - - hdr = (const struct dmcu_firmware_header_v1_0 *)adev->dm.fw_dmcu->data; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].ucode_id = AMDGPU_UCODE_ID_DMCU_ERAM; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].fw = adev->dm.fw_dmcu; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->header.ucode_size_bytes) - le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); - - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].ucode_id = AMDGPU_UCODE_ID_DMCU_INTV; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].fw = adev->dm.fw_dmcu; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); - - adev->dm.dmcu_fw_version = le32_to_cpu(hdr->header.ucode_version); - - DRM_DEBUG_KMS("PSP loading DMCU firmware\n"); - - return 0; -} - -static uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address) -{ - struct amdgpu_device *adev = ctx; - - return dm_read_reg(adev->dm.dc->ctx, address); -} - -static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address, - uint32_t value) -{ - struct amdgpu_device *adev = ctx; - - return dm_write_reg(adev->dm.dc->ctx, address, value); -} - -static int dm_dmub_sw_init(struct amdgpu_device *adev) -{ - struct dmub_srv_create_params create_params; - struct dmub_srv_region_params region_params; - struct dmub_srv_region_info region_info; - struct dmub_srv_memory_params memory_params; - struct dmub_srv_fb_info *fb_info; - struct dmub_srv *dmub_srv; - const struct dmcub_firmware_header_v1_0 *hdr; - enum dmub_asic dmub_asic; - enum dmub_status status; - static enum dmub_window_memory_type window_memory_type[DMUB_WINDOW_TOTAL] = { - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_0_INST_CONST - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_1_STACK - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_2_BSS_DATA - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_3_VBIOS - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_4_MAILBOX - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_5_TRACEBUFF - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_6_FW_STATE - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_7_SCRATCH_MEM - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_SHARED_STATE - }; - int r; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - dmub_asic = DMUB_ASIC_DCN21; - break; - case IP_VERSION(3, 0, 0): - dmub_asic = DMUB_ASIC_DCN30; - break; - case IP_VERSION(3, 0, 1): - dmub_asic = DMUB_ASIC_DCN301; - break; - case IP_VERSION(3, 0, 2): - dmub_asic = DMUB_ASIC_DCN302; - break; - case IP_VERSION(3, 0, 3): - dmub_asic = DMUB_ASIC_DCN303; - break; - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - dmub_asic = (adev->external_rev_id == YELLOW_CARP_B0) ? DMUB_ASIC_DCN31B : DMUB_ASIC_DCN31; - break; - case IP_VERSION(3, 1, 4): - dmub_asic = DMUB_ASIC_DCN314; - break; - case IP_VERSION(3, 1, 5): - dmub_asic = DMUB_ASIC_DCN315; - break; - case IP_VERSION(3, 1, 6): - dmub_asic = DMUB_ASIC_DCN316; - break; - case IP_VERSION(3, 2, 0): - dmub_asic = DMUB_ASIC_DCN32; - break; - case IP_VERSION(3, 2, 1): - dmub_asic = DMUB_ASIC_DCN321; - break; - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - dmub_asic = DMUB_ASIC_DCN35; - break; - case IP_VERSION(4, 0, 1): - dmub_asic = DMUB_ASIC_DCN401; - break; - - default: - /* ASIC doesn't support DMUB. */ - return 0; - } - - hdr = (const struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data; - adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id = - AMDGPU_UCODE_ID_DMCUB; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].fw = - adev->dm.dmub_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->inst_const_bytes), PAGE_SIZE); - - DRM_INFO("Loading DMUB firmware via PSP: version=0x%08X\n", - adev->dm.dmcub_fw_version); - } - - - adev->dm.dmub_srv = kzalloc(sizeof(*adev->dm.dmub_srv), GFP_KERNEL); - dmub_srv = adev->dm.dmub_srv; - - if (!dmub_srv) { - DRM_ERROR("Failed to allocate DMUB service!\n"); - return -ENOMEM; - } - - memset(&create_params, 0, sizeof(create_params)); - create_params.user_ctx = adev; - create_params.funcs.reg_read = amdgpu_dm_dmub_reg_read; - create_params.funcs.reg_write = amdgpu_dm_dmub_reg_write; - create_params.asic = dmub_asic; - - /* Create the DMUB service. */ - status = dmub_srv_create(dmub_srv, &create_params); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error creating DMUB service: %d\n", status); - return -EINVAL; - } - - /* Calculate the size of all the regions for the DMUB service. */ - memset(®ion_params, 0, sizeof(region_params)); - - region_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) - - PSP_HEADER_BYTES - PSP_FOOTER_BYTES; - region_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes); - region_params.vbios_size = adev->bios_size; - region_params.fw_bss_data = region_params.bss_data_size ? - adev->dm.dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - le32_to_cpu(hdr->inst_const_bytes) : NULL; - region_params.fw_inst_const = - adev->dm.dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - PSP_HEADER_BYTES; - region_params.window_memory_type = window_memory_type; - - status = dmub_srv_calc_region_info(dmub_srv, ®ion_params, - ®ion_info); - - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error calculating DMUB region info: %d\n", status); - return -EINVAL; - } - - /* - * Allocate a framebuffer based on the total size of all the regions. - * TODO: Move this into GART. - */ - r = amdgpu_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, - &adev->dm.dmub_bo, - &adev->dm.dmub_bo_gpu_addr, - &adev->dm.dmub_bo_cpu_addr); - if (r) - return r; - - /* Rebase the regions on the framebuffer address. */ - memset(&memory_params, 0, sizeof(memory_params)); - memory_params.cpu_fb_addr = adev->dm.dmub_bo_cpu_addr; - memory_params.gpu_fb_addr = adev->dm.dmub_bo_gpu_addr; - memory_params.region_info = ®ion_info; - memory_params.window_memory_type = window_memory_type; - - adev->dm.dmub_fb_info = - kzalloc(sizeof(*adev->dm.dmub_fb_info), GFP_KERNEL); - fb_info = adev->dm.dmub_fb_info; - - if (!fb_info) { - DRM_ERROR( - "Failed to allocate framebuffer info for DMUB service!\n"); - return -ENOMEM; - } - - status = dmub_srv_calc_mem_info(dmub_srv, &memory_params, fb_info); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error calculating DMUB FB info: %d\n", status); - return -EINVAL; - } - - adev->dm.bb_from_dmub = dm_dmub_get_vbios_bounding_box(adev); - - return 0; -} - -static int dm_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - adev->dm.cgs_device = amdgpu_cgs_create_device(adev); - - if (!adev->dm.cgs_device) { - DRM_ERROR("amdgpu: failed to create cgs device.\n"); - return -EINVAL; - } - - /* Moved from dm init since we need to use allocations for storing bounding box data */ - INIT_LIST_HEAD(&adev->dm.da_list); - - r = dm_dmub_sw_init(adev); - if (r) - return r; - - return load_dmcu_fw(adev); -} - -static int dm_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct dal_allocation *da; - - list_for_each_entry(da, &adev->dm.da_list, list) { - if (adev->dm.bb_from_dmub == (void *) da->cpu_ptr) { - amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr); - list_del(&da->list); - kfree(da); - break; - } - } - - adev->dm.bb_from_dmub = NULL; - - kfree(adev->dm.dmub_fb_info); - adev->dm.dmub_fb_info = NULL; - - if (adev->dm.dmub_srv) { - dmub_srv_destroy(adev->dm.dmub_srv); - kfree(adev->dm.dmub_srv); - adev->dm.dmub_srv = NULL; - } - - amdgpu_ucode_release(&adev->dm.dmub_fw); - amdgpu_ucode_release(&adev->dm.fw_dmcu); - - return 0; -} - -static int detect_mst_link_for_all_connectors(struct drm_device *dev) -{ - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - int ret = 0; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type == dc_connection_mst_branch && - aconnector->mst_mgr.aux) { - drm_dbg_kms(dev, "DM_MST: starting TM on aconnector: %p [id: %d]\n", - aconnector, - aconnector->base.base.id); - - ret = drm_dp_mst_topology_mgr_set_mst(&aconnector->mst_mgr, true); - if (ret < 0) { - drm_err(dev, "DM_MST: Failed to start MST\n"); - aconnector->dc_link->type = - dc_connection_single; - ret = dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx, - aconnector->dc_link); - break; - } - } - } - drm_connector_list_iter_end(&iter); - - return ret; -} - -static int dm_late_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - struct dmcu_iram_parameters params; - unsigned int linear_lut[16]; - int i; - struct dmcu *dmcu = NULL; - - dmcu = adev->dm.dc->res_pool->dmcu; - - for (i = 0; i < 16; i++) - linear_lut[i] = 0xFFFF * i / 15; - - params.set = 0; - params.backlight_ramping_override = false; - params.backlight_ramping_start = 0xCCCC; - params.backlight_ramping_reduction = 0xCCCCCCCC; - params.backlight_lut_array_size = 16; - params.backlight_lut_array = linear_lut; - - /* Min backlight level after ABM reduction, Don't allow below 1% - * 0xFFFF x 0.01 = 0x28F - */ - params.min_abm_backlight = 0x28F; - /* In the case where abm is implemented on dmcub, - * dmcu object will be null. - * ABM 2.4 and up are implemented on dmcub. - */ - if (dmcu) { - if (!dmcu_load_iram(dmcu, params)) - return -EINVAL; - } else if (adev->dm.dc->ctx->dmub_srv) { - struct dc_link *edp_links[MAX_NUM_EDP]; - int edp_num; - - dc_get_edp_links(adev->dm.dc, edp_links, &edp_num); - for (i = 0; i < edp_num; i++) { - if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i)) - return -EINVAL; - } - } - - return detect_mst_link_for_all_connectors(adev_to_drm(adev)); -} - -static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr) -{ - int ret; - u8 guid[16]; - u64 tmp64; - - mutex_lock(&mgr->lock); - if (!mgr->mst_primary) - goto out_fail; - - if (drm_dp_read_dpcd_caps(mgr->aux, mgr->dpcd) < 0) { - drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); - goto out_fail; - } - - ret = drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL, - DP_MST_EN | - DP_UP_REQ_EN | - DP_UPSTREAM_IS_SRC); - if (ret < 0) { - drm_dbg_kms(mgr->dev, "mst write failed - undocked during suspend?\n"); - goto out_fail; - } - - /* Some hubs forget their guids after they resume */ - ret = drm_dp_dpcd_read(mgr->aux, DP_GUID, guid, 16); - if (ret != 16) { - drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); - goto out_fail; - } - - if (memchr_inv(guid, 0, 16) == NULL) { - tmp64 = get_jiffies_64(); - memcpy(&guid[0], &tmp64, sizeof(u64)); - memcpy(&guid[8], &tmp64, sizeof(u64)); - - ret = drm_dp_dpcd_write(mgr->aux, DP_GUID, guid, 16); - - if (ret != 16) { - drm_dbg_kms(mgr->dev, "check mstb guid failed - undocked during suspend?\n"); - goto out_fail; - } - } - - memcpy(mgr->mst_primary->guid, guid, 16); - -out_fail: - mutex_unlock(&mgr->lock); -} - -static void s3_handle_mst(struct drm_device *dev, bool suspend) -{ - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct drm_dp_mst_topology_mgr *mgr; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type != dc_connection_mst_branch || - aconnector->mst_root) - continue; - - mgr = &aconnector->mst_mgr; - - if (suspend) { - drm_dp_mst_topology_mgr_suspend(mgr); - } else { - /* if extended timeout is supported in hardware, - * default to LTTPR timeout (3.2ms) first as a W/A for DP link layer - * CTS 4.2.1.1 regression introduced by CTS specs requirement update. - */ - try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD); - if (!dp_is_lttpr_present(aconnector->dc_link)) - try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD); - - /* TODO: move resume_mst_branch_status() into drm mst resume again - * once topology probing work is pulled out from mst resume into mst - * resume 2nd step. mst resume 2nd step should be called after old - * state getting restored (i.e. drm_atomic_helper_resume()). - */ - resume_mst_branch_status(mgr); - } - } - drm_connector_list_iter_end(&iter); -} - -static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev) -{ - int ret = 0; - - /* This interface is for dGPU Navi1x.Linux dc-pplib interface depends - * on window driver dc implementation. - * For Navi1x, clock settings of dcn watermarks are fixed. the settings - * should be passed to smu during boot up and resume from s3. - * boot up: dc calculate dcn watermark clock settings within dc_create, - * dcn20_resource_construct - * then call pplib functions below to pass the settings to smu: - * smu_set_watermarks_for_clock_ranges - * smu_set_watermarks_table - * navi10_set_watermarks_table - * smu_write_watermarks_table - * - * For Renoir, clock settings of dcn watermark are also fixed values. - * dc has implemented different flow for window driver: - * dc_hardware_init / dc_set_power_state - * dcn10_init_hw - * notify_wm_ranges - * set_wm_ranges - * -- Linux - * smu_set_watermarks_for_clock_ranges - * renoir_set_watermarks_table - * smu_write_watermarks_table - * - * For Linux, - * dc_hardware_init -> amdgpu_dm_init - * dc_set_power_state --> dm_resume - * - * therefore, this function apply to navi10/12/14 but not Renoir - * * - */ - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 0): - break; - default: - return 0; - } - - ret = amdgpu_dpm_write_watermarks_table(adev); - if (ret) { - DRM_ERROR("Failed to update WMTABLE!\n"); - return ret; - } - - return 0; -} - -/** - * dm_hw_init() - Initialize DC device - * @handle: The base driver device containing the amdgpu_dm device. - * - * Initialize the &struct amdgpu_display_manager device. This involves calling - * the initializers of each DM component, then populating the struct with them. - * - * Although the function implies hardware initialization, both hardware and - * software are initialized here. Splitting them out to their relevant init - * hooks is a future TODO item. - * - * Some notable things that are initialized here: - * - * - Display Core, both software and hardware - * - DC modules that we need (freesync and color management) - * - DRM software states - * - Interrupt sources and handlers - * - Vblank support - * - Debug FS entries, if enabled - */ -static int dm_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - /* Create DAL display manager */ - r = amdgpu_dm_init(adev); - if (r) - return r; - amdgpu_dm_hpd_init(adev); - - return 0; -} - -/** - * dm_hw_fini() - Teardown DC device - * @handle: The base driver device containing the amdgpu_dm device. - * - * Teardown components within &struct amdgpu_display_manager that require - * cleanup. This involves cleaning up the DRM device, DC, and any modules that - * were loaded. Also flush IRQ workqueues and disable them. - */ -static int dm_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_dm_hpd_fini(adev); - - amdgpu_dm_irq_fini(adev); - amdgpu_dm_fini(adev); - return 0; -} - - -static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, - struct dc_state *state, bool enable) -{ - enum dc_irq_source irq_source; - struct amdgpu_crtc *acrtc; - int rc = -EBUSY; - int i = 0; - - for (i = 0; i < state->stream_count; i++) { - acrtc = get_crtc_by_otg_inst( - adev, state->stream_status[i].primary_otg_inst); - - if (acrtc && state->stream_status[i].plane_count != 0) { - irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst; - rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; - if (rc) - DRM_WARN("Failed to %s pflip interrupts\n", - enable ? "enable" : "disable"); - - if (enable) { - if (amdgpu_dm_crtc_vrr_active(to_dm_crtc_state(acrtc->base.state))) - rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, true); - } else - rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, false); - - if (rc) - DRM_WARN("Failed to %sable vupdate interrupt\n", enable ? "en" : "dis"); - - irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst; - /* During gpu-reset we disable and then enable vblank irq, so - * don't use amdgpu_irq_get/put() to avoid refcount change. - */ - if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) - DRM_WARN("Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); - } - } - -} - -static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc) -{ - struct dc_state *context = NULL; - enum dc_status res = DC_ERROR_UNEXPECTED; - int i; - struct dc_stream_state *del_streams[MAX_PIPES]; - int del_streams_count = 0; - struct dc_commit_streams_params params = {}; - - memset(del_streams, 0, sizeof(del_streams)); - - context = dc_state_create_current_copy(dc); - if (context == NULL) - goto context_alloc_fail; - - /* First remove from context all streams */ - for (i = 0; i < context->stream_count; i++) { - struct dc_stream_state *stream = context->streams[i]; - - del_streams[del_streams_count++] = stream; - } - - /* Remove all planes for removed streams and then remove the streams */ - for (i = 0; i < del_streams_count; i++) { - if (!dc_state_rem_all_planes_for_stream(dc, del_streams[i], context)) { - res = DC_FAIL_DETACH_SURFACES; - goto fail; - } - - res = dc_state_remove_stream(dc, context, del_streams[i]); - if (res != DC_OK) - goto fail; - } - - params.streams = context->streams; - params.stream_count = context->stream_count; - res = dc_commit_streams(dc, ¶ms); - -fail: - dc_state_release(context); - -context_alloc_fail: - return res; -} - -static void hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm) -{ - int i; - - if (dm->hpd_rx_offload_wq) { - for (i = 0; i < dm->dc->caps.max_links; i++) - flush_workqueue(dm->hpd_rx_offload_wq[i].wq); - } -} - -static int dm_suspend(void *handle) -{ - struct amdgpu_device *adev = handle; - struct amdgpu_display_manager *dm = &adev->dm; - int ret = 0; - - if (amdgpu_in_reset(adev)) { - mutex_lock(&dm->dc_lock); - - dc_allow_idle_optimizations(adev->dm.dc, false); - - dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state); - - if (dm->cached_dc_state) - dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); - - amdgpu_dm_commit_zero_streams(dm->dc); - - amdgpu_dm_irq_suspend(adev); - - hpd_rx_irq_work_suspend(dm); - - return ret; - } - - WARN_ON(adev->dm.cached_state); - adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev)); - if (IS_ERR(adev->dm.cached_state)) - return PTR_ERR(adev->dm.cached_state); - - s3_handle_mst(adev_to_drm(adev), true); - - amdgpu_dm_irq_suspend(adev); - - hpd_rx_irq_work_suspend(dm); - - if (adev->dm.dc->caps.ips_support) - dc_allow_idle_optimizations(adev->dm.dc, true); - - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D3); - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D3); - - return 0; -} - -struct drm_connector * -amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_state *state, - struct drm_crtc *crtc) -{ - u32 i; - struct drm_connector_state *new_con_state; - struct drm_connector *connector; - struct drm_crtc *crtc_from_state; - - for_each_new_connector_in_state(state, connector, new_con_state, i) { - crtc_from_state = new_con_state->crtc; - - if (crtc_from_state == crtc) - return connector; - } - - return NULL; -} - -static void emulated_link_detect(struct dc_link *link) -{ - struct dc_sink_init_data sink_init_data = { 0 }; - struct display_sink_capability sink_caps = { 0 }; - enum dc_edid_status edid_status; - struct dc_context *dc_ctx = link->ctx; - struct drm_device *dev = adev_to_drm(dc_ctx->driver_context); - struct dc_sink *sink = NULL; - struct dc_sink *prev_sink = NULL; - - link->type = dc_connection_none; - prev_sink = link->local_sink; - - if (prev_sink) - dc_sink_release(prev_sink); - - switch (link->connector_signal) { - case SIGNAL_TYPE_HDMI_TYPE_A: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_HDMI_TYPE_A; - break; - } - - case SIGNAL_TYPE_DVI_SINGLE_LINK: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_DVI_SINGLE_LINK; - break; - } - - case SIGNAL_TYPE_DVI_DUAL_LINK: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_DVI_DUAL_LINK; - break; - } - - case SIGNAL_TYPE_LVDS: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_LVDS; - break; - } - - case SIGNAL_TYPE_EDP: { - sink_caps.transaction_type = - DDC_TRANSACTION_TYPE_I2C_OVER_AUX; - sink_caps.signal = SIGNAL_TYPE_EDP; - break; - } - - case SIGNAL_TYPE_DISPLAY_PORT: { - sink_caps.transaction_type = - DDC_TRANSACTION_TYPE_I2C_OVER_AUX; - sink_caps.signal = SIGNAL_TYPE_VIRTUAL; - break; - } - - default: - drm_err(dev, "Invalid connector type! signal:%d\n", - link->connector_signal); - return; - } - - sink_init_data.link = link; - sink_init_data.sink_signal = sink_caps.signal; - - sink = dc_sink_create(&sink_init_data); - if (!sink) { - drm_err(dev, "Failed to create sink!\n"); - return; - } - - /* dc_sink_create returns a new reference */ - link->local_sink = sink; - - edid_status = dm_helpers_read_local_edid( - link->ctx, - link, - sink); - - if (edid_status != EDID_OK) - drm_err(dev, "Failed to read EDID\n"); - -} - -static void dm_gpureset_commit_state(struct dc_state *dc_state, - struct amdgpu_display_manager *dm) -{ - struct { - struct dc_surface_update surface_updates[MAX_SURFACES]; - struct dc_plane_info plane_infos[MAX_SURFACES]; - struct dc_scaling_info scaling_infos[MAX_SURFACES]; - struct dc_flip_addrs flip_addrs[MAX_SURFACES]; - struct dc_stream_update stream_update; - } *bundle; - int k, m; - - bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - - if (!bundle) { - drm_err(dm->ddev, "Failed to allocate update bundle\n"); - goto cleanup; - } - - for (k = 0; k < dc_state->stream_count; k++) { - bundle->stream_update.stream = dc_state->streams[k]; - - for (m = 0; m < dc_state->stream_status->plane_count; m++) { - bundle->surface_updates[m].surface = - dc_state->stream_status->plane_states[m]; - bundle->surface_updates[m].surface->force_full_update = - true; - } - - update_planes_and_stream_adapter(dm->dc, - UPDATE_TYPE_FULL, - dc_state->stream_status->plane_count, - dc_state->streams[k], - &bundle->stream_update, - bundle->surface_updates); - } - -cleanup: - kfree(bundle); -} - -static int dm_resume(void *handle) -{ - struct amdgpu_device *adev = handle; - struct drm_device *ddev = adev_to_drm(adev); - struct amdgpu_display_manager *dm = &adev->dm; - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct drm_crtc *crtc; - struct drm_crtc_state *new_crtc_state; - struct dm_crtc_state *dm_new_crtc_state; - struct drm_plane *plane; - struct drm_plane_state *new_plane_state; - struct dm_plane_state *dm_new_plane_state; - struct dm_atomic_state *dm_state = to_dm_atomic_state(dm->atomic_obj.state); - enum dc_connection_type new_connection_type = dc_connection_none; - struct dc_state *dc_state; - int i, r, j, ret; - bool need_hotplug = false; - struct dc_commit_streams_params commit_params = {}; - - if (dm->dc->caps.ips_support) { - dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false); - } - - if (amdgpu_in_reset(adev)) { - dc_state = dm->cached_dc_state; - - /* - * The dc->current_state is backed up into dm->cached_dc_state - * before we commit 0 streams. - * - * DC will clear link encoder assignments on the real state - * but the changes won't propagate over to the copy we made - * before the 0 streams commit. - * - * DC expects that link encoder assignments are *not* valid - * when committing a state, so as a workaround we can copy - * off of the current state. - * - * We lose the previous assignments, but we had already - * commit 0 streams anyway. - */ - link_enc_cfg_copy(adev->dm.dc->current_state, dc_state); - - r = dm_dmub_hw_init(adev); - if (r) - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0); - - dc_resume(dm->dc); - - amdgpu_dm_irq_resume_early(adev); - - for (i = 0; i < dc_state->stream_count; i++) { - dc_state->streams[i]->mode_changed = true; - for (j = 0; j < dc_state->stream_status[i].plane_count; j++) { - dc_state->stream_status[i].plane_states[j]->update_flags.raw - = 0xffffffff; - } - } - - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - amdgpu_dm_outbox_init(adev); - dc_enable_dmub_outbox(adev->dm.dc); - } - - commit_params.streams = dc_state->streams; - commit_params.stream_count = dc_state->stream_count; - dc_exit_ips_for_hw_access(dm->dc); - WARN_ON(!dc_commit_streams(dm->dc, &commit_params)); - - dm_gpureset_commit_state(dm->cached_dc_state, dm); - - dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, true); - - dc_state_release(dm->cached_dc_state); - dm->cached_dc_state = NULL; - - amdgpu_dm_irq_resume_late(adev); - - mutex_unlock(&dm->dc_lock); - - return 0; - } - /* Recreate dc_state - DC invalidates it when setting power state to S3. */ - dc_state_release(dm_state->context); - dm_state->context = dc_state_create(dm->dc, NULL); - /* TODO: Remove dc_state->dccg, use dc->dccg directly. */ - - /* Before powering on DC we need to re-initialize DMUB. */ - dm_dmub_hw_resume(adev); - - /* Re-enable outbox interrupts for DPIA. */ - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - amdgpu_dm_outbox_init(adev); - dc_enable_dmub_outbox(adev->dm.dc); - } - - /* power on hardware */ - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0); - - /* program HPD filter */ - dc_resume(dm->dc); - - /* - * early enable HPD Rx IRQ, should be done before set mode as short - * pulse interrupts are used for MST - */ - amdgpu_dm_irq_resume_early(adev); - - /* On resume we need to rewrite the MSTM control bits to enable MST*/ - s3_handle_mst(ddev, false); - - /* Do detection*/ - drm_connector_list_iter_begin(ddev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->dc_link) - continue; - - /* - * this is the case when traversing through already created end sink - * MST connectors, should be skipped - */ - if (aconnector->mst_root) - continue; - - mutex_lock(&aconnector->hpd_lock); - if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(aconnector->dc_link); - } else { - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_link_detect(aconnector->dc_link, DETECT_REASON_RESUMEFROMS3S4); - mutex_unlock(&dm->dc_lock); - } - - if (aconnector->fake_enable && aconnector->dc_link->local_sink) - aconnector->fake_enable = false; - - if (aconnector->dc_sink) - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - amdgpu_dm_update_connector_after_detect(aconnector); - mutex_unlock(&aconnector->hpd_lock); - } - drm_connector_list_iter_end(&iter); - - /* Force mode set in atomic commit */ - for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) { - new_crtc_state->active_changed = true; - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - reset_freesync_config_for_crtc(dm_new_crtc_state); - } - - /* - * atomic_check is expected to create the dc states. We need to release - * them here, since they were duplicated as part of the suspend - * procedure. - */ - for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->stream) { - WARN_ON(kref_read(&dm_new_crtc_state->stream->refcount) > 1); - dc_stream_release(dm_new_crtc_state->stream); - dm_new_crtc_state->stream = NULL; - } - dm_new_crtc_state->base.color_mgmt_changed = true; - } - - for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) { - dm_new_plane_state = to_dm_plane_state(new_plane_state); - if (dm_new_plane_state->dc_state) { - WARN_ON(kref_read(&dm_new_plane_state->dc_state->refcount) > 1); - dc_plane_state_release(dm_new_plane_state->dc_state); - dm_new_plane_state->dc_state = NULL; - } - } - - drm_atomic_helper_resume(ddev, dm->cached_state); - - dm->cached_state = NULL; - - /* Do mst topology probing after resuming cached state*/ - drm_connector_list_iter_begin(ddev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type != dc_connection_mst_branch || - aconnector->mst_root) - continue; - - ret = drm_dp_mst_topology_mgr_resume(&aconnector->mst_mgr, true); - - if (ret < 0) { - dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx, - aconnector->dc_link); - need_hotplug = true; - } - } - drm_connector_list_iter_end(&iter); - - if (need_hotplug) - drm_kms_helper_hotplug_event(ddev); - - amdgpu_dm_irq_resume_late(adev); - - amdgpu_dm_smu_write_watermarks_table(adev); - - return 0; -} - -/** - * DOC: DM Lifecycle - * - * DM (and consequently DC) is registered in the amdgpu base driver as a IP - * block. When CONFIG_DRM_AMD_DC is enabled, the DM device IP block is added to - * the base driver's device list to be initialized and torn down accordingly. - * - * The functions to do so are provided as hooks in &struct amd_ip_funcs. - */ - -static const struct amd_ip_funcs amdgpu_dm_funcs = { - .name = "dm", - .early_init = dm_early_init, - .late_init = dm_late_init, - .sw_init = dm_sw_init, - .sw_fini = dm_sw_fini, - .early_fini = amdgpu_dm_early_fini, - .hw_init = dm_hw_init, - .hw_fini = dm_hw_fini, - .suspend = dm_suspend, - .resume = dm_resume, - .is_idle = dm_is_idle, - .wait_for_idle = dm_wait_for_idle, - .check_soft_reset = dm_check_soft_reset, - .soft_reset = dm_soft_reset, - .set_clockgating_state = dm_set_clockgating_state, - .set_powergating_state = dm_set_powergating_state, - .dump_ip_state = NULL, - .print_ip_state = NULL, -}; - -const struct amdgpu_ip_block_version dm_ip_block = { - .type = AMD_IP_BLOCK_TYPE_DCE, - .major = 1, - .minor = 0, - .rev = 0, - .funcs = &amdgpu_dm_funcs, -}; - - -/** - * DOC: atomic - * - * *WIP* - */ - -static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = { - .fb_create = amdgpu_display_user_framebuffer_create, - .get_format_info = amdgpu_dm_plane_get_format_info, - .atomic_check = amdgpu_dm_atomic_check, - .atomic_commit = drm_atomic_helper_commit, -}; - -static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = { - .atomic_commit_tail = amdgpu_dm_atomic_commit_tail, - .atomic_commit_setup = drm_dp_mst_atomic_setup_commit, -}; - -static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) -{ - struct amdgpu_dm_backlight_caps *caps; - struct drm_connector *conn_base; - struct amdgpu_device *adev; - struct drm_luminance_range_info *luminance_range; - - if (aconnector->bl_idx == -1 || - aconnector->dc_link->connector_signal != SIGNAL_TYPE_EDP) - return; - - conn_base = &aconnector->base; - adev = drm_to_adev(conn_base->dev); - - caps = &adev->dm.backlight_caps[aconnector->bl_idx]; - caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps; - caps->aux_support = false; - - if (caps->ext_caps->bits.oled == 1 - /* - * || - * caps->ext_caps->bits.sdr_aux_backlight_control == 1 || - * caps->ext_caps->bits.hdr_aux_backlight_control == 1 - */) - caps->aux_support = true; - - if (amdgpu_backlight == 0) - caps->aux_support = false; - else if (amdgpu_backlight == 1) - caps->aux_support = true; - - luminance_range = &conn_base->display_info.luminance_range; - - if (luminance_range->max_luminance) { - caps->aux_min_input_signal = luminance_range->min_luminance; - caps->aux_max_input_signal = luminance_range->max_luminance; - } else { - caps->aux_min_input_signal = 0; - caps->aux_max_input_signal = 512; - } -} - -void amdgpu_dm_update_connector_after_detect( - struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - struct dc_sink *sink; - - /* MST handled by drm_mst framework */ - if (aconnector->mst_mgr.mst_state == true) - return; - - sink = aconnector->dc_link->local_sink; - if (sink) - dc_sink_retain(sink); - - /* - * Edid mgmt connector gets first update only in mode_valid hook and then - * the connector sink is set to either fake or physical sink depends on link status. - * Skip if already done during boot. - */ - if (aconnector->base.force != DRM_FORCE_UNSPECIFIED - && aconnector->dc_em_sink) { - - /* - * For S3 resume with headless use eml_sink to fake stream - * because on resume connector->sink is set to NULL - */ - mutex_lock(&dev->mode_config.mutex); - - if (sink) { - if (aconnector->dc_sink) { - amdgpu_dm_update_freesync_caps(connector, NULL); - /* - * retain and release below are used to - * bump up refcount for sink because the link doesn't point - * to it anymore after disconnect, so on next crtc to connector - * reshuffle by UMD we will get into unwanted dc_sink release - */ - dc_sink_release(aconnector->dc_sink); - } - aconnector->dc_sink = sink; - dc_sink_retain(aconnector->dc_sink); - amdgpu_dm_update_freesync_caps(connector, - aconnector->edid); - } else { - amdgpu_dm_update_freesync_caps(connector, NULL); - if (!aconnector->dc_sink) { - aconnector->dc_sink = aconnector->dc_em_sink; - dc_sink_retain(aconnector->dc_sink); - } - } - - mutex_unlock(&dev->mode_config.mutex); - - if (sink) - dc_sink_release(sink); - return; - } - - /* - * TODO: temporary guard to look for proper fix - * if this sink is MST sink, we should not do anything - */ - if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - dc_sink_release(sink); - return; - } - - if (aconnector->dc_sink == sink) { - /* - * We got a DP short pulse (Link Loss, DP CTS, etc...). - * Do nothing!! - */ - drm_dbg_kms(dev, "DCHPD: connector_id=%d: dc_sink didn't change.\n", - aconnector->connector_id); - if (sink) - dc_sink_release(sink); - return; - } - - drm_dbg_kms(dev, "DCHPD: connector_id=%d: Old sink=%p New sink=%p\n", - aconnector->connector_id, aconnector->dc_sink, sink); - - mutex_lock(&dev->mode_config.mutex); - - /* - * 1. Update status of the drm connector - * 2. Send an event and let userspace tell us what to do - */ - if (sink) { - /* - * TODO: check if we still need the S3 mode update workaround. - * If yes, put it here. - */ - if (aconnector->dc_sink) { - amdgpu_dm_update_freesync_caps(connector, NULL); - dc_sink_release(aconnector->dc_sink); - } - - aconnector->dc_sink = sink; - dc_sink_retain(aconnector->dc_sink); - if (sink->dc_edid.length == 0) { - aconnector->edid = NULL; - if (aconnector->dc_link->aux_mode) { - drm_dp_cec_unset_edid( - &aconnector->dm_dp_aux.aux); - } - } else { - aconnector->edid = - (struct edid *)sink->dc_edid.raw_edid; - - if (aconnector->dc_link->aux_mode) - drm_dp_cec_set_edid(&aconnector->dm_dp_aux.aux, - aconnector->edid); - } - - if (!aconnector->timing_requested) { - aconnector->timing_requested = - kzalloc(sizeof(struct dc_crtc_timing), GFP_KERNEL); - if (!aconnector->timing_requested) - drm_err(dev, - "failed to create aconnector->requested_timing\n"); - } - - drm_connector_update_edid_property(connector, aconnector->edid); - amdgpu_dm_update_freesync_caps(connector, aconnector->edid); - update_connector_ext_caps(aconnector); - } else { - drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); - amdgpu_dm_update_freesync_caps(connector, NULL); - drm_connector_update_edid_property(connector, NULL); - aconnector->num_modes = 0; - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - aconnector->edid = NULL; - kfree(aconnector->timing_requested); - aconnector->timing_requested = NULL; - /* Set CP to DESIRED if it was ENABLED, so we can re-enable it again on hotplug */ - if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) - connector->state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - } - - mutex_unlock(&dev->mode_config.mutex); - - update_subconnector_property(aconnector); - - if (sink) - dc_sink_release(sink); -} - -static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - enum dc_connection_type new_connection_type = dc_connection_none; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); - struct dc *dc = aconnector->dc_link->ctx->dc; - bool ret = false; - - if (adev->dm.disable_hpd_irq) - return; - - /* - * In case of failure or MST no need to update connector status or notify the OS - * since (for MST case) MST does this in its own context. - */ - mutex_lock(&aconnector->hpd_lock); - - if (adev->dm.hdcp_workqueue) { - hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); - dm_con_state->update_hdcp = true; - } - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - aconnector->timing_changed = false; - - if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(aconnector->dc_link); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED) - drm_kms_helper_connector_hotplug_event(connector); - } else { - mutex_lock(&adev->dm.dc_lock); - dc_exit_ips_for_hw_access(dc); - ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD); - mutex_unlock(&adev->dm.dc_lock); - if (ret) { - amdgpu_dm_update_connector_after_detect(aconnector); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED) - drm_kms_helper_connector_hotplug_event(connector); - } - } - mutex_unlock(&aconnector->hpd_lock); - -} - -static void handle_hpd_irq(void *param) -{ - struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; - - handle_hpd_irq_helper(aconnector); - -} - -static void schedule_hpd_rx_offload_work(struct hpd_rx_irq_offload_work_queue *offload_wq, - union hpd_irq_data hpd_irq_data) -{ - struct hpd_rx_irq_offload_work *offload_work = - kzalloc(sizeof(*offload_work), GFP_KERNEL); - - if (!offload_work) { - DRM_ERROR("Failed to allocate hpd_rx_irq_offload_work.\n"); - return; - } - - INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work); - offload_work->data = hpd_irq_data; - offload_work->offload_wq = offload_wq; - - queue_work(offload_wq->wq, &offload_work->work); - DRM_DEBUG_KMS("queue work to handle hpd_rx offload work"); -} - -static void handle_hpd_rx_irq(void *param) -{ - struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - struct dc_link *dc_link = aconnector->dc_link; - bool is_mst_root_connector = aconnector->mst_mgr.mst_state; - bool result = false; - enum dc_connection_type new_connection_type = dc_connection_none; - struct amdgpu_device *adev = drm_to_adev(dev); - union hpd_irq_data hpd_irq_data; - bool link_loss = false; - bool has_left_work = false; - int idx = dc_link->link_index; - struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx]; - struct dc *dc = aconnector->dc_link->ctx->dc; - - memset(&hpd_irq_data, 0, sizeof(hpd_irq_data)); - - if (adev->dm.disable_hpd_irq) - return; - - /* - * TODO:Temporary add mutex to protect hpd interrupt not have a gpio - * conflict, after implement i2c helper, this mutex should be - * retired. - */ - mutex_lock(&aconnector->hpd_lock); - - result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, - &link_loss, true, &has_left_work); - - if (!has_left_work) - goto out; - - if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) { - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - goto out; - } - - if (dc_link_dp_allow_hpd_rx_irq(dc_link)) { - if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || - hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { - bool skip = false; - - /* - * DOWN_REP_MSG_RDY is also handled by polling method - * mgr->cbs->poll_hpd_irq() - */ - spin_lock(&offload_wq->offload_lock); - skip = offload_wq->is_handling_mst_msg_rdy_event; - - if (!skip) - offload_wq->is_handling_mst_msg_rdy_event = true; - - spin_unlock(&offload_wq->offload_lock); - - if (!skip) - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - - goto out; - } - - if (link_loss) { - bool skip = false; - - spin_lock(&offload_wq->offload_lock); - skip = offload_wq->is_handling_link_loss; - - if (!skip) - offload_wq->is_handling_link_loss = true; - - spin_unlock(&offload_wq->offload_lock); - - if (!skip) - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - - goto out; - } - } - -out: - if (result && !is_mst_root_connector) { - /* Downstream Port status changed. */ - if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(dc_link); - - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - amdgpu_dm_update_connector_after_detect(aconnector); - - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - drm_kms_helper_connector_hotplug_event(connector); - } else { - bool ret = false; - - mutex_lock(&adev->dm.dc_lock); - dc_exit_ips_for_hw_access(dc); - ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX); - mutex_unlock(&adev->dm.dc_lock); - - if (ret) { - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - amdgpu_dm_update_connector_after_detect(aconnector); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - drm_kms_helper_connector_hotplug_event(connector); - } - } - } - if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) { - if (adev->dm.hdcp_workqueue) - hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index); - } - - if (dc_link->type != dc_connection_mst_branch) - drm_dp_cec_irq(&aconnector->dm_dp_aux.aux); - - mutex_unlock(&aconnector->hpd_lock); -} - -static int register_hpd_handlers(struct amdgpu_device *adev) -{ - struct drm_device *dev = adev_to_drm(adev); - struct drm_connector *connector; - struct amdgpu_dm_connector *aconnector; - const struct dc_link *dc_link; - struct dc_interrupt_params int_params = {0}; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD, - dmub_hpd_callback, true)) { - DRM_ERROR("amdgpu: fail to register dmub hpd callback"); - return -EINVAL; - } - - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ, - dmub_hpd_callback, true)) { - DRM_ERROR("amdgpu: fail to register dmub hpd callback"); - return -EINVAL; - } - } - - list_for_each_entry(connector, - &dev->mode_config.connector_list, head) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - dc_link = aconnector->dc_link; - - if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = dc_link->irq_source_hpd; - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_HPD1 || - int_params.irq_source > DC_IRQ_SOURCE_HPD6) { - DRM_ERROR("Failed to register hpd irq!\n"); - return -EINVAL; - } - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - handle_hpd_irq, (void *) aconnector)) - return -ENOMEM; - } - - if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { - - /* Also register for DP short pulse (hpd_rx). */ - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = dc_link->irq_source_hpd_rx; - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_HPD1RX || - int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) { - DRM_ERROR("Failed to register hpd rx irq!\n"); - return -EINVAL; - } - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - handle_hpd_rx_irq, (void *) aconnector)) - return -ENOMEM; - } - } - return 0; -} - -#if defined(CONFIG_DRM_AMD_DC_SI) -/* Register IRQ sources and initialize IRQ callbacks */ -static int dce60_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; - unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VBLANK interrupt */ - for (i = 0; i < adev->mode_info.num_crtc; i++) { - r = amdgpu_irq_add_id(adev, client_id, i + 1, &adev->crtc_irq); - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i + 1, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; - i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, client_id, - VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} -#endif - -/* Register IRQ sources and initialize IRQ callbacks */ -static int dce110_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; - unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; - - if (adev->family >= AMDGPU_FAMILY_AI) - client_id = SOC15_IH_CLIENTID_DCE; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VBLANK interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0; i <= VISLANDS30_IV_SRCID_D6_VERTICAL_INTERRUPT0; i++) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->crtc_irq); - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use VUPDATE interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT; i <= VISLANDS30_IV_SRCID_D6_V_UPDATE_INT; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->vupdate_irq); - if (r) { - DRM_ERROR("Failed to add vupdate irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || - int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { - DRM_ERROR("Failed to register vupdate irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_vupdate_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; - i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, client_id, - VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} - -/* Register IRQ sources and initialize IRQ callbacks */ -static int dcn10_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - static const unsigned int vrtl_int_srcid[] = { - DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL - }; -#endif - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VSTARTUP interrupt */ - for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP; - i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq); - - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use otg vertical line interrupt */ -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, - vrtl_int_srcid[i], &adev->vline0_irq); - - if (r) { - DRM_ERROR("Failed to add vline0 irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 || - int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) { - DRM_ERROR("Failed to register vline0 irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vline0_params[int_params.irq_source - - DC_IRQ_SOURCE_DC1_VLINE0]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_dcn_vertical_interrupt0_high_irq, - c_irq_params)) - return -ENOMEM; - } -#endif - - /* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to - * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx - * to trigger at end of each vblank, regardless of state of the lock, - * matching DCE behaviour. - */ - for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT; - i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq); - - if (r) { - DRM_ERROR("Failed to add vupdate irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || - int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { - DRM_ERROR("Failed to register vupdate irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_vupdate_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT; - i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, - &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} -/* Register Outbox IRQ sources and initialize IRQ callbacks */ -static int register_outbox_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r, i; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT, - &adev->dmub_outbox_irq); - if (r) { - DRM_ERROR("Failed to add outbox irq id!\n"); - return r; - } - - if (dc->ctx->dmub_srv) { - i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT; - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - c_irq_params = &adev->dm.dmub_outbox_params[0]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_dmub_outbox1_low_irq, c_irq_params)) - return -ENOMEM; - } - - return 0; -} - -/* - * Acquires the lock for the atomic state object and returns - * the new atomic state. - * - * This should only be called during atomic check. - */ -int dm_atomic_get_state(struct drm_atomic_state *state, - struct dm_atomic_state **dm_state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_private_state *priv_state; - - if (*dm_state) - return 0; - - priv_state = drm_atomic_get_private_obj_state(state, &dm->atomic_obj); - if (IS_ERR(priv_state)) - return PTR_ERR(priv_state); - - *dm_state = to_dm_atomic_state(priv_state); - - return 0; -} - -static struct dm_atomic_state * -dm_atomic_get_new_state(struct drm_atomic_state *state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_private_obj *obj; - struct drm_private_state *new_obj_state; - int i; - - for_each_new_private_obj_in_state(state, obj, new_obj_state, i) { - if (obj->funcs == dm->atomic_obj.funcs) - return to_dm_atomic_state(new_obj_state); - } - - return NULL; -} - -static struct drm_private_state * -dm_atomic_duplicate_state(struct drm_private_obj *obj) -{ - struct dm_atomic_state *old_state, *new_state; - - new_state = kzalloc(sizeof(*new_state), GFP_KERNEL); - if (!new_state) - return NULL; - - __drm_atomic_helper_private_obj_duplicate_state(obj, &new_state->base); - - old_state = to_dm_atomic_state(obj->state); - - if (old_state && old_state->context) - new_state->context = dc_state_create_copy(old_state->context); - - if (!new_state->context) { - kfree(new_state); - return NULL; - } - - return &new_state->base; -} - -static void dm_atomic_destroy_state(struct drm_private_obj *obj, - struct drm_private_state *state) -{ - struct dm_atomic_state *dm_state = to_dm_atomic_state(state); - - if (dm_state && dm_state->context) - dc_state_release(dm_state->context); - - kfree(dm_state); -} - -static struct drm_private_state_funcs dm_atomic_state_funcs = { - .atomic_duplicate_state = dm_atomic_duplicate_state, - .atomic_destroy_state = dm_atomic_destroy_state, -}; - -static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) -{ - struct dm_atomic_state *state; - int r; - - adev->mode_info.mode_config_initialized = true; - - adev_to_drm(adev)->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs; - adev_to_drm(adev)->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs; - - adev_to_drm(adev)->mode_config.max_width = 16384; - adev_to_drm(adev)->mode_config.max_height = 16384; - - adev_to_drm(adev)->mode_config.preferred_depth = 24; - if (adev->asic_type == CHIP_HAWAII) - /* disable prefer shadow for now due to hibernation issues */ - adev_to_drm(adev)->mode_config.prefer_shadow = 0; - else - adev_to_drm(adev)->mode_config.prefer_shadow = 1; - /* indicates support for immediate flip */ - adev_to_drm(adev)->mode_config.async_page_flip = true; - - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - - state->context = dc_state_create_current_copy(adev->dm.dc); - if (!state->context) { - kfree(state); - return -ENOMEM; - } - - drm_atomic_private_obj_init(adev_to_drm(adev), - &adev->dm.atomic_obj, - &state->base, - &dm_atomic_state_funcs); - - r = amdgpu_display_modeset_create_props(adev); - if (r) { - dc_state_release(state->context); - kfree(state); - return r; - } - -#ifdef AMD_PRIVATE_COLOR - if (amdgpu_dm_create_color_properties(adev)) { - dc_state_release(state->context); - kfree(state); - return -ENOMEM; - } -#endif - - r = amdgpu_dm_audio_init(adev); - if (r) { - dc_state_release(state->context); - kfree(state); - return r; - } - - return 0; -} - -#define AMDGPU_DM_DEFAULT_MIN_BACKLIGHT 12 -#define AMDGPU_DM_DEFAULT_MAX_BACKLIGHT 255 -#define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50 - -static void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm, - int bl_idx) -{ -#if defined(CONFIG_ACPI) - struct amdgpu_dm_backlight_caps caps; - - memset(&caps, 0, sizeof(caps)); - - if (dm->backlight_caps[bl_idx].caps_valid) - return; - - amdgpu_acpi_get_backlight_caps(&caps); - if (caps.caps_valid) { - dm->backlight_caps[bl_idx].caps_valid = true; - if (caps.aux_support) - return; - dm->backlight_caps[bl_idx].min_input_signal = caps.min_input_signal; - dm->backlight_caps[bl_idx].max_input_signal = caps.max_input_signal; - } else { - dm->backlight_caps[bl_idx].min_input_signal = - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT; - dm->backlight_caps[bl_idx].max_input_signal = - AMDGPU_DM_DEFAULT_MAX_BACKLIGHT; - } -#else - if (dm->backlight_caps[bl_idx].aux_support) - return; - - dm->backlight_caps[bl_idx].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT; - dm->backlight_caps[bl_idx].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT; -#endif -} - -static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, - unsigned int *min, unsigned int *max) -{ - if (!caps) - return 0; - - if (caps->aux_support) { - // Firmware limits are in nits, DC API wants millinits. - *max = 1000 * caps->aux_max_input_signal; - *min = 1000 * caps->aux_min_input_signal; - } else { - // Firmware limits are 8-bit, PWM control is 16-bit. - *max = 0x101 * caps->max_input_signal; - *min = 0x101 * caps->min_input_signal; - } - return 1; -} - -static u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps, - uint32_t brightness) -{ - unsigned int min, max; - - if (!get_brightness_range(caps, &min, &max)) - return brightness; - - // Rescale 0..255 to min..max - return min + DIV_ROUND_CLOSEST((max - min) * brightness, - AMDGPU_MAX_BL_LEVEL); -} - -static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps, - uint32_t brightness) -{ - unsigned int min, max; - - if (!get_brightness_range(caps, &min, &max)) - return brightness; - - if (brightness < min) - return 0; - // Rescale min..max to 0..255 - return DIV_ROUND_CLOSEST(AMDGPU_MAX_BL_LEVEL * (brightness - min), - max - min); -} - -static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, - int bl_idx, - u32 user_brightness) -{ - struct amdgpu_dm_backlight_caps caps; - struct dc_link *link; - u32 brightness; - bool rc; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - caps = dm->backlight_caps[bl_idx]; - - dm->brightness[bl_idx] = user_brightness; - /* update scratch register */ - if (bl_idx == 0) - amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]); - brightness = convert_brightness_from_user(&caps, dm->brightness[bl_idx]); - link = (struct dc_link *)dm->backlight_link[bl_idx]; - - /* Change brightness based on AUX property */ - if (caps.aux_support) { - rc = dc_link_set_backlight_level_nits(link, true, brightness, - AUX_BL_DEFAULT_TRANSITION_TIME_MS); - if (!rc) - DRM_DEBUG("DM: Failed to update backlight via AUX on eDP[%d]\n", bl_idx); - } else { - rc = dc_link_set_backlight_level(link, brightness, 0); - if (!rc) - DRM_DEBUG("DM: Failed to update backlight on eDP[%d]\n", bl_idx); - } - - if (rc) - dm->actual_brightness[bl_idx] = user_brightness; -} - -static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) -{ - struct amdgpu_display_manager *dm = bl_get_data(bd); - int i; - - for (i = 0; i < dm->num_of_edps; i++) { - if (bd == dm->backlight_dev[i]) - break; - } - if (i >= AMDGPU_DM_MAX_NUM_EDP) - i = 0; - amdgpu_dm_backlight_set_level(dm, i, bd->props.brightness); - - return 0; -} - -static u32 amdgpu_dm_backlight_get_level(struct amdgpu_display_manager *dm, - int bl_idx) -{ - int ret; - struct amdgpu_dm_backlight_caps caps; - struct dc_link *link = (struct dc_link *)dm->backlight_link[bl_idx]; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - caps = dm->backlight_caps[bl_idx]; - - if (caps.aux_support) { - u32 avg, peak; - bool rc; - - rc = dc_link_get_backlight_level_nits(link, &avg, &peak); - if (!rc) - return dm->brightness[bl_idx]; - return convert_brightness_to_user(&caps, avg); - } - - ret = dc_link_get_backlight_level(link); - - if (ret == DC_ERROR_UNEXPECTED) - return dm->brightness[bl_idx]; - - return convert_brightness_to_user(&caps, ret); -} - -static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) -{ - struct amdgpu_display_manager *dm = bl_get_data(bd); - int i; - - for (i = 0; i < dm->num_of_edps; i++) { - if (bd == dm->backlight_dev[i]) - break; - } - if (i >= AMDGPU_DM_MAX_NUM_EDP) - i = 0; - return amdgpu_dm_backlight_get_level(dm, i); -} - -static const struct backlight_ops amdgpu_dm_backlight_ops = { - .options = BL_CORE_SUSPENDRESUME, - .get_brightness = amdgpu_dm_backlight_get_brightness, - .update_status = amdgpu_dm_backlight_update_status, -}; - -static void -amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) -{ - struct drm_device *drm = aconnector->base.dev; - struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm; - struct backlight_properties props = { 0 }; - struct amdgpu_dm_backlight_caps caps = { 0 }; - char bl_name[16]; - - if (aconnector->bl_idx == -1) - return; - - if (!acpi_video_backlight_use_native()) { - drm_info(drm, "Skipping amdgpu DM backlight registration\n"); - /* Try registering an ACPI video backlight device instead. */ - acpi_video_register_backlight(); - return; - } - - amdgpu_acpi_get_backlight_caps(&caps); - if (caps.caps_valid) { - if (power_supply_is_system_supplied() > 0) - props.brightness = caps.ac_level; - else - props.brightness = caps.dc_level; - } else - props.brightness = AMDGPU_MAX_BL_LEVEL; - - props.max_brightness = AMDGPU_MAX_BL_LEVEL; - props.type = BACKLIGHT_RAW; - - snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d", - drm->primary->index + aconnector->bl_idx); - - dm->backlight_dev[aconnector->bl_idx] = - backlight_device_register(bl_name, aconnector->base.kdev, dm, - &amdgpu_dm_backlight_ops, &props); - - if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) { - DRM_ERROR("DM: Backlight registration failed!\n"); - dm->backlight_dev[aconnector->bl_idx] = NULL; - } else - DRM_DEBUG_DRIVER("DM: Registered Backlight device: %s\n", bl_name); -} - -static int initialize_plane(struct amdgpu_display_manager *dm, - struct amdgpu_mode_info *mode_info, int plane_id, - enum drm_plane_type plane_type, - const struct dc_plane_cap *plane_cap) -{ - struct drm_plane *plane; - unsigned long possible_crtcs; - int ret = 0; - - plane = kzalloc(sizeof(struct drm_plane), GFP_KERNEL); - if (!plane) { - DRM_ERROR("KMS: Failed to allocate plane\n"); - return -ENOMEM; - } - plane->type = plane_type; - - /* - * HACK: IGT tests expect that the primary plane for a CRTC - * can only have one possible CRTC. Only expose support for - * any CRTC if they're not going to be used as a primary plane - * for a CRTC - like overlay or underlay planes. - */ - possible_crtcs = 1 << plane_id; - if (plane_id >= dm->dc->caps.max_streams) - possible_crtcs = 0xff; - - ret = amdgpu_dm_plane_init(dm, plane, possible_crtcs, plane_cap); - - if (ret) { - DRM_ERROR("KMS: Failed to initialize plane\n"); - kfree(plane); - return ret; - } - - if (mode_info) - mode_info->planes[plane_id] = plane; - - return ret; -} - - -static void setup_backlight_device(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = aconnector->dc_link; - int bl_idx = dm->num_of_edps; - - if (!(link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) || - link->type == dc_connection_none) - return; - - if (dm->num_of_edps >= AMDGPU_DM_MAX_NUM_EDP) { - drm_warn(adev_to_drm(dm->adev), "Too much eDP connections, skipping backlight setup for additional eDPs\n"); - return; - } - - aconnector->bl_idx = bl_idx; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - dm->brightness[bl_idx] = AMDGPU_MAX_BL_LEVEL; - dm->backlight_link[bl_idx] = link; - dm->num_of_edps++; - - update_connector_ext_caps(aconnector); -} - -static void amdgpu_set_panel_orientation(struct drm_connector *connector); - -/* - * In this architecture, the association - * connector -> encoder -> crtc - * id not really requried. The crtc and connector will hold the - * display_index as an abstraction to use with DAL component - * - * Returns 0 on success - */ -static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) -{ - struct amdgpu_display_manager *dm = &adev->dm; - s32 i; - struct amdgpu_dm_connector *aconnector = NULL; - struct amdgpu_encoder *aencoder = NULL; - struct amdgpu_mode_info *mode_info = &adev->mode_info; - u32 link_cnt; - s32 primary_planes; - enum dc_connection_type new_connection_type = dc_connection_none; - const struct dc_plane_cap *plane; - bool psr_feature_enabled = false; - bool replay_feature_enabled = false; - int max_overlay = dm->dc->caps.max_slave_planes; - - dm->display_indexes_num = dm->dc->caps.max_streams; - /* Update the actual used number of crtc */ - adev->mode_info.num_crtc = adev->dm.display_indexes_num; - - amdgpu_dm_set_irq_funcs(adev); - - link_cnt = dm->dc->caps.max_links; - if (amdgpu_dm_mode_config_init(dm->adev)) { - DRM_ERROR("DM: Failed to initialize mode config\n"); - return -EINVAL; - } - - /* There is one primary plane per CRTC */ - primary_planes = dm->dc->caps.max_streams; - if (primary_planes > AMDGPU_MAX_PLANES) { - DRM_ERROR("DM: Plane nums out of 6 planes\n"); - return -EINVAL; - } - - /* - * Initialize primary planes, implicit planes for legacy IOCTLS. - * Order is reversed to match iteration order in atomic check. - */ - for (i = (primary_planes - 1); i >= 0; i--) { - plane = &dm->dc->caps.planes[i]; - - if (initialize_plane(dm, mode_info, i, - DRM_PLANE_TYPE_PRIMARY, plane)) { - DRM_ERROR("KMS: Failed to initialize primary plane\n"); - goto fail; - } - } - - /* - * Initialize overlay planes, index starting after primary planes. - * These planes have a higher DRM index than the primary planes since - * they should be considered as having a higher z-order. - * Order is reversed to match iteration order in atomic check. - * - * Only support DCN for now, and only expose one so we don't encourage - * userspace to use up all the pipes. - */ - for (i = 0; i < dm->dc->caps.max_planes; ++i) { - struct dc_plane_cap *plane = &dm->dc->caps.planes[i]; - - /* Do not create overlay if MPO disabled */ - if (amdgpu_dc_debug_mask & DC_DISABLE_MPO) - break; - - if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL) - continue; - - if (!plane->pixel_format_support.argb8888) - continue; - - if (max_overlay-- == 0) - break; - - if (initialize_plane(dm, NULL, primary_planes + i, - DRM_PLANE_TYPE_OVERLAY, plane)) { - DRM_ERROR("KMS: Failed to initialize overlay plane\n"); - goto fail; - } - } - - for (i = 0; i < dm->dc->caps.max_streams; i++) - if (amdgpu_dm_crtc_init(dm, mode_info->planes[i], i)) { - DRM_ERROR("KMS: Failed to initialize crtc\n"); - goto fail; - } - - /* Use Outbox interrupt */ - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - if (register_outbox_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - DRM_DEBUG_KMS("Unsupported DCN IP version for outbox: 0x%X\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - } - - /* Determine whether to enable PSR support by default. */ - if (!(amdgpu_dc_debug_mask & DC_DISABLE_PSR)) { - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - psr_feature_enabled = true; - break; - default: - psr_feature_enabled = amdgpu_dc_feature_mask & DC_PSR_MASK; - break; - } - } - - /* Determine whether to enable Replay support by default. */ - if (!(amdgpu_dc_debug_mask & DC_DISABLE_REPLAY)) { - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - replay_feature_enabled = true; - break; - - default: - replay_feature_enabled = amdgpu_dc_feature_mask & DC_REPLAY_MASK; - break; - } - } - - if (link_cnt > MAX_LINKS) { - DRM_ERROR( - "KMS: Cannot support more than %d display indexes\n", - MAX_LINKS); - goto fail; - } - - /* loops over all connectors on the board */ - for (i = 0; i < link_cnt; i++) { - struct dc_link *link = NULL; - - link = dc_get_link_at_index(dm->dc, i); - - if (link->connector_signal == SIGNAL_TYPE_VIRTUAL) { - struct amdgpu_dm_wb_connector *wbcon = kzalloc(sizeof(*wbcon), GFP_KERNEL); - - if (!wbcon) { - DRM_ERROR("KMS: Failed to allocate writeback connector\n"); - continue; - } - - if (amdgpu_dm_wb_connector_init(dm, wbcon, i)) { - DRM_ERROR("KMS: Failed to initialize writeback connector\n"); - kfree(wbcon); - continue; - } - - link->psr_settings.psr_feature_enabled = false; - link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; - - continue; - } - - aconnector = kzalloc(sizeof(*aconnector), GFP_KERNEL); - if (!aconnector) - goto fail; - - aencoder = kzalloc(sizeof(*aencoder), GFP_KERNEL); - if (!aencoder) - goto fail; - - if (amdgpu_dm_encoder_init(dm->ddev, aencoder, i)) { - DRM_ERROR("KMS: Failed to initialize encoder\n"); - goto fail; - } - - if (amdgpu_dm_connector_init(dm, aconnector, i, aencoder)) { - DRM_ERROR("KMS: Failed to initialize connector\n"); - goto fail; - } - - if (dm->hpd_rx_offload_wq) - dm->hpd_rx_offload_wq[aconnector->base.index].aconnector = - aconnector; - - if (!dc_link_detect_connection_type(link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(link); - amdgpu_dm_update_connector_after_detect(aconnector); - } else { - bool ret = false; - - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - ret = dc_link_detect(link, DETECT_REASON_BOOT); - mutex_unlock(&dm->dc_lock); - - if (ret) { - amdgpu_dm_update_connector_after_detect(aconnector); - setup_backlight_device(dm, aconnector); - - /* Disable PSR if Replay can be enabled */ - if (replay_feature_enabled) - if (amdgpu_dm_set_replay_caps(link, aconnector)) - psr_feature_enabled = false; - - if (psr_feature_enabled) - amdgpu_dm_set_psr_caps(link); - } - } - amdgpu_set_panel_orientation(&aconnector->base); - } - - /* Software is initialized. Now we can register interrupt handlers. */ - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - case CHIP_OLAND: - if (dce60_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - case CHIP_TONGA: - case CHIP_FIJI: - case CHIP_CARRIZO: - case CHIP_STONEY: - case CHIP_POLARIS11: - case CHIP_POLARIS10: - case CHIP_POLARIS12: - case CHIP_VEGAM: - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - if (dce110_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(1, 0, 0): - case IP_VERSION(1, 0, 1): - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 3): - case IP_VERSION(2, 0, 0): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 0, 2): - case IP_VERSION(3, 0, 3): - case IP_VERSION(3, 0, 1): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - if (dcn10_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - DRM_ERROR("Unsupported DCE IP versions: 0x%X\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - goto fail; - } - break; - } - - return 0; -fail: - kfree(aencoder); - kfree(aconnector); - - return -EINVAL; -} - -static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm) -{ - drm_atomic_private_obj_fini(&dm->atomic_obj); -} - -/****************************************************************************** - * amdgpu_display_funcs functions - *****************************************************************************/ - -/* - * dm_bandwidth_update - program display watermarks - * - * @adev: amdgpu_device pointer - * - * Calculate and program the display watermarks and line buffer allocation. - */ -static void dm_bandwidth_update(struct amdgpu_device *adev) -{ - /* TODO: implement later */ -} - -static const struct amdgpu_display_funcs dm_display_funcs = { - .bandwidth_update = dm_bandwidth_update, /* called unconditionally */ - .vblank_get_counter = dm_vblank_get_counter,/* called unconditionally */ - .backlight_set_level = NULL, /* never called for DC */ - .backlight_get_level = NULL, /* never called for DC */ - .hpd_sense = NULL,/* called unconditionally */ - .hpd_set_polarity = NULL, /* called unconditionally */ - .hpd_get_gpio_reg = NULL, /* VBIOS parsing. DAL does it. */ - .page_flip_get_scanoutpos = - dm_crtc_get_scanoutpos,/* called unconditionally */ - .add_encoder = NULL, /* VBIOS parsing. DAL does it. */ - .add_connector = NULL, /* VBIOS parsing. DAL does it. */ -}; - -#if defined(CONFIG_DEBUG_KERNEL_DC) - -static ssize_t s3_debug_store(struct device *device, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - int ret; - int s3_state; - struct drm_device *drm_dev = dev_get_drvdata(device); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - - ret = kstrtoint(buf, 0, &s3_state); - - if (ret == 0) { - if (s3_state) { - dm_resume(adev); - drm_kms_helper_hotplug_event(adev_to_drm(adev)); - } else - dm_suspend(adev); - } - - return ret == 0 ? count : 0; -} - -DEVICE_ATTR_WO(s3_debug); - -#endif - -static int dm_init_microcode(struct amdgpu_device *adev) -{ - char *fw_name_dmub; - int r; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - fw_name_dmub = FIRMWARE_RENOIR_DMUB; - if (ASICREV_IS_GREEN_SARDINE(adev->external_rev_id)) - fw_name_dmub = FIRMWARE_GREEN_SARDINE_DMUB; - break; - case IP_VERSION(3, 0, 0): - if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 0)) - fw_name_dmub = FIRMWARE_SIENNA_CICHLID_DMUB; - else - fw_name_dmub = FIRMWARE_NAVY_FLOUNDER_DMUB; - break; - case IP_VERSION(3, 0, 1): - fw_name_dmub = FIRMWARE_VANGOGH_DMUB; - break; - case IP_VERSION(3, 0, 2): - fw_name_dmub = FIRMWARE_DIMGREY_CAVEFISH_DMUB; - break; - case IP_VERSION(3, 0, 3): - fw_name_dmub = FIRMWARE_BEIGE_GOBY_DMUB; - break; - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - fw_name_dmub = FIRMWARE_YELLOW_CARP_DMUB; - break; - case IP_VERSION(3, 1, 4): - fw_name_dmub = FIRMWARE_DCN_314_DMUB; - break; - case IP_VERSION(3, 1, 5): - fw_name_dmub = FIRMWARE_DCN_315_DMUB; - break; - case IP_VERSION(3, 1, 6): - fw_name_dmub = FIRMWARE_DCN316_DMUB; - break; - case IP_VERSION(3, 2, 0): - fw_name_dmub = FIRMWARE_DCN_V3_2_0_DMCUB; - break; - case IP_VERSION(3, 2, 1): - fw_name_dmub = FIRMWARE_DCN_V3_2_1_DMCUB; - break; - case IP_VERSION(3, 5, 0): - fw_name_dmub = FIRMWARE_DCN_35_DMUB; - break; - case IP_VERSION(3, 5, 1): - fw_name_dmub = FIRMWARE_DCN_351_DMUB; - break; - case IP_VERSION(4, 0, 1): - fw_name_dmub = FIRMWARE_DCN_401_DMUB; - break; - default: - /* ASIC doesn't support DMUB. */ - return 0; - } - r = amdgpu_ucode_request(adev, &adev->dm.dmub_fw, "%s", fw_name_dmub); - return r; -} - -static int dm_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_mode_info *mode_info = &adev->mode_info; - struct atom_context *ctx = mode_info->atom_context; - int index = GetIndexIntoMasterTable(DATA, Object_Header); - u16 data_offset; - - /* if there is no object header, skip DM */ - if (!amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) { - adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK; - dev_info(adev->dev, "No object header, skipping DM\n"); - return -ENOENT; - } - - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_OLAND: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 2; - adev->mode_info.num_dig = 2; - break; -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_KAVERI: - adev->mode_info.num_crtc = 4; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 7; - break; - case CHIP_KABINI: - case CHIP_MULLINS: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_FIJI: - case CHIP_TONGA: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 7; - break; - case CHIP_CARRIZO: - adev->mode_info.num_crtc = 3; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 9; - break; - case CHIP_STONEY: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 9; - break; - case CHIP_POLARIS11: - case CHIP_POLARIS12: - adev->mode_info.num_crtc = 5; - adev->mode_info.num_hpd = 5; - adev->mode_info.num_dig = 5; - break; - case CHIP_POLARIS10: - case CHIP_VEGAM: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - default: - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(3, 0, 0): - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case IP_VERSION(2, 0, 0): - case IP_VERSION(3, 0, 2): - adev->mode_info.num_crtc = 5; - adev->mode_info.num_hpd = 5; - adev->mode_info.num_dig = 5; - break; - case IP_VERSION(2, 0, 3): - case IP_VERSION(3, 0, 3): - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 2; - adev->mode_info.num_dig = 2; - break; - case IP_VERSION(1, 0, 0): - case IP_VERSION(1, 0, 1): - case IP_VERSION(3, 0, 1): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - adev->mode_info.num_crtc = 4; - adev->mode_info.num_hpd = 4; - adev->mode_info.num_dig = 4; - break; - default: - DRM_ERROR("Unsupported DCE IP versions: 0x%x\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - return -EINVAL; - } - break; - } - - if (adev->mode_info.funcs == NULL) - adev->mode_info.funcs = &dm_display_funcs; - - /* - * Note: Do NOT change adev->audio_endpt_rreg and - * adev->audio_endpt_wreg because they are initialised in - * amdgpu_device_init() - */ -#if defined(CONFIG_DEBUG_KERNEL_DC) - device_create_file( - adev_to_drm(adev)->dev, - &dev_attr_s3_debug); -#endif - adev->dc_enabled = true; - - return dm_init_microcode(adev); -} - -static bool modereset_required(struct drm_crtc_state *crtc_state) -{ - return !crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state); -} - -static void amdgpu_dm_encoder_destroy(struct drm_encoder *encoder) -{ - drm_encoder_cleanup(encoder); - kfree(encoder); -} - -static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = { - .destroy = amdgpu_dm_encoder_destroy, -}; - -static int -fill_plane_color_attributes(const struct drm_plane_state *plane_state, - const enum surface_pixel_format format, - enum dc_color_space *color_space) -{ - bool full_range; - - *color_space = COLOR_SPACE_SRGB; - - /* DRM color properties only affect non-RGB formats. */ - if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) - return 0; - - full_range = (plane_state->color_range == DRM_COLOR_YCBCR_FULL_RANGE); - - switch (plane_state->color_encoding) { - case DRM_COLOR_YCBCR_BT601: - if (full_range) - *color_space = COLOR_SPACE_YCBCR601; - else - *color_space = COLOR_SPACE_YCBCR601_LIMITED; - break; - - case DRM_COLOR_YCBCR_BT709: - if (full_range) - *color_space = COLOR_SPACE_YCBCR709; - else - *color_space = COLOR_SPACE_YCBCR709_LIMITED; - break; - - case DRM_COLOR_YCBCR_BT2020: - if (full_range) - *color_space = COLOR_SPACE_2020_YCBCR; - else - return -EINVAL; - break; - - default: - return -EINVAL; - } - - return 0; -} - -static int -fill_dc_plane_info_and_addr(struct amdgpu_device *adev, - const struct drm_plane_state *plane_state, - const u64 tiling_flags, - struct dc_plane_info *plane_info, - struct dc_plane_address *address, - bool tmz_surface, - bool force_disable_dcc) -{ - const struct drm_framebuffer *fb = plane_state->fb; - const struct amdgpu_framebuffer *afb = - to_amdgpu_framebuffer(plane_state->fb); - int ret; - - memset(plane_info, 0, sizeof(*plane_info)); - - switch (fb->format->format) { - case DRM_FORMAT_C8: - plane_info->format = - SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS; - break; - case DRM_FORMAT_RGB565: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565; - break; - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888; - break; - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_ARGB2101010: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010; - break; - case DRM_FORMAT_XBGR2101010: - case DRM_FORMAT_ABGR2101010: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010; - break; - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888; - break; - case DRM_FORMAT_NV21: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr; - break; - case DRM_FORMAT_NV12: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb; - break; - case DRM_FORMAT_P010: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb; - break; - case DRM_FORMAT_XRGB16161616F: - case DRM_FORMAT_ARGB16161616F: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F; - break; - case DRM_FORMAT_XBGR16161616F: - case DRM_FORMAT_ABGR16161616F: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F; - break; - case DRM_FORMAT_XRGB16161616: - case DRM_FORMAT_ARGB16161616: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616; - break; - case DRM_FORMAT_XBGR16161616: - case DRM_FORMAT_ABGR16161616: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616; - break; - default: - DRM_ERROR( - "Unsupported screen format %p4cc\n", - &fb->format->format); - return -EINVAL; - } - - switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) { - case DRM_MODE_ROTATE_0: - plane_info->rotation = ROTATION_ANGLE_0; - break; - case DRM_MODE_ROTATE_90: - plane_info->rotation = ROTATION_ANGLE_90; - break; - case DRM_MODE_ROTATE_180: - plane_info->rotation = ROTATION_ANGLE_180; - break; - case DRM_MODE_ROTATE_270: - plane_info->rotation = ROTATION_ANGLE_270; - break; - default: - plane_info->rotation = ROTATION_ANGLE_0; - break; - } - - - plane_info->visible = true; - plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE; - - plane_info->layer_index = plane_state->normalized_zpos; - - ret = fill_plane_color_attributes(plane_state, plane_info->format, - &plane_info->color_space); - if (ret) - return ret; - - ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, plane_info->format, - plane_info->rotation, tiling_flags, - &plane_info->tiling_info, - &plane_info->plane_size, - &plane_info->dcc, address, - tmz_surface, force_disable_dcc); - if (ret) - return ret; - - amdgpu_dm_plane_fill_blending_from_plane_state( - plane_state, &plane_info->per_pixel_alpha, &plane_info->pre_multiplied_alpha, - &plane_info->global_alpha, &plane_info->global_alpha_value); - - return 0; -} - -static int fill_dc_plane_attributes(struct amdgpu_device *adev, - struct dc_plane_state *dc_plane_state, - struct drm_plane_state *plane_state, - struct drm_crtc_state *crtc_state) -{ - struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); - struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)plane_state->fb; - struct dc_scaling_info scaling_info; - struct dc_plane_info plane_info; - int ret; - bool force_disable_dcc = false; - - ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, plane_state, &scaling_info); - if (ret) - return ret; - - dc_plane_state->src_rect = scaling_info.src_rect; - dc_plane_state->dst_rect = scaling_info.dst_rect; - dc_plane_state->clip_rect = scaling_info.clip_rect; - dc_plane_state->scaling_quality = scaling_info.scaling_quality; - - force_disable_dcc = adev->asic_type == CHIP_RAVEN && adev->in_suspend; - ret = fill_dc_plane_info_and_addr(adev, plane_state, - afb->tiling_flags, - &plane_info, - &dc_plane_state->address, - afb->tmz_surface, - force_disable_dcc); - if (ret) - return ret; - - dc_plane_state->format = plane_info.format; - dc_plane_state->color_space = plane_info.color_space; - dc_plane_state->format = plane_info.format; - dc_plane_state->plane_size = plane_info.plane_size; - dc_plane_state->rotation = plane_info.rotation; - dc_plane_state->horizontal_mirror = plane_info.horizontal_mirror; - dc_plane_state->stereo_format = plane_info.stereo_format; - dc_plane_state->tiling_info = plane_info.tiling_info; - dc_plane_state->visible = plane_info.visible; - dc_plane_state->per_pixel_alpha = plane_info.per_pixel_alpha; - dc_plane_state->pre_multiplied_alpha = plane_info.pre_multiplied_alpha; - dc_plane_state->global_alpha = plane_info.global_alpha; - dc_plane_state->global_alpha_value = plane_info.global_alpha_value; - dc_plane_state->dcc = plane_info.dcc; - dc_plane_state->layer_index = plane_info.layer_index; - dc_plane_state->flip_int_enabled = true; - - /* - * Always set input transfer function, since plane state is refreshed - * every time. - */ - ret = amdgpu_dm_update_plane_color_mgmt(dm_crtc_state, - plane_state, - dc_plane_state); - if (ret) - return ret; - - return 0; -} - -static inline void fill_dc_dirty_rect(struct drm_plane *plane, - struct rect *dirty_rect, int32_t x, - s32 y, s32 width, s32 height, - int *i, bool ffu) -{ - WARN_ON(*i >= DC_MAX_DIRTY_RECTS); - - dirty_rect->x = x; - dirty_rect->y = y; - dirty_rect->width = width; - dirty_rect->height = height; - - if (ffu) - drm_dbg(plane->dev, - "[PLANE:%d] PSR FFU dirty rect size (%d, %d)\n", - plane->base.id, width, height); - else - drm_dbg(plane->dev, - "[PLANE:%d] PSR SU dirty rect at (%d, %d) size (%d, %d)", - plane->base.id, x, y, width, height); - - (*i)++; -} - -/** - * fill_dc_dirty_rects() - Fill DC dirty regions for PSR selective updates - * - * @plane: DRM plane containing dirty regions that need to be flushed to the eDP - * remote fb - * @old_plane_state: Old state of @plane - * @new_plane_state: New state of @plane - * @crtc_state: New state of CRTC connected to the @plane - * @flip_addrs: DC flip tracking struct, which also tracts dirty rects - * @is_psr_su: Flag indicating whether Panel Self Refresh Selective Update (PSR SU) is enabled. - * If PSR SU is enabled and damage clips are available, only the regions of the screen - * that have changed will be updated. If PSR SU is not enabled, - * or if damage clips are not available, the entire screen will be updated. - * @dirty_regions_changed: dirty regions changed - * - * For PSR SU, DC informs the DMUB uController of dirty rectangle regions - * (referred to as "damage clips" in DRM nomenclature) that require updating on - * the eDP remote buffer. The responsibility of specifying the dirty regions is - * amdgpu_dm's. - * - * A damage-aware DRM client should fill the FB_DAMAGE_CLIPS property on the - * plane with regions that require flushing to the eDP remote buffer. In - * addition, certain use cases - such as cursor and multi-plane overlay (MPO) - - * implicitly provide damage clips without any client support via the plane - * bounds. - */ -static void fill_dc_dirty_rects(struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state, - struct drm_crtc_state *crtc_state, - struct dc_flip_addrs *flip_addrs, - bool is_psr_su, - bool *dirty_regions_changed) -{ - struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); - struct rect *dirty_rects = flip_addrs->dirty_rects; - u32 num_clips; - struct drm_mode_rect *clips; - bool bb_changed; - bool fb_changed; - u32 i = 0; - *dirty_regions_changed = false; - - /* - * Cursor plane has it's own dirty rect update interface. See - * dcn10_dmub_update_cursor_data and dmub_cmd_update_cursor_info_data - */ - if (plane->type == DRM_PLANE_TYPE_CURSOR) - return; - - if (new_plane_state->rotation != DRM_MODE_ROTATE_0) - goto ffu; - - num_clips = drm_plane_get_damage_clips_count(new_plane_state); - clips = drm_plane_get_damage_clips(new_plane_state); - - if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 && - is_psr_su))) - goto ffu; - - if (!dm_crtc_state->mpo_requested) { - if (!num_clips || num_clips > DC_MAX_DIRTY_RECTS) - goto ffu; - - for (; flip_addrs->dirty_rect_count < num_clips; clips++) - fill_dc_dirty_rect(new_plane_state->plane, - &dirty_rects[flip_addrs->dirty_rect_count], - clips->x1, clips->y1, - clips->x2 - clips->x1, clips->y2 - clips->y1, - &flip_addrs->dirty_rect_count, - false); - return; - } - - /* - * MPO is requested. Add entire plane bounding box to dirty rects if - * flipped to or damaged. - * - * If plane is moved or resized, also add old bounding box to dirty - * rects. - */ - fb_changed = old_plane_state->fb->base.id != - new_plane_state->fb->base.id; - bb_changed = (old_plane_state->crtc_x != new_plane_state->crtc_x || - old_plane_state->crtc_y != new_plane_state->crtc_y || - old_plane_state->crtc_w != new_plane_state->crtc_w || - old_plane_state->crtc_h != new_plane_state->crtc_h); - - drm_dbg(plane->dev, - "[PLANE:%d] PSR bb_changed:%d fb_changed:%d num_clips:%d\n", - new_plane_state->plane->base.id, - bb_changed, fb_changed, num_clips); - - *dirty_regions_changed = bb_changed; - - if ((num_clips + (bb_changed ? 2 : 0)) > DC_MAX_DIRTY_RECTS) - goto ffu; - - if (bb_changed) { - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - new_plane_state->crtc_x, - new_plane_state->crtc_y, - new_plane_state->crtc_w, - new_plane_state->crtc_h, &i, false); - - /* Add old plane bounding-box if plane is moved or resized */ - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - old_plane_state->crtc_x, - old_plane_state->crtc_y, - old_plane_state->crtc_w, - old_plane_state->crtc_h, &i, false); - } - - if (num_clips) { - for (; i < num_clips; clips++) - fill_dc_dirty_rect(new_plane_state->plane, - &dirty_rects[i], clips->x1, - clips->y1, clips->x2 - clips->x1, - clips->y2 - clips->y1, &i, false); - } else if (fb_changed && !bb_changed) { - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - new_plane_state->crtc_x, - new_plane_state->crtc_y, - new_plane_state->crtc_w, - new_plane_state->crtc_h, &i, false); - } - - flip_addrs->dirty_rect_count = i; - return; - -ffu: - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[0], 0, 0, - dm_crtc_state->base.mode.crtc_hdisplay, - dm_crtc_state->base.mode.crtc_vdisplay, - &flip_addrs->dirty_rect_count, true); -} - -static void update_stream_scaling_settings(const struct drm_display_mode *mode, - const struct dm_connector_state *dm_state, - struct dc_stream_state *stream) -{ - enum amdgpu_rmx_type rmx_type; - - struct rect src = { 0 }; /* viewport in composition space*/ - struct rect dst = { 0 }; /* stream addressable area */ - - /* no mode. nothing to be done */ - if (!mode) - return; - - /* Full screen scaling by default */ - src.width = mode->hdisplay; - src.height = mode->vdisplay; - dst.width = stream->timing.h_addressable; - dst.height = stream->timing.v_addressable; - - if (dm_state) { - rmx_type = dm_state->scaling; - if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) { - if (src.width * dst.height < - src.height * dst.width) { - /* height needs less upscaling/more downscaling */ - dst.width = src.width * - dst.height / src.height; - } else { - /* width needs less upscaling/more downscaling */ - dst.height = src.height * - dst.width / src.width; - } - } else if (rmx_type == RMX_CENTER) { - dst = src; - } - - dst.x = (stream->timing.h_addressable - dst.width) / 2; - dst.y = (stream->timing.v_addressable - dst.height) / 2; - - if (dm_state->underscan_enable) { - dst.x += dm_state->underscan_hborder / 2; - dst.y += dm_state->underscan_vborder / 2; - dst.width -= dm_state->underscan_hborder; - dst.height -= dm_state->underscan_vborder; - } - } - - stream->src = src; - stream->dst = dst; - - DRM_DEBUG_KMS("Destination Rectangle x:%d y:%d width:%d height:%d\n", - dst.x, dst.y, dst.width, dst.height); - -} - -static enum dc_color_depth -convert_color_depth_from_display_info(const struct drm_connector *connector, - bool is_y420, int requested_bpc) -{ - u8 bpc; - - if (is_y420) { - bpc = 8; - - /* Cap display bpc based on HDMI 2.0 HF-VSDB */ - if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48) - bpc = 16; - else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36) - bpc = 12; - else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30) - bpc = 10; - } else { - bpc = (uint8_t)connector->display_info.bpc; - /* Assume 8 bpc by default if no bpc is specified. */ - bpc = bpc ? bpc : 8; - } - - if (requested_bpc > 0) { - /* - * Cap display bpc based on the user requested value. - * - * The value for state->max_bpc may not correctly updated - * depending on when the connector gets added to the state - * or if this was called outside of atomic check, so it - * can't be used directly. - */ - bpc = min_t(u8, bpc, requested_bpc); - - /* Round down to the nearest even number. */ - bpc = bpc - (bpc & 1); - } - - switch (bpc) { - case 0: - /* - * Temporary Work around, DRM doesn't parse color depth for - * EDID revision before 1.4 - * TODO: Fix edid parsing - */ - return COLOR_DEPTH_888; - case 6: - return COLOR_DEPTH_666; - case 8: - return COLOR_DEPTH_888; - case 10: - return COLOR_DEPTH_101010; - case 12: - return COLOR_DEPTH_121212; - case 14: - return COLOR_DEPTH_141414; - case 16: - return COLOR_DEPTH_161616; - default: - return COLOR_DEPTH_UNDEFINED; - } -} - -static enum dc_aspect_ratio -get_aspect_ratio(const struct drm_display_mode *mode_in) -{ - /* 1-1 mapping, since both enums follow the HDMI spec. */ - return (enum dc_aspect_ratio) mode_in->picture_aspect_ratio; -} - -static enum dc_color_space -get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, - const struct drm_connector_state *connector_state) -{ - enum dc_color_space color_space = COLOR_SPACE_SRGB; - - switch (connector_state->colorspace) { - case DRM_MODE_COLORIMETRY_BT601_YCC: - if (dc_crtc_timing->flags.Y_ONLY) - color_space = COLOR_SPACE_YCBCR601_LIMITED; - else - color_space = COLOR_SPACE_YCBCR601; - break; - case DRM_MODE_COLORIMETRY_BT709_YCC: - if (dc_crtc_timing->flags.Y_ONLY) - color_space = COLOR_SPACE_YCBCR709_LIMITED; - else - color_space = COLOR_SPACE_YCBCR709; - break; - case DRM_MODE_COLORIMETRY_OPRGB: - color_space = COLOR_SPACE_ADOBERGB; - break; - case DRM_MODE_COLORIMETRY_BT2020_RGB: - case DRM_MODE_COLORIMETRY_BT2020_YCC: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) - color_space = COLOR_SPACE_2020_RGB_FULLRANGE; - else - color_space = COLOR_SPACE_2020_YCBCR; - break; - case DRM_MODE_COLORIMETRY_DEFAULT: // ITU601 - default: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { - color_space = COLOR_SPACE_SRGB; - /* - * 27030khz is the separation point between HDTV and SDTV - * according to HDMI spec, we use YCbCr709 and YCbCr601 - * respectively - */ - } else if (dc_crtc_timing->pix_clk_100hz > 270300) { - if (dc_crtc_timing->flags.Y_ONLY) - color_space = - COLOR_SPACE_YCBCR709_LIMITED; - else - color_space = COLOR_SPACE_YCBCR709; - } else { - if (dc_crtc_timing->flags.Y_ONLY) - color_space = - COLOR_SPACE_YCBCR601_LIMITED; - else - color_space = COLOR_SPACE_YCBCR601; - } - break; - } - - return color_space; -} - -static enum display_content_type -get_output_content_type(const struct drm_connector_state *connector_state) -{ - switch (connector_state->content_type) { - default: - case DRM_MODE_CONTENT_TYPE_NO_DATA: - return DISPLAY_CONTENT_TYPE_NO_DATA; - case DRM_MODE_CONTENT_TYPE_GRAPHICS: - return DISPLAY_CONTENT_TYPE_GRAPHICS; - case DRM_MODE_CONTENT_TYPE_PHOTO: - return DISPLAY_CONTENT_TYPE_PHOTO; - case DRM_MODE_CONTENT_TYPE_CINEMA: - return DISPLAY_CONTENT_TYPE_CINEMA; - case DRM_MODE_CONTENT_TYPE_GAME: - return DISPLAY_CONTENT_TYPE_GAME; - } -} - -static bool adjust_colour_depth_from_display_info( - struct dc_crtc_timing *timing_out, - const struct drm_display_info *info) -{ - enum dc_color_depth depth = timing_out->display_color_depth; - int normalized_clk; - - do { - normalized_clk = timing_out->pix_clk_100hz / 10; - /* YCbCr 4:2:0 requires additional adjustment of 1/2 */ - if (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420) - normalized_clk /= 2; - /* Adjusting pix clock following on HDMI spec based on colour depth */ - switch (depth) { - case COLOR_DEPTH_888: - break; - case COLOR_DEPTH_101010: - normalized_clk = (normalized_clk * 30) / 24; - break; - case COLOR_DEPTH_121212: - normalized_clk = (normalized_clk * 36) / 24; - break; - case COLOR_DEPTH_161616: - normalized_clk = (normalized_clk * 48) / 24; - break; - default: - /* The above depths are the only ones valid for HDMI. */ - return false; - } - if (normalized_clk <= info->max_tmds_clock) { - timing_out->display_color_depth = depth; - return true; - } - } while (--depth > COLOR_DEPTH_666); - return false; -} - -static void fill_stream_properties_from_drm_display_mode( - struct dc_stream_state *stream, - const struct drm_display_mode *mode_in, - const struct drm_connector *connector, - const struct drm_connector_state *connector_state, - const struct dc_stream_state *old_stream, - int requested_bpc) -{ - struct dc_crtc_timing *timing_out = &stream->timing; - const struct drm_display_info *info = &connector->display_info; - struct amdgpu_dm_connector *aconnector = NULL; - struct hdmi_vendor_infoframe hv_frame; - struct hdmi_avi_infoframe avi_frame; - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - aconnector = to_amdgpu_dm_connector(connector); - - memset(&hv_frame, 0, sizeof(hv_frame)); - memset(&avi_frame, 0, sizeof(avi_frame)); - - timing_out->h_border_left = 0; - timing_out->h_border_right = 0; - timing_out->v_border_top = 0; - timing_out->v_border_bottom = 0; - /* TODO: un-hardcode */ - if (drm_mode_is_420_only(info, mode_in) - && stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - else if (drm_mode_is_420_also(info, mode_in) - && aconnector - && aconnector->force_yuv420_output) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - else if ((connector->display_info.color_formats & DRM_COLOR_FORMAT_YCBCR444) - && stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR444; - else - timing_out->pixel_encoding = PIXEL_ENCODING_RGB; - - timing_out->timing_3d_format = TIMING_3D_FORMAT_NONE; - timing_out->display_color_depth = convert_color_depth_from_display_info( - connector, - (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420), - requested_bpc); - timing_out->scan_type = SCANNING_TYPE_NODATA; - timing_out->hdmi_vic = 0; - - if (old_stream) { - timing_out->vic = old_stream->timing.vic; - timing_out->flags.HSYNC_POSITIVE_POLARITY = old_stream->timing.flags.HSYNC_POSITIVE_POLARITY; - timing_out->flags.VSYNC_POSITIVE_POLARITY = old_stream->timing.flags.VSYNC_POSITIVE_POLARITY; - } else { - timing_out->vic = drm_match_cea_mode(mode_in); - if (mode_in->flags & DRM_MODE_FLAG_PHSYNC) - timing_out->flags.HSYNC_POSITIVE_POLARITY = 1; - if (mode_in->flags & DRM_MODE_FLAG_PVSYNC) - timing_out->flags.VSYNC_POSITIVE_POLARITY = 1; - } - - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) { - drm_hdmi_avi_infoframe_from_display_mode(&avi_frame, (struct drm_connector *)connector, mode_in); - timing_out->vic = avi_frame.video_code; - drm_hdmi_vendor_infoframe_from_display_mode(&hv_frame, (struct drm_connector *)connector, mode_in); - timing_out->hdmi_vic = hv_frame.vic; - } - - if (aconnector && is_freesync_video_mode(mode_in, aconnector)) { - timing_out->h_addressable = mode_in->hdisplay; - timing_out->h_total = mode_in->htotal; - timing_out->h_sync_width = mode_in->hsync_end - mode_in->hsync_start; - timing_out->h_front_porch = mode_in->hsync_start - mode_in->hdisplay; - timing_out->v_total = mode_in->vtotal; - timing_out->v_addressable = mode_in->vdisplay; - timing_out->v_front_porch = mode_in->vsync_start - mode_in->vdisplay; - timing_out->v_sync_width = mode_in->vsync_end - mode_in->vsync_start; - timing_out->pix_clk_100hz = mode_in->clock * 10; - } else { - timing_out->h_addressable = mode_in->crtc_hdisplay; - timing_out->h_total = mode_in->crtc_htotal; - timing_out->h_sync_width = mode_in->crtc_hsync_end - mode_in->crtc_hsync_start; - timing_out->h_front_porch = mode_in->crtc_hsync_start - mode_in->crtc_hdisplay; - timing_out->v_total = mode_in->crtc_vtotal; - timing_out->v_addressable = mode_in->crtc_vdisplay; - timing_out->v_front_porch = mode_in->crtc_vsync_start - mode_in->crtc_vdisplay; - timing_out->v_sync_width = mode_in->crtc_vsync_end - mode_in->crtc_vsync_start; - timing_out->pix_clk_100hz = mode_in->crtc_clock * 10; - } - - timing_out->aspect_ratio = get_aspect_ratio(mode_in); - - stream->out_transfer_func.type = TF_TYPE_PREDEFINED; - stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB; - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) { - if (!adjust_colour_depth_from_display_info(timing_out, info) && - drm_mode_is_420_also(info, mode_in) && - timing_out->pixel_encoding != PIXEL_ENCODING_YCBCR420) { - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - adjust_colour_depth_from_display_info(timing_out, info); - } - } - - stream->output_color_space = get_output_color_space(timing_out, connector_state); - stream->content_type = get_output_content_type(connector_state); -} - -static void fill_audio_info(struct audio_info *audio_info, - const struct drm_connector *drm_connector, - const struct dc_sink *dc_sink) -{ - int i = 0; - int cea_revision = 0; - const struct dc_edid_caps *edid_caps = &dc_sink->edid_caps; - - audio_info->manufacture_id = edid_caps->manufacturer_id; - audio_info->product_id = edid_caps->product_id; - - cea_revision = drm_connector->display_info.cea_rev; - - strscpy(audio_info->display_name, - edid_caps->display_name, - AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS); - - if (cea_revision >= 3) { - audio_info->mode_count = edid_caps->audio_mode_count; - - for (i = 0; i < audio_info->mode_count; ++i) { - audio_info->modes[i].format_code = - (enum audio_format_code) - (edid_caps->audio_modes[i].format_code); - audio_info->modes[i].channel_count = - edid_caps->audio_modes[i].channel_count; - audio_info->modes[i].sample_rates.all = - edid_caps->audio_modes[i].sample_rate; - audio_info->modes[i].sample_size = - edid_caps->audio_modes[i].sample_size; - } - } - - audio_info->flags.all = edid_caps->speaker_flags; - - /* TODO: We only check for the progressive mode, check for interlace mode too */ - if (drm_connector->latency_present[0]) { - audio_info->video_latency = drm_connector->video_latency[0]; - audio_info->audio_latency = drm_connector->audio_latency[0]; - } - - /* TODO: For DP, video and audio latency should be calculated from DPCD caps */ - -} - -static void -copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode, - struct drm_display_mode *dst_mode) -{ - dst_mode->crtc_hdisplay = src_mode->crtc_hdisplay; - dst_mode->crtc_vdisplay = src_mode->crtc_vdisplay; - dst_mode->crtc_clock = src_mode->crtc_clock; - dst_mode->crtc_hblank_start = src_mode->crtc_hblank_start; - dst_mode->crtc_hblank_end = src_mode->crtc_hblank_end; - dst_mode->crtc_hsync_start = src_mode->crtc_hsync_start; - dst_mode->crtc_hsync_end = src_mode->crtc_hsync_end; - dst_mode->crtc_htotal = src_mode->crtc_htotal; - dst_mode->crtc_hskew = src_mode->crtc_hskew; - dst_mode->crtc_vblank_start = src_mode->crtc_vblank_start; - dst_mode->crtc_vblank_end = src_mode->crtc_vblank_end; - dst_mode->crtc_vsync_start = src_mode->crtc_vsync_start; - dst_mode->crtc_vsync_end = src_mode->crtc_vsync_end; - dst_mode->crtc_vtotal = src_mode->crtc_vtotal; -} - -static void -decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode, - const struct drm_display_mode *native_mode, - bool scale_enabled) -{ - if (scale_enabled) { - copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode); - } else if (native_mode->clock == drm_mode->clock && - native_mode->htotal == drm_mode->htotal && - native_mode->vtotal == drm_mode->vtotal) { - copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode); - } else { - /* no scaling nor amdgpu inserted, no need to patch */ - } -} - -static struct dc_sink * -create_fake_sink(struct dc_link *link) -{ - struct dc_sink_init_data sink_init_data = { 0 }; - struct dc_sink *sink = NULL; - - sink_init_data.link = link; - sink_init_data.sink_signal = link->connector_signal; - - sink = dc_sink_create(&sink_init_data); - if (!sink) { - DRM_ERROR("Failed to create sink!\n"); - return NULL; - } - sink->sink_signal = SIGNAL_TYPE_VIRTUAL; - - return sink; -} - -static void set_multisync_trigger_params( - struct dc_stream_state *stream) -{ - struct dc_stream_state *master = NULL; - - if (stream->triggered_crtc_reset.enabled) { - master = stream->triggered_crtc_reset.event_source; - stream->triggered_crtc_reset.event = - master->timing.flags.VSYNC_POSITIVE_POLARITY ? - CRTC_EVENT_VSYNC_RISING : CRTC_EVENT_VSYNC_FALLING; - stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_PIXEL; - } -} - -static void set_master_stream(struct dc_stream_state *stream_set[], - int stream_count) -{ - int j, highest_rfr = 0, master_stream = 0; - - for (j = 0; j < stream_count; j++) { - if (stream_set[j] && stream_set[j]->triggered_crtc_reset.enabled) { - int refresh_rate = 0; - - refresh_rate = (stream_set[j]->timing.pix_clk_100hz*100)/ - (stream_set[j]->timing.h_total*stream_set[j]->timing.v_total); - if (refresh_rate > highest_rfr) { - highest_rfr = refresh_rate; - master_stream = j; - } - } - } - for (j = 0; j < stream_count; j++) { - if (stream_set[j]) - stream_set[j]->triggered_crtc_reset.event_source = stream_set[master_stream]; - } -} - -static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context) -{ - int i = 0; - struct dc_stream_state *stream; - - if (context->stream_count < 2) - return; - for (i = 0; i < context->stream_count ; i++) { - if (!context->streams[i]) - continue; - /* - * TODO: add a function to read AMD VSDB bits and set - * crtc_sync_master.multi_sync_enabled flag - * For now it's set to false - */ - } - - set_master_stream(context->streams, context->stream_count); - - for (i = 0; i < context->stream_count ; i++) { - stream = context->streams[i]; - - if (!stream) - continue; - - set_multisync_trigger_params(stream); - } -} - -/** - * DOC: FreeSync Video - * - * When a userspace application wants to play a video, the content follows a - * standard format definition that usually specifies the FPS for that format. - * The below list illustrates some video format and the expected FPS, - * respectively: - * - * - TV/NTSC (23.976 FPS) - * - Cinema (24 FPS) - * - TV/PAL (25 FPS) - * - TV/NTSC (29.97 FPS) - * - TV/NTSC (30 FPS) - * - Cinema HFR (48 FPS) - * - TV/PAL (50 FPS) - * - Commonly used (60 FPS) - * - Multiples of 24 (48,72,96 FPS) - * - * The list of standards video format is not huge and can be added to the - * connector modeset list beforehand. With that, userspace can leverage - * FreeSync to extends the front porch in order to attain the target refresh - * rate. Such a switch will happen seamlessly, without screen blanking or - * reprogramming of the output in any other way. If the userspace requests a - * modesetting change compatible with FreeSync modes that only differ in the - * refresh rate, DC will skip the full update and avoid blink during the - * transition. For example, the video player can change the modesetting from - * 60Hz to 30Hz for playing TV/NTSC content when it goes full screen without - * causing any display blink. This same concept can be applied to a mode - * setting change. - */ -static struct drm_display_mode * -get_highest_refresh_rate_mode(struct amdgpu_dm_connector *aconnector, - bool use_probed_modes) -{ - struct drm_display_mode *m, *m_pref = NULL; - u16 current_refresh, highest_refresh; - struct list_head *list_head = use_probed_modes ? - &aconnector->base.probed_modes : - &aconnector->base.modes; - - if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return NULL; - - if (aconnector->freesync_vid_base.clock != 0) - return &aconnector->freesync_vid_base; - - /* Find the preferred mode */ - list_for_each_entry(m, list_head, head) { - if (m->type & DRM_MODE_TYPE_PREFERRED) { - m_pref = m; - break; - } - } - - if (!m_pref) { - /* Probably an EDID with no preferred mode. Fallback to first entry */ - m_pref = list_first_entry_or_null( - &aconnector->base.modes, struct drm_display_mode, head); - if (!m_pref) { - DRM_DEBUG_DRIVER("No preferred mode found in EDID\n"); - return NULL; - } - } - - highest_refresh = drm_mode_vrefresh(m_pref); - - /* - * Find the mode with highest refresh rate with same resolution. - * For some monitors, preferred mode is not the mode with highest - * supported refresh rate. - */ - list_for_each_entry(m, list_head, head) { - current_refresh = drm_mode_vrefresh(m); - - if (m->hdisplay == m_pref->hdisplay && - m->vdisplay == m_pref->vdisplay && - highest_refresh < current_refresh) { - highest_refresh = current_refresh; - m_pref = m; - } - } - - drm_mode_copy(&aconnector->freesync_vid_base, m_pref); - return m_pref; -} - -static bool is_freesync_video_mode(const struct drm_display_mode *mode, - struct amdgpu_dm_connector *aconnector) -{ - struct drm_display_mode *high_mode; - int timing_diff; - - high_mode = get_highest_refresh_rate_mode(aconnector, false); - if (!high_mode || !mode) - return false; - - timing_diff = high_mode->vtotal - mode->vtotal; - - if (high_mode->clock == 0 || high_mode->clock != mode->clock || - high_mode->hdisplay != mode->hdisplay || - high_mode->vdisplay != mode->vdisplay || - high_mode->hsync_start != mode->hsync_start || - high_mode->hsync_end != mode->hsync_end || - high_mode->htotal != mode->htotal || - high_mode->hskew != mode->hskew || - high_mode->vscan != mode->vscan || - high_mode->vsync_start - mode->vsync_start != timing_diff || - high_mode->vsync_end - mode->vsync_end != timing_diff) - return false; - else - return true; -} - -#if defined(CONFIG_DRM_AMD_DC_FP) -static void update_dsc_caps(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps) -{ - stream->timing.flags.DSC = 0; - dsc_caps->is_dsc_supported = false; - - if (aconnector->dc_link && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { - if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE || - sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) - dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc, - aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw, - aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw, - dsc_caps); - } -} - -static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps, - uint32_t max_dsc_target_bpp_limit_override) -{ - const struct dc_link_settings *verified_link_cap = NULL; - u32 link_bw_in_kbps; - u32 edp_min_bpp_x16, edp_max_bpp_x16; - struct dc *dc = sink->ctx->dc; - struct dc_dsc_bw_range bw_range = {0}; - struct dc_dsc_config dsc_cfg = {0}; - struct dc_dsc_config_options dsc_options = {0}; - - dc_dsc_get_default_config_option(dc, &dsc_options); - dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16; - - verified_link_cap = dc_link_get_link_cap(stream->link); - link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap); - edp_min_bpp_x16 = 8 * 16; - edp_max_bpp_x16 = 8 * 16; - - if (edp_max_bpp_x16 > dsc_caps->edp_max_bits_per_pixel) - edp_max_bpp_x16 = dsc_caps->edp_max_bits_per_pixel; - - if (edp_max_bpp_x16 < edp_min_bpp_x16) - edp_min_bpp_x16 = edp_max_bpp_x16; - - if (dc_dsc_compute_bandwidth_range(dc->res_pool->dscs[0], - dc->debug.dsc_min_slice_height_override, - edp_min_bpp_x16, edp_max_bpp_x16, - dsc_caps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &bw_range)) { - - if (bw_range.max_kbps < link_bw_in_kbps) { - if (dc_dsc_compute_config(dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - 0, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &dsc_cfg)) { - stream->timing.dsc_cfg = dsc_cfg; - stream->timing.flags.DSC = 1; - stream->timing.dsc_cfg.bits_per_pixel = edp_max_bpp_x16; - } - return; - } - } - - if (dc_dsc_compute_config(dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - link_bw_in_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &dsc_cfg)) { - stream->timing.dsc_cfg = dsc_cfg; - stream->timing.flags.DSC = 1; - } -} - -static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps) -{ - struct drm_connector *drm_connector = &aconnector->base; - u32 link_bandwidth_kbps; - struct dc *dc = sink->ctx->dc; - u32 max_supported_bw_in_kbps, timing_bw_in_kbps; - u32 dsc_max_supported_bw_in_kbps; - u32 max_dsc_target_bpp_limit_override = - drm_connector->display_info.max_dsc_bpp; - struct dc_dsc_config_options dsc_options = {0}; - - dc_dsc_get_default_config_option(dc, &dsc_options); - dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16; - - link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link, - dc_link_get_link_cap(aconnector->dc_link)); - - /* Set DSC policy according to dsc_clock_en */ - dc_dsc_policy_set_enable_dsc_when_not_needed( - aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE); - - if (sink->sink_signal == SIGNAL_TYPE_EDP && - !aconnector->dc_link->panel_config.dsc.disable_dsc_edp && - dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) { - - apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override); - - } else if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) { - if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) { - if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - link_bandwidth_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &stream->timing.dsc_cfg)) { - stream->timing.flags.DSC = 1; - DRM_DEBUG_DRIVER("%s: [%s] DSC is selected from SST RX\n", __func__, drm_connector->name); - } - } else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) { - timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link)); - max_supported_bw_in_kbps = link_bandwidth_kbps; - dsc_max_supported_bw_in_kbps = link_bandwidth_kbps; - - if (timing_bw_in_kbps > max_supported_bw_in_kbps && - max_supported_bw_in_kbps > 0 && - dsc_max_supported_bw_in_kbps > 0) - if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - dsc_max_supported_bw_in_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &stream->timing.dsc_cfg)) { - stream->timing.flags.DSC = 1; - DRM_DEBUG_DRIVER("%s: [%s] DSC is selected from DP-HDMI PCON\n", - __func__, drm_connector->name); - } - } - } - - /* Overwrite the stream flag if DSC is enabled through debugfs */ - if (aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE) - stream->timing.flags.DSC = 1; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_h) - stream->timing.dsc_cfg.num_slices_h = aconnector->dsc_settings.dsc_num_slices_h; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_v) - stream->timing.dsc_cfg.num_slices_v = aconnector->dsc_settings.dsc_num_slices_v; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel) - stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel; -} -#endif - -static struct dc_stream_state * -create_stream_for_sink(struct drm_connector *connector, - const struct drm_display_mode *drm_mode, - const struct dm_connector_state *dm_state, - const struct dc_stream_state *old_stream, - int requested_bpc) -{ - struct amdgpu_dm_connector *aconnector = NULL; - struct drm_display_mode *preferred_mode = NULL; - const struct drm_connector_state *con_state = &dm_state->base; - struct dc_stream_state *stream = NULL; - struct drm_display_mode mode; - struct drm_display_mode saved_mode; - struct drm_display_mode *freesync_mode = NULL; - bool native_mode_found = false; - bool recalculate_timing = false; - bool scale = dm_state->scaling != RMX_OFF; - int mode_refresh; - int preferred_refresh = 0; - enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN; -#if defined(CONFIG_DRM_AMD_DC_FP) - struct dsc_dec_dpcd_caps dsc_caps; -#endif - struct dc_link *link = NULL; - struct dc_sink *sink = NULL; - - drm_mode_init(&mode, drm_mode); - memset(&saved_mode, 0, sizeof(saved_mode)); - - if (connector == NULL) { - DRM_ERROR("connector is NULL!\n"); - return stream; - } - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) { - aconnector = NULL; - aconnector = to_amdgpu_dm_connector(connector); - link = aconnector->dc_link; - } else { - struct drm_writeback_connector *wbcon = NULL; - struct amdgpu_dm_wb_connector *dm_wbcon = NULL; - - wbcon = drm_connector_to_writeback(connector); - dm_wbcon = to_amdgpu_dm_wb_connector(wbcon); - link = dm_wbcon->link; - } - - if (!aconnector || !aconnector->dc_sink) { - sink = create_fake_sink(link); - if (!sink) - return stream; - - } else { - sink = aconnector->dc_sink; - dc_sink_retain(sink); - } - - stream = dc_create_stream_for_sink(sink); - - if (stream == NULL) { - DRM_ERROR("Failed to create stream for sink!\n"); - goto finish; - } - - /* We leave this NULL for writeback connectors */ - stream->dm_stream_context = aconnector; - - stream->timing.flags.LTE_340MCSC_SCRAMBLE = - connector->display_info.hdmi.scdc.scrambling.low_rates; - - list_for_each_entry(preferred_mode, &connector->modes, head) { - /* Search for preferred mode */ - if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) { - native_mode_found = true; - break; - } - } - if (!native_mode_found) - preferred_mode = list_first_entry_or_null( - &connector->modes, - struct drm_display_mode, - head); - - mode_refresh = drm_mode_vrefresh(&mode); - - if (preferred_mode == NULL) { - /* - * This may not be an error, the use case is when we have no - * usermode calls to reset and set mode upon hotplug. In this - * case, we call set mode ourselves to restore the previous mode - * and the modelist may not be filled in time. - */ - DRM_DEBUG_DRIVER("No preferred mode found\n"); - } else if (aconnector) { - recalculate_timing = amdgpu_freesync_vid_mode && - is_freesync_video_mode(&mode, aconnector); - if (recalculate_timing) { - freesync_mode = get_highest_refresh_rate_mode(aconnector, false); - drm_mode_copy(&saved_mode, &mode); - saved_mode.picture_aspect_ratio = mode.picture_aspect_ratio; - drm_mode_copy(&mode, freesync_mode); - mode.picture_aspect_ratio = saved_mode.picture_aspect_ratio; - } else { - decide_crtc_timing_for_drm_display_mode( - &mode, preferred_mode, scale); - - preferred_refresh = drm_mode_vrefresh(preferred_mode); - } - } - - if (recalculate_timing) - drm_mode_set_crtcinfo(&saved_mode, 0); - - /* - * If scaling is enabled and refresh rate didn't change - * we copy the vic and polarities of the old timings - */ - if (!scale || mode_refresh != preferred_refresh) - fill_stream_properties_from_drm_display_mode( - stream, &mode, connector, con_state, NULL, - requested_bpc); - else - fill_stream_properties_from_drm_display_mode( - stream, &mode, connector, con_state, old_stream, - requested_bpc); - - /* The rest isn't needed for writeback connectors */ - if (!aconnector) - goto finish; - - if (aconnector->timing_changed) { - drm_dbg(aconnector->base.dev, - "overriding timing for automated test, bpc %d, changing to %d\n", - stream->timing.display_color_depth, - aconnector->timing_requested->display_color_depth); - stream->timing = *aconnector->timing_requested; - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - /* SST DSC determination policy */ - update_dsc_caps(aconnector, sink, stream, &dsc_caps); - if (aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE && dsc_caps.is_dsc_supported) - apply_dsc_policy_for_stream(aconnector, sink, stream, &dsc_caps); -#endif - - update_stream_scaling_settings(&mode, dm_state, stream); - - fill_audio_info( - &stream->audio_info, - connector, - sink); - - update_stream_signal(stream, sink); - - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket); - - if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || - stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST || - stream->signal == SIGNAL_TYPE_EDP) { - // - // should decide stream support vsc sdp colorimetry capability - // before building vsc info packet - // - stream->use_vsc_sdp_for_colorimetry = stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 && - stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED; - - if (stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) - tf = TRANSFER_FUNC_GAMMA_22; - mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf); - aconnector->psr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY; - - } -finish: - dc_sink_release(sink); - - return stream; -} - -static enum drm_connector_status -amdgpu_dm_connector_detect(struct drm_connector *connector, bool force) -{ - bool connected; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - /* - * Notes: - * 1. This interface is NOT called in context of HPD irq. - * 2. This interface *is called* in context of user-mode ioctl. Which - * makes it a bad place for *any* MST-related activity. - */ - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED && - !aconnector->fake_enable) - connected = (aconnector->dc_sink != NULL); - else - connected = (aconnector->base.force == DRM_FORCE_ON || - aconnector->base.force == DRM_FORCE_ON_DIGITAL); - - update_subconnector_property(aconnector); - - return (connected ? connector_status_connected : - connector_status_disconnected); -} - -int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector, - struct drm_connector_state *connector_state, - struct drm_property *property, - uint64_t val) -{ - struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_old_state = - to_dm_connector_state(connector->state); - struct dm_connector_state *dm_new_state = - to_dm_connector_state(connector_state); - - int ret = -EINVAL; - - if (property == dev->mode_config.scaling_mode_property) { - enum amdgpu_rmx_type rmx_type; - - switch (val) { - case DRM_MODE_SCALE_CENTER: - rmx_type = RMX_CENTER; - break; - case DRM_MODE_SCALE_ASPECT: - rmx_type = RMX_ASPECT; - break; - case DRM_MODE_SCALE_FULLSCREEN: - rmx_type = RMX_FULL; - break; - case DRM_MODE_SCALE_NONE: - default: - rmx_type = RMX_OFF; - break; - } - - if (dm_old_state->scaling == rmx_type) - return 0; - - dm_new_state->scaling = rmx_type; - ret = 0; - } else if (property == adev->mode_info.underscan_hborder_property) { - dm_new_state->underscan_hborder = val; - ret = 0; - } else if (property == adev->mode_info.underscan_vborder_property) { - dm_new_state->underscan_vborder = val; - ret = 0; - } else if (property == adev->mode_info.underscan_property) { - dm_new_state->underscan_enable = val; - ret = 0; - } - - return ret; -} - -int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector, - const struct drm_connector_state *state, - struct drm_property *property, - uint64_t *val) -{ - struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_state = - to_dm_connector_state(state); - int ret = -EINVAL; - - if (property == dev->mode_config.scaling_mode_property) { - switch (dm_state->scaling) { - case RMX_CENTER: - *val = DRM_MODE_SCALE_CENTER; - break; - case RMX_ASPECT: - *val = DRM_MODE_SCALE_ASPECT; - break; - case RMX_FULL: - *val = DRM_MODE_SCALE_FULLSCREEN; - break; - case RMX_OFF: - default: - *val = DRM_MODE_SCALE_NONE; - break; - } - ret = 0; - } else if (property == adev->mode_info.underscan_hborder_property) { - *val = dm_state->underscan_hborder; - ret = 0; - } else if (property == adev->mode_info.underscan_vborder_property) { - *val = dm_state->underscan_vborder; - ret = 0; - } else if (property == adev->mode_info.underscan_property) { - *val = dm_state->underscan_enable; - ret = 0; - } - - return ret; -} - -/** - * DOC: panel power savings - * - * The display manager allows you to set your desired **panel power savings** - * level (between 0-4, with 0 representing off), e.g. using the following:: - * - * # echo 3 > /sys/class/drm/card0-eDP-1/amdgpu/panel_power_savings - * - * Modifying this value can have implications on color accuracy, so tread - * carefully. - */ - -static ssize_t panel_power_savings_show(struct device *device, - struct device_attribute *attr, - char *buf) -{ - struct drm_connector *connector = dev_get_drvdata(device); - struct drm_device *dev = connector->dev; - u8 val; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - val = to_dm_connector_state(connector->state)->abm_level == - ABM_LEVEL_IMMEDIATE_DISABLE ? 0 : - to_dm_connector_state(connector->state)->abm_level; - drm_modeset_unlock(&dev->mode_config.connection_mutex); - - return sysfs_emit(buf, "%u\n", val); -} - -static ssize_t panel_power_savings_store(struct device *device, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct drm_connector *connector = dev_get_drvdata(device); - struct drm_device *dev = connector->dev; - long val; - int ret; - - ret = kstrtol(buf, 0, &val); - - if (ret) - return ret; - - if (val < 0 || val > 4) - return -EINVAL; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - to_dm_connector_state(connector->state)->abm_level = val ?: - ABM_LEVEL_IMMEDIATE_DISABLE; - drm_modeset_unlock(&dev->mode_config.connection_mutex); - - drm_kms_helper_hotplug_event(dev); - - return count; -} - -static DEVICE_ATTR_RW(panel_power_savings); - -static struct attribute *amdgpu_attrs[] = { - &dev_attr_panel_power_savings.attr, - NULL -}; - -static const struct attribute_group amdgpu_group = { - .name = "amdgpu", - .attrs = amdgpu_attrs -}; - -static bool -amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *amdgpu_dm_connector) -{ - if (amdgpu_dm_abm_level >= 0) - return false; - - if (amdgpu_dm_connector->base.connector_type != DRM_MODE_CONNECTOR_eDP) - return false; - - /* check for OLED panels */ - if (amdgpu_dm_connector->bl_idx >= 0) { - struct drm_device *drm = amdgpu_dm_connector->base.dev; - struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm; - struct amdgpu_dm_backlight_caps *caps; - - caps = &dm->backlight_caps[amdgpu_dm_connector->bl_idx]; - if (caps->aux_support) - return false; - } - - return true; -} - -static void amdgpu_dm_connector_unregister(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector); - - if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) - sysfs_remove_group(&connector->kdev->kobj, &amdgpu_group); - - drm_dp_aux_unregister(&amdgpu_dm_connector->dm_dp_aux.aux); -} - -static void amdgpu_dm_connector_destroy(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct amdgpu_display_manager *dm = &adev->dm; - - /* - * Call only if mst_mgr was initialized before since it's not done - * for all connector types. - */ - if (aconnector->mst_mgr.dev) - drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr); - - if (aconnector->bl_idx != -1) { - backlight_device_unregister(dm->backlight_dev[aconnector->bl_idx]); - dm->backlight_dev[aconnector->bl_idx] = NULL; - } - - if (aconnector->dc_em_sink) - dc_sink_release(aconnector->dc_em_sink); - aconnector->dc_em_sink = NULL; - if (aconnector->dc_sink) - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - - drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux); - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - if (aconnector->i2c) { - i2c_del_adapter(&aconnector->i2c->base); - kfree(aconnector->i2c); - } - kfree(aconnector->dm_dp_aux.aux.name); - - kfree(connector); -} - -void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector) -{ - struct dm_connector_state *state = - to_dm_connector_state(connector->state); - - if (connector->state) - __drm_atomic_helper_connector_destroy_state(connector->state); - - kfree(state); - - state = kzalloc(sizeof(*state), GFP_KERNEL); - - if (state) { - state->scaling = RMX_OFF; - state->underscan_enable = false; - state->underscan_hborder = 0; - state->underscan_vborder = 0; - state->base.max_requested_bpc = 8; - state->vcpi_slots = 0; - state->pbn = 0; - - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { - if (amdgpu_dm_abm_level <= 0) - state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE; - else - state->abm_level = amdgpu_dm_abm_level; - } - - __drm_atomic_helper_connector_reset(connector, &state->base); - } -} - -struct drm_connector_state * -amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector) -{ - struct dm_connector_state *state = - to_dm_connector_state(connector->state); - - struct dm_connector_state *new_state = - kmemdup(state, sizeof(*state), GFP_KERNEL); - - if (!new_state) - return NULL; - - __drm_atomic_helper_connector_duplicate_state(connector, &new_state->base); - - new_state->freesync_capable = state->freesync_capable; - new_state->abm_level = state->abm_level; - new_state->scaling = state->scaling; - new_state->underscan_enable = state->underscan_enable; - new_state->underscan_hborder = state->underscan_hborder; - new_state->underscan_vborder = state->underscan_vborder; - new_state->vcpi_slots = state->vcpi_slots; - new_state->pbn = state->pbn; - return &new_state->base; -} - -static int -amdgpu_dm_connector_late_register(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - int r; - - if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) { - r = sysfs_create_group(&connector->kdev->kobj, - &amdgpu_group); - if (r) - return r; - } - - amdgpu_dm_register_backlight_device(amdgpu_dm_connector); - - if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) || - (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) { - amdgpu_dm_connector->dm_dp_aux.aux.dev = connector->kdev; - r = drm_dp_aux_register(&amdgpu_dm_connector->dm_dp_aux.aux); - if (r) - return r; - } - -#if defined(CONFIG_DEBUG_FS) - connector_debugfs_init(amdgpu_dm_connector); -#endif - - return 0; -} - -static void amdgpu_dm_connector_funcs_force(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dc_link *dc_link = aconnector->dc_link; - struct dc_sink *dc_em_sink = aconnector->dc_em_sink; - struct edid *edid; - struct i2c_adapter *ddc; - - if (dc_link && dc_link->aux_mode) - ddc = &aconnector->dm_dp_aux.aux.ddc; - else - ddc = &aconnector->i2c->base; - - /* - * Note: drm_get_edid gets edid in the following order: - * 1) override EDID if set via edid_override debugfs, - * 2) firmware EDID if set via edid_firmware module parameter - * 3) regular DDC read. - */ - edid = drm_get_edid(connector, ddc); - if (!edid) { - DRM_ERROR("No EDID found on connector: %s.\n", connector->name); - return; - } - - aconnector->edid = edid; - - /* Update emulated (virtual) sink's EDID */ - if (dc_em_sink && dc_link) { - memset(&dc_em_sink->edid_caps, 0, sizeof(struct dc_edid_caps)); - memmove(dc_em_sink->dc_edid.raw_edid, edid, (edid->extensions + 1) * EDID_LENGTH); - dm_helpers_parse_edid_caps( - dc_link, - &dc_em_sink->dc_edid, - &dc_em_sink->edid_caps); - } -} - -static const struct drm_connector_funcs amdgpu_dm_connector_funcs = { - .reset = amdgpu_dm_connector_funcs_reset, - .detect = amdgpu_dm_connector_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = amdgpu_dm_connector_destroy, - .atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_set_property = amdgpu_dm_connector_atomic_set_property, - .atomic_get_property = amdgpu_dm_connector_atomic_get_property, - .late_register = amdgpu_dm_connector_late_register, - .early_unregister = amdgpu_dm_connector_unregister, - .force = amdgpu_dm_connector_funcs_force -}; - -static int get_modes(struct drm_connector *connector) -{ - return amdgpu_dm_connector_get_modes(connector); -} - -static void create_eml_sink(struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct dc_link *dc_link = aconnector->dc_link; - struct dc_sink_init_data init_params = { - .link = aconnector->dc_link, - .sink_signal = SIGNAL_TYPE_VIRTUAL - }; - struct edid *edid; - struct i2c_adapter *ddc; - - if (dc_link->aux_mode) - ddc = &aconnector->dm_dp_aux.aux.ddc; - else - ddc = &aconnector->i2c->base; - - /* - * Note: drm_get_edid gets edid in the following order: - * 1) override EDID if set via edid_override debugfs, - * 2) firmware EDID if set via edid_firmware module parameter - * 3) regular DDC read. - */ - edid = drm_get_edid(connector, ddc); - if (!edid) { - DRM_ERROR("No EDID found on connector: %s.\n", connector->name); - return; - } - - if (drm_detect_hdmi_monitor(edid)) - init_params.sink_signal = SIGNAL_TYPE_HDMI_TYPE_A; - - aconnector->edid = edid; - - aconnector->dc_em_sink = dc_link_add_remote_sink( - aconnector->dc_link, - (uint8_t *)edid, - (edid->extensions + 1) * EDID_LENGTH, - &init_params); - - if (aconnector->base.force == DRM_FORCE_ON) { - aconnector->dc_sink = aconnector->dc_link->local_sink ? - aconnector->dc_link->local_sink : - aconnector->dc_em_sink; - if (aconnector->dc_sink) - dc_sink_retain(aconnector->dc_sink); - } -} - -static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = (struct dc_link *)aconnector->dc_link; - - /* - * In case of headless boot with force on for DP managed connector - * Those settings have to be != 0 to get initial modeset - */ - if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) { - link->verified_link_cap.lane_count = LANE_COUNT_FOUR; - link->verified_link_cap.link_rate = LINK_RATE_HIGH2; - } - - create_eml_sink(aconnector); -} - -static enum dc_status dm_validate_stream_and_context(struct dc *dc, - struct dc_stream_state *stream) -{ - enum dc_status dc_result = DC_ERROR_UNEXPECTED; - struct dc_plane_state *dc_plane_state = NULL; - struct dc_state *dc_state = NULL; - - if (!stream) - goto cleanup; - - dc_plane_state = dc_create_plane_state(dc); - if (!dc_plane_state) - goto cleanup; - - dc_state = dc_state_create(dc, NULL); - if (!dc_state) - goto cleanup; - - /* populate stream to plane */ - dc_plane_state->src_rect.height = stream->src.height; - dc_plane_state->src_rect.width = stream->src.width; - dc_plane_state->dst_rect.height = stream->src.height; - dc_plane_state->dst_rect.width = stream->src.width; - dc_plane_state->clip_rect.height = stream->src.height; - dc_plane_state->clip_rect.width = stream->src.width; - dc_plane_state->plane_size.surface_pitch = ((stream->src.width + 255) / 256) * 256; - dc_plane_state->plane_size.surface_size.height = stream->src.height; - dc_plane_state->plane_size.surface_size.width = stream->src.width; - dc_plane_state->plane_size.chroma_size.height = stream->src.height; - dc_plane_state->plane_size.chroma_size.width = stream->src.width; - dc_plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888; - dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN; - dc_plane_state->rotation = ROTATION_ANGLE_0; - dc_plane_state->is_tiling_rotated = false; - dc_plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_LINEAR_GENERAL; - - dc_result = dc_validate_stream(dc, stream); - if (dc_result == DC_OK) - dc_result = dc_validate_plane(dc, dc_plane_state); - - if (dc_result == DC_OK) - dc_result = dc_state_add_stream(dc, dc_state, stream); - - if (dc_result == DC_OK && !dc_state_add_plane( - dc, - stream, - dc_plane_state, - dc_state)) - dc_result = DC_FAIL_ATTACH_SURFACES; - - if (dc_result == DC_OK) - dc_result = dc_validate_global_state(dc, dc_state, true); - -cleanup: - if (dc_state) - dc_state_release(dc_state); - - if (dc_plane_state) - dc_plane_state_release(dc_plane_state); - - return dc_result; -} - -struct dc_stream_state * -create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, - const struct drm_display_mode *drm_mode, - const struct dm_connector_state *dm_state, - const struct dc_stream_state *old_stream) -{ - struct drm_connector *connector = &aconnector->base; - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct dc_stream_state *stream; - const struct drm_connector_state *drm_state = dm_state ? &dm_state->base : NULL; - int requested_bpc = drm_state ? drm_state->max_requested_bpc : 8; - enum dc_status dc_result = DC_OK; - - if (!dm_state) - return NULL; - - do { - stream = create_stream_for_sink(connector, drm_mode, - dm_state, old_stream, - requested_bpc); - if (stream == NULL) { - DRM_ERROR("Failed to create stream for sink!\n"); - break; - } - - if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return stream; - - dc_result = dc_validate_stream(adev->dm.dc, stream); - if (dc_result == DC_OK && stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) - dc_result = dm_dp_mst_is_port_support_mode(aconnector, stream); - - if (dc_result == DC_OK) - dc_result = dm_validate_stream_and_context(adev->dm.dc, stream); - - if (dc_result != DC_OK) { - DRM_DEBUG_KMS("Mode %dx%d (clk %d) failed DC validation with error %d (%s)\n", - drm_mode->hdisplay, - drm_mode->vdisplay, - drm_mode->clock, - dc_result, - dc_status_to_str(dc_result)); - - dc_stream_release(stream); - stream = NULL; - requested_bpc -= 2; /* lower bpc to retry validation */ - } - - } while (stream == NULL && requested_bpc >= 6); - - if (dc_result == DC_FAIL_ENC_VALIDATE && !aconnector->force_yuv420_output) { - DRM_DEBUG_KMS("Retry forcing YCbCr420 encoding\n"); - - aconnector->force_yuv420_output = true; - stream = create_validate_stream_for_sink(aconnector, drm_mode, - dm_state, old_stream); - aconnector->force_yuv420_output = false; - } - - return stream; -} - -enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - int result = MODE_ERROR; - struct dc_sink *dc_sink; - /* TODO: Unhardcode stream count */ - struct dc_stream_state *stream; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - if ((mode->flags & DRM_MODE_FLAG_INTERLACE) || - (mode->flags & DRM_MODE_FLAG_DBLSCAN)) - return result; - - /* - * Only run this the first time mode_valid is called to initilialize - * EDID mgmt - */ - if (aconnector->base.force != DRM_FORCE_UNSPECIFIED && - !aconnector->dc_em_sink) - handle_edid_mgmt(aconnector); - - dc_sink = to_amdgpu_dm_connector(connector)->dc_sink; - - if (dc_sink == NULL && aconnector->base.force != DRM_FORCE_ON_DIGITAL && - aconnector->base.force != DRM_FORCE_ON) { - DRM_ERROR("dc_sink is NULL!\n"); - goto fail; - } - - drm_mode_set_crtcinfo(mode, 0); - - stream = create_validate_stream_for_sink(aconnector, mode, - to_dm_connector_state(connector->state), - NULL); - if (stream) { - dc_stream_release(stream); - result = MODE_OK; - } - -fail: - /* TODO: error handling*/ - return result; -} - -static int fill_hdr_info_packet(const struct drm_connector_state *state, - struct dc_info_packet *out) -{ - struct hdmi_drm_infoframe frame; - unsigned char buf[30]; /* 26 + 4 */ - ssize_t len; - int ret, i; - - memset(out, 0, sizeof(*out)); - - if (!state->hdr_output_metadata) - return 0; - - ret = drm_hdmi_infoframe_set_hdr_metadata(&frame, state); - if (ret) - return ret; - - len = hdmi_drm_infoframe_pack_only(&frame, buf, sizeof(buf)); - if (len < 0) - return (int)len; - - /* Static metadata is a fixed 26 bytes + 4 byte header. */ - if (len != 30) - return -EINVAL; - - /* Prepare the infopacket for DC. */ - switch (state->connector->connector_type) { - case DRM_MODE_CONNECTOR_HDMIA: - out->hb0 = 0x87; /* type */ - out->hb1 = 0x01; /* version */ - out->hb2 = 0x1A; /* length */ - out->sb[0] = buf[3]; /* checksum */ - i = 1; - break; - - case DRM_MODE_CONNECTOR_DisplayPort: - case DRM_MODE_CONNECTOR_eDP: - out->hb0 = 0x00; /* sdp id, zero */ - out->hb1 = 0x87; /* type */ - out->hb2 = 0x1D; /* payload len - 1 */ - out->hb3 = (0x13 << 2); /* sdp version */ - out->sb[0] = 0x01; /* version */ - out->sb[1] = 0x1A; /* length */ - i = 2; - break; - - default: - return -EINVAL; - } - - memcpy(&out->sb[i], &buf[4], 26); - out->valid = true; - - print_hex_dump(KERN_DEBUG, "HDR SB:", DUMP_PREFIX_NONE, 16, 1, out->sb, - sizeof(out->sb), false); - - return 0; -} - -static int -amdgpu_dm_connector_atomic_check(struct drm_connector *conn, - struct drm_atomic_state *state) -{ - struct drm_connector_state *new_con_state = - drm_atomic_get_new_connector_state(state, conn); - struct drm_connector_state *old_con_state = - drm_atomic_get_old_connector_state(state, conn); - struct drm_crtc *crtc = new_con_state->crtc; - struct drm_crtc_state *new_crtc_state; - struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn); - int ret; - - trace_amdgpu_dm_connector_atomic_check(new_con_state); - - if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { - ret = drm_dp_mst_root_conn_atomic_check(new_con_state, &aconn->mst_mgr); - if (ret < 0) - return ret; - } - - if (!crtc) - return 0; - - if (new_con_state->colorspace != old_con_state->colorspace) { - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - new_crtc_state->mode_changed = true; - } - - if (new_con_state->content_type != old_con_state->content_type) { - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - new_crtc_state->mode_changed = true; - } - - if (!drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state)) { - struct dc_info_packet hdr_infopacket; - - ret = fill_hdr_info_packet(new_con_state, &hdr_infopacket); - if (ret) - return ret; - - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - /* - * DC considers the stream backends changed if the - * static metadata changes. Forcing the modeset also - * gives a simple way for userspace to switch from - * 8bpc to 10bpc when setting the metadata to enter - * or exit HDR. - * - * Changing the static metadata after it's been - * set is permissible, however. So only force a - * modeset if we're entering or exiting HDR. - */ - new_crtc_state->mode_changed = new_crtc_state->mode_changed || - !old_con_state->hdr_output_metadata || - !new_con_state->hdr_output_metadata; - } - - return 0; -} - -static const struct drm_connector_helper_funcs -amdgpu_dm_connector_helper_funcs = { - /* - * If hotplugging a second bigger display in FB Con mode, bigger resolution - * modes will be filtered by drm_mode_validate_size(), and those modes - * are missing after user start lightdm. So we need to renew modes list. - * in get_modes call back, not just return the modes count - */ - .get_modes = get_modes, - .mode_valid = amdgpu_dm_connector_mode_valid, - .atomic_check = amdgpu_dm_connector_atomic_check, -}; - -static void dm_encoder_helper_disable(struct drm_encoder *encoder) -{ - -} - -int convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth) -{ - switch (display_color_depth) { - case COLOR_DEPTH_666: - return 6; - case COLOR_DEPTH_888: - return 8; - case COLOR_DEPTH_101010: - return 10; - case COLOR_DEPTH_121212: - return 12; - case COLOR_DEPTH_141414: - return 14; - case COLOR_DEPTH_161616: - return 16; - default: - break; - } - return 0; -} - -static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder, - struct drm_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct drm_atomic_state *state = crtc_state->state; - struct drm_connector *connector = conn_state->connector; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_new_connector_state = to_dm_connector_state(conn_state); - const struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode; - struct drm_dp_mst_topology_mgr *mst_mgr; - struct drm_dp_mst_port *mst_port; - struct drm_dp_mst_topology_state *mst_state; - enum dc_color_depth color_depth; - int clock, bpp = 0; - bool is_y420 = false; - - if (!aconnector->mst_output_port) - return 0; - - mst_port = aconnector->mst_output_port; - mst_mgr = &aconnector->mst_root->mst_mgr; - - if (!crtc_state->connectors_changed && !crtc_state->mode_changed) - return 0; - - mst_state = drm_atomic_get_mst_topology_state(state, mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - mst_state->pbn_div.full = dfixed_const(dm_mst_get_pbn_divider(aconnector->mst_root->dc_link)); - - if (!state->duplicated) { - int max_bpc = conn_state->max_requested_bpc; - - is_y420 = drm_mode_is_420_also(&connector->display_info, adjusted_mode) && - aconnector->force_yuv420_output; - color_depth = convert_color_depth_from_display_info(connector, - is_y420, - max_bpc); - bpp = convert_dc_color_depth_into_bpc(color_depth) * 3; - clock = adjusted_mode->clock; - dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp << 4); - } - - dm_new_connector_state->vcpi_slots = - drm_dp_atomic_find_time_slots(state, mst_mgr, mst_port, - dm_new_connector_state->pbn); - if (dm_new_connector_state->vcpi_slots < 0) { - DRM_DEBUG_ATOMIC("failed finding vcpi slots: %d\n", (int)dm_new_connector_state->vcpi_slots); - return dm_new_connector_state->vcpi_slots; - } - return 0; -} - -const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = { - .disable = dm_encoder_helper_disable, - .atomic_check = dm_encoder_helper_atomic_check -}; - -static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state, - struct dc_state *dc_state, - struct dsc_mst_fairness_vars *vars) -{ - struct dc_stream_state *stream = NULL; - struct drm_connector *connector; - struct drm_connector_state *new_con_state; - struct amdgpu_dm_connector *aconnector; - struct dm_connector_state *dm_conn_state; - int i, j, ret; - int vcpi, pbn_div, pbn = 0, slot_num = 0; - - for_each_new_connector_in_state(state, connector, new_con_state, i) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->mst_output_port) - continue; - - if (!new_con_state || !new_con_state->crtc) - continue; - - dm_conn_state = to_dm_connector_state(new_con_state); - - for (j = 0; j < dc_state->stream_count; j++) { - stream = dc_state->streams[j]; - if (!stream) - continue; - - if ((struct amdgpu_dm_connector *)stream->dm_stream_context == aconnector) - break; - - stream = NULL; - } - - if (!stream) - continue; - - pbn_div = dm_mst_get_pbn_divider(stream->link); - /* pbn is calculated by compute_mst_dsc_configs_for_state*/ - for (j = 0; j < dc_state->stream_count; j++) { - if (vars[j].aconnector == aconnector) { - pbn = vars[j].pbn; - break; - } - } - - if (j == dc_state->stream_count || pbn_div == 0) - continue; - - slot_num = DIV_ROUND_UP(pbn, pbn_div); - - if (stream->timing.flags.DSC != 1) { - dm_conn_state->pbn = pbn; - dm_conn_state->vcpi_slots = slot_num; - - ret = drm_dp_mst_atomic_enable_dsc(state, aconnector->mst_output_port, - dm_conn_state->pbn, false); - if (ret < 0) - return ret; - - continue; - } - - vcpi = drm_dp_mst_atomic_enable_dsc(state, aconnector->mst_output_port, pbn, true); - if (vcpi < 0) - return vcpi; - - dm_conn_state->pbn = pbn; - dm_conn_state->vcpi_slots = vcpi; - } - return 0; -} - -static int to_drm_connector_type(enum signal_type st) -{ - switch (st) { - case SIGNAL_TYPE_HDMI_TYPE_A: - return DRM_MODE_CONNECTOR_HDMIA; - case SIGNAL_TYPE_EDP: - return DRM_MODE_CONNECTOR_eDP; - case SIGNAL_TYPE_LVDS: - return DRM_MODE_CONNECTOR_LVDS; - case SIGNAL_TYPE_RGB: - return DRM_MODE_CONNECTOR_VGA; - case SIGNAL_TYPE_DISPLAY_PORT: - case SIGNAL_TYPE_DISPLAY_PORT_MST: - return DRM_MODE_CONNECTOR_DisplayPort; - case SIGNAL_TYPE_DVI_DUAL_LINK: - case SIGNAL_TYPE_DVI_SINGLE_LINK: - return DRM_MODE_CONNECTOR_DVID; - case SIGNAL_TYPE_VIRTUAL: - return DRM_MODE_CONNECTOR_VIRTUAL; - - default: - return DRM_MODE_CONNECTOR_Unknown; - } -} - -static struct drm_encoder *amdgpu_dm_connector_to_encoder(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - - /* There is only one encoder per connector */ - drm_connector_for_each_possible_encoder(connector, encoder) - return encoder; - - return NULL; -} - -static void amdgpu_dm_get_native_mode(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - struct amdgpu_encoder *amdgpu_encoder; - - encoder = amdgpu_dm_connector_to_encoder(connector); - - if (encoder == NULL) - return; - - amdgpu_encoder = to_amdgpu_encoder(encoder); - - amdgpu_encoder->native_mode.clock = 0; - - if (!list_empty(&connector->probed_modes)) { - struct drm_display_mode *preferred_mode = NULL; - - list_for_each_entry(preferred_mode, - &connector->probed_modes, - head) { - if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) - amdgpu_encoder->native_mode = *preferred_mode; - - break; - } - - } -} - -static struct drm_display_mode * -amdgpu_dm_create_common_mode(struct drm_encoder *encoder, - char *name, - int hdisplay, int vdisplay) -{ - struct drm_device *dev = encoder->dev; - struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); - struct drm_display_mode *mode = NULL; - struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; - - mode = drm_mode_duplicate(dev, native_mode); - - if (mode == NULL) - return NULL; - - mode->hdisplay = hdisplay; - mode->vdisplay = vdisplay; - mode->type &= ~DRM_MODE_TYPE_PREFERRED; - strscpy(mode->name, name, DRM_DISPLAY_MODE_LEN); - - return mode; - -} - -static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder, - struct drm_connector *connector) -{ - struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); - struct drm_display_mode *mode = NULL; - struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - int i; - int n; - struct mode_size { - char name[DRM_DISPLAY_MODE_LEN]; - int w; - int h; - } common_modes[] = { - { "640x480", 640, 480}, - { "800x600", 800, 600}, - { "1024x768", 1024, 768}, - { "1280x720", 1280, 720}, - { "1280x800", 1280, 800}, - {"1280x1024", 1280, 1024}, - { "1440x900", 1440, 900}, - {"1680x1050", 1680, 1050}, - {"1600x1200", 1600, 1200}, - {"1920x1080", 1920, 1080}, - {"1920x1200", 1920, 1200} - }; - - n = ARRAY_SIZE(common_modes); - - for (i = 0; i < n; i++) { - struct drm_display_mode *curmode = NULL; - bool mode_existed = false; - - if (common_modes[i].w > native_mode->hdisplay || - common_modes[i].h > native_mode->vdisplay || - (common_modes[i].w == native_mode->hdisplay && - common_modes[i].h == native_mode->vdisplay)) - continue; - - list_for_each_entry(curmode, &connector->probed_modes, head) { - if (common_modes[i].w == curmode->hdisplay && - common_modes[i].h == curmode->vdisplay) { - mode_existed = true; - break; - } - } - - if (mode_existed) - continue; - - mode = amdgpu_dm_create_common_mode(encoder, - common_modes[i].name, common_modes[i].w, - common_modes[i].h); - if (!mode) - continue; - - drm_mode_probed_add(connector, mode); - amdgpu_dm_connector->num_modes++; - } -} - -static void amdgpu_set_panel_orientation(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - struct amdgpu_encoder *amdgpu_encoder; - const struct drm_display_mode *native_mode; - - if (connector->connector_type != DRM_MODE_CONNECTOR_eDP && - connector->connector_type != DRM_MODE_CONNECTOR_LVDS) - return; - - mutex_lock(&connector->dev->mode_config.mutex); - amdgpu_dm_connector_get_modes(connector); - mutex_unlock(&connector->dev->mode_config.mutex); - - encoder = amdgpu_dm_connector_to_encoder(connector); - if (!encoder) - return; - - amdgpu_encoder = to_amdgpu_encoder(encoder); - - native_mode = &amdgpu_encoder->native_mode; - if (native_mode->hdisplay == 0 || native_mode->vdisplay == 0) - return; - - drm_connector_set_panel_orientation_with_quirk(connector, - DRM_MODE_PANEL_ORIENTATION_UNKNOWN, - native_mode->hdisplay, - native_mode->vdisplay); -} - -static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector, - struct edid *edid) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - - if (edid) { - /* empty probed_modes */ - INIT_LIST_HEAD(&connector->probed_modes); - amdgpu_dm_connector->num_modes = - drm_add_edid_modes(connector, edid); - - /* sorting the probed modes before calling function - * amdgpu_dm_get_native_mode() since EDID can have - * more than one preferred mode. The modes that are - * later in the probed mode list could be of higher - * and preferred resolution. For example, 3840x2160 - * resolution in base EDID preferred timing and 4096x2160 - * preferred resolution in DID extension block later. - */ - drm_mode_sort(&connector->probed_modes); - amdgpu_dm_get_native_mode(connector); - - /* Freesync capabilities are reset by calling - * drm_add_edid_modes() and need to be - * restored here. - */ - amdgpu_dm_update_freesync_caps(connector, edid); - } else { - amdgpu_dm_connector->num_modes = 0; - } -} - -static bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector, - struct drm_display_mode *mode) -{ - struct drm_display_mode *m; - - list_for_each_entry(m, &aconnector->base.probed_modes, head) { - if (drm_mode_equal(m, mode)) - return true; - } - - return false; -} - -static uint add_fs_modes(struct amdgpu_dm_connector *aconnector) -{ - const struct drm_display_mode *m; - struct drm_display_mode *new_mode; - uint i; - u32 new_modes_count = 0; - - /* Standard FPS values - * - * 23.976 - TV/NTSC - * 24 - Cinema - * 25 - TV/PAL - * 29.97 - TV/NTSC - * 30 - TV/NTSC - * 48 - Cinema HFR - * 50 - TV/PAL - * 60 - Commonly used - * 48,72,96,120 - Multiples of 24 - */ - static const u32 common_rates[] = { - 23976, 24000, 25000, 29970, 30000, - 48000, 50000, 60000, 72000, 96000, 120000 - }; - - /* - * Find mode with highest refresh rate with the same resolution - * as the preferred mode. Some monitors report a preferred mode - * with lower resolution than the highest refresh rate supported. - */ - - m = get_highest_refresh_rate_mode(aconnector, true); - if (!m) - return 0; - - for (i = 0; i < ARRAY_SIZE(common_rates); i++) { - u64 target_vtotal, target_vtotal_diff; - u64 num, den; - - if (drm_mode_vrefresh(m) * 1000 < common_rates[i]) - continue; - - if (common_rates[i] < aconnector->min_vfreq * 1000 || - common_rates[i] > aconnector->max_vfreq * 1000) - continue; - - num = (unsigned long long)m->clock * 1000 * 1000; - den = common_rates[i] * (unsigned long long)m->htotal; - target_vtotal = div_u64(num, den); - target_vtotal_diff = target_vtotal - m->vtotal; - - /* Check for illegal modes */ - if (m->vsync_start + target_vtotal_diff < m->vdisplay || - m->vsync_end + target_vtotal_diff < m->vsync_start || - m->vtotal + target_vtotal_diff < m->vsync_end) - continue; - - new_mode = drm_mode_duplicate(aconnector->base.dev, m); - if (!new_mode) - goto out; - - new_mode->vtotal += (u16)target_vtotal_diff; - new_mode->vsync_start += (u16)target_vtotal_diff; - new_mode->vsync_end += (u16)target_vtotal_diff; - new_mode->type &= ~DRM_MODE_TYPE_PREFERRED; - new_mode->type |= DRM_MODE_TYPE_DRIVER; - - if (!is_duplicate_mode(aconnector, new_mode)) { - drm_mode_probed_add(&aconnector->base, new_mode); - new_modes_count += 1; - } else - drm_mode_destroy(aconnector->base.dev, new_mode); - } - out: - return new_modes_count; -} - -static void amdgpu_dm_connector_add_freesync_modes(struct drm_connector *connector, - struct edid *edid) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - - if (!(amdgpu_freesync_vid_mode && edid)) - return; - - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - amdgpu_dm_connector->num_modes += - add_fs_modes(amdgpu_dm_connector); -} - -static int amdgpu_dm_connector_get_modes(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - struct drm_encoder *encoder; - struct edid *edid = amdgpu_dm_connector->edid; - struct dc_link_settings *verified_link_cap = - &amdgpu_dm_connector->dc_link->verified_link_cap; - const struct dc *dc = amdgpu_dm_connector->dc_link->dc; - - encoder = amdgpu_dm_connector_to_encoder(connector); - - if (!drm_edid_is_valid(edid)) { - amdgpu_dm_connector->num_modes = - drm_add_modes_noedid(connector, 640, 480); - if (dc->link_srv->dp_get_encoding_format(verified_link_cap) == DP_128b_132b_ENCODING) - amdgpu_dm_connector->num_modes += - drm_add_modes_noedid(connector, 1920, 1080); - } else { - amdgpu_dm_connector_ddc_get_modes(connector, edid); - if (encoder) - amdgpu_dm_connector_add_common_modes(encoder, connector); - amdgpu_dm_connector_add_freesync_modes(connector, edid); - } - amdgpu_dm_fbc_init(connector); - - return amdgpu_dm_connector->num_modes; -} - -static const u32 supported_colorspaces = - BIT(DRM_MODE_COLORIMETRY_BT709_YCC) | - BIT(DRM_MODE_COLORIMETRY_OPRGB) | - BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) | - BIT(DRM_MODE_COLORIMETRY_BT2020_YCC); - -void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector, - int connector_type, - struct dc_link *link, - int link_index) -{ - struct amdgpu_device *adev = drm_to_adev(dm->ddev); - - /* - * Some of the properties below require access to state, like bpc. - * Allocate some default initial connector state with our reset helper. - */ - if (aconnector->base.funcs->reset) - aconnector->base.funcs->reset(&aconnector->base); - - aconnector->connector_id = link_index; - aconnector->bl_idx = -1; - aconnector->dc_link = link; - aconnector->base.interlace_allowed = false; - aconnector->base.doublescan_allowed = false; - aconnector->base.stereo_allowed = false; - aconnector->base.dpms = DRM_MODE_DPMS_OFF; - aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */ - aconnector->audio_inst = -1; - aconnector->pack_sdp_v1_3 = false; - aconnector->as_type = ADAPTIVE_SYNC_TYPE_NONE; - memset(&aconnector->vsdb_info, 0, sizeof(aconnector->vsdb_info)); - mutex_init(&aconnector->hpd_lock); - mutex_init(&aconnector->handle_mst_msg_ready); - - /* - * configure support HPD hot plug connector_>polled default value is 0 - * which means HPD hot plug not supported - */ - switch (connector_type) { - case DRM_MODE_CONNECTOR_HDMIA: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - aconnector->base.ycbcr_420_allowed = - link->link_enc->features.hdmi_ycbcr420_supported ? true : false; - break; - case DRM_MODE_CONNECTOR_DisplayPort: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - link->link_enc = link_enc_cfg_get_link_enc(link); - ASSERT(link->link_enc); - if (link->link_enc) - aconnector->base.ycbcr_420_allowed = - link->link_enc->features.dp_ycbcr420_supported ? true : false; - break; - case DRM_MODE_CONNECTOR_DVID: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - break; - default: - break; - } - - drm_object_attach_property(&aconnector->base.base, - dm->ddev->mode_config.scaling_mode_property, - DRM_MODE_SCALE_NONE); - - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_property, - UNDERSCAN_OFF); - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_hborder_property, - 0); - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_vborder_property, - 0); - - if (!aconnector->mst_root) - drm_connector_attach_max_bpc_property(&aconnector->base, 8, 16); - - aconnector->base.state->max_bpc = 16; - aconnector->base.state->max_requested_bpc = aconnector->base.state->max_bpc; - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA) { - /* Content Type is currently only implemented for HDMI. */ - drm_connector_attach_content_type_property(&aconnector->base); - } - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA) { - if (!drm_mode_create_hdmi_colorspace_property(&aconnector->base, supported_colorspaces)) - drm_connector_attach_colorspace_property(&aconnector->base); - } else if ((connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root) || - connector_type == DRM_MODE_CONNECTOR_eDP) { - if (!drm_mode_create_dp_colorspace_property(&aconnector->base, supported_colorspaces)) - drm_connector_attach_colorspace_property(&aconnector->base); - } - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA || - connector_type == DRM_MODE_CONNECTOR_DisplayPort || - connector_type == DRM_MODE_CONNECTOR_eDP) { - drm_connector_attach_hdr_output_metadata_property(&aconnector->base); - - if (!aconnector->mst_root) - drm_connector_attach_vrr_capable_property(&aconnector->base); - - if (adev->dm.hdcp_workqueue) - drm_connector_attach_content_protection_property(&aconnector->base, true); - } -} - -static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msgs, int num) -{ - struct amdgpu_i2c_adapter *i2c = i2c_get_adapdata(i2c_adap); - struct ddc_service *ddc_service = i2c->ddc_service; - struct i2c_command cmd; - int i; - int result = -EIO; - - if (!ddc_service->ddc_pin || !ddc_service->ddc_pin->hw_info.hw_supported) - return result; - - cmd.payloads = kcalloc(num, sizeof(struct i2c_payload), GFP_KERNEL); - - if (!cmd.payloads) - return result; - - cmd.number_of_payloads = num; - cmd.engine = I2C_COMMAND_ENGINE_DEFAULT; - cmd.speed = 100; - - for (i = 0; i < num; i++) { - cmd.payloads[i].write = !(msgs[i].flags & I2C_M_RD); - cmd.payloads[i].address = msgs[i].addr; - cmd.payloads[i].length = msgs[i].len; - cmd.payloads[i].data = msgs[i].buf; - } - - if (dc_submit_i2c( - ddc_service->ctx->dc, - ddc_service->link->link_index, - &cmd)) - result = num; - - kfree(cmd.payloads); - return result; -} - -static u32 amdgpu_dm_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm amdgpu_dm_i2c_algo = { - .master_xfer = amdgpu_dm_i2c_xfer, - .functionality = amdgpu_dm_i2c_func, -}; - -static struct amdgpu_i2c_adapter * -create_i2c(struct ddc_service *ddc_service, - int link_index, - int *res) -{ - struct amdgpu_device *adev = ddc_service->ctx->driver_context; - struct amdgpu_i2c_adapter *i2c; - - i2c = kzalloc(sizeof(struct amdgpu_i2c_adapter), GFP_KERNEL); - if (!i2c) - return NULL; - i2c->base.owner = THIS_MODULE; - i2c->base.dev.parent = &adev->pdev->dev; - i2c->base.algo = &amdgpu_dm_i2c_algo; - snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c hw bus %d", link_index); - i2c_set_adapdata(&i2c->base, i2c); - i2c->ddc_service = ddc_service; - - return i2c; -} - - -/* - * Note: this function assumes that dc_link_detect() was called for the - * dc_link which will be represented by this aconnector. - */ -static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector, - u32 link_index, - struct amdgpu_encoder *aencoder) -{ - int res = 0; - int connector_type; - struct dc *dc = dm->dc; - struct dc_link *link = dc_get_link_at_index(dc, link_index); - struct amdgpu_i2c_adapter *i2c; - - /* Not needed for writeback connector */ - link->priv = aconnector; - - - i2c = create_i2c(link->ddc, link->link_index, &res); - if (!i2c) { - DRM_ERROR("Failed to create i2c adapter data\n"); - return -ENOMEM; - } - - aconnector->i2c = i2c; - res = i2c_add_adapter(&i2c->base); - - if (res) { - DRM_ERROR("Failed to register hw i2c %d\n", link->link_index); - goto out_free; - } - - connector_type = to_drm_connector_type(link->connector_signal); - - res = drm_connector_init_with_ddc( - dm->ddev, - &aconnector->base, - &amdgpu_dm_connector_funcs, - connector_type, - &i2c->base); - - if (res) { - DRM_ERROR("connector_init failed\n"); - aconnector->connector_id = -1; - goto out_free; - } - - drm_connector_helper_add( - &aconnector->base, - &amdgpu_dm_connector_helper_funcs); - - amdgpu_dm_connector_init_helper( - dm, - aconnector, - connector_type, - link, - link_index); - - drm_connector_attach_encoder( - &aconnector->base, &aencoder->base); - - if (connector_type == DRM_MODE_CONNECTOR_DisplayPort - || connector_type == DRM_MODE_CONNECTOR_eDP) - amdgpu_dm_initialize_dp_connector(dm, aconnector, link->link_index); - -out_free: - if (res) { - kfree(i2c); - aconnector->i2c = NULL; - } - return res; -} - -int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev) -{ - switch (adev->mode_info.num_crtc) { - case 1: - return 0x1; - case 2: - return 0x3; - case 3: - return 0x7; - case 4: - return 0xf; - case 5: - return 0x1f; - case 6: - default: - return 0x3f; - } -} - -static int amdgpu_dm_encoder_init(struct drm_device *dev, - struct amdgpu_encoder *aencoder, - uint32_t link_index) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - - int res = drm_encoder_init(dev, - &aencoder->base, - &amdgpu_dm_encoder_funcs, - DRM_MODE_ENCODER_TMDS, - NULL); - - aencoder->base.possible_crtcs = amdgpu_dm_get_encoder_crtc_mask(adev); - - if (!res) - aencoder->encoder_id = link_index; - else - aencoder->encoder_id = -1; - - drm_encoder_helper_add(&aencoder->base, &amdgpu_dm_encoder_helper_funcs); - - return res; -} - -static void manage_dm_interrupts(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc, - struct dm_crtc_state *acrtc_state) -{ - /* - * We have no guarantee that the frontend index maps to the same - * backend index - some even map to more than one. - * - * TODO: Use a different interrupt or check DC itself for the mapping. - */ - int irq_type = - amdgpu_display_crtc_idx_to_irq_type( - adev, - acrtc->crtc_id); - struct drm_vblank_crtc_config config = {0}; - struct dc_crtc_timing *timing; - int offdelay; - - if (acrtc_state) { - if (amdgpu_ip_version(adev, DCE_HWIP, 0) < - IP_VERSION(3, 5, 0) || - acrtc_state->stream->link->psr_settings.psr_version < - DC_PSR_VERSION_UNSUPPORTED) { - timing = &acrtc_state->stream->timing; - - /* at least 2 frames */ - offdelay = DIV64_U64_ROUND_UP((u64)20 * - timing->v_total * - timing->h_total, - timing->pix_clk_100hz); - - config.offdelay_ms = offdelay ?: 30; - } else { - config.disable_immediate = true; - } - - drm_crtc_vblank_on_config(&acrtc->base, - &config); - - amdgpu_irq_get( - adev, - &adev->pageflip_irq, - irq_type); -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - amdgpu_irq_get( - adev, - &adev->vline0_irq, - irq_type); -#endif - } else { -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - amdgpu_irq_put( - adev, - &adev->vline0_irq, - irq_type); -#endif - amdgpu_irq_put( - adev, - &adev->pageflip_irq, - irq_type); - drm_crtc_vblank_off(&acrtc->base); - } -} - -static void dm_update_pflip_irq_state(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc) -{ - int irq_type = - amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); - - /** - * This reads the current state for the IRQ and force reapplies - * the setting to hardware. - */ - amdgpu_irq_update(adev, &adev->pageflip_irq, irq_type); -} - -static bool -is_scaling_state_different(const struct dm_connector_state *dm_state, - const struct dm_connector_state *old_dm_state) -{ - if (dm_state->scaling != old_dm_state->scaling) - return true; - if (!dm_state->underscan_enable && old_dm_state->underscan_enable) { - if (old_dm_state->underscan_hborder != 0 && old_dm_state->underscan_vborder != 0) - return true; - } else if (dm_state->underscan_enable && !old_dm_state->underscan_enable) { - if (dm_state->underscan_hborder != 0 && dm_state->underscan_vborder != 0) - return true; - } else if (dm_state->underscan_hborder != old_dm_state->underscan_hborder || - dm_state->underscan_vborder != old_dm_state->underscan_vborder) - return true; - return false; -} - -static bool is_content_protection_different(struct drm_crtc_state *new_crtc_state, - struct drm_crtc_state *old_crtc_state, - struct drm_connector_state *new_conn_state, - struct drm_connector_state *old_conn_state, - const struct drm_connector *connector, - struct hdcp_workqueue *hdcp_w) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); - - pr_debug("[HDCP_DM] connector->index: %x connect_status: %x dpms: %x\n", - connector->index, connector->status, connector->dpms); - pr_debug("[HDCP_DM] state protection old: %x new: %x\n", - old_conn_state->content_protection, new_conn_state->content_protection); - - if (old_crtc_state) - pr_debug("[HDCP_DM] old crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - old_crtc_state->enable, - old_crtc_state->active, - old_crtc_state->mode_changed, - old_crtc_state->active_changed, - old_crtc_state->connectors_changed); - - if (new_crtc_state) - pr_debug("[HDCP_DM] NEW crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* hdcp content type change */ - if (old_conn_state->hdcp_content_type != new_conn_state->hdcp_content_type && - new_conn_state->content_protection != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - pr_debug("[HDCP_DM] Type0/1 change %s :true\n", __func__); - return true; - } - - /* CP is being re enabled, ignore this */ - if (old_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) { - if (new_crtc_state && new_crtc_state->mode_changed) { - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - pr_debug("[HDCP_DM] ENABLED->DESIRED & mode_changed %s :true\n", __func__); - return true; - } - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_ENABLED; - pr_debug("[HDCP_DM] ENABLED -> DESIRED %s :false\n", __func__); - return false; - } - - /* S3 resume case, since old state will always be 0 (UNDESIRED) and the restored state will be ENABLED - * - * Handles: UNDESIRED -> ENABLED - */ - if (old_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_UNDESIRED && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - - /* Stream removed and re-enabled - * - * Can sometimes overlap with the HPD case, - * thus set update_hdcp to false to avoid - * setting HDCP multiple times. - * - * Handles: DESIRED -> DESIRED (Special case) - */ - if (!(old_conn_state->crtc && old_conn_state->crtc->enabled) && - new_conn_state->crtc && new_conn_state->crtc->enabled && - connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) { - dm_con_state->update_hdcp = false; - pr_debug("[HDCP_DM] DESIRED->DESIRED (Stream removed and re-enabled) %s :true\n", - __func__); - return true; - } - - /* Hot-plug, headless s3, dpms - * - * Only start HDCP if the display is connected/enabled. - * update_hdcp flag will be set to false until the next - * HPD comes in. - * - * Handles: DESIRED -> DESIRED (Special case) - */ - if (dm_con_state->update_hdcp && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED && - connector->dpms == DRM_MODE_DPMS_ON && aconnector->dc_sink != NULL) { - dm_con_state->update_hdcp = false; - pr_debug("[HDCP_DM] DESIRED->DESIRED (Hot-plug, headless s3, dpms) %s :true\n", - __func__); - return true; - } - - if (old_conn_state->content_protection == new_conn_state->content_protection) { - if (new_conn_state->content_protection >= DRM_MODE_CONTENT_PROTECTION_DESIRED) { - if (new_crtc_state && new_crtc_state->mode_changed) { - pr_debug("[HDCP_DM] DESIRED->DESIRED or ENABLE->ENABLE mode_change %s :true\n", - __func__); - return true; - } - pr_debug("[HDCP_DM] DESIRED->DESIRED & ENABLE->ENABLE %s :false\n", - __func__); - return false; - } - - pr_debug("[HDCP_DM] UNDESIRED->UNDESIRED %s :false\n", __func__); - return false; - } - - if (new_conn_state->content_protection != DRM_MODE_CONTENT_PROTECTION_ENABLED) { - pr_debug("[HDCP_DM] UNDESIRED->DESIRED or DESIRED->UNDESIRED or ENABLED->UNDESIRED %s :true\n", - __func__); - return true; - } - - pr_debug("[HDCP_DM] DESIRED->ENABLED %s :false\n", __func__); - return false; -} - -static void remove_stream(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc, - struct dc_stream_state *stream) -{ - /* this is the update mode case */ - - acrtc->otg_inst = -1; - acrtc->enabled = false; -} - -static void prepare_flip_isr(struct amdgpu_crtc *acrtc) -{ - - assert_spin_locked(&acrtc->base.dev->event_lock); - WARN_ON(acrtc->event); - - acrtc->event = acrtc->base.state->event; - - /* Set the flip status */ - acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED; - - /* Mark this event as consumed */ - acrtc->base.state->event = NULL; - - drm_dbg_state(acrtc->base.dev, - "crtc:%d, pflip_stat:AMDGPU_FLIP_SUBMITTED\n", - acrtc->crtc_id); -} - -static void update_freesync_state_on_stream( - struct amdgpu_display_manager *dm, - struct dm_crtc_state *new_crtc_state, - struct dc_stream_state *new_stream, - struct dc_plane_state *surface, - u32 flip_timestamp_in_us) -{ - struct mod_vrr_params vrr_params; - struct dc_info_packet vrr_infopacket = {0}; - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(new_crtc_state->base.crtc); - unsigned long flags; - bool pack_sdp_v1_3 = false; - struct amdgpu_dm_connector *aconn; - enum vrr_packet_type packet_type = PACKET_TYPE_VRR; - - if (!new_stream) - return; - - /* - * TODO: Determine why min/max totals and vrefresh can be 0 here. - * For now it's sufficient to just guard against these conditions. - */ - - if (!new_stream->timing.h_total || !new_stream->timing.v_total) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - vrr_params = acrtc->dm_irq_params.vrr_params; - - if (surface) { - mod_freesync_handle_preflip( - dm->freesync_module, - surface, - new_stream, - flip_timestamp_in_us, - &vrr_params); - - if (adev->family < AMDGPU_FAMILY_AI && - amdgpu_dm_crtc_vrr_active(new_crtc_state)) { - mod_freesync_handle_v_update(dm->freesync_module, - new_stream, &vrr_params); - - /* Need to call this before the frame ends. */ - dc_stream_adjust_vmin_vmax(dm->dc, - new_crtc_state->stream, - &vrr_params.adjust); - } - } - - aconn = (struct amdgpu_dm_connector *)new_stream->dm_stream_context; - - if (aconn && (aconn->as_type == FREESYNC_TYPE_PCON_IN_WHITELIST || aconn->vsdb_info.replay_mode)) { - pack_sdp_v1_3 = aconn->pack_sdp_v1_3; - - if (aconn->vsdb_info.amd_vsdb_version == 1) - packet_type = PACKET_TYPE_FS_V1; - else if (aconn->vsdb_info.amd_vsdb_version == 2) - packet_type = PACKET_TYPE_FS_V2; - else if (aconn->vsdb_info.amd_vsdb_version == 3) - packet_type = PACKET_TYPE_FS_V3; - - mod_build_adaptive_sync_infopacket(new_stream, aconn->as_type, NULL, - &new_stream->adaptive_sync_infopacket); - } - - mod_freesync_build_vrr_infopacket( - dm->freesync_module, - new_stream, - &vrr_params, - packet_type, - TRANSFER_FUNC_UNKNOWN, - &vrr_infopacket, - pack_sdp_v1_3); - - new_crtc_state->freesync_vrr_info_changed |= - (memcmp(&new_crtc_state->vrr_infopacket, - &vrr_infopacket, - sizeof(vrr_infopacket)) != 0); - - acrtc->dm_irq_params.vrr_params = vrr_params; - new_crtc_state->vrr_infopacket = vrr_infopacket; - - new_stream->vrr_infopacket = vrr_infopacket; - new_stream->allow_freesync = mod_freesync_get_freesync_enabled(&vrr_params); - - if (new_crtc_state->freesync_vrr_info_changed) - DRM_DEBUG_KMS("VRR packet update: crtc=%u enabled=%d state=%d", - new_crtc_state->base.crtc->base.id, - (int)new_crtc_state->base.vrr_enabled, - (int)vrr_params.state); - - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -static void update_stream_irq_parameters( - struct amdgpu_display_manager *dm, - struct dm_crtc_state *new_crtc_state) -{ - struct dc_stream_state *new_stream = new_crtc_state->stream; - struct mod_vrr_params vrr_params; - struct mod_freesync_config config = new_crtc_state->freesync_config; - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(new_crtc_state->base.crtc); - unsigned long flags; - - if (!new_stream) - return; - - /* - * TODO: Determine why min/max totals and vrefresh can be 0 here. - * For now it's sufficient to just guard against these conditions. - */ - if (!new_stream->timing.h_total || !new_stream->timing.v_total) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - vrr_params = acrtc->dm_irq_params.vrr_params; - - if (new_crtc_state->vrr_supported && - config.min_refresh_in_uhz && - config.max_refresh_in_uhz) { - /* - * if freesync compatible mode was set, config.state will be set - * in atomic check - */ - if (config.state == VRR_STATE_ACTIVE_FIXED && config.fixed_refresh_in_uhz && - (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) || - new_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED)) { - vrr_params.max_refresh_in_uhz = config.max_refresh_in_uhz; - vrr_params.min_refresh_in_uhz = config.min_refresh_in_uhz; - vrr_params.fixed_refresh_in_uhz = config.fixed_refresh_in_uhz; - vrr_params.state = VRR_STATE_ACTIVE_FIXED; - } else { - config.state = new_crtc_state->base.vrr_enabled ? - VRR_STATE_ACTIVE_VARIABLE : - VRR_STATE_INACTIVE; - } - } else { - config.state = VRR_STATE_UNSUPPORTED; - } - - mod_freesync_build_vrr_params(dm->freesync_module, - new_stream, - &config, &vrr_params); - - new_crtc_state->freesync_config = config; - /* Copy state for access from DM IRQ handler */ - acrtc->dm_irq_params.freesync_config = config; - acrtc->dm_irq_params.active_planes = new_crtc_state->active_planes; - acrtc->dm_irq_params.vrr_params = vrr_params; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, - struct dm_crtc_state *new_state) -{ - bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state); - bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state); - - if (!old_vrr_active && new_vrr_active) { - /* Transition VRR inactive -> active: - * While VRR is active, we must not disable vblank irq, as a - * reenable after disable would compute bogus vblank/pflip - * timestamps if it likely happened inside display front-porch. - * - * We also need vupdate irq for the actual core vblank handling - * at end of vblank. - */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); - WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0); - DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n", - __func__, new_state->base.crtc->base.id); - } else if (old_vrr_active && !new_vrr_active) { - /* Transition VRR active -> inactive: - * Allow vblank irq disable again for fixed refresh rate. - */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); - drm_crtc_vblank_put(new_state->base.crtc); - DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n", - __func__, new_state->base.crtc->base.id); - } -} - -static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state) -{ - struct drm_plane *plane; - struct drm_plane_state *old_plane_state; - int i; - - /* - * TODO: Make this per-stream so we don't issue redundant updates for - * commits with multiple streams. - */ - for_each_old_plane_in_state(state, plane, old_plane_state, i) - if (plane->type == DRM_PLANE_TYPE_CURSOR) - amdgpu_dm_plane_handle_cursor_update(plane, old_plane_state); -} - -static inline uint32_t get_mem_type(struct drm_framebuffer *fb) -{ - struct amdgpu_bo *abo = gem_to_amdgpu_bo(fb->obj[0]); - - return abo->tbo.resource ? abo->tbo.resource->mem_type : 0; -} - -static void amdgpu_dm_update_cursor(struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct dc_stream_update *update) -{ - struct amdgpu_device *adev = drm_to_adev(plane->dev); - struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(plane->state->fb); - struct drm_crtc *crtc = afb ? plane->state->crtc : old_plane_state->crtc; - struct dm_crtc_state *crtc_state = crtc ? to_dm_crtc_state(crtc->state) : NULL; - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - uint64_t address = afb ? afb->address : 0; - struct dc_cursor_position position = {0}; - struct dc_cursor_attributes attributes; - int ret; - - if (!plane->state->fb && !old_plane_state->fb) - return; - - drm_dbg_atomic(plane->dev, "crtc_id=%d with size %d to %d\n", - amdgpu_crtc->crtc_id, plane->state->crtc_w, - plane->state->crtc_h); - - ret = amdgpu_dm_plane_get_cursor_position(plane, crtc, &position); - if (ret) - return; - - if (!position.enable) { - /* turn off cursor */ - if (crtc_state && crtc_state->stream) { - dc_stream_set_cursor_position(crtc_state->stream, - &position); - update->cursor_position = &crtc_state->stream->cursor_position; - } - return; - } - - amdgpu_crtc->cursor_width = plane->state->crtc_w; - amdgpu_crtc->cursor_height = plane->state->crtc_h; - - memset(&attributes, 0, sizeof(attributes)); - attributes.address.high_part = upper_32_bits(address); - attributes.address.low_part = lower_32_bits(address); - attributes.width = plane->state->crtc_w; - attributes.height = plane->state->crtc_h; - attributes.color_format = CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA; - attributes.rotation_angle = 0; - attributes.attribute_flags.value = 0; - - /* Enable cursor degamma ROM on DCN3+ for implicit sRGB degamma in DRM - * legacy gamma setup. - */ - if (crtc_state->cm_is_degamma_srgb && - adev->dm.dc->caps.color.dpp.gamma_corr) - attributes.attribute_flags.bits.ENABLE_CURSOR_DEGAMMA = 1; - - if (afb) - attributes.pitch = afb->base.pitches[0] / afb->base.format->cpp[0]; - - if (crtc_state->stream) { - if (!dc_stream_set_cursor_attributes(crtc_state->stream, - &attributes)) - DRM_ERROR("DC failed to set cursor attributes\n"); - - update->cursor_attributes = &crtc_state->stream->cursor_attributes; - - if (!dc_stream_set_cursor_position(crtc_state->stream, - &position)) - DRM_ERROR("DC failed to set cursor position\n"); - - update->cursor_position = &crtc_state->stream->cursor_position; - } -} - -static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, - struct drm_device *dev, - struct amdgpu_display_manager *dm, - struct drm_crtc *pcrtc, - bool wait_for_vblank) -{ - u32 i; - u64 timestamp_ns = ktime_get_ns(); - struct drm_plane *plane; - struct drm_plane_state *old_plane_state, *new_plane_state; - struct amdgpu_crtc *acrtc_attach = to_amdgpu_crtc(pcrtc); - struct drm_crtc_state *new_pcrtc_state = - drm_atomic_get_new_crtc_state(state, pcrtc); - struct dm_crtc_state *acrtc_state = to_dm_crtc_state(new_pcrtc_state); - struct dm_crtc_state *dm_old_crtc_state = - to_dm_crtc_state(drm_atomic_get_old_crtc_state(state, pcrtc)); - int planes_count = 0, vpos, hpos; - unsigned long flags; - u32 target_vblank, last_flip_vblank; - bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state); - bool cursor_update = false; - bool pflip_present = false; - bool dirty_rects_changed = false; - bool updated_planes_and_streams = false; - struct { - struct dc_surface_update surface_updates[MAX_SURFACES]; - struct dc_plane_info plane_infos[MAX_SURFACES]; - struct dc_scaling_info scaling_infos[MAX_SURFACES]; - struct dc_flip_addrs flip_addrs[MAX_SURFACES]; - struct dc_stream_update stream_update; - } *bundle; - - bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - - if (!bundle) { - drm_err(dev, "Failed to allocate update bundle\n"); - goto cleanup; - } - - /* - * Disable the cursor first if we're disabling all the planes. - * It'll remain on the screen after the planes are re-enabled - * if we don't. - * - * If the cursor is transitioning from native to overlay mode, the - * native cursor needs to be disabled first. - */ - if (acrtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE && - dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) { - struct dc_cursor_position cursor_position = {0}; - - if (!dc_stream_set_cursor_position(acrtc_state->stream, - &cursor_position)) - drm_err(dev, "DC failed to disable native cursor\n"); - - bundle->stream_update.cursor_position = - &acrtc_state->stream->cursor_position; - } - - if (acrtc_state->active_planes == 0 && - dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) - amdgpu_dm_commit_cursors(state); - - /* update planes when needed */ - for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { - struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *new_crtc_state; - struct drm_framebuffer *fb = new_plane_state->fb; - struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)fb; - bool plane_needs_flip; - struct dc_plane_state *dc_plane; - struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state); - - /* Cursor plane is handled after stream updates */ - if (plane->type == DRM_PLANE_TYPE_CURSOR && - acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) { - if ((fb && crtc == pcrtc) || - (old_plane_state->fb && old_plane_state->crtc == pcrtc)) { - cursor_update = true; - if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) - amdgpu_dm_update_cursor(plane, old_plane_state, &bundle->stream_update); - } - - continue; - } - - if (!fb || !crtc || pcrtc != crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - if (!new_crtc_state->active) - continue; - - dc_plane = dm_new_plane_state->dc_state; - if (!dc_plane) - continue; - - bundle->surface_updates[planes_count].surface = dc_plane; - if (new_pcrtc_state->color_mgmt_changed) { - bundle->surface_updates[planes_count].gamma = &dc_plane->gamma_correction; - bundle->surface_updates[planes_count].in_transfer_func = &dc_plane->in_transfer_func; - bundle->surface_updates[planes_count].gamut_remap_matrix = &dc_plane->gamut_remap_matrix; - bundle->surface_updates[planes_count].hdr_mult = dc_plane->hdr_mult; - bundle->surface_updates[planes_count].func_shaper = &dc_plane->in_shaper_func; - bundle->surface_updates[planes_count].lut3d_func = &dc_plane->lut3d_func; - bundle->surface_updates[planes_count].blend_tf = &dc_plane->blend_tf; - } - - amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state, - &bundle->scaling_infos[planes_count]); - - bundle->surface_updates[planes_count].scaling_info = - &bundle->scaling_infos[planes_count]; - - plane_needs_flip = old_plane_state->fb && new_plane_state->fb; - - pflip_present = pflip_present || plane_needs_flip; - - if (!plane_needs_flip) { - planes_count += 1; - continue; - } - - fill_dc_plane_info_and_addr( - dm->adev, new_plane_state, - afb->tiling_flags, - &bundle->plane_infos[planes_count], - &bundle->flip_addrs[planes_count].address, - afb->tmz_surface, false); - - drm_dbg_state(state->dev, "plane: id=%d dcc_en=%d\n", - new_plane_state->plane->index, - bundle->plane_infos[planes_count].dcc.enable); - - bundle->surface_updates[planes_count].plane_info = - &bundle->plane_infos[planes_count]; - - if (acrtc_state->stream->link->psr_settings.psr_feature_enabled || - acrtc_state->stream->link->replay_settings.replay_feature_enabled) { - fill_dc_dirty_rects(plane, old_plane_state, - new_plane_state, new_crtc_state, - &bundle->flip_addrs[planes_count], - acrtc_state->stream->link->psr_settings.psr_version == - DC_PSR_VERSION_SU_1, - &dirty_rects_changed); - - /* - * If the dirty regions changed, PSR-SU need to be disabled temporarily - * and enabled it again after dirty regions are stable to avoid video glitch. - * PSR-SU will be enabled in vblank_control_worker() if user pause the video - * during the PSR-SU was disabled. - */ - if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 && - acrtc_attach->dm_irq_params.allow_psr_entry && -#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY - !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) && -#endif - dirty_rects_changed) { - mutex_lock(&dm->dc_lock); - acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns = - timestamp_ns; - if (acrtc_state->stream->link->psr_settings.psr_allow_active) - amdgpu_dm_psr_disable(acrtc_state->stream); - mutex_unlock(&dm->dc_lock); - } - } - - /* - * Only allow immediate flips for fast updates that don't - * change memory domain, FB pitch, DCC state, rotation or - * mirroring. - * - * dm_crtc_helper_atomic_check() only accepts async flips with - * fast updates. - */ - if (crtc->state->async_flip && - (acrtc_state->update_type != UPDATE_TYPE_FAST || - get_mem_type(old_plane_state->fb) != get_mem_type(fb))) - drm_warn_once(state->dev, - "[PLANE:%d:%s] async flip with non-fast update\n", - plane->base.id, plane->name); - - bundle->flip_addrs[planes_count].flip_immediate = - crtc->state->async_flip && - acrtc_state->update_type == UPDATE_TYPE_FAST && - get_mem_type(old_plane_state->fb) == get_mem_type(fb); - - timestamp_ns = ktime_get_ns(); - bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000); - bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count]; - bundle->surface_updates[planes_count].surface = dc_plane; - - if (!bundle->surface_updates[planes_count].surface) { - DRM_ERROR("No surface for CRTC: id=%d\n", - acrtc_attach->crtc_id); - continue; - } - - if (plane == pcrtc->primary) - update_freesync_state_on_stream( - dm, - acrtc_state, - acrtc_state->stream, - dc_plane, - bundle->flip_addrs[planes_count].flip_timestamp_in_us); - - drm_dbg_state(state->dev, "%s Flipping to hi: 0x%x, low: 0x%x\n", - __func__, - bundle->flip_addrs[planes_count].address.grph.addr.high_part, - bundle->flip_addrs[planes_count].address.grph.addr.low_part); - - planes_count += 1; - - } - - if (pflip_present) { - if (!vrr_active) { - /* Use old throttling in non-vrr fixed refresh rate mode - * to keep flip scheduling based on target vblank counts - * working in a backwards compatible way, e.g., for - * clients using the GLX_OML_sync_control extension or - * DRI3/Present extension with defined target_msc. - */ - last_flip_vblank = amdgpu_get_vblank_counter_kms(pcrtc); - } else { - /* For variable refresh rate mode only: - * Get vblank of last completed flip to avoid > 1 vrr - * flips per video frame by use of throttling, but allow - * flip programming anywhere in the possibly large - * variable vrr vblank interval for fine-grained flip - * timing control and more opportunity to avoid stutter - * on late submission of flips. - */ - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - last_flip_vblank = acrtc_attach->dm_irq_params.last_flip_vblank; - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - target_vblank = last_flip_vblank + wait_for_vblank; - - /* - * Wait until we're out of the vertical blank period before the one - * targeted by the flip - */ - while ((acrtc_attach->enabled && - (amdgpu_display_get_crtc_scanoutpos(dm->ddev, acrtc_attach->crtc_id, - 0, &vpos, &hpos, NULL, - NULL, &pcrtc->hwmode) - & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) == - (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) && - (int)(target_vblank - - amdgpu_get_vblank_counter_kms(pcrtc)) > 0)) { - usleep_range(1000, 1100); - } - - /** - * Prepare the flip event for the pageflip interrupt to handle. - * - * This only works in the case where we've already turned on the - * appropriate hardware blocks (eg. HUBP) so in the transition case - * from 0 -> n planes we have to skip a hardware generated event - * and rely on sending it from software. - */ - if (acrtc_attach->base.state->event && - acrtc_state->active_planes > 0) { - drm_crtc_vblank_get(pcrtc); - - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - - WARN_ON(acrtc_attach->pflip_status != AMDGPU_FLIP_NONE); - prepare_flip_isr(acrtc_attach); - - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - if (acrtc_state->stream) { - if (acrtc_state->freesync_vrr_info_changed) - bundle->stream_update.vrr_infopacket = - &acrtc_state->stream->vrr_infopacket; - } - } else if (cursor_update && acrtc_state->active_planes > 0) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - if (acrtc_attach->base.state->event) { - drm_crtc_vblank_get(pcrtc); - acrtc_attach->event = acrtc_attach->base.state->event; - acrtc_attach->base.state->event = NULL; - } - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - /* Update the planes if changed or disable if we don't have any. */ - if ((planes_count || acrtc_state->active_planes == 0) && - acrtc_state->stream) { - /* - * If PSR or idle optimizations are enabled then flush out - * any pending work before hardware programming. - */ - if (dm->vblank_control_workqueue) - flush_workqueue(dm->vblank_control_workqueue); - - bundle->stream_update.stream = acrtc_state->stream; - if (new_pcrtc_state->mode_changed) { - bundle->stream_update.src = acrtc_state->stream->src; - bundle->stream_update.dst = acrtc_state->stream->dst; - } - - if (new_pcrtc_state->color_mgmt_changed) { - /* - * TODO: This isn't fully correct since we've actually - * already modified the stream in place. - */ - bundle->stream_update.gamut_remap = - &acrtc_state->stream->gamut_remap_matrix; - bundle->stream_update.output_csc_transform = - &acrtc_state->stream->csc_color_matrix; - bundle->stream_update.out_transfer_func = - &acrtc_state->stream->out_transfer_func; - bundle->stream_update.lut3d_func = - (struct dc_3dlut *) acrtc_state->stream->lut3d_func; - bundle->stream_update.func_shaper = - (struct dc_transfer_func *) acrtc_state->stream->func_shaper; - } - - acrtc_state->stream->abm_level = acrtc_state->abm_level; - if (acrtc_state->abm_level != dm_old_crtc_state->abm_level) - bundle->stream_update.abm_level = &acrtc_state->abm_level; - - mutex_lock(&dm->dc_lock); - if ((acrtc_state->update_type > UPDATE_TYPE_FAST) && - acrtc_state->stream->link->psr_settings.psr_allow_active) - amdgpu_dm_psr_disable(acrtc_state->stream); - mutex_unlock(&dm->dc_lock); - - /* - * If FreeSync state on the stream has changed then we need to - * re-adjust the min/max bounds now that DC doesn't handle this - * as part of commit. - */ - if (is_dc_timing_adjust_needed(dm_old_crtc_state, acrtc_state)) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - dc_stream_adjust_vmin_vmax( - dm->dc, acrtc_state->stream, - &acrtc_attach->dm_irq_params.vrr_params.adjust); - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - mutex_lock(&dm->dc_lock); - update_planes_and_stream_adapter(dm->dc, - acrtc_state->update_type, - planes_count, - acrtc_state->stream, - &bundle->stream_update, - bundle->surface_updates); - updated_planes_and_streams = true; - - /** - * Enable or disable the interrupts on the backend. - * - * Most pipes are put into power gating when unused. - * - * When power gating is enabled on a pipe we lose the - * interrupt enablement state when power gating is disabled. - * - * So we need to update the IRQ control state in hardware - * whenever the pipe turns on (since it could be previously - * power gated) or off (since some pipes can't be power gated - * on some ASICs). - */ - if (dm_old_crtc_state->active_planes != acrtc_state->active_planes) - dm_update_pflip_irq_state(drm_to_adev(dev), - acrtc_attach); - - if (acrtc_state->update_type > UPDATE_TYPE_FAST) { - if (acrtc_state->stream->link->replay_settings.config.replay_supported && - !acrtc_state->stream->link->replay_settings.replay_feature_enabled) { - struct amdgpu_dm_connector *aconn = - (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; - amdgpu_dm_link_setup_replay(acrtc_state->stream->link, aconn); - } else if (acrtc_state->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED && - !acrtc_state->stream->link->psr_settings.psr_feature_enabled) { - - struct amdgpu_dm_connector *aconn = (struct amdgpu_dm_connector *) - acrtc_state->stream->dm_stream_context; - - if (!aconn->disallow_edp_enter_psr) - amdgpu_dm_link_setup_psr(acrtc_state->stream); - } - } - - /* Decrement skip count when PSR is enabled and we're doing fast updates. */ - if (acrtc_state->update_type == UPDATE_TYPE_FAST && - acrtc_state->stream->link->psr_settings.psr_feature_enabled) { - struct amdgpu_dm_connector *aconn = - (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; - - if (aconn->psr_skip_count > 0) - aconn->psr_skip_count--; - - /* Allow PSR when skip count is 0. */ - acrtc_attach->dm_irq_params.allow_psr_entry = !aconn->psr_skip_count; - - /* - * If sink supports PSR SU, there is no need to rely on - * a vblank event disable request to enable PSR. PSR SU - * can be enabled immediately once OS demonstrates an - * adequate number of fast atomic commits to notify KMD - * of update events. See `vblank_control_worker()`. - */ - if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 && - acrtc_attach->dm_irq_params.allow_psr_entry && -#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY - !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) && -#endif - !acrtc_state->stream->link->psr_settings.psr_allow_active && - !aconn->disallow_edp_enter_psr && - (timestamp_ns - - acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns) > - 500000000) - amdgpu_dm_psr_enable(acrtc_state->stream); - } else { - acrtc_attach->dm_irq_params.allow_psr_entry = false; - } - - mutex_unlock(&dm->dc_lock); - } - - /* - * Update cursor state *after* programming all the planes. - * This avoids redundant programming in the case where we're going - * to be disabling a single plane - those pipes are being disabled. - */ - if (acrtc_state->active_planes && - (!updated_planes_and_streams || amdgpu_ip_version(dm->adev, DCE_HWIP, 0) == 0) && - acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) - amdgpu_dm_commit_cursors(state); - -cleanup: - kfree(bundle); -} - -static void amdgpu_dm_commit_audio(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct drm_crtc_state *new_crtc_state; - struct dm_crtc_state *new_dm_crtc_state; - const struct dc_stream_status *status; - int i, inst; - - /* Notify device removals. */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - if (old_con_state->crtc != new_con_state->crtc) { - /* CRTC changes require notification. */ - goto notify; - } - - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_con_state->crtc); - - if (!new_crtc_state) - continue; - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - -notify: - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - mutex_lock(&adev->dm.audio_lock); - inst = aconnector->audio_inst; - aconnector->audio_inst = -1; - mutex_unlock(&adev->dm.audio_lock); - - amdgpu_dm_audio_eld_notify(adev, inst); - } - - /* Notify audio device additions. */ - for_each_new_connector_in_state(state, connector, new_con_state, i) { - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_con_state->crtc); - - if (!new_crtc_state) - continue; - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - - new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); - if (!new_dm_crtc_state->stream) - continue; - - status = dc_stream_get_status(new_dm_crtc_state->stream); - if (!status) - continue; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - mutex_lock(&adev->dm.audio_lock); - inst = status->audio_inst; - aconnector->audio_inst = inst; - mutex_unlock(&adev->dm.audio_lock); - - amdgpu_dm_audio_eld_notify(adev, inst); - } -} - -/* - * amdgpu_dm_crtc_copy_transient_flags - copy mirrored flags from DRM to DC - * @crtc_state: the DRM CRTC state - * @stream_state: the DC stream state. - * - * Copy the mirrored transient state flags from DRM, to DC. It is used to bring - * a dc_stream_state's flags in sync with a drm_crtc_state's flags. - */ -static void amdgpu_dm_crtc_copy_transient_flags(struct drm_crtc_state *crtc_state, - struct dc_stream_state *stream_state) -{ - stream_state->mode_changed = drm_atomic_crtc_needs_modeset(crtc_state); -} - -static void dm_clear_writeback(struct amdgpu_display_manager *dm, - struct dm_crtc_state *crtc_state) -{ - dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0); -} - -static void amdgpu_dm_commit_streams(struct drm_atomic_state *state, - struct dc_state *dc_state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct drm_connector_state *old_con_state; - struct drm_connector *connector; - bool mode_set_reset_required = false; - u32 i; - struct dc_commit_streams_params params = {dc_state->streams, dc_state->stream_count}; - - /* Disable writeback */ - for_each_old_connector_in_state(state, connector, old_con_state, i) { - struct dm_connector_state *dm_old_con_state; - struct amdgpu_crtc *acrtc; - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - old_crtc_state = NULL; - - dm_old_con_state = to_dm_connector_state(old_con_state); - if (!dm_old_con_state->base.crtc) - continue; - - acrtc = to_amdgpu_crtc(dm_old_con_state->base.crtc); - if (acrtc) - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - - if (!acrtc || !acrtc->wb_enabled) - continue; - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - dm_clear_writeback(dm, dm_old_crtc_state); - acrtc->wb_enabled = false; - } - - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (old_crtc_state->active && - (!new_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - manage_dm_interrupts(adev, acrtc, NULL); - dc_stream_release(dm_old_crtc_state->stream); - } - } - - drm_atomic_helper_calc_timestamping_constants(state); - - /* update changed items */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - drm_dbg_state(state->dev, - "amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, planes_changed:%d, mode_changed:%d,active_changed:%d,connectors_changed:%d\n", - acrtc->crtc_id, - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->planes_changed, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* Disable cursor if disabling crtc */ - if (old_crtc_state->active && !new_crtc_state->active) { - struct dc_cursor_position position; - - memset(&position, 0, sizeof(position)); - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_stream_program_cursor_position(dm_old_crtc_state->stream, &position); - mutex_unlock(&dm->dc_lock); - } - - /* Copy all transient state flags into dc state */ - if (dm_new_crtc_state->stream) { - amdgpu_dm_crtc_copy_transient_flags(&dm_new_crtc_state->base, - dm_new_crtc_state->stream); - } - - /* handles headless hotplug case, updating new_state and - * aconnector as needed - */ - - if (amdgpu_dm_crtc_modeset_required(new_crtc_state, dm_new_crtc_state->stream, dm_old_crtc_state->stream)) { - - drm_dbg_atomic(dev, - "Atomic commit: SET crtc id %d: [%p]\n", - acrtc->crtc_id, acrtc); - - if (!dm_new_crtc_state->stream) { - /* - * this could happen because of issues with - * userspace notifications delivery. - * In this case userspace tries to set mode on - * display which is disconnected in fact. - * dc_sink is NULL in this case on aconnector. - * We expect reset mode will come soon. - * - * This can also happen when unplug is done - * during resume sequence ended - * - * In this case, we want to pretend we still - * have a sink to keep the pipe running so that - * hw state is consistent with the sw state - */ - drm_dbg_atomic(dev, - "Failed to create new stream for crtc %d\n", - acrtc->base.base.id); - continue; - } - - if (dm_old_crtc_state->stream) - remove_stream(adev, acrtc, dm_old_crtc_state->stream); - - pm_runtime_get_noresume(dev->dev); - - acrtc->enabled = true; - acrtc->hw_mode = new_crtc_state->mode; - crtc->hwmode = new_crtc_state->mode; - mode_set_reset_required = true; - } else if (modereset_required(new_crtc_state)) { - drm_dbg_atomic(dev, - "Atomic commit: RESET. crtc id %d:[%p]\n", - acrtc->crtc_id, acrtc); - /* i.e. reset mode */ - if (dm_old_crtc_state->stream) - remove_stream(adev, acrtc, dm_old_crtc_state->stream); - - mode_set_reset_required = true; - } - } /* for_each_crtc_in_state() */ - - /* if there mode set or reset, disable eDP PSR, Replay */ - if (mode_set_reset_required) { - if (dm->vblank_control_workqueue) - flush_workqueue(dm->vblank_control_workqueue); - - amdgpu_dm_replay_disable_all(dm); - amdgpu_dm_psr_disable_all(dm); - } - - dm_enable_per_frame_crtc_master_sync(dc_state); - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - WARN_ON(!dc_commit_streams(dm->dc, ¶ms)); - - /* Allow idle optimization when vblank count is 0 for display off */ - if (dm->active_vblank_irq_count == 0) - dc_allow_idle_optimizations(dm->dc, true); - mutex_unlock(&dm->dc_lock); - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state->stream != NULL) { - const struct dc_stream_status *status = - dc_stream_get_status(dm_new_crtc_state->stream); - - if (!status) - status = dc_state_get_stream_status(dc_state, - dm_new_crtc_state->stream); - if (!status) - drm_err(dev, - "got no status for stream %p on acrtc%p\n", - dm_new_crtc_state->stream, acrtc); - else - acrtc->otg_inst = status->primary_otg_inst; - } - } -} - -static void dm_set_writeback(struct amdgpu_display_manager *dm, - struct dm_crtc_state *crtc_state, - struct drm_connector *connector, - struct drm_connector_state *new_con_state) -{ - struct drm_writeback_connector *wb_conn = drm_connector_to_writeback(connector); - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc; - struct dc_writeback_info *wb_info; - struct pipe_ctx *pipe = NULL; - struct amdgpu_framebuffer *afb; - int i = 0; - - wb_info = kzalloc(sizeof(*wb_info), GFP_KERNEL); - if (!wb_info) { - DRM_ERROR("Failed to allocate wb_info\n"); - return; - } - - acrtc = to_amdgpu_crtc(wb_conn->encoder.crtc); - if (!acrtc) { - DRM_ERROR("no amdgpu_crtc found\n"); - kfree(wb_info); - return; - } - - afb = to_amdgpu_framebuffer(new_con_state->writeback_job->fb); - if (!afb) { - DRM_ERROR("No amdgpu_framebuffer found\n"); - kfree(wb_info); - return; - } - - for (i = 0; i < MAX_PIPES; i++) { - if (dm->dc->current_state->res_ctx.pipe_ctx[i].stream == crtc_state->stream) { - pipe = &dm->dc->current_state->res_ctx.pipe_ctx[i]; - break; - } - } - - /* fill in wb_info */ - wb_info->wb_enabled = true; - - wb_info->dwb_pipe_inst = 0; - wb_info->dwb_params.dwbscl_black_color = 0; - wb_info->dwb_params.hdr_mult = 0x1F000; - wb_info->dwb_params.csc_params.gamut_adjust_type = CM_GAMUT_ADJUST_TYPE_BYPASS; - wb_info->dwb_params.csc_params.gamut_coef_format = CM_GAMUT_REMAP_COEF_FORMAT_S2_13; - wb_info->dwb_params.output_depth = DWB_OUTPUT_PIXEL_DEPTH_10BPC; - wb_info->dwb_params.cnv_params.cnv_out_bpc = DWB_CNV_OUT_BPC_10BPC; - - /* width & height from crtc */ - wb_info->dwb_params.cnv_params.src_width = acrtc->base.mode.crtc_hdisplay; - wb_info->dwb_params.cnv_params.src_height = acrtc->base.mode.crtc_vdisplay; - wb_info->dwb_params.dest_width = acrtc->base.mode.crtc_hdisplay; - wb_info->dwb_params.dest_height = acrtc->base.mode.crtc_vdisplay; - - wb_info->dwb_params.cnv_params.crop_en = false; - wb_info->dwb_params.stereo_params.stereo_enabled = false; - - wb_info->dwb_params.cnv_params.out_max_pix_val = 0x3ff; // 10 bits - wb_info->dwb_params.cnv_params.out_min_pix_val = 0; - wb_info->dwb_params.cnv_params.fc_out_format = DWB_OUT_FORMAT_32BPP_ARGB; - wb_info->dwb_params.cnv_params.out_denorm_mode = DWB_OUT_DENORM_BYPASS; - - wb_info->dwb_params.out_format = dwb_scaler_mode_bypass444; - - wb_info->dwb_params.capture_rate = dwb_capture_rate_0; - - wb_info->dwb_params.scaler_taps.h_taps = 4; - wb_info->dwb_params.scaler_taps.v_taps = 4; - wb_info->dwb_params.scaler_taps.h_taps_c = 2; - wb_info->dwb_params.scaler_taps.v_taps_c = 2; - wb_info->dwb_params.subsample_position = DWB_INTERSTITIAL_SUBSAMPLING; - - wb_info->mcif_buf_params.luma_pitch = afb->base.pitches[0]; - wb_info->mcif_buf_params.chroma_pitch = afb->base.pitches[1]; - - for (i = 0; i < DWB_MCIF_BUF_COUNT; i++) { - wb_info->mcif_buf_params.luma_address[i] = afb->address; - wb_info->mcif_buf_params.chroma_address[i] = 0; - } - - wb_info->mcif_buf_params.p_vmid = 1; - if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0)) { - wb_info->mcif_warmup_params.start_address.quad_part = afb->address; - wb_info->mcif_warmup_params.region_size = - wb_info->mcif_buf_params.luma_pitch * wb_info->dwb_params.dest_height; - } - wb_info->mcif_warmup_params.p_vmid = 1; - wb_info->writeback_source_plane = pipe->plane_state; - - dc_stream_add_writeback(dm->dc, crtc_state->stream, wb_info); - - acrtc->wb_pending = true; - acrtc->wb_conn = wb_conn; - drm_writeback_queue_job(wb_conn, new_con_state); -} - -/** - * amdgpu_dm_atomic_commit_tail() - AMDgpu DM's commit tail implementation. - * @state: The atomic state to commit - * - * This will tell DC to commit the constructed DC state from atomic_check, - * programming the hardware. Any failures here implies a hardware failure, since - * atomic check should have filtered anything non-kosher. - */ -static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct dm_atomic_state *dm_state; - struct dc_state *dc_state = NULL; - u32 i, j; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - unsigned long flags; - bool wait_for_vblank = true; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - int crtc_disable_count = 0; - - trace_amdgpu_dm_atomic_commit_tail_begin(state); - - drm_atomic_helper_update_legacy_modeset_state(dev, state); - drm_dp_mst_atomic_wait_for_dependencies(state); - - dm_state = dm_atomic_get_new_state(state); - if (dm_state && dm_state->context) { - dc_state = dm_state->context; - amdgpu_dm_commit_streams(state, dc_state); - } - - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct amdgpu_dm_connector *aconnector; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!adev->dm.hdcp_workqueue) - continue; - - pr_debug("[HDCP_DM] -------------- i : %x ----------\n", i); - - if (!connector) - continue; - - pr_debug("[HDCP_DM] connector->index: %x connect_status: %x dpms: %x\n", - connector->index, connector->status, connector->dpms); - pr_debug("[HDCP_DM] state protection old: %x new: %x\n", - old_con_state->content_protection, new_con_state->content_protection); - - if (aconnector->dc_sink) { - if (aconnector->dc_sink->sink_signal != SIGNAL_TYPE_VIRTUAL && - aconnector->dc_sink->sink_signal != SIGNAL_TYPE_NONE) { - pr_debug("[HDCP_DM] pipe_ctx dispname=%s\n", - aconnector->dc_sink->edid_caps.display_name); - } - } - - new_crtc_state = NULL; - old_crtc_state = NULL; - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - if (old_crtc_state) - pr_debug("old crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - old_crtc_state->enable, - old_crtc_state->active, - old_crtc_state->mode_changed, - old_crtc_state->active_changed, - old_crtc_state->connectors_changed); - - if (new_crtc_state) - pr_debug("NEW crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - } - - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - if (!adev->dm.hdcp_workqueue) - continue; - - new_crtc_state = NULL; - old_crtc_state = NULL; - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state && dm_new_crtc_state->stream == NULL && - connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) { - hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); - new_con_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - dm_new_con_state->update_hdcp = true; - continue; - } - - if (is_content_protection_different(new_crtc_state, old_crtc_state, new_con_state, - old_con_state, connector, adev->dm.hdcp_workqueue)) { - /* when display is unplugged from mst hub, connctor will - * be destroyed within dm_dp_mst_connector_destroy. connector - * hdcp perperties, like type, undesired, desired, enabled, - * will be lost. So, save hdcp properties into hdcp_work within - * amdgpu_dm_atomic_commit_tail. if the same display is - * plugged back with same display index, its hdcp properties - * will be retrieved from hdcp_work within dm_dp_mst_get_modes - */ - - bool enable_encryption = false; - - if (new_con_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) - enable_encryption = true; - - if (aconnector->dc_link && aconnector->dc_sink && - aconnector->dc_link->type == dc_connection_mst_branch) { - struct hdcp_workqueue *hdcp_work = adev->dm.hdcp_workqueue; - struct hdcp_workqueue *hdcp_w = - &hdcp_work[aconnector->dc_link->link_index]; - - hdcp_w->hdcp_content_type[connector->index] = - new_con_state->hdcp_content_type; - hdcp_w->content_protection[connector->index] = - new_con_state->content_protection; - } - - if (new_crtc_state && new_crtc_state->mode_changed && - new_con_state->content_protection >= DRM_MODE_CONTENT_PROTECTION_DESIRED) - enable_encryption = true; - - DRM_INFO("[HDCP_DM] hdcp_update_display enable_encryption = %x\n", enable_encryption); - - if (aconnector->dc_link) - hdcp_update_display( - adev->dm.hdcp_workqueue, aconnector->dc_link->link_index, aconnector, - new_con_state->hdcp_content_type, enable_encryption); - } - } - - /* Handle connector state changes */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct dc_surface_update *dummy_updates; - struct dc_stream_update stream_update; - struct dc_info_packet hdr_packet; - struct dc_stream_status *status = NULL; - bool abm_changed, hdr_changed, scaling_changed; - - memset(&stream_update, 0, sizeof(stream_update)); - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - /* Skip any modesets/resets */ - if (!acrtc || drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - scaling_changed = is_scaling_state_different(dm_new_con_state, - dm_old_con_state); - - abm_changed = dm_new_crtc_state->abm_level != - dm_old_crtc_state->abm_level; - - hdr_changed = - !drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state); - - if (!scaling_changed && !abm_changed && !hdr_changed) - continue; - - stream_update.stream = dm_new_crtc_state->stream; - if (scaling_changed) { - update_stream_scaling_settings(&dm_new_con_state->base.crtc->mode, - dm_new_con_state, dm_new_crtc_state->stream); - - stream_update.src = dm_new_crtc_state->stream->src; - stream_update.dst = dm_new_crtc_state->stream->dst; - } - - if (abm_changed) { - dm_new_crtc_state->stream->abm_level = dm_new_crtc_state->abm_level; - - stream_update.abm_level = &dm_new_crtc_state->abm_level; - } - - if (hdr_changed) { - fill_hdr_info_packet(new_con_state, &hdr_packet); - stream_update.hdr_static_metadata = &hdr_packet; - } - - status = dc_stream_get_status(dm_new_crtc_state->stream); - - if (WARN_ON(!status)) - continue; - - WARN_ON(!status->plane_count); - - /* - * TODO: DC refuses to perform stream updates without a dc_surface_update. - * Here we create an empty update on each plane. - * To fix this, DC should permit updating only stream properties. - */ - dummy_updates = kzalloc(sizeof(struct dc_surface_update) * MAX_SURFACES, GFP_ATOMIC); - if (!dummy_updates) { - DRM_ERROR("Failed to allocate memory for dummy_updates.\n"); - continue; - } - for (j = 0; j < status->plane_count; j++) - dummy_updates[j].surface = status->plane_states[0]; - - sort(dummy_updates, status->plane_count, - sizeof(*dummy_updates), dm_plane_layer_index_cmp, NULL); - - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_update_planes_and_stream(dm->dc, - dummy_updates, - status->plane_count, - dm_new_crtc_state->stream, - &stream_update); - mutex_unlock(&dm->dc_lock); - kfree(dummy_updates); - } - - /** - * Enable interrupts for CRTCs that are newly enabled or went through - * a modeset. It was intentionally deferred until after the front end - * state was modified to wait until the OTG was on and so the IRQ - * handlers didn't access stale or invalid state. - */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); -#ifdef CONFIG_DEBUG_FS - enum amdgpu_dm_pipe_crc_source cur_crc_src; -#endif - /* Count number of newly disabled CRTCs for dropping PM refs later. */ - if (old_crtc_state->active && !new_crtc_state->active) - crtc_disable_count++; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - /* For freesync config update on crtc state and params for irq */ - update_stream_irq_parameters(dm, dm_new_crtc_state); - -#ifdef CONFIG_DEBUG_FS - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - cur_crc_src = acrtc->dm_irq_params.crc_src; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -#endif - - if (new_crtc_state->active && - (!old_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - dc_stream_retain(dm_new_crtc_state->stream); - acrtc->dm_irq_params.stream = dm_new_crtc_state->stream; - manage_dm_interrupts(adev, acrtc, dm_new_crtc_state); - } - /* Handle vrr on->off / off->on transitions */ - amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state); - -#ifdef CONFIG_DEBUG_FS - if (new_crtc_state->active && - (!old_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - /** - * Frontend may have changed so reapply the CRC capture - * settings for the stream. - */ - if (amdgpu_dm_is_valid_crc_source(cur_crc_src)) { -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - if (amdgpu_dm_crc_window_is_activated(crtc)) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - acrtc->dm_irq_params.window_param.update_win = true; - - /** - * It takes 2 frames for HW to stably generate CRC when - * resuming from suspend, so we set skip_frame_cnt 2. - */ - acrtc->dm_irq_params.window_param.skip_frame_cnt = 2; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } -#endif - if (amdgpu_dm_crtc_configure_crc_source( - crtc, dm_new_crtc_state, cur_crc_src)) - drm_dbg_atomic(dev, "Failed to configure crc source"); - } - } -#endif - } - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) - if (new_crtc_state->async_flip) - wait_for_vblank = false; - - /* update planes when needed per crtc*/ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state->stream) - amdgpu_dm_commit_planes(state, dev, dm, crtc, wait_for_vblank); - } - - /* Enable writeback */ - for_each_new_connector_in_state(state, connector, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - if (!new_con_state->writeback_job) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - - if (!new_crtc_state) - continue; - - if (acrtc->wb_enabled) - continue; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - dm_set_writeback(dm, dm_new_crtc_state, connector, new_con_state); - acrtc->wb_enabled = true; - } - - /* Update audio instances for each connector. */ - amdgpu_dm_commit_audio(dev, state); - - /* restore the backlight level */ - for (i = 0; i < dm->num_of_edps; i++) { - if (dm->backlight_dev[i] && - (dm->actual_brightness[i] != dm->brightness[i])) - amdgpu_dm_backlight_set_level(dm, i, dm->brightness[i]); - } - - /* - * send vblank event on all events not handled in flip and - * mark consumed event for drm_atomic_helper_commit_hw_done - */ - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - - if (new_crtc_state->event) - drm_send_event_locked(dev, &new_crtc_state->event->base); - - new_crtc_state->event = NULL; - } - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - - /* Signal HW programming completion */ - drm_atomic_helper_commit_hw_done(state); - - if (wait_for_vblank) - drm_atomic_helper_wait_for_flip_done(dev, state); - - drm_atomic_helper_cleanup_planes(dev, state); - - /* Don't free the memory if we are hitting this as part of suspend. - * This way we don't free any memory during suspend; see - * amdgpu_bo_free_kernel(). The memory will be freed in the first - * non-suspend modeset or when the driver is torn down. - */ - if (!adev->in_suspend) { - /* return the stolen vga memory back to VRAM */ - if (!adev->mman.keep_stolen_vga_memory) - amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); - amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); - } - - /* - * Finally, drop a runtime PM reference for each newly disabled CRTC, - * so we can put the GPU into runtime suspend if we're not driving any - * displays anymore - */ - for (i = 0; i < crtc_disable_count; i++) - pm_runtime_put_autosuspend(dev->dev); - pm_runtime_mark_last_busy(dev->dev); -} - -static int dm_force_atomic_commit(struct drm_connector *connector) -{ - int ret = 0; - struct drm_device *ddev = connector->dev; - struct drm_atomic_state *state = drm_atomic_state_alloc(ddev); - struct amdgpu_crtc *disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc); - struct drm_plane *plane = disconnected_acrtc->base.primary; - struct drm_connector_state *conn_state; - struct drm_crtc_state *crtc_state; - struct drm_plane_state *plane_state; - - if (!state) - return -ENOMEM; - - state->acquire_ctx = ddev->mode_config.acquire_ctx; - - /* Construct an atomic state to restore previous display setting */ - - /* - * Attach connectors to drm_atomic_state - */ - conn_state = drm_atomic_get_connector_state(state, connector); - - ret = PTR_ERR_OR_ZERO(conn_state); - if (ret) - goto out; - - /* Attach crtc to drm_atomic_state*/ - crtc_state = drm_atomic_get_crtc_state(state, &disconnected_acrtc->base); - - ret = PTR_ERR_OR_ZERO(crtc_state); - if (ret) - goto out; - - /* force a restore */ - crtc_state->mode_changed = true; - - /* Attach plane to drm_atomic_state */ - plane_state = drm_atomic_get_plane_state(state, plane); - - ret = PTR_ERR_OR_ZERO(plane_state); - if (ret) - goto out; - - /* Call commit internally with the state we just constructed */ - ret = drm_atomic_commit(state); - -out: - drm_atomic_state_put(state); - if (ret) - DRM_ERROR("Restoring old state failed with %i\n", ret); - - return ret; -} - -/* - * This function handles all cases when set mode does not come upon hotplug. - * This includes when a display is unplugged then plugged back into the - * same port and when running without usermode desktop manager supprot - */ -void dm_restore_drm_connector_state(struct drm_device *dev, - struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector; - struct amdgpu_crtc *disconnected_acrtc; - struct dm_crtc_state *acrtc_state; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->dc_sink || !connector->state || !connector->encoder) - return; - - disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc); - if (!disconnected_acrtc) - return; - - acrtc_state = to_dm_crtc_state(disconnected_acrtc->base.state); - if (!acrtc_state->stream) - return; - - /* - * If the previous sink is not released and different from the current, - * we deduce we are in a state where we can not rely on usermode call - * to turn on the display, so we do it here - */ - if (acrtc_state->stream->sink != aconnector->dc_sink) - dm_force_atomic_commit(&aconnector->base); -} - -/* - * Grabs all modesetting locks to serialize against any blocking commits, - * Waits for completion of all non blocking commits. - */ -static int do_aquire_global_lock(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct drm_crtc *crtc; - struct drm_crtc_commit *commit; - long ret; - - /* - * Adding all modeset locks to aquire_ctx will - * ensure that when the framework release it the - * extra locks we are locking here will get released to - */ - ret = drm_modeset_lock_all_ctx(dev, state->acquire_ctx); - if (ret) - return ret; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - spin_lock(&crtc->commit_lock); - commit = list_first_entry_or_null(&crtc->commit_list, - struct drm_crtc_commit, commit_entry); - if (commit) - drm_crtc_commit_get(commit); - spin_unlock(&crtc->commit_lock); - - if (!commit) - continue; - - /* - * Make sure all pending HW programming completed and - * page flips done - */ - ret = wait_for_completion_interruptible_timeout(&commit->hw_done, 10*HZ); - - if (ret > 0) - ret = wait_for_completion_interruptible_timeout( - &commit->flip_done, 10*HZ); - - if (ret == 0) - DRM_ERROR("[CRTC:%d:%s] hw_done or flip_done timed out\n", - crtc->base.id, crtc->name); - - drm_crtc_commit_put(commit); - } - - return ret < 0 ? ret : 0; -} - -static void get_freesync_config_for_crtc( - struct dm_crtc_state *new_crtc_state, - struct dm_connector_state *new_con_state) -{ - struct mod_freesync_config config = {0}; - struct amdgpu_dm_connector *aconnector; - struct drm_display_mode *mode = &new_crtc_state->base.mode; - int vrefresh = drm_mode_vrefresh(mode); - bool fs_vid_mode = false; - - if (new_con_state->base.connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return; - - aconnector = to_amdgpu_dm_connector(new_con_state->base.connector); - - new_crtc_state->vrr_supported = new_con_state->freesync_capable && - vrefresh >= aconnector->min_vfreq && - vrefresh <= aconnector->max_vfreq; - - if (new_crtc_state->vrr_supported) { - new_crtc_state->stream->ignore_msa_timing_param = true; - fs_vid_mode = new_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED; - - config.min_refresh_in_uhz = aconnector->min_vfreq * 1000000; - config.max_refresh_in_uhz = aconnector->max_vfreq * 1000000; - config.vsif_supported = true; - config.btr = true; - - if (fs_vid_mode) { - config.state = VRR_STATE_ACTIVE_FIXED; - config.fixed_refresh_in_uhz = new_crtc_state->freesync_config.fixed_refresh_in_uhz; - goto out; - } else if (new_crtc_state->base.vrr_enabled) { - config.state = VRR_STATE_ACTIVE_VARIABLE; - } else { - config.state = VRR_STATE_INACTIVE; - } - } -out: - new_crtc_state->freesync_config = config; -} - -static void reset_freesync_config_for_crtc( - struct dm_crtc_state *new_crtc_state) -{ - new_crtc_state->vrr_supported = false; - - memset(&new_crtc_state->vrr_infopacket, 0, - sizeof(new_crtc_state->vrr_infopacket)); -} - -static bool -is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state) -{ - const struct drm_display_mode *old_mode, *new_mode; - - if (!old_crtc_state || !new_crtc_state) - return false; - - old_mode = &old_crtc_state->mode; - new_mode = &new_crtc_state->mode; - - if (old_mode->clock == new_mode->clock && - old_mode->hdisplay == new_mode->hdisplay && - old_mode->vdisplay == new_mode->vdisplay && - old_mode->htotal == new_mode->htotal && - old_mode->vtotal != new_mode->vtotal && - old_mode->hsync_start == new_mode->hsync_start && - old_mode->vsync_start != new_mode->vsync_start && - old_mode->hsync_end == new_mode->hsync_end && - old_mode->vsync_end != new_mode->vsync_end && - old_mode->hskew == new_mode->hskew && - old_mode->vscan == new_mode->vscan && - (old_mode->vsync_end - old_mode->vsync_start) == - (new_mode->vsync_end - new_mode->vsync_start)) - return true; - - return false; -} - -static void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state) -{ - u64 num, den, res; - struct drm_crtc_state *new_crtc_state = &dm_new_crtc_state->base; - - dm_new_crtc_state->freesync_config.state = VRR_STATE_ACTIVE_FIXED; - - num = (unsigned long long)new_crtc_state->mode.clock * 1000 * 1000000; - den = (unsigned long long)new_crtc_state->mode.htotal * - (unsigned long long)new_crtc_state->mode.vtotal; - - res = div_u64(num, den); - dm_new_crtc_state->freesync_config.fixed_refresh_in_uhz = res; -} - -static int dm_update_crtc_state(struct amdgpu_display_manager *dm, - struct drm_atomic_state *state, - struct drm_crtc *crtc, - struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state, - bool enable, - bool *lock_and_validation_needed) -{ - struct dm_atomic_state *dm_state = NULL; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct dc_stream_state *new_stream; - int ret = 0; - - /* - * TODO Move this code into dm_crtc_atomic_check once we get rid of dc_validation_set - * update changed items - */ - struct amdgpu_crtc *acrtc = NULL; - struct drm_connector *connector = NULL; - struct amdgpu_dm_connector *aconnector = NULL; - struct drm_connector_state *drm_new_conn_state = NULL, *drm_old_conn_state = NULL; - struct dm_connector_state *dm_new_conn_state = NULL, *dm_old_conn_state = NULL; - - new_stream = NULL; - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - acrtc = to_amdgpu_crtc(crtc); - connector = amdgpu_dm_find_first_crtc_matching_connector(state, crtc); - if (connector) - aconnector = to_amdgpu_dm_connector(connector); - - /* TODO This hack should go away */ - if (connector && enable) { - /* Make sure fake sink is created in plug-in scenario */ - drm_new_conn_state = drm_atomic_get_new_connector_state(state, - connector); - drm_old_conn_state = drm_atomic_get_old_connector_state(state, - connector); - - if (IS_ERR(drm_new_conn_state)) { - ret = PTR_ERR_OR_ZERO(drm_new_conn_state); - goto fail; - } - - dm_new_conn_state = to_dm_connector_state(drm_new_conn_state); - dm_old_conn_state = to_dm_connector_state(drm_old_conn_state); - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - goto skip_modeset; - - new_stream = create_validate_stream_for_sink(aconnector, - &new_crtc_state->mode, - dm_new_conn_state, - dm_old_crtc_state->stream); - - /* - * we can have no stream on ACTION_SET if a display - * was disconnected during S3, in this case it is not an - * error, the OS will be updated after detection, and - * will do the right thing on next atomic commit - */ - - if (!new_stream) { - DRM_DEBUG_DRIVER("%s: Failed to create new stream for crtc %d\n", - __func__, acrtc->base.base.id); - ret = -ENOMEM; - goto fail; - } - - /* - * TODO: Check VSDB bits to decide whether this should - * be enabled or not. - */ - new_stream->triggered_crtc_reset.enabled = - dm->force_timing_sync; - - dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level; - - ret = fill_hdr_info_packet(drm_new_conn_state, - &new_stream->hdr_static_metadata); - if (ret) - goto fail; - - /* - * If we already removed the old stream from the context - * (and set the new stream to NULL) then we can't reuse - * the old stream even if the stream and scaling are unchanged. - * We'll hit the BUG_ON and black screen. - * - * TODO: Refactor this function to allow this check to work - * in all conditions. - */ - if (amdgpu_freesync_vid_mode && - dm_new_crtc_state->stream && - is_timing_unchanged_for_freesync(new_crtc_state, old_crtc_state)) - goto skip_modeset; - - if (dm_new_crtc_state->stream && - dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) && - dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream)) { - new_crtc_state->mode_changed = false; - DRM_DEBUG_DRIVER("Mode change not required, setting mode_changed to %d", - new_crtc_state->mode_changed); - } - } - - /* mode_changed flag may get updated above, need to check again */ - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - goto skip_modeset; - - drm_dbg_state(state->dev, - "amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, planes_changed:%d, mode_changed:%d,active_changed:%d,connectors_changed:%d\n", - acrtc->crtc_id, - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->planes_changed, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* Remove stream for any changed/disabled CRTC */ - if (!enable) { - - if (!dm_old_crtc_state->stream) - goto skip_modeset; - - /* Unset freesync video if it was active before */ - if (dm_old_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED) { - dm_new_crtc_state->freesync_config.state = VRR_STATE_INACTIVE; - dm_new_crtc_state->freesync_config.fixed_refresh_in_uhz = 0; - } - - /* Now check if we should set freesync video mode */ - if (amdgpu_freesync_vid_mode && dm_new_crtc_state->stream && - dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) && - dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream) && - is_timing_unchanged_for_freesync(new_crtc_state, - old_crtc_state)) { - new_crtc_state->mode_changed = false; - DRM_DEBUG_DRIVER( - "Mode change not required for front porch change, setting mode_changed to %d", - new_crtc_state->mode_changed); - - set_freesync_fixed_config(dm_new_crtc_state); - - goto skip_modeset; - } else if (amdgpu_freesync_vid_mode && aconnector && - is_freesync_video_mode(&new_crtc_state->mode, - aconnector)) { - struct drm_display_mode *high_mode; - - high_mode = get_highest_refresh_rate_mode(aconnector, false); - if (!drm_mode_equal(&new_crtc_state->mode, high_mode)) - set_freesync_fixed_config(dm_new_crtc_state); - } - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - goto fail; - - DRM_DEBUG_DRIVER("Disabling DRM crtc: %d\n", - crtc->base.id); - - /* i.e. reset mode */ - if (dc_state_remove_stream( - dm->dc, - dm_state->context, - dm_old_crtc_state->stream) != DC_OK) { - ret = -EINVAL; - goto fail; - } - - dc_stream_release(dm_old_crtc_state->stream); - dm_new_crtc_state->stream = NULL; - - reset_freesync_config_for_crtc(dm_new_crtc_state); - - *lock_and_validation_needed = true; - - } else {/* Add stream for any updated/enabled CRTC */ - /* - * Quick fix to prevent NULL pointer on new_stream when - * added MST connectors not found in existing crtc_state in the chained mode - * TODO: need to dig out the root cause of that - */ - if (!connector) - goto skip_modeset; - - if (modereset_required(new_crtc_state)) - goto skip_modeset; - - if (amdgpu_dm_crtc_modeset_required(new_crtc_state, new_stream, - dm_old_crtc_state->stream)) { - - WARN_ON(dm_new_crtc_state->stream); - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - goto fail; - - dm_new_crtc_state->stream = new_stream; - - dc_stream_retain(new_stream); - - DRM_DEBUG_ATOMIC("Enabling DRM crtc: %d\n", - crtc->base.id); - - if (dc_state_add_stream( - dm->dc, - dm_state->context, - dm_new_crtc_state->stream) != DC_OK) { - ret = -EINVAL; - goto fail; - } - - *lock_and_validation_needed = true; - } - } - -skip_modeset: - /* Release extra reference */ - if (new_stream) - dc_stream_release(new_stream); - - /* - * We want to do dc stream updates that do not require a - * full modeset below. - */ - if (!(enable && connector && new_crtc_state->active)) - return 0; - /* - * Given above conditions, the dc state cannot be NULL because: - * 1. We're in the process of enabling CRTCs (just been added - * to the dc context, or already is on the context) - * 2. Has a valid connector attached, and - * 3. Is currently active and enabled. - * => The dc stream state currently exists. - */ - BUG_ON(dm_new_crtc_state->stream == NULL); - - /* Scaling or underscan settings */ - if (is_scaling_state_different(dm_old_conn_state, dm_new_conn_state) || - drm_atomic_crtc_needs_modeset(new_crtc_state)) - update_stream_scaling_settings( - &new_crtc_state->mode, dm_new_conn_state, dm_new_crtc_state->stream); - - /* ABM settings */ - dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level; - - /* - * Color management settings. We also update color properties - * when a modeset is needed, to ensure it gets reprogrammed. - */ - if (dm_new_crtc_state->base.color_mgmt_changed || - dm_old_crtc_state->regamma_tf != dm_new_crtc_state->regamma_tf || - drm_atomic_crtc_needs_modeset(new_crtc_state)) { - ret = amdgpu_dm_update_crtc_color_mgmt(dm_new_crtc_state); - if (ret) - goto fail; - } - - /* Update Freesync settings. */ - get_freesync_config_for_crtc(dm_new_crtc_state, - dm_new_conn_state); - - return ret; - -fail: - if (new_stream) - dc_stream_release(new_stream); - return ret; -} - -static bool should_reset_plane(struct drm_atomic_state *state, - struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state) -{ - struct drm_plane *other; - struct drm_plane_state *old_other_state, *new_other_state; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *old_dm_crtc_state, *new_dm_crtc_state; - struct amdgpu_device *adev = drm_to_adev(plane->dev); - int i; - - /* - * TODO: Remove this hack for all asics once it proves that the - * fast updates works fine on DCN3.2+. - */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 2, 0) && - state->allow_modeset) - return true; - - /* Exit early if we know that we're adding or removing the plane. */ - if (old_plane_state->crtc != new_plane_state->crtc) - return true; - - /* old crtc == new_crtc == NULL, plane not in context. */ - if (!new_plane_state->crtc) - return false; - - new_crtc_state = - drm_atomic_get_new_crtc_state(state, new_plane_state->crtc); - old_crtc_state = - drm_atomic_get_old_crtc_state(state, old_plane_state->crtc); - - if (!new_crtc_state) - return true; - - /* - * A change in cursor mode means a new dc pipe needs to be acquired or - * released from the state - */ - old_dm_crtc_state = to_dm_crtc_state(old_crtc_state); - new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); - if (plane->type == DRM_PLANE_TYPE_CURSOR && - old_dm_crtc_state != NULL && - old_dm_crtc_state->cursor_mode != new_dm_crtc_state->cursor_mode) { - return true; - } - - /* CRTC Degamma changes currently require us to recreate planes. */ - if (new_crtc_state->color_mgmt_changed) - return true; - - /* - * On zpos change, planes need to be reordered by removing and re-adding - * them one by one to the dc state, in order of descending zpos. - * - * TODO: We can likely skip bandwidth validation if the only thing that - * changed about the plane was it'z z-ordering. - */ - if (new_crtc_state->zpos_changed) - return true; - - if (drm_atomic_crtc_needs_modeset(new_crtc_state)) - return true; - - /* - * If there are any new primary or overlay planes being added or - * removed then the z-order can potentially change. To ensure - * correct z-order and pipe acquisition the current DC architecture - * requires us to remove and recreate all existing planes. - * - * TODO: Come up with a more elegant solution for this. - */ - for_each_oldnew_plane_in_state(state, other, old_other_state, new_other_state, i) { - struct amdgpu_framebuffer *old_afb, *new_afb; - struct dm_plane_state *dm_new_other_state, *dm_old_other_state; - - dm_new_other_state = to_dm_plane_state(new_other_state); - dm_old_other_state = to_dm_plane_state(old_other_state); - - if (other->type == DRM_PLANE_TYPE_CURSOR) - continue; - - if (old_other_state->crtc != new_plane_state->crtc && - new_other_state->crtc != new_plane_state->crtc) - continue; - - if (old_other_state->crtc != new_other_state->crtc) - return true; - - /* Src/dst size and scaling updates. */ - if (old_other_state->src_w != new_other_state->src_w || - old_other_state->src_h != new_other_state->src_h || - old_other_state->crtc_w != new_other_state->crtc_w || - old_other_state->crtc_h != new_other_state->crtc_h) - return true; - - /* Rotation / mirroring updates. */ - if (old_other_state->rotation != new_other_state->rotation) - return true; - - /* Blending updates. */ - if (old_other_state->pixel_blend_mode != - new_other_state->pixel_blend_mode) - return true; - - /* Alpha updates. */ - if (old_other_state->alpha != new_other_state->alpha) - return true; - - /* Colorspace changes. */ - if (old_other_state->color_range != new_other_state->color_range || - old_other_state->color_encoding != new_other_state->color_encoding) - return true; - - /* HDR/Transfer Function changes. */ - if (dm_old_other_state->degamma_tf != dm_new_other_state->degamma_tf || - dm_old_other_state->degamma_lut != dm_new_other_state->degamma_lut || - dm_old_other_state->hdr_mult != dm_new_other_state->hdr_mult || - dm_old_other_state->ctm != dm_new_other_state->ctm || - dm_old_other_state->shaper_lut != dm_new_other_state->shaper_lut || - dm_old_other_state->shaper_tf != dm_new_other_state->shaper_tf || - dm_old_other_state->lut3d != dm_new_other_state->lut3d || - dm_old_other_state->blend_lut != dm_new_other_state->blend_lut || - dm_old_other_state->blend_tf != dm_new_other_state->blend_tf) - return true; - - /* Framebuffer checks fall at the end. */ - if (!old_other_state->fb || !new_other_state->fb) - continue; - - /* Pixel format changes can require bandwidth updates. */ - if (old_other_state->fb->format != new_other_state->fb->format) - return true; - - old_afb = (struct amdgpu_framebuffer *)old_other_state->fb; - new_afb = (struct amdgpu_framebuffer *)new_other_state->fb; - - /* Tiling and DCC changes also require bandwidth updates. */ - if (old_afb->tiling_flags != new_afb->tiling_flags || - old_afb->base.modifier != new_afb->base.modifier) - return true; - } - - return false; -} - -static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc, - struct drm_plane_state *new_plane_state, - struct drm_framebuffer *fb) -{ - struct amdgpu_device *adev = drm_to_adev(new_acrtc->base.dev); - struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(fb); - unsigned int pitch; - bool linear; - - if (fb->width > new_acrtc->max_cursor_width || - fb->height > new_acrtc->max_cursor_height) { - DRM_DEBUG_ATOMIC("Bad cursor FB size %dx%d\n", - new_plane_state->fb->width, - new_plane_state->fb->height); - return -EINVAL; - } - if (new_plane_state->src_w != fb->width << 16 || - new_plane_state->src_h != fb->height << 16) { - DRM_DEBUG_ATOMIC("Cropping not supported for cursor plane\n"); - return -EINVAL; - } - - /* Pitch in pixels */ - pitch = fb->pitches[0] / fb->format->cpp[0]; - - if (fb->width != pitch) { - DRM_DEBUG_ATOMIC("Cursor FB width %d doesn't match pitch %d", - fb->width, pitch); - return -EINVAL; - } - - switch (pitch) { - case 64: - case 128: - case 256: - /* FB pitch is supported by cursor plane */ - break; - default: - DRM_DEBUG_ATOMIC("Bad cursor FB pitch %d px\n", pitch); - return -EINVAL; - } - - /* Core DRM takes care of checking FB modifiers, so we only need to - * check tiling flags when the FB doesn't have a modifier. - */ - if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) { - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { - linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0; - } else if (adev->family >= AMDGPU_FAMILY_AI) { - linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0; - } else { - linear = AMDGPU_TILING_GET(afb->tiling_flags, ARRAY_MODE) != DC_ARRAY_2D_TILED_THIN1 && - AMDGPU_TILING_GET(afb->tiling_flags, ARRAY_MODE) != DC_ARRAY_1D_TILED_THIN1 && - AMDGPU_TILING_GET(afb->tiling_flags, MICRO_TILE_MODE) == 0; - } - if (!linear) { - DRM_DEBUG_ATOMIC("Cursor FB not linear"); - return -EINVAL; - } - } - - return 0; -} - -/* - * Helper function for checking the cursor in native mode - */ -static int dm_check_native_cursor_state(struct drm_crtc *new_plane_crtc, - struct drm_plane *plane, - struct drm_plane_state *new_plane_state, - bool enable) -{ - - struct amdgpu_crtc *new_acrtc; - int ret; - - if (!enable || !new_plane_crtc || - drm_atomic_plane_disabling(plane->state, new_plane_state)) - return 0; - - new_acrtc = to_amdgpu_crtc(new_plane_crtc); - - if (new_plane_state->src_x != 0 || new_plane_state->src_y != 0) { - DRM_DEBUG_ATOMIC("Cropping not supported for cursor plane\n"); - return -EINVAL; - } - - if (new_plane_state->fb) { - ret = dm_check_cursor_fb(new_acrtc, new_plane_state, - new_plane_state->fb); - if (ret) - return ret; - } - - return 0; -} - -static bool dm_should_update_native_cursor(struct drm_atomic_state *state, - struct drm_crtc *old_plane_crtc, - struct drm_crtc *new_plane_crtc, - bool enable) -{ - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - - if (!enable) { - if (old_plane_crtc == NULL) - return true; - - old_crtc_state = drm_atomic_get_old_crtc_state( - state, old_plane_crtc); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - return dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE; - } else { - if (new_plane_crtc == NULL) - return true; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_plane_crtc); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - return dm_new_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE; - } -} - -static int dm_update_plane_state(struct dc *dc, - struct drm_atomic_state *state, - struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state, - bool enable, - bool *lock_and_validation_needed, - bool *is_top_most_overlay) -{ - - struct dm_atomic_state *dm_state = NULL; - struct drm_crtc *new_plane_crtc, *old_plane_crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_new_crtc_state, *dm_old_crtc_state; - struct dm_plane_state *dm_new_plane_state, *dm_old_plane_state; - bool needs_reset, update_native_cursor; - int ret = 0; - - - new_plane_crtc = new_plane_state->crtc; - old_plane_crtc = old_plane_state->crtc; - dm_new_plane_state = to_dm_plane_state(new_plane_state); - dm_old_plane_state = to_dm_plane_state(old_plane_state); - - update_native_cursor = dm_should_update_native_cursor(state, - old_plane_crtc, - new_plane_crtc, - enable); - - if (plane->type == DRM_PLANE_TYPE_CURSOR && update_native_cursor) { - ret = dm_check_native_cursor_state(new_plane_crtc, plane, - new_plane_state, enable); - if (ret) - return ret; - - return 0; - } - - needs_reset = should_reset_plane(state, plane, old_plane_state, - new_plane_state); - - /* Remove any changed/removed planes */ - if (!enable) { - if (!needs_reset) - return 0; - - if (!old_plane_crtc) - return 0; - - old_crtc_state = drm_atomic_get_old_crtc_state( - state, old_plane_crtc); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (!dm_old_crtc_state->stream) - return 0; - - DRM_DEBUG_ATOMIC("Disabling DRM plane: %d on DRM crtc %d\n", - plane->base.id, old_plane_crtc->base.id); - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - return ret; - - if (!dc_state_remove_plane( - dc, - dm_old_crtc_state->stream, - dm_old_plane_state->dc_state, - dm_state->context)) { - - return -EINVAL; - } - - if (dm_old_plane_state->dc_state) - dc_plane_state_release(dm_old_plane_state->dc_state); - - dm_new_plane_state->dc_state = NULL; - - *lock_and_validation_needed = true; - - } else { /* Add new planes */ - struct dc_plane_state *dc_new_plane_state; - - if (drm_atomic_plane_disabling(plane->state, new_plane_state)) - return 0; - - if (!new_plane_crtc) - return 0; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, new_plane_crtc); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (!dm_new_crtc_state->stream) - return 0; - - if (!needs_reset) - return 0; - - ret = amdgpu_dm_plane_helper_check_state(new_plane_state, new_crtc_state); - if (ret) - goto out; - - WARN_ON(dm_new_plane_state->dc_state); - - dc_new_plane_state = dc_create_plane_state(dc); - if (!dc_new_plane_state) { - ret = -ENOMEM; - goto out; - } - - DRM_DEBUG_ATOMIC("Enabling DRM plane: %d on DRM crtc %d\n", - plane->base.id, new_plane_crtc->base.id); - - ret = fill_dc_plane_attributes( - drm_to_adev(new_plane_crtc->dev), - dc_new_plane_state, - new_plane_state, - new_crtc_state); - if (ret) { - dc_plane_state_release(dc_new_plane_state); - goto out; - } - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) { - dc_plane_state_release(dc_new_plane_state); - goto out; - } - - /* - * Any atomic check errors that occur after this will - * not need a release. The plane state will be attached - * to the stream, and therefore part of the atomic - * state. It'll be released when the atomic state is - * cleaned. - */ - if (!dc_state_add_plane( - dc, - dm_new_crtc_state->stream, - dc_new_plane_state, - dm_state->context)) { - - dc_plane_state_release(dc_new_plane_state); - ret = -EINVAL; - goto out; - } - - dm_new_plane_state->dc_state = dc_new_plane_state; - - dm_new_crtc_state->mpo_requested |= (plane->type == DRM_PLANE_TYPE_OVERLAY); - - /* Tell DC to do a full surface update every time there - * is a plane change. Inefficient, but works for now. - */ - dm_new_plane_state->dc_state->update_flags.bits.full_update = 1; - - *lock_and_validation_needed = true; - } - -out: - /* If enabling cursor overlay failed, attempt fallback to native mode */ - if (enable && ret == -EINVAL && plane->type == DRM_PLANE_TYPE_CURSOR) { - ret = dm_check_native_cursor_state(new_plane_crtc, plane, - new_plane_state, enable); - if (ret) - return ret; - - dm_new_crtc_state->cursor_mode = DM_CURSOR_NATIVE_MODE; - } - - return ret; -} - -static void dm_get_oriented_plane_size(struct drm_plane_state *plane_state, - int *src_w, int *src_h) -{ - switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) { - case DRM_MODE_ROTATE_90: - case DRM_MODE_ROTATE_270: - *src_w = plane_state->src_h >> 16; - *src_h = plane_state->src_w >> 16; - break; - case DRM_MODE_ROTATE_0: - case DRM_MODE_ROTATE_180: - default: - *src_w = plane_state->src_w >> 16; - *src_h = plane_state->src_h >> 16; - break; - } -} - -static void -dm_get_plane_scale(struct drm_plane_state *plane_state, - int *out_plane_scale_w, int *out_plane_scale_h) -{ - int plane_src_w, plane_src_h; - - dm_get_oriented_plane_size(plane_state, &plane_src_w, &plane_src_h); - *out_plane_scale_w = plane_state->crtc_w * 1000 / plane_src_w; - *out_plane_scale_h = plane_state->crtc_h * 1000 / plane_src_h; -} - -/* - * The normalized_zpos value cannot be used by this iterator directly. It's only - * calculated for enabled planes, potentially causing normalized_zpos collisions - * between enabled/disabled planes in the atomic state. We need a unique value - * so that the iterator will not generate the same object twice, or loop - * indefinitely. - */ -static inline struct __drm_planes_state *__get_next_zpos( - struct drm_atomic_state *state, - struct __drm_planes_state *prev) -{ - unsigned int highest_zpos = 0, prev_zpos = 256; - uint32_t highest_id = 0, prev_id = UINT_MAX; - struct drm_plane_state *new_plane_state; - struct drm_plane *plane; - int i, highest_i = -1; - - if (prev != NULL) { - prev_zpos = prev->new_state->zpos; - prev_id = prev->ptr->base.id; - } - - for_each_new_plane_in_state(state, plane, new_plane_state, i) { - /* Skip planes with higher zpos than the previously returned */ - if (new_plane_state->zpos > prev_zpos || - (new_plane_state->zpos == prev_zpos && - plane->base.id >= prev_id)) - continue; - - /* Save the index of the plane with highest zpos */ - if (new_plane_state->zpos > highest_zpos || - (new_plane_state->zpos == highest_zpos && - plane->base.id > highest_id)) { - highest_zpos = new_plane_state->zpos; - highest_id = plane->base.id; - highest_i = i; - } - } - - if (highest_i < 0) - return NULL; - - return &state->planes[highest_i]; -} - -/* - * Use the uniqueness of the plane's (zpos, drm obj ID) combination to iterate - * by descending zpos, as read from the new plane state. This is the same - * ordering as defined by drm_atomic_normalize_zpos(). - */ -#define for_each_oldnew_plane_in_descending_zpos(__state, plane, old_plane_state, new_plane_state) \ - for (struct __drm_planes_state *__i = __get_next_zpos((__state), NULL); \ - __i != NULL; __i = __get_next_zpos((__state), __i)) \ - for_each_if(((plane) = __i->ptr, \ - (void)(plane) /* Only to avoid unused-but-set-variable warning */, \ - (old_plane_state) = __i->old_state, \ - (new_plane_state) = __i->new_state, 1)) - -static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc) -{ - struct drm_connector *connector; - struct drm_connector_state *conn_state, *old_conn_state; - struct amdgpu_dm_connector *aconnector = NULL; - int i; - - for_each_oldnew_connector_in_state(state, connector, old_conn_state, conn_state, i) { - if (!conn_state->crtc) - conn_state = old_conn_state; - - if (conn_state->crtc != crtc) - continue; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (!aconnector->mst_output_port || !aconnector->mst_root) - aconnector = NULL; - else - break; - } - - if (!aconnector) - return 0; - - return drm_dp_mst_add_affected_dsc_crtcs(state, &aconnector->mst_root->mst_mgr); -} - -/** - * DOC: Cursor Modes - Native vs Overlay - * - * In native mode, the cursor uses a integrated cursor pipe within each DCN hw - * plane. It does not require a dedicated hw plane to enable, but it is - * subjected to the same z-order and scaling as the hw plane. It also has format - * restrictions, a RGB cursor in native mode cannot be enabled within a non-RGB - * hw plane. - * - * In overlay mode, the cursor uses a separate DCN hw plane, and thus has its - * own scaling and z-pos. It also has no blending restrictions. It lends to a - * cursor behavior more akin to a DRM client's expectations. However, it does - * occupy an extra DCN plane, and therefore will only be used if a DCN plane is - * available. - */ - -/** - * dm_crtc_get_cursor_mode() - Determine the required cursor mode on crtc - * @adev: amdgpu device - * @state: DRM atomic state - * @dm_crtc_state: amdgpu state for the CRTC containing the cursor - * @cursor_mode: Returns the required cursor mode on dm_crtc_state - * - * Get whether the cursor should be enabled in native mode, or overlay mode, on - * the dm_crtc_state. - * - * The cursor should be enabled in overlay mode if there exists an underlying - * plane - on which the cursor may be blended - that is either YUV formatted, or - * scaled differently from the cursor. - * - * Since zpos info is required, drm_atomic_normalize_zpos must be called before - * calling this function. - * - * Return: 0 on success, or an error code if getting the cursor plane state - * failed. - */ -static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, - struct drm_atomic_state *state, - struct dm_crtc_state *dm_crtc_state, - enum amdgpu_dm_cursor_mode *cursor_mode) -{ - struct drm_plane_state *old_plane_state, *plane_state, *cursor_state; - struct drm_crtc_state *crtc_state = &dm_crtc_state->base; - struct drm_plane *plane; - bool consider_mode_change = false; - bool entire_crtc_covered = false; - bool cursor_changed = false; - int underlying_scale_w, underlying_scale_h; - int cursor_scale_w, cursor_scale_h; - int i; - - /* Overlay cursor not supported on HW before DCN - * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE - */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { - *cursor_mode = DM_CURSOR_NATIVE_MODE; - return 0; - } - - /* Init cursor_mode to be the same as current */ - *cursor_mode = dm_crtc_state->cursor_mode; - - /* - * Cursor mode can change if a plane's format changes, scale changes, is - * enabled/disabled, or z-order changes. - */ - for_each_oldnew_plane_in_state(state, plane, old_plane_state, plane_state, i) { - int new_scale_w, new_scale_h, old_scale_w, old_scale_h; - - /* Only care about planes on this CRTC */ - if ((drm_plane_mask(plane) & crtc_state->plane_mask) == 0) - continue; - - if (plane->type == DRM_PLANE_TYPE_CURSOR) - cursor_changed = true; - - if (drm_atomic_plane_enabling(old_plane_state, plane_state) || - drm_atomic_plane_disabling(old_plane_state, plane_state) || - old_plane_state->fb->format != plane_state->fb->format) { - consider_mode_change = true; - break; - } - - dm_get_plane_scale(plane_state, &new_scale_w, &new_scale_h); - dm_get_plane_scale(old_plane_state, &old_scale_w, &old_scale_h); - if (new_scale_w != old_scale_w || new_scale_h != old_scale_h) { - consider_mode_change = true; - break; - } - } - - if (!consider_mode_change && !crtc_state->zpos_changed) - return 0; - - /* - * If no cursor change on this CRTC, and not enabled on this CRTC, then - * no need to set cursor mode. This avoids needlessly locking the cursor - * state. - */ - if (!cursor_changed && - !(drm_plane_mask(crtc_state->crtc->cursor) & crtc_state->plane_mask)) { - return 0; - } - - cursor_state = drm_atomic_get_plane_state(state, - crtc_state->crtc->cursor); - if (IS_ERR(cursor_state)) - return PTR_ERR(cursor_state); - - /* Cursor is disabled */ - if (!cursor_state->fb) - return 0; - - /* For all planes in descending z-order (all of which are below cursor - * as per zpos definitions), check their scaling and format - */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, plane_state) { - - /* Only care about non-cursor planes on this CRTC */ - if ((drm_plane_mask(plane) & crtc_state->plane_mask) == 0 || - plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - /* Underlying plane is YUV format - use overlay cursor */ - if (amdgpu_dm_plane_is_video_format(plane_state->fb->format->format)) { - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - return 0; - } - - dm_get_plane_scale(plane_state, - &underlying_scale_w, &underlying_scale_h); - dm_get_plane_scale(cursor_state, - &cursor_scale_w, &cursor_scale_h); - - /* Underlying plane has different scale - use overlay cursor */ - if (cursor_scale_w != underlying_scale_w && - cursor_scale_h != underlying_scale_h) { - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - return 0; - } - - /* If this plane covers the whole CRTC, no need to check planes underneath */ - if (plane_state->crtc_x <= 0 && plane_state->crtc_y <= 0 && - plane_state->crtc_x + plane_state->crtc_w >= crtc_state->mode.hdisplay && - plane_state->crtc_y + plane_state->crtc_h >= crtc_state->mode.vdisplay) { - entire_crtc_covered = true; - break; - } - } - - /* If planes do not cover the entire CRTC, use overlay mode to enable - * cursor over holes - */ - if (entire_crtc_covered) - *cursor_mode = DM_CURSOR_NATIVE_MODE; - else - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - - return 0; -} - -/** - * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM. - * - * @dev: The DRM device - * @state: The atomic state to commit - * - * Validate that the given atomic state is programmable by DC into hardware. - * This involves constructing a &struct dc_state reflecting the new hardware - * state we wish to commit, then querying DC to see if it is programmable. It's - * important not to modify the existing DC state. Otherwise, atomic_check - * may unexpectedly commit hardware changes. - * - * When validating the DC state, it's important that the right locks are - * acquired. For full updates case which removes/adds/updates streams on one - * CRTC while flipping on another CRTC, acquiring global lock will guarantee - * that any such full update commit will wait for completion of any outstanding - * flip using DRMs synchronization events. - * - * Note that DM adds the affected connectors for all CRTCs in state, when that - * might not seem necessary. This is because DC stream creation requires the - * DC sink, which is tied to the DRM connector state. Cleaning this up should - * be possible but non-trivial - a possible TODO item. - * - * Return: -Error code if validation failed. - */ -static int amdgpu_dm_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_atomic_state *dm_state = NULL; - struct dc *dc = adev->dm.dc; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct drm_plane *plane; - struct drm_plane_state *old_plane_state, *new_plane_state, *new_cursor_state; - enum dc_status status; - int ret, i; - bool lock_and_validation_needed = false; - bool is_top_most_overlay = true; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct drm_dp_mst_topology_mgr *mgr; - struct drm_dp_mst_topology_state *mst_state; - struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0}; - - trace_amdgpu_dm_atomic_check_begin(state); - - ret = drm_atomic_helper_check_modeset(dev, state); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_helper_check_modeset() failed\n"); - goto fail; - } - - /* Check connector changes */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - - /* Skip connectors that are disabled or part of modeset already. */ - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_crtc_state(state, new_con_state->crtc); - if (IS_ERR(new_crtc_state)) { - drm_dbg_atomic(dev, "drm_atomic_get_crtc_state() failed\n"); - ret = PTR_ERR(new_crtc_state); - goto fail; - } - - if (dm_old_con_state->abm_level != dm_new_con_state->abm_level || - dm_old_con_state->scaling != dm_new_con_state->scaling) - new_crtc_state->connectors_changed = true; - } - - if (dc_resource_is_dsc_encoding_supported(dc)) { - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { - ret = add_affected_mst_dsc_crtcs(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "add_affected_mst_dsc_crtcs() failed\n"); - goto fail; - } - } - } - } - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state) && - !new_crtc_state->color_mgmt_changed && - old_crtc_state->vrr_enabled == new_crtc_state->vrr_enabled && - dm_old_crtc_state->dsc_force_changed == false) - continue; - - ret = amdgpu_dm_verify_lut_sizes(new_crtc_state); - if (ret) { - drm_dbg_atomic(dev, "amdgpu_dm_verify_lut_sizes() failed\n"); - goto fail; - } - - if (!new_crtc_state->enable) - continue; - - ret = drm_atomic_add_affected_connectors(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_add_affected_connectors() failed\n"); - goto fail; - } - - ret = drm_atomic_add_affected_planes(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_add_affected_planes() failed\n"); - goto fail; - } - - if (dm_old_crtc_state->dsc_force_changed) - new_crtc_state->mode_changed = true; - } - - /* - * Add all primary and overlay planes on the CRTC to the state - * whenever a plane is enabled to maintain correct z-ordering - * and to enable fast surface updates. - */ - drm_for_each_crtc(crtc, dev) { - bool modified = false; - - for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - if (new_plane_state->crtc == crtc || - old_plane_state->crtc == crtc) { - modified = true; - break; - } - } - - if (!modified) - continue; - - drm_for_each_plane_mask(plane, state->dev, crtc->state->plane_mask) { - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - new_plane_state = - drm_atomic_get_plane_state(state, plane); - - if (IS_ERR(new_plane_state)) { - ret = PTR_ERR(new_plane_state); - drm_dbg_atomic(dev, "new_plane_state is BAD\n"); - goto fail; - } - } - } - - /* - * DC consults the zpos (layer_index in DC terminology) to determine the - * hw plane on which to enable the hw cursor (see - * `dcn10_can_pipe_disable_cursor`). By now, all modified planes are in - * atomic state, so call drm helper to normalize zpos. - */ - ret = drm_atomic_normalize_zpos(dev, state); - if (ret) { - drm_dbg(dev, "drm_atomic_normalize_zpos() failed\n"); - goto fail; - } - - /* - * Determine whether cursors on each CRTC should be enabled in native or - * overlay mode. - */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - ret = dm_crtc_get_cursor_mode(adev, state, dm_new_crtc_state, - &dm_new_crtc_state->cursor_mode); - if (ret) { - drm_dbg(dev, "Failed to determine cursor mode\n"); - goto fail; - } - } - - /* Remove exiting planes if they are modified */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, new_plane_state) { - if (old_plane_state->fb && new_plane_state->fb && - get_mem_type(old_plane_state->fb) != - get_mem_type(new_plane_state->fb)) - lock_and_validation_needed = true; - - ret = dm_update_plane_state(dc, state, plane, - old_plane_state, - new_plane_state, - false, - &lock_and_validation_needed, - &is_top_most_overlay); - if (ret) { - drm_dbg_atomic(dev, "dm_update_plane_state() failed\n"); - goto fail; - } - } - - /* Disable all crtcs which require disable */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = dm_update_crtc_state(&adev->dm, state, crtc, - old_crtc_state, - new_crtc_state, - false, - &lock_and_validation_needed); - if (ret) { - drm_dbg_atomic(dev, "DISABLE: dm_update_crtc_state() failed\n"); - goto fail; - } - } - - /* Enable all crtcs which require enable */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = dm_update_crtc_state(&adev->dm, state, crtc, - old_crtc_state, - new_crtc_state, - true, - &lock_and_validation_needed); - if (ret) { - drm_dbg_atomic(dev, "ENABLE: dm_update_crtc_state() failed\n"); - goto fail; - } - } - - /* Add new/modified planes */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, new_plane_state) { - ret = dm_update_plane_state(dc, state, plane, - old_plane_state, - new_plane_state, - true, - &lock_and_validation_needed, - &is_top_most_overlay); - if (ret) { - drm_dbg_atomic(dev, "dm_update_plane_state() failed\n"); - goto fail; - } - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - if (dc_resource_is_dsc_encoding_supported(dc)) { - ret = pre_validate_dsc(state, &dm_state, vars); - if (ret != 0) - goto fail; - } -#endif - - /* Run this here since we want to validate the streams we created */ - ret = drm_atomic_helper_check_planes(dev, state); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_helper_check_planes() failed\n"); - goto fail; - } - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->mpo_requested) - drm_dbg_atomic(dev, "MPO enablement requested on crtc:[%p]\n", crtc); - } - - /* Check cursor restrictions */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - enum amdgpu_dm_cursor_mode required_cursor_mode; - int is_rotated, is_scaled; - - /* Overlay cusor not subject to native cursor restrictions */ - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) - continue; - - /* Check if rotation or scaling is enabled on DCN401 */ - if ((drm_plane_mask(crtc->cursor) & new_crtc_state->plane_mask) && - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { - new_cursor_state = drm_atomic_get_new_plane_state(state, crtc->cursor); - - is_rotated = new_cursor_state && - ((new_cursor_state->rotation & DRM_MODE_ROTATE_MASK) != DRM_MODE_ROTATE_0); - is_scaled = new_cursor_state && ((new_cursor_state->src_w >> 16 != new_cursor_state->crtc_w) || - (new_cursor_state->src_h >> 16 != new_cursor_state->crtc_h)); - - if (is_rotated || is_scaled) { - drm_dbg_driver( - crtc->dev, - "[CRTC:%d:%s] cannot enable hardware cursor due to rotation/scaling\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - } - - /* If HW can only do native cursor, check restrictions again */ - ret = dm_crtc_get_cursor_mode(adev, state, dm_new_crtc_state, - &required_cursor_mode); - if (ret) { - drm_dbg_driver(crtc->dev, - "[CRTC:%d:%s] Checking cursor mode failed\n", - crtc->base.id, crtc->name); - goto fail; - } else if (required_cursor_mode == DM_CURSOR_OVERLAY_MODE) { - drm_dbg_driver(crtc->dev, - "[CRTC:%d:%s] Cannot enable native cursor due to scaling or YUV restrictions\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - } - - if (state->legacy_cursor_update) { - /* - * This is a fast cursor update coming from the plane update - * helper, check if it can be done asynchronously for better - * performance. - */ - state->async_update = - !drm_atomic_helper_async_check(dev, state); - - /* - * Skip the remaining global validation if this is an async - * update. Cursor updates can be done without affecting - * state or bandwidth calcs and this avoids the performance - * penalty of locking the private state object and - * allocating a new dc_state. - */ - if (state->async_update) - return 0; - } - - /* Check scaling and underscan changes*/ - /* TODO Removed scaling changes validation due to inability to commit - * new stream into context w\o causing full reset. Need to - * decide how to handle. - */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - - /* Skip any modesets/resets */ - if (!acrtc || drm_atomic_crtc_needs_modeset( - drm_atomic_get_new_crtc_state(state, &acrtc->base))) - continue; - - /* Skip any thing not scale or underscan changes */ - if (!is_scaling_state_different(dm_new_con_state, dm_old_con_state)) - continue; - - lock_and_validation_needed = true; - } - - /* set the slot info for each mst_state based on the link encoding format */ - for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) { - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - u8 link_coding_cap; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - if (connector->index == mst_state->mgr->conn_base_id) { - aconnector = to_amdgpu_dm_connector(connector); - link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(aconnector->dc_link); - drm_dp_mst_update_slots(mst_state, link_coding_cap); - - break; - } - } - drm_connector_list_iter_end(&iter); - } - - /** - * Streams and planes are reset when there are changes that affect - * bandwidth. Anything that affects bandwidth needs to go through - * DC global validation to ensure that the configuration can be applied - * to hardware. - * - * We have to currently stall out here in atomic_check for outstanding - * commits to finish in this case because our IRQ handlers reference - * DRM state directly - we can end up disabling interrupts too early - * if we don't. - * - * TODO: Remove this stall and drop DM state private objects. - */ - if (lock_and_validation_needed) { - ret = dm_atomic_get_state(state, &dm_state); - if (ret) { - drm_dbg_atomic(dev, "dm_atomic_get_state() failed\n"); - goto fail; - } - - ret = do_aquire_global_lock(dev, state); - if (ret) { - drm_dbg_atomic(dev, "do_aquire_global_lock() failed\n"); - goto fail; - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - if (dc_resource_is_dsc_encoding_supported(dc)) { - ret = compute_mst_dsc_configs_for_state(state, dm_state->context, vars); - if (ret) { - drm_dbg_atomic(dev, "compute_mst_dsc_configs_for_state() failed\n"); - ret = -EINVAL; - goto fail; - } - } -#endif - - ret = dm_update_mst_vcpi_slots_for_dsc(state, dm_state->context, vars); - if (ret) { - drm_dbg_atomic(dev, "dm_update_mst_vcpi_slots_for_dsc() failed\n"); - goto fail; - } - - /* - * Perform validation of MST topology in the state: - * We need to perform MST atomic check before calling - * dc_validate_global_state(), or there is a chance - * to get stuck in an infinite loop and hang eventually. - */ - ret = drm_dp_mst_atomic_check(state); - if (ret) { - drm_dbg_atomic(dev, "drm_dp_mst_atomic_check() failed\n"); - goto fail; - } - status = dc_validate_global_state(dc, dm_state->context, true); - if (status != DC_OK) { - drm_dbg_atomic(dev, "DC global validation failure: %s (%d)", - dc_status_to_str(status), status); - ret = -EINVAL; - goto fail; - } - } else { - /* - * The commit is a fast update. Fast updates shouldn't change - * the DC context, affect global validation, and can have their - * commit work done in parallel with other commits not touching - * the same resource. If we have a new DC context as part of - * the DM atomic state from validation we need to free it and - * retain the existing one instead. - * - * Furthermore, since the DM atomic state only contains the DC - * context and can safely be annulled, we can free the state - * and clear the associated private object now to free - * some memory and avoid a possible use-after-free later. - */ - - for (i = 0; i < state->num_private_objs; i++) { - struct drm_private_obj *obj = state->private_objs[i].ptr; - - if (obj->funcs == adev->dm.atomic_obj.funcs) { - int j = state->num_private_objs-1; - - dm_atomic_destroy_state(obj, - state->private_objs[i].state); - - /* If i is not at the end of the array then the - * last element needs to be moved to where i was - * before the array can safely be truncated. - */ - if (i != j) - state->private_objs[i] = - state->private_objs[j]; - - state->private_objs[j].ptr = NULL; - state->private_objs[j].state = NULL; - state->private_objs[j].old_state = NULL; - state->private_objs[j].new_state = NULL; - - state->num_private_objs = j; - break; - } - } - } - - /* Store the overall update type for use later in atomic check. */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - struct dm_crtc_state *dm_new_crtc_state = - to_dm_crtc_state(new_crtc_state); - - /* - * Only allow async flips for fast updates that don't change - * the FB pitch, the DCC state, rotation, etc. - */ - if (new_crtc_state->async_flip && lock_and_validation_needed) { - drm_dbg_atomic(crtc->dev, - "[CRTC:%d:%s] async flips are only supported for fast updates\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - - dm_new_crtc_state->update_type = lock_and_validation_needed ? - UPDATE_TYPE_FULL : UPDATE_TYPE_FAST; - } - - /* Must be success */ - WARN_ON(ret); - - trace_amdgpu_dm_atomic_check_finish(state, ret); - - return ret; - -fail: - if (ret == -EDEADLK) - drm_dbg_atomic(dev, "Atomic check stopped to avoid deadlock.\n"); - else if (ret == -EINTR || ret == -EAGAIN || ret == -ERESTARTSYS) - drm_dbg_atomic(dev, "Atomic check stopped due to signal.\n"); - else - drm_dbg_atomic(dev, "Atomic check failed with err: %d\n", ret); - - trace_amdgpu_dm_atomic_check_finish(state, ret); - - return ret; -} - -static bool dm_edid_parser_send_cea(struct amdgpu_display_manager *dm, - unsigned int offset, - unsigned int total_length, - u8 *data, - unsigned int length, - struct amdgpu_hdmi_vsdb_info *vsdb) -{ - bool res; - union dmub_rb_cmd cmd; - struct dmub_cmd_send_edid_cea *input; - struct dmub_cmd_edid_cea_output *output; - - if (length > DMUB_EDID_CEA_DATA_CHUNK_BYTES) - return false; - - memset(&cmd, 0, sizeof(cmd)); - - input = &cmd.edid_cea.data.input; - - cmd.edid_cea.header.type = DMUB_CMD__EDID_CEA; - cmd.edid_cea.header.sub_type = 0; - cmd.edid_cea.header.payload_bytes = - sizeof(cmd.edid_cea) - sizeof(cmd.edid_cea.header); - input->offset = offset; - input->length = length; - input->cea_total_length = total_length; - memcpy(input->payload, data, length); - - res = dc_wake_and_execute_dmub_cmd(dm->dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY); - if (!res) { - DRM_ERROR("EDID CEA parser failed\n"); - return false; - } - - output = &cmd.edid_cea.data.output; - - if (output->type == DMUB_CMD__EDID_CEA_ACK) { - if (!output->ack.success) { - DRM_ERROR("EDID CEA ack failed at offset %d\n", - output->ack.offset); - } - } else if (output->type == DMUB_CMD__EDID_CEA_AMD_VSDB) { - if (!output->amd_vsdb.vsdb_found) - return false; - - vsdb->freesync_supported = output->amd_vsdb.freesync_supported; - vsdb->amd_vsdb_version = output->amd_vsdb.amd_vsdb_version; - vsdb->min_refresh_rate_hz = output->amd_vsdb.min_frame_rate; - vsdb->max_refresh_rate_hz = output->amd_vsdb.max_frame_rate; - } else { - DRM_WARN("Unknown EDID CEA parser results\n"); - return false; - } - - return true; -} - -static bool parse_edid_cea_dmcu(struct amdgpu_display_manager *dm, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - int i; - - /* send extension block to DMCU for parsing */ - for (i = 0; i < len; i += 8) { - bool res; - int offset; - - /* send 8 bytes a time */ - if (!dc_edid_parser_send_cea(dm->dc, i, len, &edid_ext[i], 8)) - return false; - - if (i+8 == len) { - /* EDID block sent completed, expect result */ - int version, min_rate, max_rate; - - res = dc_edid_parser_recv_amd_vsdb(dm->dc, &version, &min_rate, &max_rate); - if (res) { - /* amd vsdb found */ - vsdb_info->freesync_supported = 1; - vsdb_info->amd_vsdb_version = version; - vsdb_info->min_refresh_rate_hz = min_rate; - vsdb_info->max_refresh_rate_hz = max_rate; - return true; - } - /* not amd vsdb */ - return false; - } - - /* check for ack*/ - res = dc_edid_parser_recv_cea_ack(dm->dc, &offset); - if (!res) - return false; - } - - return false; -} - -static bool parse_edid_cea_dmub(struct amdgpu_display_manager *dm, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - int i; - - /* send extension block to DMCU for parsing */ - for (i = 0; i < len; i += 8) { - /* send 8 bytes a time */ - if (!dm_edid_parser_send_cea(dm, i, len, &edid_ext[i], 8, vsdb_info)) - return false; - } - - return vsdb_info->freesync_supported; -} - -static bool parse_edid_cea(struct amdgpu_dm_connector *aconnector, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev); - bool ret; - - mutex_lock(&adev->dm.dc_lock); - if (adev->dm.dmub_srv) - ret = parse_edid_cea_dmub(&adev->dm, edid_ext, len, vsdb_info); - else - ret = parse_edid_cea_dmcu(&adev->dm, edid_ext, len, vsdb_info); - mutex_unlock(&adev->dm.dc_lock); - return ret; -} - -static void parse_edid_displayid_vrr(struct drm_connector *connector, - struct edid *edid) -{ - u8 *edid_ext = NULL; - int i; - int j = 0; - u16 min_vfreq; - u16 max_vfreq; - - if (edid == NULL || edid->extensions == 0) - return; - - /* Find DisplayID extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (void *)(edid + (i + 1)); - if (edid_ext[0] == DISPLAYID_EXT) - break; - } - - if (edid_ext == NULL) - return; - - while (j < EDID_LENGTH) { - /* Get dynamic video timing range from DisplayID if available */ - if (EDID_LENGTH - j > 13 && edid_ext[j] == 0x25 && - (edid_ext[j+1] & 0xFE) == 0 && (edid_ext[j+2] == 9)) { - min_vfreq = edid_ext[j+9]; - if (edid_ext[j+1] & 7) - max_vfreq = edid_ext[j+10] + ((edid_ext[j+11] & 3) << 8); - else - max_vfreq = edid_ext[j+10]; - - if (max_vfreq && min_vfreq) { - connector->display_info.monitor_range.max_vfreq = max_vfreq; - connector->display_info.monitor_range.min_vfreq = min_vfreq; - - return; - } - } - j++; - } -} - -static int parse_amd_vsdb(struct amdgpu_dm_connector *aconnector, - struct edid *edid, struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - u8 *edid_ext = NULL; - int i; - int j = 0; - - if (edid == NULL || edid->extensions == 0) - return -ENODEV; - - /* Find DisplayID extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (void *)(edid + (i + 1)); - if (edid_ext[0] == DISPLAYID_EXT) - break; - } - - while (j < EDID_LENGTH) { - struct amd_vsdb_block *amd_vsdb = (struct amd_vsdb_block *)&edid_ext[j]; - unsigned int ieeeId = (amd_vsdb->ieee_id[2] << 16) | (amd_vsdb->ieee_id[1] << 8) | (amd_vsdb->ieee_id[0]); - - if (ieeeId == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_IEEE_REGISTRATION_ID && - amd_vsdb->version == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3) { - vsdb_info->replay_mode = (amd_vsdb->feature_caps & AMD_VSDB_VERSION_3_FEATURECAP_REPLAYMODE) ? true : false; - vsdb_info->amd_vsdb_version = HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3; - DRM_DEBUG_KMS("Panel supports Replay Mode: %d\n", vsdb_info->replay_mode); - - return true; - } - j++; - } - - return false; -} - -static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector, - struct edid *edid, struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - u8 *edid_ext = NULL; - int i; - bool valid_vsdb_found = false; - - /*----- drm_find_cea_extension() -----*/ - /* No EDID or EDID extensions */ - if (edid == NULL || edid->extensions == 0) - return -ENODEV; - - /* Find CEA extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (uint8_t *)edid + EDID_LENGTH * (i + 1); - if (edid_ext[0] == CEA_EXT) - break; - } - - if (i == edid->extensions) - return -ENODEV; - - /*----- cea_db_offsets() -----*/ - if (edid_ext[0] != CEA_EXT) - return -ENODEV; - - valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info); - - return valid_vsdb_found ? i : -ENODEV; -} - -/** - * amdgpu_dm_update_freesync_caps - Update Freesync capabilities - * - * @connector: Connector to query. - * @edid: EDID from monitor - * - * Amdgpu supports Freesync in DP and HDMI displays, and it is required to keep - * track of some of the display information in the internal data struct used by - * amdgpu_dm. This function checks which type of connector we need to set the - * FreeSync parameters. - */ -void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, - struct edid *edid) -{ - int i = 0; - struct detailed_timing *timing; - struct detailed_non_pixel *data; - struct detailed_data_monitor_range *range; - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_con_state = NULL; - struct dc_sink *sink; - - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct amdgpu_hdmi_vsdb_info vsdb_info = {0}; - bool freesync_capable = false; - enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE; - - if (!connector->state) { - DRM_ERROR("%s - Connector has no state", __func__); - goto update; - } - - sink = amdgpu_dm_connector->dc_sink ? - amdgpu_dm_connector->dc_sink : - amdgpu_dm_connector->dc_em_sink; - - if (!edid || !sink) { - dm_con_state = to_dm_connector_state(connector->state); - - amdgpu_dm_connector->min_vfreq = 0; - amdgpu_dm_connector->max_vfreq = 0; - connector->display_info.monitor_range.min_vfreq = 0; - connector->display_info.monitor_range.max_vfreq = 0; - freesync_capable = false; - - goto update; - } - - dm_con_state = to_dm_connector_state(connector->state); - - if (!adev->dm.freesync_module) - goto update; - - /* Some eDP panels only have the refresh rate range info in DisplayID */ - if ((connector->display_info.monitor_range.min_vfreq == 0 || - connector->display_info.monitor_range.max_vfreq == 0)) - parse_edid_displayid_vrr(connector, edid); - - if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { - bool edid_check_required = false; - - if (amdgpu_dm_connector->dc_link && - amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) { - if (edid->features & DRM_EDID_FEATURE_CONTINUOUS_FREQ) { - amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq; - amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq; - if (amdgpu_dm_connector->max_vfreq - - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - } else { - edid_check_required = edid->version > 1 || - (edid->version == 1 && - edid->revision > 1); - } - } - - if (edid_check_required) { - for (i = 0; i < 4; i++) { - - timing = &edid->detailed_timings[i]; - data = &timing->data.other_data; - range = &data->data.range; - /* - * Check if monitor has continuous frequency mode - */ - if (data->type != EDID_DETAIL_MONITOR_RANGE) - continue; - /* - * Check for flag range limits only. If flag == 1 then - * no additional timing information provided. - * Default GTF, GTF Secondary curve and CVT are not - * supported - */ - if (range->flags != 1) - continue; - - connector->display_info.monitor_range.min_vfreq = range->min_vfreq; - connector->display_info.monitor_range.max_vfreq = range->max_vfreq; - - if (edid->revision >= 4) { - if (data->pad2 & DRM_EDID_RANGE_OFFSET_MIN_VFREQ) - connector->display_info.monitor_range.min_vfreq += 255; - if (data->pad2 & DRM_EDID_RANGE_OFFSET_MAX_VFREQ) - connector->display_info.monitor_range.max_vfreq += 255; - } - - amdgpu_dm_connector->min_vfreq = - connector->display_info.monitor_range.min_vfreq; - amdgpu_dm_connector->max_vfreq = - connector->display_info.monitor_range.max_vfreq; - - break; - } - - if (amdgpu_dm_connector->max_vfreq - - amdgpu_dm_connector->min_vfreq > 10) { - - freesync_capable = true; - } - } - parse_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - - if (vsdb_info.replay_mode) { - amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode; - amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version; - amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP; - } - - } else if (edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0 && vsdb_info.freesync_supported) { - timing = &edid->detailed_timings[i]; - data = &timing->data.other_data; - - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } - } - - if (amdgpu_dm_connector->dc_link) - as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link); - - if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0 && vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) { - - amdgpu_dm_connector->pack_sdp_v1_3 = true; - amdgpu_dm_connector->as_type = as_type; - amdgpu_dm_connector->vsdb_info = vsdb_info; - - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } - } - -update: - if (dm_con_state) - dm_con_state->freesync_capable = freesync_capable; - - if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable && - amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) { - amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false; - amdgpu_dm_connector->dc_link->replay_settings.replay_feature_enabled = false; - } - - if (connector->vrr_capable_property) - drm_connector_set_vrr_capable_property(connector, - freesync_capable); -} - -void amdgpu_dm_trigger_timing_sync(struct drm_device *dev) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct dc *dc = adev->dm.dc; - int i; - - mutex_lock(&adev->dm.dc_lock); - if (dc->current_state) { - for (i = 0; i < dc->current_state->stream_count; ++i) - dc->current_state->streams[i] - ->triggered_crtc_reset.enabled = - adev->dm.force_timing_sync; - - dm_enable_per_frame_crtc_master_sync(dc->current_state); - dc_trigger_sync(dc, dc->current_state); - } - mutex_unlock(&adev->dm.dc_lock); -} - -static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc) -{ - if (dc->ctx->dmub_srv && !dc->ctx->dmub_srv->idle_exit_counter) - dc_exit_ips_for_hw_access(dc); -} - -void dm_write_reg_func(const struct dc_context *ctx, uint32_t address, - u32 value, const char *func_name) -{ -#ifdef DM_CHECK_ADDR_0 - if (address == 0) { - drm_err(adev_to_drm(ctx->driver_context), - "invalid register write. address = 0"); - return; - } -#endif - - amdgpu_dm_exit_ips_for_hw_access(ctx->dc); - cgs_write_register(ctx->cgs_device, address, value); - trace_amdgpu_dc_wreg(&ctx->perf_trace->write_count, address, value); -} - -uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address, - const char *func_name) -{ - u32 value; -#ifdef DM_CHECK_ADDR_0 - if (address == 0) { - drm_err(adev_to_drm(ctx->driver_context), - "invalid register read; address = 0\n"); - return 0; - } -#endif - - if (ctx->dmub_srv && - ctx->dmub_srv->reg_helper_offload.gather_in_progress && - !ctx->dmub_srv->reg_helper_offload.should_burst_write) { - ASSERT(false); - return 0; - } - - amdgpu_dm_exit_ips_for_hw_access(ctx->dc); - - value = cgs_read_register(ctx->cgs_device, address); - - trace_amdgpu_dc_rreg(&ctx->perf_trace->read_count, address, value); - - return value; -} - -int amdgpu_dm_process_dmub_aux_transfer_sync( - struct dc_context *ctx, - unsigned int link_index, - struct aux_payload *payload, - enum aux_return_code_type *operation_result) -{ - struct amdgpu_device *adev = ctx->driver_context; - struct dmub_notification *p_notify = adev->dm.dmub_notify; - int ret = -1; - - mutex_lock(&adev->dm.dpia_aux_lock); - if (!dc_process_dmub_aux_transfer_async(ctx->dc, link_index, payload)) { - *operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE; - goto out; - } - - if (!wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) { - DRM_ERROR("wait_for_completion_timeout timeout!"); - *operation_result = AUX_RET_ERROR_TIMEOUT; - goto out; - } - - if (p_notify->result != AUX_RET_SUCCESS) { - /* - * Transient states before tunneling is enabled could - * lead to this error. We can ignore this for now. - */ - if (p_notify->result != AUX_RET_ERROR_PROTOCOL_ERROR) { - DRM_WARN("DPIA AUX failed on 0x%x(%d), error %d\n", - payload->address, payload->length, - p_notify->result); - } - *operation_result = AUX_RET_ERROR_INVALID_REPLY; - goto out; - } - - - payload->reply[0] = adev->dm.dmub_notify->aux_reply.command; - if (!payload->write && p_notify->aux_reply.length && - (payload->reply[0] == AUX_TRANSACTION_REPLY_AUX_ACK)) { - - if (payload->length != p_notify->aux_reply.length) { - DRM_WARN("invalid read length %d from DPIA AUX 0x%x(%d)!\n", - p_notify->aux_reply.length, - payload->address, payload->length); - *operation_result = AUX_RET_ERROR_INVALID_REPLY; - goto out; - } - - memcpy(payload->data, p_notify->aux_reply.data, - p_notify->aux_reply.length); - } - - /* success */ - ret = p_notify->aux_reply.length; - *operation_result = p_notify->result; -out: - reinit_completion(&adev->dm.dmub_aux_transfer_done); - mutex_unlock(&adev->dm.dpia_aux_lock); - return ret; -} - -int amdgpu_dm_process_dmub_set_config_sync( - struct dc_context *ctx, - unsigned int link_index, - struct set_config_cmd_payload *payload, - enum set_config_status *operation_result) -{ - struct amdgpu_device *adev = ctx->driver_context; - bool is_cmd_complete; - int ret; - - mutex_lock(&adev->dm.dpia_aux_lock); - is_cmd_complete = dc_process_dmub_set_config_async(ctx->dc, - link_index, payload, adev->dm.dmub_notify); - - if (is_cmd_complete || wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) { - ret = 0; - *operation_result = adev->dm.dmub_notify->sc_status; - } else { - DRM_ERROR("wait_for_completion_timeout timeout!"); - ret = -1; - *operation_result = SET_CONFIG_UNKNOWN_ERROR; - } - - if (!is_cmd_complete) - reinit_completion(&adev->dm.dmub_aux_transfer_done); - mutex_unlock(&adev->dm.dpia_aux_lock); - return ret; -} - -bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type) -{ - return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type); -} - -bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type) -{ - return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type); -} diff --git a/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/preimage b/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/preimage deleted file mode 100644 index c18b34f873fd..000000000000 --- a/rr-cache/184514ddd9867f1a8f27d1e7de47d36242b6ad90/preimage +++ /dev/null @@ -1,12348 +0,0 @@ -/* - * Copyright 2015 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: AMD - * - */ - -/* The caprices of the preprocessor require that this be declared right here */ -#define CREATE_TRACE_POINTS - -#include "dm_services_types.h" -#include "dc.h" -#include "link_enc_cfg.h" -#include "dc/inc/core_types.h" -#include "dal_asic_id.h" -#include "dmub/dmub_srv.h" -#include "dc/inc/hw/dmcu.h" -#include "dc/inc/hw/abm.h" -#include "dc/dc_dmub_srv.h" -#include "dc/dc_edid_parser.h" -#include "dc/dc_stat.h" -#include "dc/dc_state.h" -#include "amdgpu_dm_trace.h" -#include "dpcd_defs.h" -#include "link/protocols/link_dpcd.h" -#include "link_service_types.h" -#include "link/protocols/link_dp_capability.h" -#include "link/protocols/link_ddc.h" - -#include "vid.h" -#include "amdgpu.h" -#include "amdgpu_display.h" -#include "amdgpu_ucode.h" -#include "atom.h" -#include "amdgpu_dm.h" -#include "amdgpu_dm_plane.h" -#include "amdgpu_dm_crtc.h" -#include "amdgpu_dm_hdcp.h" -#include <drm/display/drm_hdcp_helper.h> -#include "amdgpu_dm_wb.h" -#include "amdgpu_pm.h" -#include "amdgpu_atombios.h" - -#include "amd_shared.h" -#include "amdgpu_dm_irq.h" -#include "dm_helpers.h" -#include "amdgpu_dm_mst_types.h" -#if defined(CONFIG_DEBUG_FS) -#include "amdgpu_dm_debugfs.h" -#endif -#include "amdgpu_dm_psr.h" -#include "amdgpu_dm_replay.h" - -#include "ivsrcid/ivsrcid_vislands30.h" - -#include <linux/backlight.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/types.h> -#include <linux/pm_runtime.h> -#include <linux/pci.h> -#include <linux/power_supply.h> -#include <linux/firmware.h> -#include <linux/component.h> -#include <linux/dmi.h> -#include <linux/sort.h> - -#include <drm/display/drm_dp_mst_helper.h> -#include <drm/display/drm_hdmi_helper.h> -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_uapi.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_blend.h> -#include <drm/drm_fixed.h> -#include <drm/drm_fourcc.h> -#include <drm/drm_edid.h> -#include <drm/drm_eld.h> -#include <drm/drm_vblank.h> -#include <drm/drm_audio_component.h> -#include <drm/drm_gem_atomic_helper.h> - -#include <acpi/video.h> - -#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h" - -#include "dcn/dcn_1_0_offset.h" -#include "dcn/dcn_1_0_sh_mask.h" -#include "soc15_hw_ip.h" -#include "soc15_common.h" -#include "vega10_ip_offset.h" - -#include "gc/gc_11_0_0_offset.h" -#include "gc/gc_11_0_0_sh_mask.h" - -#include "modules/inc/mod_freesync.h" -#include "modules/power/power_helpers.h" - -#define FIRMWARE_RENOIR_DMUB "amdgpu/renoir_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_RENOIR_DMUB); -#define FIRMWARE_SIENNA_CICHLID_DMUB "amdgpu/sienna_cichlid_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID_DMUB); -#define FIRMWARE_NAVY_FLOUNDER_DMUB "amdgpu/navy_flounder_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER_DMUB); -#define FIRMWARE_GREEN_SARDINE_DMUB "amdgpu/green_sardine_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE_DMUB); -#define FIRMWARE_VANGOGH_DMUB "amdgpu/vangogh_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_VANGOGH_DMUB); -#define FIRMWARE_DIMGREY_CAVEFISH_DMUB "amdgpu/dimgrey_cavefish_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH_DMUB); -#define FIRMWARE_BEIGE_GOBY_DMUB "amdgpu/beige_goby_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY_DMUB); -#define FIRMWARE_YELLOW_CARP_DMUB "amdgpu/yellow_carp_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP_DMUB); -#define FIRMWARE_DCN_314_DMUB "amdgpu/dcn_3_1_4_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_314_DMUB); -#define FIRMWARE_DCN_315_DMUB "amdgpu/dcn_3_1_5_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_315_DMUB); -#define FIRMWARE_DCN316_DMUB "amdgpu/dcn_3_1_6_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN316_DMUB); - -#define FIRMWARE_DCN_V3_2_0_DMCUB "amdgpu/dcn_3_2_0_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_0_DMCUB); -#define FIRMWARE_DCN_V3_2_1_DMCUB "amdgpu/dcn_3_2_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_1_DMCUB); - -#define FIRMWARE_RAVEN_DMCU "amdgpu/raven_dmcu.bin" -MODULE_FIRMWARE(FIRMWARE_RAVEN_DMCU); - -#define FIRMWARE_NAVI12_DMCU "amdgpu/navi12_dmcu.bin" -MODULE_FIRMWARE(FIRMWARE_NAVI12_DMCU); - -#define FIRMWARE_DCN_35_DMUB "amdgpu/dcn_3_5_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_35_DMUB); - -#define FIRMWARE_DCN_351_DMUB "amdgpu/dcn_3_5_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_351_DMUB); - -#define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin" -MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB); - -/* Number of bytes in PSP header for firmware. */ -#define PSP_HEADER_BYTES 0x100 - -/* Number of bytes in PSP footer for firmware. */ -#define PSP_FOOTER_BYTES 0x100 - -/** - * DOC: overview - * - * The AMDgpu display manager, **amdgpu_dm** (or even simpler, - * **dm**) sits between DRM and DC. It acts as a liaison, converting DRM - * requests into DC requests, and DC responses into DRM responses. - * - * The root control structure is &struct amdgpu_display_manager. - */ - -/* basic init/fini API */ -static int amdgpu_dm_init(struct amdgpu_device *adev); -static void amdgpu_dm_fini(struct amdgpu_device *adev); -static bool is_freesync_video_mode(const struct drm_display_mode *mode, struct amdgpu_dm_connector *aconnector); -static void reset_freesync_config_for_crtc(struct dm_crtc_state *new_crtc_state); - -static enum drm_mode_subconnector get_subconnector_type(struct dc_link *link) -{ - switch (link->dpcd_caps.dongle_type) { - case DISPLAY_DONGLE_NONE: - return DRM_MODE_SUBCONNECTOR_Native; - case DISPLAY_DONGLE_DP_VGA_CONVERTER: - return DRM_MODE_SUBCONNECTOR_VGA; - case DISPLAY_DONGLE_DP_DVI_CONVERTER: - case DISPLAY_DONGLE_DP_DVI_DONGLE: - return DRM_MODE_SUBCONNECTOR_DVID; - case DISPLAY_DONGLE_DP_HDMI_CONVERTER: - case DISPLAY_DONGLE_DP_HDMI_DONGLE: - return DRM_MODE_SUBCONNECTOR_HDMIA; - case DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE: - default: - return DRM_MODE_SUBCONNECTOR_Unknown; - } -} - -static void update_subconnector_property(struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = aconnector->dc_link; - struct drm_connector *connector = &aconnector->base; - enum drm_mode_subconnector subconnector = DRM_MODE_SUBCONNECTOR_Unknown; - - if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) - return; - - if (aconnector->dc_sink) - subconnector = get_subconnector_type(link); - - drm_object_property_set_value(&connector->base, - connector->dev->mode_config.dp_subconnector_property, - subconnector); -} - -/* - * initializes drm_device display related structures, based on the information - * provided by DAL. The drm strcutures are: drm_crtc, drm_connector, - * drm_encoder, drm_mode_config - * - * Returns 0 on success - */ -static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev); -/* removes and deallocates the drm structures, created by the above function */ -static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm); - -static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *amdgpu_dm_connector, - u32 link_index, - struct amdgpu_encoder *amdgpu_encoder); -static int amdgpu_dm_encoder_init(struct drm_device *dev, - struct amdgpu_encoder *aencoder, - uint32_t link_index); - -static int amdgpu_dm_connector_get_modes(struct drm_connector *connector); - -static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state); - -static int amdgpu_dm_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state); - -static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector); -static void handle_hpd_rx_irq(void *param); - -static bool -is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state); -/* - * dm_vblank_get_counter - * - * @brief - * Get counter for number of vertical blanks - * - * @param - * struct amdgpu_device *adev - [in] desired amdgpu device - * int disp_idx - [in] which CRTC to get the counter from - * - * @return - * Counter for vertical blanks - */ -static u32 dm_vblank_get_counter(struct amdgpu_device *adev, int crtc) -{ - struct amdgpu_crtc *acrtc = NULL; - - if (crtc >= adev->mode_info.num_crtc) - return 0; - - acrtc = adev->mode_info.crtcs[crtc]; - - if (!acrtc->dm_irq_params.stream) { - DRM_ERROR("dc_stream_state is NULL for crtc '%d'!\n", - crtc); - return 0; - } - - return dc_stream_get_vblank_counter(acrtc->dm_irq_params.stream); -} - -static int dm_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc, - u32 *vbl, u32 *position) -{ - u32 v_blank_start = 0, v_blank_end = 0, h_position = 0, v_position = 0; - struct amdgpu_crtc *acrtc = NULL; - struct dc *dc = adev->dm.dc; - - if ((crtc < 0) || (crtc >= adev->mode_info.num_crtc)) - return -EINVAL; - - acrtc = adev->mode_info.crtcs[crtc]; - - if (!acrtc->dm_irq_params.stream) { - DRM_ERROR("dc_stream_state is NULL for crtc '%d'!\n", - crtc); - return 0; - } - - if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed) - dc_allow_idle_optimizations(dc, false); - - /* - * TODO rework base driver to use values directly. - * for now parse it back into reg-format - */ - dc_stream_get_scanoutpos(acrtc->dm_irq_params.stream, - &v_blank_start, - &v_blank_end, - &h_position, - &v_position); - - *position = v_position | (h_position << 16); - *vbl = v_blank_start | (v_blank_end << 16); - - return 0; -} - -static bool dm_is_idle(void *handle) -{ - /* XXX todo */ - return true; -} - -static int dm_wait_for_idle(void *handle) -{ - /* XXX todo */ - return 0; -} - -static bool dm_check_soft_reset(void *handle) -{ - return false; -} - -static int dm_soft_reset(void *handle) -{ - /* XXX todo */ - return 0; -} - -static struct amdgpu_crtc * -get_crtc_by_otg_inst(struct amdgpu_device *adev, - int otg_inst) -{ - struct drm_device *dev = adev_to_drm(adev); - struct drm_crtc *crtc; - struct amdgpu_crtc *amdgpu_crtc; - - if (WARN_ON(otg_inst == -1)) - return adev->mode_info.crtcs[0]; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - amdgpu_crtc = to_amdgpu_crtc(crtc); - - if (amdgpu_crtc->otg_inst == otg_inst) - return amdgpu_crtc; - } - - return NULL; -} - -static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state, - struct dm_crtc_state *new_state) -{ - if (new_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED) - return true; - else if (amdgpu_dm_crtc_vrr_active(old_state) != amdgpu_dm_crtc_vrr_active(new_state)) - return true; - else - return false; -} - -/* - * DC will program planes with their z-order determined by their ordering - * in the dc_surface_updates array. This comparator is used to sort them - * by descending zpos. - */ -static int dm_plane_layer_index_cmp(const void *a, const void *b) -{ - const struct dc_surface_update *sa = (struct dc_surface_update *)a; - const struct dc_surface_update *sb = (struct dc_surface_update *)b; - - /* Sort by descending dc_plane layer_index (i.e. normalized_zpos) */ - return sb->surface->layer_index - sa->surface->layer_index; -} - -/** - * update_planes_and_stream_adapter() - Send planes to be updated in DC - * - * DC has a generic way to update planes and stream via - * dc_update_planes_and_stream function; however, DM might need some - * adjustments and preparation before calling it. This function is a wrapper - * for the dc_update_planes_and_stream that does any required configuration - * before passing control to DC. - * - * @dc: Display Core control structure - * @update_type: specify whether it is FULL/MEDIUM/FAST update - * @planes_count: planes count to update - * @stream: stream state - * @stream_update: stream update - * @array_of_surface_update: dc surface update pointer - * - */ -static inline bool update_planes_and_stream_adapter(struct dc *dc, - int update_type, - int planes_count, - struct dc_stream_state *stream, - struct dc_stream_update *stream_update, - struct dc_surface_update *array_of_surface_update) -{ - sort(array_of_surface_update, planes_count, - sizeof(*array_of_surface_update), dm_plane_layer_index_cmp, NULL); - - /* - * Previous frame finished and HW is ready for optimization. - */ - if (update_type == UPDATE_TYPE_FAST) - dc_post_update_surfaces_to_stream(dc); - - return dc_update_planes_and_stream(dc, - array_of_surface_update, - planes_count, - stream, - stream_update); -} - -/** - * dm_pflip_high_irq() - Handle pageflip interrupt - * @interrupt_params: ignored - * - * Handles the pageflip interrupt by notifying all interested parties - * that the pageflip has been completed. - */ -static void dm_pflip_high_irq(void *interrupt_params) -{ - struct amdgpu_crtc *amdgpu_crtc; - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct drm_device *dev = adev_to_drm(adev); - unsigned long flags; - struct drm_pending_vblank_event *e; - u32 vpos, hpos, v_blank_start, v_blank_end; - bool vrr_active; - - amdgpu_crtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP); - - /* IRQ could occur when in initial stage */ - /* TODO work and BO cleanup */ - if (amdgpu_crtc == NULL) { - drm_dbg_state(dev, "CRTC is null, returning.\n"); - return; - } - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - - if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { - drm_dbg_state(dev, - "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n", - amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED, - amdgpu_crtc->crtc_id, amdgpu_crtc); - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - return; - } - - /* page flip completed. */ - e = amdgpu_crtc->event; - amdgpu_crtc->event = NULL; - - WARN_ON(!e); - - vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc); - - /* Fixed refresh rate, or VRR scanout position outside front-porch? */ - if (!vrr_active || - !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start, - &v_blank_end, &hpos, &vpos) || - (vpos < v_blank_start)) { - /* Update to correct count and vblank timestamp if racing with - * vblank irq. This also updates to the correct vblank timestamp - * even in VRR mode, as scanout is past the front-porch atm. - */ - drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); - - /* Wake up userspace by sending the pageflip event with proper - * count and timestamp of vblank of flip completion. - */ - if (e) { - drm_crtc_send_vblank_event(&amdgpu_crtc->base, e); - - /* Event sent, so done with vblank for this flip */ - drm_crtc_vblank_put(&amdgpu_crtc->base); - } - } else if (e) { - /* VRR active and inside front-porch: vblank count and - * timestamp for pageflip event will only be up to date after - * drm_crtc_handle_vblank() has been executed from late vblank - * irq handler after start of back-porch (vline 0). We queue the - * pageflip event for send-out by drm_crtc_handle_vblank() with - * updated timestamp and count, once it runs after us. - * - * We need to open-code this instead of using the helper - * drm_crtc_arm_vblank_event(), as that helper would - * call drm_crtc_accurate_vblank_count(), which we must - * not call in VRR mode while we are in front-porch! - */ - - /* sequence will be replaced by real count during send-out. */ - e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); - e->pipe = amdgpu_crtc->crtc_id; - - list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); - e = NULL; - } - - /* Keep track of vblank of this flip for flip throttling. We use the - * cooked hw counter, as that one incremented at start of this vblank - * of pageflip completion, so last_flip_vblank is the forbidden count - * for queueing new pageflips if vsync + VRR is enabled. - */ - amdgpu_crtc->dm_irq_params.last_flip_vblank = - amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); - - amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - - drm_dbg_state(dev, - "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", - amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e); -} - -static void dm_vupdate_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - struct drm_device *drm_dev; - struct drm_vblank_crtc *vblank; - ktime_t frame_duration_ns, previous_timestamp; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); - - if (acrtc) { - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - drm_dev = acrtc->base.dev; - vblank = drm_crtc_vblank_crtc(&acrtc->base); - previous_timestamp = atomic64_read(&irq_params->previous_timestamp); - frame_duration_ns = vblank->time - previous_timestamp; - - if (frame_duration_ns > 0) { - trace_amdgpu_refresh_rate_track(acrtc->base.index, - frame_duration_ns, - ktime_divns(NSEC_PER_SEC, frame_duration_ns)); - atomic64_set(&irq_params->previous_timestamp, vblank->time); - } - - drm_dbg_vbl(drm_dev, - "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, - vrr_active); - - /* Core vblank handling is done here after end of front-porch in - * vrr mode, as vblank timestamping will give valid results - * while now done after front-porch. This will also deliver - * page-flip completion events that have been queued to us - * if a pageflip happened inside front-porch. - */ - if (vrr_active) { - amdgpu_dm_crtc_handle_vblank(acrtc); - - /* BTR processing for pre-DCE12 ASICs */ - if (acrtc->dm_irq_params.stream && - adev->family < AMDGPU_FAMILY_AI) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - mod_freesync_handle_v_update( - adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - dc_stream_adjust_vmin_vmax( - adev->dm.dc, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } - } - } -} - -/** - * dm_crtc_high_irq() - Handles CRTC interrupt - * @interrupt_params: used for determining the CRTC instance - * - * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK - * event handler. - */ -static void dm_crtc_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct drm_writeback_job *job; - struct amdgpu_crtc *acrtc; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); - if (!acrtc) - return; - - if (acrtc->wb_conn) { - spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags); - - if (acrtc->wb_pending) { - job = list_first_entry_or_null(&acrtc->wb_conn->job_queue, - struct drm_writeback_job, - list_entry); - acrtc->wb_pending = false; - spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); - - if (job) { - unsigned int v_total, refresh_hz; - struct dc_stream_state *stream = acrtc->dm_irq_params.stream; - - v_total = stream->adjust.v_total_max ? - stream->adjust.v_total_max : stream->timing.v_total; - refresh_hz = div_u64((uint64_t) stream->timing.pix_clk_100hz * - 100LL, (v_total * stream->timing.h_total)); - mdelay(1000 / refresh_hz); - - drm_writeback_signal_completion(acrtc->wb_conn, 0); - dc_stream_fc_disable_writeback(adev->dm.dc, - acrtc->dm_irq_params.stream, 0); - } - } else - spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); - } - - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - - drm_dbg_vbl(adev_to_drm(adev), - "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id, - vrr_active, acrtc->dm_irq_params.active_planes); - - /** - * Core vblank handling at start of front-porch is only possible - * in non-vrr mode, as only there vblank timestamping will give - * valid results while done in front-porch. Otherwise defer it - * to dm_vupdate_high_irq after end of front-porch. - */ - if (!vrr_active) - amdgpu_dm_crtc_handle_vblank(acrtc); - - /** - * Following stuff must happen at start of vblank, for crc - * computation and below-the-range btr support in vrr mode. - */ - amdgpu_dm_crtc_handle_crc_irq(&acrtc->base); - - /* BTR updates need to happen before VUPDATE on Vega and above. */ - if (adev->family < AMDGPU_FAMILY_AI) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - - if (acrtc->dm_irq_params.stream && - acrtc->dm_irq_params.vrr_params.supported && - acrtc->dm_irq_params.freesync_config.state == - VRR_STATE_ACTIVE_VARIABLE) { - mod_freesync_handle_v_update(adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - dc_stream_adjust_vmin_vmax(adev->dm.dc, acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - } - - /* - * If there aren't any active_planes then DCH HUBP may be clock-gated. - * In that case, pageflip completion interrupts won't fire and pageflip - * completion events won't get delivered. Prevent this by sending - * pending pageflip events from here if a flip is still pending. - * - * If any planes are enabled, use dm_pflip_high_irq() instead, to - * avoid race conditions between flip programming and completion, - * which could cause too early flip completion events. - */ - if (adev->family >= AMDGPU_FAMILY_RV && - acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && - acrtc->dm_irq_params.active_planes == 0) { - if (acrtc->event) { - drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); - acrtc->event = NULL; - drm_crtc_vblank_put(&acrtc->base); - } - acrtc->pflip_status = AMDGPU_FLIP_NONE; - } - - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -/** - * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for - * DCN generation ASICs - * @interrupt_params: interrupt parameters - * - * Used to set crc window/read out crc value at vertical line 0 position - */ -static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0); - - if (!acrtc) - return; - - amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base); -} -#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ - -/** - * dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command. - * @adev: amdgpu_device pointer - * @notify: dmub notification structure - * - * Dmub AUX or SET_CONFIG command completion processing callback - * Copies dmub notification to DM which is to be read by AUX command. - * issuing thread and also signals the event to wake up the thread. - */ -static void dmub_aux_setconfig_callback(struct amdgpu_device *adev, - struct dmub_notification *notify) -{ - if (adev->dm.dmub_notify) - memcpy(adev->dm.dmub_notify, notify, sizeof(struct dmub_notification)); - if (notify->type == DMUB_NOTIFICATION_AUX_REPLY) - complete(&adev->dm.dmub_aux_transfer_done); -} - -/** - * dmub_hpd_callback - DMUB HPD interrupt processing callback. - * @adev: amdgpu_device pointer - * @notify: dmub notification structure - * - * Dmub Hpd interrupt processing callback. Gets displayindex through the - * ink index and calls helper to do the processing. - */ -static void dmub_hpd_callback(struct amdgpu_device *adev, - struct dmub_notification *notify) -{ - struct amdgpu_dm_connector *aconnector; - struct amdgpu_dm_connector *hpd_aconnector = NULL; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct dc_link *link; - u8 link_index = 0; - struct drm_device *dev; - - if (adev == NULL) - return; - - if (notify == NULL) { - DRM_ERROR("DMUB HPD callback notification was NULL"); - return; - } - - if (notify->link_index > adev->dm.dc->link_count) { - DRM_ERROR("DMUB HPD index (%u)is abnormal", notify->link_index); - return; - } - - link_index = notify->link_index; - link = adev->dm.dc->links[link_index]; - dev = adev->dm.ddev; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (link && aconnector->dc_link == link) { - if (notify->type == DMUB_NOTIFICATION_HPD) - DRM_INFO("DMUB HPD IRQ callback: link_index=%u\n", link_index); - else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) - DRM_INFO("DMUB HPD RX IRQ callback: link_index=%u\n", link_index); - else - DRM_WARN("DMUB Unknown HPD callback type %d, link_index=%u\n", - notify->type, link_index); - - hpd_aconnector = aconnector; - break; - } - } - drm_connector_list_iter_end(&iter); - - if (hpd_aconnector) { - if (notify->type == DMUB_NOTIFICATION_HPD) { - if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG)) - DRM_WARN("DMUB reported hpd status unchanged. link_index=%u\n", link_index); - handle_hpd_irq_helper(hpd_aconnector); - } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) { - handle_hpd_rx_irq(hpd_aconnector); - } - } -} - -/** - * register_dmub_notify_callback - Sets callback for DMUB notify - * @adev: amdgpu_device pointer - * @type: Type of dmub notification - * @callback: Dmub interrupt callback function - * @dmub_int_thread_offload: offload indicator - * - * API to register a dmub callback handler for a dmub notification - * Also sets indicator whether callback processing to be offloaded. - * to dmub interrupt handling thread - * Return: true if successfully registered, false if there is existing registration - */ -static bool register_dmub_notify_callback(struct amdgpu_device *adev, - enum dmub_notification_type type, - dmub_notify_interrupt_callback_t callback, - bool dmub_int_thread_offload) -{ - if (callback != NULL && type < ARRAY_SIZE(adev->dm.dmub_thread_offload)) { - adev->dm.dmub_callback[type] = callback; - adev->dm.dmub_thread_offload[type] = dmub_int_thread_offload; - } else - return false; - - return true; -} - -static void dm_handle_hpd_work(struct work_struct *work) -{ - struct dmub_hpd_work *dmub_hpd_wrk; - - dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work); - - if (!dmub_hpd_wrk->dmub_notify) { - DRM_ERROR("dmub_hpd_wrk dmub_notify is NULL"); - return; - } - - if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) { - dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev, - dmub_hpd_wrk->dmub_notify); - } - - kfree(dmub_hpd_wrk->dmub_notify); - kfree(dmub_hpd_wrk); - -} - -#define DMUB_TRACE_MAX_READ 64 -/** - * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt - * @interrupt_params: used for determining the Outbox instance - * - * Handles the Outbox Interrupt - * event handler. - */ -static void dm_dmub_outbox1_low_irq(void *interrupt_params) -{ - struct dmub_notification notify = {0}; - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_display_manager *dm = &adev->dm; - struct dmcub_trace_buf_entry entry = { 0 }; - u32 count = 0; - struct dmub_hpd_work *dmub_hpd_wrk; - static const char *const event_type[] = { - "NO_DATA", - "AUX_REPLY", - "HPD", - "HPD_IRQ", - "SET_CONFIGC_REPLY", - "DPIA_NOTIFICATION", - "HPD_SENSE_NOTIFY", - }; - - do { - if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) { - trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count, - entry.param0, entry.param1); - - DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n", - entry.trace_code, entry.tick_count, entry.param0, entry.param1); - } else - break; - - count++; - - } while (count <= DMUB_TRACE_MAX_READ); - - if (count > DMUB_TRACE_MAX_READ) - DRM_DEBUG_DRIVER("Warning : count > DMUB_TRACE_MAX_READ"); - - if (dc_enable_dmub_notifications(adev->dm.dc) && - irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) { - - do { - dc_stat_get_dmub_notification(adev->dm.dc, ¬ify); - if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) { - DRM_ERROR("DM: notify type %d invalid!", notify.type); - continue; - } - if (!dm->dmub_callback[notify.type]) { - DRM_WARN("DMUB notification skipped due to no handler: type=%s\n", - event_type[notify.type]); - continue; - } - if (dm->dmub_thread_offload[notify.type] == true) { - dmub_hpd_wrk = kzalloc(sizeof(*dmub_hpd_wrk), GFP_ATOMIC); - if (!dmub_hpd_wrk) { - DRM_ERROR("Failed to allocate dmub_hpd_wrk"); - return; - } - dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification), - GFP_ATOMIC); - if (!dmub_hpd_wrk->dmub_notify) { - kfree(dmub_hpd_wrk); - DRM_ERROR("Failed to allocate dmub_hpd_wrk->dmub_notify"); - return; - } - INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work); - dmub_hpd_wrk->adev = adev; - queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work); - } else { - dm->dmub_callback[notify.type](adev, ¬ify); - } - } while (notify.pending_notification); - } -} - -static int dm_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - return 0; -} - -static int dm_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -/* Prototypes of private functions */ -static int dm_early_init(void *handle); - -/* Allocate memory for FBC compressed data */ -static void amdgpu_dm_fbc_init(struct drm_connector *connector) -{ - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct dm_compressor_info *compressor = &adev->dm.compressor; - struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector); - struct drm_display_mode *mode; - unsigned long max_size = 0; - - if (adev->dm.dc->fbc_compressor == NULL) - return; - - if (aconn->dc_link->connector_signal != SIGNAL_TYPE_EDP) - return; - - if (compressor->bo_ptr) - return; - - - list_for_each_entry(mode, &connector->modes, head) { - if (max_size < (unsigned long) mode->htotal * mode->vtotal) - max_size = (unsigned long) mode->htotal * mode->vtotal; - } - - if (max_size) { - int r = amdgpu_bo_create_kernel(adev, max_size * 4, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_GTT, &compressor->bo_ptr, - &compressor->gpu_addr, &compressor->cpu_addr); - - if (r) - DRM_ERROR("DM: Failed to initialize FBC\n"); - else { - adev->dm.dc->ctx->fbc_gpu_addr = compressor->gpu_addr; - DRM_INFO("DM: FBC alloc %lu\n", max_size*4); - } - - } - -} - -static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port, - int pipe, bool *enabled, - unsigned char *buf, int max_bytes) -{ - struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = drm_to_adev(dev); - struct drm_connector *connector; - struct drm_connector_list_iter conn_iter; - struct amdgpu_dm_connector *aconnector; - int ret = 0; - - *enabled = false; - - mutex_lock(&adev->dm.audio_lock); - - drm_connector_list_iter_begin(dev, &conn_iter); - drm_for_each_connector_iter(connector, &conn_iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->audio_inst != port) - continue; - - *enabled = true; - ret = drm_eld_size(connector->eld); - memcpy(buf, connector->eld, min(max_bytes, ret)); - - break; - } - drm_connector_list_iter_end(&conn_iter); - - mutex_unlock(&adev->dm.audio_lock); - - DRM_DEBUG_KMS("Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled); - - return ret; -} - -static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = { - .get_eld = amdgpu_dm_audio_component_get_eld, -}; - -static int amdgpu_dm_audio_component_bind(struct device *kdev, - struct device *hda_kdev, void *data) -{ - struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = drm_to_adev(dev); - struct drm_audio_component *acomp = data; - - acomp->ops = &amdgpu_dm_audio_component_ops; - acomp->dev = kdev; - adev->dm.audio_component = acomp; - - return 0; -} - -static void amdgpu_dm_audio_component_unbind(struct device *kdev, - struct device *hda_kdev, void *data) -{ - struct amdgpu_device *adev = drm_to_adev(dev_get_drvdata(kdev)); - struct drm_audio_component *acomp = data; - - acomp->ops = NULL; - acomp->dev = NULL; - adev->dm.audio_component = NULL; -} - -static const struct component_ops amdgpu_dm_audio_component_bind_ops = { - .bind = amdgpu_dm_audio_component_bind, - .unbind = amdgpu_dm_audio_component_unbind, -}; - -static int amdgpu_dm_audio_init(struct amdgpu_device *adev) -{ - int i, ret; - - if (!amdgpu_audio) - return 0; - - adev->mode_info.audio.enabled = true; - - adev->mode_info.audio.num_pins = adev->dm.dc->res_pool->audio_count; - - for (i = 0; i < adev->mode_info.audio.num_pins; i++) { - adev->mode_info.audio.pin[i].channels = -1; - adev->mode_info.audio.pin[i].rate = -1; - adev->mode_info.audio.pin[i].bits_per_sample = -1; - adev->mode_info.audio.pin[i].status_bits = 0; - adev->mode_info.audio.pin[i].category_code = 0; - adev->mode_info.audio.pin[i].connected = false; - adev->mode_info.audio.pin[i].id = - adev->dm.dc->res_pool->audios[i]->inst; - adev->mode_info.audio.pin[i].offset = 0; - } - - ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops); - if (ret < 0) - return ret; - - adev->dm.audio_registered = true; - - return 0; -} - -static void amdgpu_dm_audio_fini(struct amdgpu_device *adev) -{ - if (!amdgpu_audio) - return; - - if (!adev->mode_info.audio.enabled) - return; - - if (adev->dm.audio_registered) { - component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops); - adev->dm.audio_registered = false; - } - - /* TODO: Disable audio? */ - - adev->mode_info.audio.enabled = false; -} - -static void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin) -{ - struct drm_audio_component *acomp = adev->dm.audio_component; - - if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) { - DRM_DEBUG_KMS("Notify ELD: %d\n", pin); - - acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, - pin, -1); - } -} - -static int dm_dmub_hw_init(struct amdgpu_device *adev) -{ - const struct dmcub_firmware_header_v1_0 *hdr; - struct dmub_srv *dmub_srv = adev->dm.dmub_srv; - struct dmub_srv_fb_info *fb_info = adev->dm.dmub_fb_info; - const struct firmware *dmub_fw = adev->dm.dmub_fw; - struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu; - struct abm *abm = adev->dm.dc->res_pool->abm; - struct dc_context *ctx = adev->dm.dc->ctx; - struct dmub_srv_hw_params hw_params; - enum dmub_status status; - const unsigned char *fw_inst_const, *fw_bss_data; - u32 i, fw_inst_const_size, fw_bss_data_size; - bool has_hw_support; - - if (!dmub_srv) - /* DMUB isn't supported on the ASIC. */ - return 0; - - if (!fb_info) { - DRM_ERROR("No framebuffer info for DMUB service.\n"); - return -EINVAL; - } - - if (!dmub_fw) { - /* Firmware required for DMUB support. */ - DRM_ERROR("No firmware provided for DMUB.\n"); - return -EINVAL; - } - - /* initialize register offsets for ASICs with runtime initialization available */ - if (dmub_srv->hw_funcs.init_reg_offsets) - dmub_srv->hw_funcs.init_reg_offsets(dmub_srv, ctx); - - status = dmub_srv_has_hw_support(dmub_srv, &has_hw_support); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error checking HW support for DMUB: %d\n", status); - return -EINVAL; - } - - if (!has_hw_support) { - DRM_INFO("DMUB unsupported on ASIC\n"); - return 0; - } - - /* Reset DMCUB if it was previously running - before we overwrite its memory. */ - status = dmub_srv_hw_reset(dmub_srv); - if (status != DMUB_STATUS_OK) - DRM_WARN("Error resetting DMUB HW: %d\n", status); - - hdr = (const struct dmcub_firmware_header_v1_0 *)dmub_fw->data; - - fw_inst_const = dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - PSP_HEADER_BYTES; - - fw_bss_data = dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - le32_to_cpu(hdr->inst_const_bytes); - - /* Copy firmware and bios info into FB memory. */ - fw_inst_const_size = le32_to_cpu(hdr->inst_const_bytes) - - PSP_HEADER_BYTES - PSP_FOOTER_BYTES; - - fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes); - - /* if adev->firmware.load_type == AMDGPU_FW_LOAD_PSP, - * amdgpu_ucode_init_single_fw will load dmub firmware - * fw_inst_const part to cw0; otherwise, the firmware back door load - * will be done by dm_dmub_hw_init - */ - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { - memcpy(fb_info->fb[DMUB_WINDOW_0_INST_CONST].cpu_addr, fw_inst_const, - fw_inst_const_size); - } - - if (fw_bss_data_size) - memcpy(fb_info->fb[DMUB_WINDOW_2_BSS_DATA].cpu_addr, - fw_bss_data, fw_bss_data_size); - - /* Copy firmware bios info into FB memory. */ - memcpy(fb_info->fb[DMUB_WINDOW_3_VBIOS].cpu_addr, adev->bios, - adev->bios_size); - - /* Reset regions that need to be reset. */ - memset(fb_info->fb[DMUB_WINDOW_4_MAILBOX].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_4_MAILBOX].size); - - memset(fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size); - - memset(fb_info->fb[DMUB_WINDOW_6_FW_STATE].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_6_FW_STATE].size); - - memset(fb_info->fb[DMUB_WINDOW_SHARED_STATE].cpu_addr, 0, - fb_info->fb[DMUB_WINDOW_SHARED_STATE].size); - - /* Initialize hardware. */ - memset(&hw_params, 0, sizeof(hw_params)); - hw_params.fb_base = adev->gmc.fb_start; - hw_params.fb_offset = adev->vm_manager.vram_base_offset; - - /* backdoor load firmware and trigger dmub running */ - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - hw_params.load_inst_const = true; - - if (dmcu) - hw_params.psp_version = dmcu->psp_version; - - for (i = 0; i < fb_info->num_fb; ++i) - hw_params.fb[i] = &fb_info->fb[i]; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - hw_params.dpia_supported = true; - hw_params.disable_dpia = adev->dm.dc->debug.dpia_debug.bits.disable_dpia; - break; - default: - break; - } - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - hw_params.ips_sequential_ono = adev->external_rev_id > 0x10; - break; - default: - break; - } - - status = dmub_srv_hw_init(dmub_srv, &hw_params); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error initializing DMUB HW: %d\n", status); - return -EINVAL; - } - - /* Wait for firmware load to finish. */ - status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); - if (status != DMUB_STATUS_OK) - DRM_WARN("Wait for DMUB auto-load failed: %d\n", status); - - /* Init DMCU and ABM if available. */ - if (dmcu && abm) { - dmcu->funcs->dmcu_init(dmcu); - abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu); - } - - if (!adev->dm.dc->ctx->dmub_srv) - adev->dm.dc->ctx->dmub_srv = dc_dmub_srv_create(adev->dm.dc, dmub_srv); - if (!adev->dm.dc->ctx->dmub_srv) { - DRM_ERROR("Couldn't allocate DC DMUB server!\n"); - return -ENOMEM; - } - - DRM_INFO("DMUB hardware initialized: version=0x%08X\n", - adev->dm.dmcub_fw_version); - - return 0; -} - -static void dm_dmub_hw_resume(struct amdgpu_device *adev) -{ - struct dmub_srv *dmub_srv = adev->dm.dmub_srv; - enum dmub_status status; - bool init; - int r; - - if (!dmub_srv) { - /* DMUB isn't supported on the ASIC. */ - return; - } - - status = dmub_srv_is_hw_init(dmub_srv, &init); - if (status != DMUB_STATUS_OK) - DRM_WARN("DMUB hardware init check failed: %d\n", status); - - if (status == DMUB_STATUS_OK && init) { - /* Wait for firmware load to finish. */ - status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); - if (status != DMUB_STATUS_OK) - DRM_WARN("Wait for DMUB auto-load failed: %d\n", status); - } else { - /* Perform the full hardware initialization. */ - r = dm_dmub_hw_init(adev); - if (r) - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - } -} - -static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config) -{ - u64 pt_base; - u32 logical_addr_low; - u32 logical_addr_high; - u32 agp_base, agp_bot, agp_top; - PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base; - - memset(pa_config, 0, sizeof(*pa_config)); - - agp_base = 0; - agp_bot = adev->gmc.agp_start >> 24; - agp_top = adev->gmc.agp_end >> 24; - - /* AGP aperture is disabled */ - if (agp_bot > agp_top) { - logical_addr_low = adev->gmc.fb_start >> 18; - if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | - AMD_APU_IS_RENOIR | - AMD_APU_IS_GREEN_SARDINE)) - /* - * Raven2 has a HW issue that it is unable to use the vram which - * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the - * workaround that increase system aperture high address (add 1) - * to get rid of the VM fault and hardware hang. - */ - logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1; - else - logical_addr_high = adev->gmc.fb_end >> 18; - } else { - logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18; - if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | - AMD_APU_IS_RENOIR | - AMD_APU_IS_GREEN_SARDINE)) - /* - * Raven2 has a HW issue that it is unable to use the vram which - * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the - * workaround that increase system aperture high address (add 1) - * to get rid of the VM fault and hardware hang. - */ - logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18); - else - logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18; - } - - pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); - - page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >> - AMDGPU_GPU_PAGE_SHIFT); - page_table_base.high_part = upper_32_bits(pt_base); - page_table_base.low_part = lower_32_bits(pt_base); - - pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18; - pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18; - - pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24; - pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24; - pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24; - - pa_config->system_aperture.fb_base = adev->gmc.fb_start; - pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset; - pa_config->system_aperture.fb_top = adev->gmc.fb_end; - - pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12; - pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12; - pa_config->gart_config.page_table_base_addr = page_table_base.quad_part; - - pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support; - -} - -static void force_connector_state( - struct amdgpu_dm_connector *aconnector, - enum drm_connector_force force_state) -{ - struct drm_connector *connector = &aconnector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - aconnector->base.force = force_state; - mutex_unlock(&connector->dev->mode_config.mutex); - - mutex_lock(&aconnector->hpd_lock); - drm_kms_helper_connector_hotplug_event(connector); - mutex_unlock(&aconnector->hpd_lock); -} - -static void dm_handle_hpd_rx_offload_work(struct work_struct *work) -{ - struct hpd_rx_irq_offload_work *offload_work; - struct amdgpu_dm_connector *aconnector; - struct dc_link *dc_link; - struct amdgpu_device *adev; - enum dc_connection_type new_connection_type = dc_connection_none; - unsigned long flags; - union test_response test_response; - - memset(&test_response, 0, sizeof(test_response)); - - offload_work = container_of(work, struct hpd_rx_irq_offload_work, work); - aconnector = offload_work->offload_wq->aconnector; - - if (!aconnector) { - DRM_ERROR("Can't retrieve aconnector in hpd_rx_irq_offload_work"); - goto skip; - } - - adev = drm_to_adev(aconnector->base.dev); - dc_link = aconnector->dc_link; - - mutex_lock(&aconnector->hpd_lock); - if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - mutex_unlock(&aconnector->hpd_lock); - - if (new_connection_type == dc_connection_none) - goto skip; - - if (amdgpu_in_reset(adev)) - goto skip; - - if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || - offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { - dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT); - spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); - offload_work->offload_wq->is_handling_mst_msg_rdy_event = false; - spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); - goto skip; - } - - mutex_lock(&adev->dm.dc_lock); - if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) { - dc_link_dp_handle_automated_test(dc_link); - - if (aconnector->timing_changed) { - /* force connector disconnect and reconnect */ - force_connector_state(aconnector, DRM_FORCE_OFF); - msleep(100); - force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED); - } - - test_response.bits.ACK = 1; - - core_link_write_dpcd( - dc_link, - DP_TEST_RESPONSE, - &test_response.raw, - sizeof(test_response)); - } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) && - dc_link_check_link_loss_status(dc_link, &offload_work->data) && - dc_link_dp_allow_hpd_rx_irq(dc_link)) { - /* offload_work->data is from handle_hpd_rx_irq-> - * schedule_hpd_rx_offload_work.this is defer handle - * for hpd short pulse. upon here, link status may be - * changed, need get latest link status from dpcd - * registers. if link status is good, skip run link - * training again. - */ - union hpd_irq_data irq_data; - - memset(&irq_data, 0, sizeof(irq_data)); - - /* before dc_link_dp_handle_link_loss, allow new link lost handle - * request be added to work queue if link lost at end of dc_link_ - * dp_handle_link_loss - */ - spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); - offload_work->offload_wq->is_handling_link_loss = false; - spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); - - if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) && - dc_link_check_link_loss_status(dc_link, &irq_data)) - dc_link_dp_handle_link_loss(dc_link); - } - mutex_unlock(&adev->dm.dc_lock); - -skip: - kfree(offload_work); - -} - -static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct dc *dc) -{ - int max_caps = dc->caps.max_links; - int i = 0; - struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL; - - hpd_rx_offload_wq = kcalloc(max_caps, sizeof(*hpd_rx_offload_wq), GFP_KERNEL); - - if (!hpd_rx_offload_wq) - return NULL; - - - for (i = 0; i < max_caps; i++) { - hpd_rx_offload_wq[i].wq = - create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq"); - - if (hpd_rx_offload_wq[i].wq == NULL) { - DRM_ERROR("create amdgpu_dm_hpd_rx_offload_wq fail!"); - goto out_err; - } - - spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); - } - - return hpd_rx_offload_wq; - -out_err: - for (i = 0; i < max_caps; i++) { - if (hpd_rx_offload_wq[i].wq) - destroy_workqueue(hpd_rx_offload_wq[i].wq); - } - kfree(hpd_rx_offload_wq); - return NULL; -} - -struct amdgpu_stutter_quirk { - u16 chip_vendor; - u16 chip_device; - u16 subsys_vendor; - u16 subsys_device; - u8 revision; -}; - -static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = { - /* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */ - { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, - { 0, 0, 0, 0, 0 }, -}; - -static bool dm_should_disable_stutter(struct pci_dev *pdev) -{ - const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list; - - while (p && p->chip_device != 0) { - if (pdev->vendor == p->chip_vendor && - pdev->device == p->chip_device && - pdev->subsystem_vendor == p->subsys_vendor && - pdev->subsystem_device == p->subsys_device && - pdev->revision == p->revision) { - return true; - } - ++p; - } - return false; -} - -static const struct dmi_system_id hpd_disconnect_quirk_table[] = { - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3660"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3260"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3460"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Tower Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Tower 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex SFF Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex SFF 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Micro Plus 7010"), - }, - }, - { - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex Micro 7010"), - }, - }, - {} - /* TODO: refactor this from a fixed table to a dynamic option */ -}; - -static void retrieve_dmi_info(struct amdgpu_display_manager *dm) -{ - const struct dmi_system_id *dmi_id; - - dm->aux_hpd_discon_quirk = false; - - dmi_id = dmi_first_match(hpd_disconnect_quirk_table); - if (dmi_id) { - dm->aux_hpd_discon_quirk = true; - DRM_INFO("aux_hpd_discon_quirk attached\n"); - } -} - -void* -dm_allocate_gpu_mem( - struct amdgpu_device *adev, - enum dc_gpu_mem_alloc_type type, - size_t size, - long long *addr) -{ - struct dal_allocation *da; - u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ? - AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM; - int ret; - - da = kzalloc(sizeof(struct dal_allocation), GFP_KERNEL); - if (!da) - return NULL; - - ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, - domain, &da->bo, - &da->gpu_addr, &da->cpu_ptr); - - *addr = da->gpu_addr; - - if (ret) { - kfree(da); - return NULL; - } - - /* add da to list in dm */ - list_add(&da->list, &adev->dm.da_list); - - return da->cpu_ptr; -} - -static enum dmub_status -dm_dmub_send_vbios_gpint_command(struct amdgpu_device *adev, - enum dmub_gpint_command command_code, - uint16_t param, - uint32_t timeout_us) -{ - union dmub_gpint_data_register reg, test; - uint32_t i; - - /* Assume that VBIOS DMUB is ready to take commands */ - - reg.bits.status = 1; - reg.bits.command_code = command_code; - reg.bits.param = param; - - cgs_write_register(adev->dm.cgs_device, 0x34c0 + 0x01f8, reg.all); - - for (i = 0; i < timeout_us; ++i) { - udelay(1); - - /* Check if our GPINT got acked */ - reg.bits.status = 0; - test = (union dmub_gpint_data_register) - cgs_read_register(adev->dm.cgs_device, 0x34c0 + 0x01f8); - - if (test.all == reg.all) - return DMUB_STATUS_OK; - } - - return DMUB_STATUS_TIMEOUT; -} - -static struct dml2_soc_bb *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev) -{ - struct dml2_soc_bb *bb; - long long addr; - int i = 0; - uint16_t chunk; - enum dmub_gpint_command send_addrs[] = { - DMUB_GPINT__SET_BB_ADDR_WORD0, - DMUB_GPINT__SET_BB_ADDR_WORD1, - DMUB_GPINT__SET_BB_ADDR_WORD2, - DMUB_GPINT__SET_BB_ADDR_WORD3, - }; - enum dmub_status ret; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(4, 0, 1): - break; - default: - return NULL; - } - - bb = dm_allocate_gpu_mem(adev, - DC_MEM_ALLOC_TYPE_GART, - sizeof(struct dml2_soc_bb), - &addr); - if (!bb) - return NULL; - - for (i = 0; i < 4; i++) { - /* Extract 16-bit chunk */ - chunk = ((uint64_t) addr >> (i * 16)) & 0xFFFF; - /* Send the chunk */ - ret = dm_dmub_send_vbios_gpint_command(adev, send_addrs[i], chunk, 30000); - if (ret != DMUB_STATUS_OK) - /* No need to free bb here since it shall be done in dm_sw_fini() */ - return NULL; - } - - /* Now ask DMUB to copy the bb */ - ret = dm_dmub_send_vbios_gpint_command(adev, DMUB_GPINT__BB_COPY, 1, 200000); - if (ret != DMUB_STATUS_OK) - return NULL; - - return bb; -} - -static int amdgpu_dm_init(struct amdgpu_device *adev) -{ - struct dc_init_data init_data; - struct dc_callback_init init_params; - int r; - - adev->dm.ddev = adev_to_drm(adev); - adev->dm.adev = adev; - - /* Zero all the fields */ - memset(&init_data, 0, sizeof(init_data)); - memset(&init_params, 0, sizeof(init_params)); - - mutex_init(&adev->dm.dpia_aux_lock); - mutex_init(&adev->dm.dc_lock); - mutex_init(&adev->dm.audio_lock); - - if (amdgpu_dm_irq_init(adev)) { - DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); - goto error; - } - - init_data.asic_id.chip_family = adev->family; - - init_data.asic_id.pci_revision_id = adev->pdev->revision; - init_data.asic_id.hw_internal_rev = adev->external_rev_id; - init_data.asic_id.chip_id = adev->pdev->device; - - init_data.asic_id.vram_width = adev->gmc.vram_width; - /* TODO: initialize init_data.asic_id.vram_type here!!!! */ - init_data.asic_id.atombios_base_address = - adev->mode_info.atom_context->bios; - - init_data.driver = adev; - - /* cgs_device was created in dm_sw_init() */ - init_data.cgs_device = adev->dm.cgs_device; - - init_data.dce_environment = DCE_ENV_PRODUCTION_DRV; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - switch (adev->dm.dmcub_fw_version) { - case 0: /* development */ - case 0x1: /* linux-firmware.git hash 6d9f399 */ - case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */ - init_data.flags.disable_dmcu = false; - break; - default: - init_data.flags.disable_dmcu = true; - } - break; - case IP_VERSION(2, 0, 3): - init_data.flags.disable_dmcu = true; - break; - default: - break; - } - - /* APU support S/G display by default except: - * ASICs before Carrizo, - * RAVEN1 (Users reported stability issue) - */ - - if (adev->asic_type < CHIP_CARRIZO) { - init_data.flags.gpu_vm_support = false; - } else if (adev->asic_type == CHIP_RAVEN) { - if (adev->apu_flags & AMD_APU_IS_RAVEN) - init_data.flags.gpu_vm_support = false; - else - init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0); - } else { - init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU); - } - - adev->mode_info.gpu_vm_support = init_data.flags.gpu_vm_support; - - if (amdgpu_dc_feature_mask & DC_FBC_MASK) - init_data.flags.fbc_support = true; - - if (amdgpu_dc_feature_mask & DC_MULTI_MON_PP_MCLK_SWITCH_MASK) - init_data.flags.multi_mon_pp_mclk_switch = true; - - if (amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK) - init_data.flags.disable_fractional_pwm = true; - - if (amdgpu_dc_feature_mask & DC_EDP_NO_POWER_SEQUENCING) - init_data.flags.edp_no_power_sequencing = true; - - if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP1_4A) - init_data.flags.allow_lttpr_non_transparent_mode.bits.DP1_4A = true; - if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP2_0) - init_data.flags.allow_lttpr_non_transparent_mode.bits.DP2_0 = true; - - init_data.flags.seamless_boot_edp_requested = false; - - if (amdgpu_device_seamless_boot_supported(adev)) { - init_data.flags.seamless_boot_edp_requested = true; - init_data.flags.allow_seamless_boot_optimization = true; - DRM_INFO("Seamless boot condition check passed\n"); - } - - init_data.flags.enable_mipi_converter_optimization = true; - - init_data.dcn_reg_offsets = adev->reg_offset[DCE_HWIP][0]; - init_data.nbio_reg_offsets = adev->reg_offset[NBIO_HWIP][0]; - init_data.clk_reg_offsets = adev->reg_offset[CLK_HWIP][0]; - - if (amdgpu_dc_debug_mask & DC_DISABLE_IPS) - init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL; - else - init_data.flags.disable_ips = DMUB_IPS_ENABLE; - - init_data.flags.disable_ips_in_vpb = 0; - - /* Enable DWB for tested platforms only */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0)) - init_data.num_virtual_links = 1; - - retrieve_dmi_info(&adev->dm); - - if (adev->dm.bb_from_dmub) - init_data.bb_from_dmub = adev->dm.bb_from_dmub; - else - init_data.bb_from_dmub = NULL; - - /* Display Core create. */ - adev->dm.dc = dc_create(&init_data); - - if (adev->dm.dc) { - DRM_INFO("Display Core v%s initialized on %s\n", DC_VER, - dce_version_to_string(adev->dm.dc->ctx->dce_version)); - } else { - DRM_INFO("Display Core failed to initialize with v%s!\n", DC_VER); - goto error; - } - - if (amdgpu_dc_debug_mask & DC_DISABLE_PIPE_SPLIT) { - adev->dm.dc->debug.force_single_disp_pipe_split = false; - adev->dm.dc->debug.pipe_split_policy = MPC_SPLIT_AVOID; - } - - if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY) - adev->dm.dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true; - if (dm_should_disable_stutter(adev->pdev)) - adev->dm.dc->debug.disable_stutter = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER) - adev->dm.dc->debug.disable_stutter = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) - adev->dm.dc->debug.disable_dsc = true; - - if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING) - adev->dm.dc->debug.disable_clock_gate = true; - - if (amdgpu_dc_debug_mask & DC_FORCE_SUBVP_MCLK_SWITCH) - adev->dm.dc->debug.force_subvp_mclk_switch = true; - - if (amdgpu_dc_debug_mask & DC_ENABLE_DML2) { - adev->dm.dc->debug.using_dml2 = true; - adev->dm.dc->debug.using_dml21 = true; - } - - adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; - - /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ - adev->dm.dc->debug.ignore_cable_id = true; - - if (adev->dm.dc->caps.dp_hdmi21_pcon_support) - DRM_INFO("DP-HDMI FRL PCON supported\n"); - - r = dm_dmub_hw_init(adev); - if (r) { - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - goto error; - } - - dc_hardware_init(adev->dm.dc); - - adev->dm.hpd_rx_offload_wq = hpd_rx_irq_create_workqueue(adev->dm.dc); - if (!adev->dm.hpd_rx_offload_wq) { - DRM_ERROR("amdgpu: failed to create hpd rx offload workqueue.\n"); - goto error; - } - - if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) { - struct dc_phy_addr_space_config pa_config; - - mmhub_read_system_context(adev, &pa_config); - - // Call the DC init_memory func - dc_setup_system_context(adev->dm.dc, &pa_config); - } - - adev->dm.freesync_module = mod_freesync_create(adev->dm.dc); - if (!adev->dm.freesync_module) { - DRM_ERROR( - "amdgpu: failed to initialize freesync_module.\n"); - } else - DRM_DEBUG_DRIVER("amdgpu: freesync_module init done %p.\n", - adev->dm.freesync_module); - - amdgpu_dm_init_color_mod(); - - if (adev->dm.dc->caps.max_links > 0) { - adev->dm.vblank_control_workqueue = - create_singlethread_workqueue("dm_vblank_control_workqueue"); - if (!adev->dm.vblank_control_workqueue) - DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); - } - - if (adev->dm.dc->caps.ips_support && adev->dm.dc->config.disable_ips == DMUB_IPS_ENABLE) - adev->dm.idle_workqueue = idle_create_workqueue(adev); - - if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) { - adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); - - if (!adev->dm.hdcp_workqueue) - DRM_ERROR("amdgpu: failed to initialize hdcp_workqueue.\n"); - else - DRM_DEBUG_DRIVER("amdgpu: hdcp_workqueue init done %p.\n", adev->dm.hdcp_workqueue); - - dc_init_callbacks(adev->dm.dc, &init_params); - } - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - init_completion(&adev->dm.dmub_aux_transfer_done); - adev->dm.dmub_notify = kzalloc(sizeof(struct dmub_notification), GFP_KERNEL); - if (!adev->dm.dmub_notify) { - DRM_INFO("amdgpu: fail to allocate adev->dm.dmub_notify"); - goto error; - } - - adev->dm.delayed_hpd_wq = create_singlethread_workqueue("amdgpu_dm_hpd_wq"); - if (!adev->dm.delayed_hpd_wq) { - DRM_ERROR("amdgpu: failed to create hpd offload workqueue.\n"); - goto error; - } - - amdgpu_dm_outbox_init(adev); - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_AUX_REPLY, - dmub_aux_setconfig_callback, false)) { - DRM_ERROR("amdgpu: fail to register dmub aux callback"); - goto error; - } - /* Enable outbox notification only after IRQ handlers are registered and DMUB is alive. - * It is expected that DMUB will resend any pending notifications at this point. Note - * that hpd and hpd_irq handler registration are deferred to register_hpd_handlers() to - * align legacy interface initialization sequence. Connection status will be proactivly - * detected once in the amdgpu_dm_initialize_drm_device. - */ - dc_enable_dmub_outbox(adev->dm.dc); - - /* DPIA trace goes to dmesg logs only if outbox is enabled */ - if (amdgpu_dc_debug_mask & DC_ENABLE_DPIA_TRACE) - dc_dmub_srv_enable_dpia_trace(adev->dm.dc); - } - - if (amdgpu_dm_initialize_drm_device(adev)) { - DRM_ERROR( - "amdgpu: failed to initialize sw for display support.\n"); - goto error; - } - - /* create fake encoders for MST */ - dm_dp_create_fake_mst_encoders(adev); - - /* TODO: Add_display_info? */ - - /* TODO use dynamic cursor width */ - adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size; - adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size; - - if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) { - DRM_ERROR( - "amdgpu: failed to initialize sw for display support.\n"); - goto error; - } - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - adev->dm.secure_display_ctxs = amdgpu_dm_crtc_secure_display_create_contexts(adev); - if (!adev->dm.secure_display_ctxs) - DRM_ERROR("amdgpu: failed to initialize secure display contexts.\n"); -#endif - - DRM_DEBUG_DRIVER("KMS initialized.\n"); - - return 0; -error: - amdgpu_dm_fini(adev); - - return -EINVAL; -} - -static int amdgpu_dm_early_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_dm_audio_fini(adev); - - return 0; -} - -static void amdgpu_dm_fini(struct amdgpu_device *adev) -{ - int i; - - if (adev->dm.vblank_control_workqueue) { - destroy_workqueue(adev->dm.vblank_control_workqueue); - adev->dm.vblank_control_workqueue = NULL; - } - - if (adev->dm.idle_workqueue) { - if (adev->dm.idle_workqueue->running) { - adev->dm.idle_workqueue->enable = false; - flush_work(&adev->dm.idle_workqueue->work); - } - - kfree(adev->dm.idle_workqueue); - adev->dm.idle_workqueue = NULL; - } - - amdgpu_dm_destroy_drm_device(&adev->dm); - -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - if (adev->dm.secure_display_ctxs) { - for (i = 0; i < adev->mode_info.num_crtc; i++) { - if (adev->dm.secure_display_ctxs[i].crtc) { - flush_work(&adev->dm.secure_display_ctxs[i].notify_ta_work); - flush_work(&adev->dm.secure_display_ctxs[i].forward_roi_work); - } - } - kfree(adev->dm.secure_display_ctxs); - adev->dm.secure_display_ctxs = NULL; - } -#endif - if (adev->dm.hdcp_workqueue) { - hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue); - adev->dm.hdcp_workqueue = NULL; - } - - if (adev->dm.dc) { - dc_deinit_callbacks(adev->dm.dc); - dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv); - if (dc_enable_dmub_notifications(adev->dm.dc)) { - kfree(adev->dm.dmub_notify); - adev->dm.dmub_notify = NULL; - destroy_workqueue(adev->dm.delayed_hpd_wq); - adev->dm.delayed_hpd_wq = NULL; - } - } - - if (adev->dm.dmub_bo) - amdgpu_bo_free_kernel(&adev->dm.dmub_bo, - &adev->dm.dmub_bo_gpu_addr, - &adev->dm.dmub_bo_cpu_addr); - - if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) { - for (i = 0; i < adev->dm.dc->caps.max_links; i++) { - if (adev->dm.hpd_rx_offload_wq[i].wq) { - destroy_workqueue(adev->dm.hpd_rx_offload_wq[i].wq); - adev->dm.hpd_rx_offload_wq[i].wq = NULL; - } - } - - kfree(adev->dm.hpd_rx_offload_wq); - adev->dm.hpd_rx_offload_wq = NULL; - } - - /* DC Destroy TODO: Replace destroy DAL */ - if (adev->dm.dc) - dc_destroy(&adev->dm.dc); - /* - * TODO: pageflip, vlank interrupt - * - * amdgpu_dm_irq_fini(adev); - */ - - if (adev->dm.cgs_device) { - amdgpu_cgs_destroy_device(adev->dm.cgs_device); - adev->dm.cgs_device = NULL; - } - if (adev->dm.freesync_module) { - mod_freesync_destroy(adev->dm.freesync_module); - adev->dm.freesync_module = NULL; - } - - mutex_destroy(&adev->dm.audio_lock); - mutex_destroy(&adev->dm.dc_lock); - mutex_destroy(&adev->dm.dpia_aux_lock); -} - -static int load_dmcu_fw(struct amdgpu_device *adev) -{ - const char *fw_name_dmcu = NULL; - int r; - const struct dmcu_firmware_header_v1_0 *hdr; - - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - case CHIP_OLAND: -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - case CHIP_TONGA: - case CHIP_FIJI: - case CHIP_CARRIZO: - case CHIP_STONEY: - case CHIP_POLARIS11: - case CHIP_POLARIS10: - case CHIP_POLARIS12: - case CHIP_VEGAM: - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - return 0; - case CHIP_NAVI12: - fw_name_dmcu = FIRMWARE_NAVI12_DMCU; - break; - case CHIP_RAVEN: - if (ASICREV_IS_PICASSO(adev->external_rev_id)) - fw_name_dmcu = FIRMWARE_RAVEN_DMCU; - else if (ASICREV_IS_RAVEN2(adev->external_rev_id)) - fw_name_dmcu = FIRMWARE_RAVEN_DMCU; - else - return 0; - break; - default: - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 3): - case IP_VERSION(2, 0, 0): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 0, 2): - case IP_VERSION(3, 0, 3): - case IP_VERSION(3, 0, 1): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - return 0; - default: - break; - } - DRM_ERROR("Unsupported ASIC type: 0x%X\n", adev->asic_type); - return -EINVAL; - } - - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { - DRM_DEBUG_KMS("dm: DMCU firmware not supported on direct or SMU loading\n"); - return 0; - } - - r = amdgpu_ucode_request(adev, &adev->dm.fw_dmcu, "%s", fw_name_dmcu); - if (r == -ENODEV) { - /* DMCU firmware is not necessary, so don't raise a fuss if it's missing */ - DRM_DEBUG_KMS("dm: DMCU firmware not found\n"); - adev->dm.fw_dmcu = NULL; - return 0; - } - if (r) { - dev_err(adev->dev, "amdgpu_dm: Can't validate firmware \"%s\"\n", - fw_name_dmcu); - amdgpu_ucode_release(&adev->dm.fw_dmcu); - return r; - } - - hdr = (const struct dmcu_firmware_header_v1_0 *)adev->dm.fw_dmcu->data; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].ucode_id = AMDGPU_UCODE_ID_DMCU_ERAM; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].fw = adev->dm.fw_dmcu; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->header.ucode_size_bytes) - le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); - - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].ucode_id = AMDGPU_UCODE_ID_DMCU_INTV; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].fw = adev->dm.fw_dmcu; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); - - adev->dm.dmcu_fw_version = le32_to_cpu(hdr->header.ucode_version); - - DRM_DEBUG_KMS("PSP loading DMCU firmware\n"); - - return 0; -} - -static uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address) -{ - struct amdgpu_device *adev = ctx; - - return dm_read_reg(adev->dm.dc->ctx, address); -} - -static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address, - uint32_t value) -{ - struct amdgpu_device *adev = ctx; - - return dm_write_reg(adev->dm.dc->ctx, address, value); -} - -static int dm_dmub_sw_init(struct amdgpu_device *adev) -{ - struct dmub_srv_create_params create_params; - struct dmub_srv_region_params region_params; - struct dmub_srv_region_info region_info; - struct dmub_srv_memory_params memory_params; - struct dmub_srv_fb_info *fb_info; - struct dmub_srv *dmub_srv; - const struct dmcub_firmware_header_v1_0 *hdr; - enum dmub_asic dmub_asic; - enum dmub_status status; - static enum dmub_window_memory_type window_memory_type[DMUB_WINDOW_TOTAL] = { - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_0_INST_CONST - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_1_STACK - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_2_BSS_DATA - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_3_VBIOS - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_4_MAILBOX - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_5_TRACEBUFF - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_6_FW_STATE - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_7_SCRATCH_MEM - DMUB_WINDOW_MEMORY_TYPE_FB, //DMUB_WINDOW_SHARED_STATE - }; - int r; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - dmub_asic = DMUB_ASIC_DCN21; - break; - case IP_VERSION(3, 0, 0): - dmub_asic = DMUB_ASIC_DCN30; - break; - case IP_VERSION(3, 0, 1): - dmub_asic = DMUB_ASIC_DCN301; - break; - case IP_VERSION(3, 0, 2): - dmub_asic = DMUB_ASIC_DCN302; - break; - case IP_VERSION(3, 0, 3): - dmub_asic = DMUB_ASIC_DCN303; - break; - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - dmub_asic = (adev->external_rev_id == YELLOW_CARP_B0) ? DMUB_ASIC_DCN31B : DMUB_ASIC_DCN31; - break; - case IP_VERSION(3, 1, 4): - dmub_asic = DMUB_ASIC_DCN314; - break; - case IP_VERSION(3, 1, 5): - dmub_asic = DMUB_ASIC_DCN315; - break; - case IP_VERSION(3, 1, 6): - dmub_asic = DMUB_ASIC_DCN316; - break; - case IP_VERSION(3, 2, 0): - dmub_asic = DMUB_ASIC_DCN32; - break; - case IP_VERSION(3, 2, 1): - dmub_asic = DMUB_ASIC_DCN321; - break; - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - dmub_asic = DMUB_ASIC_DCN35; - break; - case IP_VERSION(4, 0, 1): - dmub_asic = DMUB_ASIC_DCN401; - break; - - default: - /* ASIC doesn't support DMUB. */ - return 0; - } - - hdr = (const struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data; - adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id = - AMDGPU_UCODE_ID_DMCUB; - adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].fw = - adev->dm.dmub_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(hdr->inst_const_bytes), PAGE_SIZE); - - DRM_INFO("Loading DMUB firmware via PSP: version=0x%08X\n", - adev->dm.dmcub_fw_version); - } - - - adev->dm.dmub_srv = kzalloc(sizeof(*adev->dm.dmub_srv), GFP_KERNEL); - dmub_srv = adev->dm.dmub_srv; - - if (!dmub_srv) { - DRM_ERROR("Failed to allocate DMUB service!\n"); - return -ENOMEM; - } - - memset(&create_params, 0, sizeof(create_params)); - create_params.user_ctx = adev; - create_params.funcs.reg_read = amdgpu_dm_dmub_reg_read; - create_params.funcs.reg_write = amdgpu_dm_dmub_reg_write; - create_params.asic = dmub_asic; - - /* Create the DMUB service. */ - status = dmub_srv_create(dmub_srv, &create_params); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error creating DMUB service: %d\n", status); - return -EINVAL; - } - - /* Calculate the size of all the regions for the DMUB service. */ - memset(®ion_params, 0, sizeof(region_params)); - - region_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) - - PSP_HEADER_BYTES - PSP_FOOTER_BYTES; - region_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes); - region_params.vbios_size = adev->bios_size; - region_params.fw_bss_data = region_params.bss_data_size ? - adev->dm.dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - le32_to_cpu(hdr->inst_const_bytes) : NULL; - region_params.fw_inst_const = - adev->dm.dmub_fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes) + - PSP_HEADER_BYTES; - region_params.window_memory_type = window_memory_type; - - status = dmub_srv_calc_region_info(dmub_srv, ®ion_params, - ®ion_info); - - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error calculating DMUB region info: %d\n", status); - return -EINVAL; - } - - /* - * Allocate a framebuffer based on the total size of all the regions. - * TODO: Move this into GART. - */ - r = amdgpu_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, - &adev->dm.dmub_bo, - &adev->dm.dmub_bo_gpu_addr, - &adev->dm.dmub_bo_cpu_addr); - if (r) - return r; - - /* Rebase the regions on the framebuffer address. */ - memset(&memory_params, 0, sizeof(memory_params)); - memory_params.cpu_fb_addr = adev->dm.dmub_bo_cpu_addr; - memory_params.gpu_fb_addr = adev->dm.dmub_bo_gpu_addr; - memory_params.region_info = ®ion_info; - memory_params.window_memory_type = window_memory_type; - - adev->dm.dmub_fb_info = - kzalloc(sizeof(*adev->dm.dmub_fb_info), GFP_KERNEL); - fb_info = adev->dm.dmub_fb_info; - - if (!fb_info) { - DRM_ERROR( - "Failed to allocate framebuffer info for DMUB service!\n"); - return -ENOMEM; - } - - status = dmub_srv_calc_mem_info(dmub_srv, &memory_params, fb_info); - if (status != DMUB_STATUS_OK) { - DRM_ERROR("Error calculating DMUB FB info: %d\n", status); - return -EINVAL; - } - - adev->dm.bb_from_dmub = dm_dmub_get_vbios_bounding_box(adev); - - return 0; -} - -static int dm_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - adev->dm.cgs_device = amdgpu_cgs_create_device(adev); - - if (!adev->dm.cgs_device) { - DRM_ERROR("amdgpu: failed to create cgs device.\n"); - return -EINVAL; - } - - /* Moved from dm init since we need to use allocations for storing bounding box data */ - INIT_LIST_HEAD(&adev->dm.da_list); - - r = dm_dmub_sw_init(adev); - if (r) - return r; - - return load_dmcu_fw(adev); -} - -static int dm_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct dal_allocation *da; - - list_for_each_entry(da, &adev->dm.da_list, list) { - if (adev->dm.bb_from_dmub == (void *) da->cpu_ptr) { - amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr); - list_del(&da->list); - kfree(da); - break; - } - } - - adev->dm.bb_from_dmub = NULL; - - kfree(adev->dm.dmub_fb_info); - adev->dm.dmub_fb_info = NULL; - - if (adev->dm.dmub_srv) { - dmub_srv_destroy(adev->dm.dmub_srv); - kfree(adev->dm.dmub_srv); - adev->dm.dmub_srv = NULL; - } - - amdgpu_ucode_release(&adev->dm.dmub_fw); - amdgpu_ucode_release(&adev->dm.fw_dmcu); - - return 0; -} - -static int detect_mst_link_for_all_connectors(struct drm_device *dev) -{ - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - int ret = 0; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type == dc_connection_mst_branch && - aconnector->mst_mgr.aux) { - drm_dbg_kms(dev, "DM_MST: starting TM on aconnector: %p [id: %d]\n", - aconnector, - aconnector->base.base.id); - - ret = drm_dp_mst_topology_mgr_set_mst(&aconnector->mst_mgr, true); - if (ret < 0) { - drm_err(dev, "DM_MST: Failed to start MST\n"); - aconnector->dc_link->type = - dc_connection_single; - ret = dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx, - aconnector->dc_link); - break; - } - } - } - drm_connector_list_iter_end(&iter); - - return ret; -} - -static int dm_late_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - struct dmcu_iram_parameters params; - unsigned int linear_lut[16]; - int i; - struct dmcu *dmcu = NULL; - - dmcu = adev->dm.dc->res_pool->dmcu; - - for (i = 0; i < 16; i++) - linear_lut[i] = 0xFFFF * i / 15; - - params.set = 0; - params.backlight_ramping_override = false; - params.backlight_ramping_start = 0xCCCC; - params.backlight_ramping_reduction = 0xCCCCCCCC; - params.backlight_lut_array_size = 16; - params.backlight_lut_array = linear_lut; - - /* Min backlight level after ABM reduction, Don't allow below 1% - * 0xFFFF x 0.01 = 0x28F - */ - params.min_abm_backlight = 0x28F; - /* In the case where abm is implemented on dmcub, - * dmcu object will be null. - * ABM 2.4 and up are implemented on dmcub. - */ - if (dmcu) { - if (!dmcu_load_iram(dmcu, params)) - return -EINVAL; - } else if (adev->dm.dc->ctx->dmub_srv) { - struct dc_link *edp_links[MAX_NUM_EDP]; - int edp_num; - - dc_get_edp_links(adev->dm.dc, edp_links, &edp_num); - for (i = 0; i < edp_num; i++) { - if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i)) - return -EINVAL; - } - } - - return detect_mst_link_for_all_connectors(adev_to_drm(adev)); -} - -static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr) -{ - int ret; - u8 guid[16]; - u64 tmp64; - - mutex_lock(&mgr->lock); - if (!mgr->mst_primary) - goto out_fail; - - if (drm_dp_read_dpcd_caps(mgr->aux, mgr->dpcd) < 0) { - drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); - goto out_fail; - } - - ret = drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL, - DP_MST_EN | - DP_UP_REQ_EN | - DP_UPSTREAM_IS_SRC); - if (ret < 0) { - drm_dbg_kms(mgr->dev, "mst write failed - undocked during suspend?\n"); - goto out_fail; - } - - /* Some hubs forget their guids after they resume */ - ret = drm_dp_dpcd_read(mgr->aux, DP_GUID, guid, 16); - if (ret != 16) { - drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); - goto out_fail; - } - - if (memchr_inv(guid, 0, 16) == NULL) { - tmp64 = get_jiffies_64(); - memcpy(&guid[0], &tmp64, sizeof(u64)); - memcpy(&guid[8], &tmp64, sizeof(u64)); - - ret = drm_dp_dpcd_write(mgr->aux, DP_GUID, guid, 16); - - if (ret != 16) { - drm_dbg_kms(mgr->dev, "check mstb guid failed - undocked during suspend?\n"); - goto out_fail; - } - } - - memcpy(mgr->mst_primary->guid, guid, 16); - -out_fail: - mutex_unlock(&mgr->lock); -} - -static void s3_handle_mst(struct drm_device *dev, bool suspend) -{ - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct drm_dp_mst_topology_mgr *mgr; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type != dc_connection_mst_branch || - aconnector->mst_root) - continue; - - mgr = &aconnector->mst_mgr; - - if (suspend) { - drm_dp_mst_topology_mgr_suspend(mgr); - } else { - /* if extended timeout is supported in hardware, - * default to LTTPR timeout (3.2ms) first as a W/A for DP link layer - * CTS 4.2.1.1 regression introduced by CTS specs requirement update. - */ - try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD); - if (!dp_is_lttpr_present(aconnector->dc_link)) - try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD); - - /* TODO: move resume_mst_branch_status() into drm mst resume again - * once topology probing work is pulled out from mst resume into mst - * resume 2nd step. mst resume 2nd step should be called after old - * state getting restored (i.e. drm_atomic_helper_resume()). - */ - resume_mst_branch_status(mgr); - } - } - drm_connector_list_iter_end(&iter); -} - -static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev) -{ - int ret = 0; - - /* This interface is for dGPU Navi1x.Linux dc-pplib interface depends - * on window driver dc implementation. - * For Navi1x, clock settings of dcn watermarks are fixed. the settings - * should be passed to smu during boot up and resume from s3. - * boot up: dc calculate dcn watermark clock settings within dc_create, - * dcn20_resource_construct - * then call pplib functions below to pass the settings to smu: - * smu_set_watermarks_for_clock_ranges - * smu_set_watermarks_table - * navi10_set_watermarks_table - * smu_write_watermarks_table - * - * For Renoir, clock settings of dcn watermark are also fixed values. - * dc has implemented different flow for window driver: - * dc_hardware_init / dc_set_power_state - * dcn10_init_hw - * notify_wm_ranges - * set_wm_ranges - * -- Linux - * smu_set_watermarks_for_clock_ranges - * renoir_set_watermarks_table - * smu_write_watermarks_table - * - * For Linux, - * dc_hardware_init -> amdgpu_dm_init - * dc_set_power_state --> dm_resume - * - * therefore, this function apply to navi10/12/14 but not Renoir - * * - */ - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 0): - break; - default: - return 0; - } - - ret = amdgpu_dpm_write_watermarks_table(adev); - if (ret) { - DRM_ERROR("Failed to update WMTABLE!\n"); - return ret; - } - - return 0; -} - -/** - * dm_hw_init() - Initialize DC device - * @handle: The base driver device containing the amdgpu_dm device. - * - * Initialize the &struct amdgpu_display_manager device. This involves calling - * the initializers of each DM component, then populating the struct with them. - * - * Although the function implies hardware initialization, both hardware and - * software are initialized here. Splitting them out to their relevant init - * hooks is a future TODO item. - * - * Some notable things that are initialized here: - * - * - Display Core, both software and hardware - * - DC modules that we need (freesync and color management) - * - DRM software states - * - Interrupt sources and handlers - * - Vblank support - * - Debug FS entries, if enabled - */ -static int dm_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - /* Create DAL display manager */ - r = amdgpu_dm_init(adev); - if (r) - return r; - amdgpu_dm_hpd_init(adev); - - return 0; -} - -/** - * dm_hw_fini() - Teardown DC device - * @handle: The base driver device containing the amdgpu_dm device. - * - * Teardown components within &struct amdgpu_display_manager that require - * cleanup. This involves cleaning up the DRM device, DC, and any modules that - * were loaded. Also flush IRQ workqueues and disable them. - */ -static int dm_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_dm_hpd_fini(adev); - - amdgpu_dm_irq_fini(adev); - amdgpu_dm_fini(adev); - return 0; -} - - -static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, - struct dc_state *state, bool enable) -{ - enum dc_irq_source irq_source; - struct amdgpu_crtc *acrtc; - int rc = -EBUSY; - int i = 0; - - for (i = 0; i < state->stream_count; i++) { - acrtc = get_crtc_by_otg_inst( - adev, state->stream_status[i].primary_otg_inst); - - if (acrtc && state->stream_status[i].plane_count != 0) { - irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst; - rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; - if (rc) - DRM_WARN("Failed to %s pflip interrupts\n", - enable ? "enable" : "disable"); - - if (enable) { - if (amdgpu_dm_crtc_vrr_active(to_dm_crtc_state(acrtc->base.state))) - rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, true); - } else - rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, false); - - if (rc) - DRM_WARN("Failed to %sable vupdate interrupt\n", enable ? "en" : "dis"); - - irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst; - /* During gpu-reset we disable and then enable vblank irq, so - * don't use amdgpu_irq_get/put() to avoid refcount change. - */ - if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) - DRM_WARN("Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); - } - } - -} - -static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc) -{ - struct dc_state *context = NULL; - enum dc_status res = DC_ERROR_UNEXPECTED; - int i; - struct dc_stream_state *del_streams[MAX_PIPES]; - int del_streams_count = 0; - struct dc_commit_streams_params params = {}; - - memset(del_streams, 0, sizeof(del_streams)); - - context = dc_state_create_current_copy(dc); - if (context == NULL) - goto context_alloc_fail; - - /* First remove from context all streams */ - for (i = 0; i < context->stream_count; i++) { - struct dc_stream_state *stream = context->streams[i]; - - del_streams[del_streams_count++] = stream; - } - - /* Remove all planes for removed streams and then remove the streams */ - for (i = 0; i < del_streams_count; i++) { - if (!dc_state_rem_all_planes_for_stream(dc, del_streams[i], context)) { - res = DC_FAIL_DETACH_SURFACES; - goto fail; - } - - res = dc_state_remove_stream(dc, context, del_streams[i]); - if (res != DC_OK) - goto fail; - } - - params.streams = context->streams; - params.stream_count = context->stream_count; - res = dc_commit_streams(dc, ¶ms); - -fail: - dc_state_release(context); - -context_alloc_fail: - return res; -} - -static void hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm) -{ - int i; - - if (dm->hpd_rx_offload_wq) { - for (i = 0; i < dm->dc->caps.max_links; i++) - flush_workqueue(dm->hpd_rx_offload_wq[i].wq); - } -} - -static int dm_suspend(void *handle) -{ - struct amdgpu_device *adev = handle; - struct amdgpu_display_manager *dm = &adev->dm; - int ret = 0; - - if (amdgpu_in_reset(adev)) { - mutex_lock(&dm->dc_lock); - - dc_allow_idle_optimizations(adev->dm.dc, false); - - dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state); - - if (dm->cached_dc_state) - dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); - - amdgpu_dm_commit_zero_streams(dm->dc); - - amdgpu_dm_irq_suspend(adev); - - hpd_rx_irq_work_suspend(dm); - - return ret; - } - - WARN_ON(adev->dm.cached_state); - adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev)); - if (IS_ERR(adev->dm.cached_state)) - return PTR_ERR(adev->dm.cached_state); - - s3_handle_mst(adev_to_drm(adev), true); - - amdgpu_dm_irq_suspend(adev); - - hpd_rx_irq_work_suspend(dm); - - if (adev->dm.dc->caps.ips_support) - dc_allow_idle_optimizations(adev->dm.dc, true); - - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D3); - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D3); - - return 0; -} - -struct drm_connector * -amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_state *state, - struct drm_crtc *crtc) -{ - u32 i; - struct drm_connector_state *new_con_state; - struct drm_connector *connector; - struct drm_crtc *crtc_from_state; - - for_each_new_connector_in_state(state, connector, new_con_state, i) { - crtc_from_state = new_con_state->crtc; - - if (crtc_from_state == crtc) - return connector; - } - - return NULL; -} - -static void emulated_link_detect(struct dc_link *link) -{ - struct dc_sink_init_data sink_init_data = { 0 }; - struct display_sink_capability sink_caps = { 0 }; - enum dc_edid_status edid_status; - struct dc_context *dc_ctx = link->ctx; - struct drm_device *dev = adev_to_drm(dc_ctx->driver_context); - struct dc_sink *sink = NULL; - struct dc_sink *prev_sink = NULL; - - link->type = dc_connection_none; - prev_sink = link->local_sink; - - if (prev_sink) - dc_sink_release(prev_sink); - - switch (link->connector_signal) { - case SIGNAL_TYPE_HDMI_TYPE_A: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_HDMI_TYPE_A; - break; - } - - case SIGNAL_TYPE_DVI_SINGLE_LINK: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_DVI_SINGLE_LINK; - break; - } - - case SIGNAL_TYPE_DVI_DUAL_LINK: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_DVI_DUAL_LINK; - break; - } - - case SIGNAL_TYPE_LVDS: { - sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C; - sink_caps.signal = SIGNAL_TYPE_LVDS; - break; - } - - case SIGNAL_TYPE_EDP: { - sink_caps.transaction_type = - DDC_TRANSACTION_TYPE_I2C_OVER_AUX; - sink_caps.signal = SIGNAL_TYPE_EDP; - break; - } - - case SIGNAL_TYPE_DISPLAY_PORT: { - sink_caps.transaction_type = - DDC_TRANSACTION_TYPE_I2C_OVER_AUX; - sink_caps.signal = SIGNAL_TYPE_VIRTUAL; - break; - } - - default: - drm_err(dev, "Invalid connector type! signal:%d\n", - link->connector_signal); - return; - } - - sink_init_data.link = link; - sink_init_data.sink_signal = sink_caps.signal; - - sink = dc_sink_create(&sink_init_data); - if (!sink) { - drm_err(dev, "Failed to create sink!\n"); - return; - } - - /* dc_sink_create returns a new reference */ - link->local_sink = sink; - - edid_status = dm_helpers_read_local_edid( - link->ctx, - link, - sink); - - if (edid_status != EDID_OK) - drm_err(dev, "Failed to read EDID\n"); - -} - -static void dm_gpureset_commit_state(struct dc_state *dc_state, - struct amdgpu_display_manager *dm) -{ - struct { - struct dc_surface_update surface_updates[MAX_SURFACES]; - struct dc_plane_info plane_infos[MAX_SURFACES]; - struct dc_scaling_info scaling_infos[MAX_SURFACES]; - struct dc_flip_addrs flip_addrs[MAX_SURFACES]; - struct dc_stream_update stream_update; - } *bundle; - int k, m; - - bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - - if (!bundle) { - drm_err(dm->ddev, "Failed to allocate update bundle\n"); - goto cleanup; - } - - for (k = 0; k < dc_state->stream_count; k++) { - bundle->stream_update.stream = dc_state->streams[k]; - - for (m = 0; m < dc_state->stream_status->plane_count; m++) { - bundle->surface_updates[m].surface = - dc_state->stream_status->plane_states[m]; - bundle->surface_updates[m].surface->force_full_update = - true; - } - - update_planes_and_stream_adapter(dm->dc, - UPDATE_TYPE_FULL, - dc_state->stream_status->plane_count, - dc_state->streams[k], - &bundle->stream_update, - bundle->surface_updates); - } - -cleanup: - kfree(bundle); -} - -static int dm_resume(void *handle) -{ - struct amdgpu_device *adev = handle; - struct drm_device *ddev = adev_to_drm(adev); - struct amdgpu_display_manager *dm = &adev->dm; - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - struct drm_crtc *crtc; - struct drm_crtc_state *new_crtc_state; - struct dm_crtc_state *dm_new_crtc_state; - struct drm_plane *plane; - struct drm_plane_state *new_plane_state; - struct dm_plane_state *dm_new_plane_state; - struct dm_atomic_state *dm_state = to_dm_atomic_state(dm->atomic_obj.state); - enum dc_connection_type new_connection_type = dc_connection_none; - struct dc_state *dc_state; - int i, r, j, ret; - bool need_hotplug = false; - struct dc_commit_streams_params commit_params = {}; - - if (dm->dc->caps.ips_support) { - dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false); - } - - if (amdgpu_in_reset(adev)) { - dc_state = dm->cached_dc_state; - - /* - * The dc->current_state is backed up into dm->cached_dc_state - * before we commit 0 streams. - * - * DC will clear link encoder assignments on the real state - * but the changes won't propagate over to the copy we made - * before the 0 streams commit. - * - * DC expects that link encoder assignments are *not* valid - * when committing a state, so as a workaround we can copy - * off of the current state. - * - * We lose the previous assignments, but we had already - * commit 0 streams anyway. - */ - link_enc_cfg_copy(adev->dm.dc->current_state, dc_state); - - r = dm_dmub_hw_init(adev); - if (r) - DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); - - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0); - - dc_resume(dm->dc); - - amdgpu_dm_irq_resume_early(adev); - - for (i = 0; i < dc_state->stream_count; i++) { - dc_state->streams[i]->mode_changed = true; - for (j = 0; j < dc_state->stream_status[i].plane_count; j++) { - dc_state->stream_status[i].plane_states[j]->update_flags.raw - = 0xffffffff; - } - } - - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - amdgpu_dm_outbox_init(adev); - dc_enable_dmub_outbox(adev->dm.dc); - } - - commit_params.streams = dc_state->streams; - commit_params.stream_count = dc_state->stream_count; - dc_exit_ips_for_hw_access(dm->dc); - WARN_ON(!dc_commit_streams(dm->dc, &commit_params)); - - dm_gpureset_commit_state(dm->cached_dc_state, dm); - - dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, true); - - dc_state_release(dm->cached_dc_state); - dm->cached_dc_state = NULL; - - amdgpu_dm_irq_resume_late(adev); - - mutex_unlock(&dm->dc_lock); - - return 0; - } - /* Recreate dc_state - DC invalidates it when setting power state to S3. */ - dc_state_release(dm_state->context); - dm_state->context = dc_state_create(dm->dc, NULL); - /* TODO: Remove dc_state->dccg, use dc->dccg directly. */ - - /* Before powering on DC we need to re-initialize DMUB. */ - dm_dmub_hw_resume(adev); - - /* Re-enable outbox interrupts for DPIA. */ - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - amdgpu_dm_outbox_init(adev); - dc_enable_dmub_outbox(adev->dm.dc); - } - - /* power on hardware */ - dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); - dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0); - - /* program HPD filter */ - dc_resume(dm->dc); - - /* - * early enable HPD Rx IRQ, should be done before set mode as short - * pulse interrupts are used for MST - */ - amdgpu_dm_irq_resume_early(adev); - - /* On resume we need to rewrite the MSTM control bits to enable MST*/ - s3_handle_mst(ddev, false); - - /* Do detection*/ - drm_connector_list_iter_begin(ddev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->dc_link) - continue; - - /* - * this is the case when traversing through already created end sink - * MST connectors, should be skipped - */ - if (aconnector->mst_root) - continue; - - mutex_lock(&aconnector->hpd_lock); - if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(aconnector->dc_link); - } else { - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_link_detect(aconnector->dc_link, DETECT_REASON_RESUMEFROMS3S4); - mutex_unlock(&dm->dc_lock); - } - - if (aconnector->fake_enable && aconnector->dc_link->local_sink) - aconnector->fake_enable = false; - - if (aconnector->dc_sink) - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - amdgpu_dm_update_connector_after_detect(aconnector); - mutex_unlock(&aconnector->hpd_lock); - } - drm_connector_list_iter_end(&iter); - - /* Force mode set in atomic commit */ - for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) { - new_crtc_state->active_changed = true; - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - reset_freesync_config_for_crtc(dm_new_crtc_state); - } - - /* - * atomic_check is expected to create the dc states. We need to release - * them here, since they were duplicated as part of the suspend - * procedure. - */ - for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->stream) { - WARN_ON(kref_read(&dm_new_crtc_state->stream->refcount) > 1); - dc_stream_release(dm_new_crtc_state->stream); - dm_new_crtc_state->stream = NULL; - } - dm_new_crtc_state->base.color_mgmt_changed = true; - } - - for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) { - dm_new_plane_state = to_dm_plane_state(new_plane_state); - if (dm_new_plane_state->dc_state) { - WARN_ON(kref_read(&dm_new_plane_state->dc_state->refcount) > 1); - dc_plane_state_release(dm_new_plane_state->dc_state); - dm_new_plane_state->dc_state = NULL; - } - } - - drm_atomic_helper_resume(ddev, dm->cached_state); - - dm->cached_state = NULL; - - /* Do mst topology probing after resuming cached state*/ - drm_connector_list_iter_begin(ddev, &iter); - drm_for_each_connector_iter(connector, &iter) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type != dc_connection_mst_branch || - aconnector->mst_root) - continue; - - ret = drm_dp_mst_topology_mgr_resume(&aconnector->mst_mgr, true); - - if (ret < 0) { - dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx, - aconnector->dc_link); - need_hotplug = true; - } - } - drm_connector_list_iter_end(&iter); - - if (need_hotplug) - drm_kms_helper_hotplug_event(ddev); - - amdgpu_dm_irq_resume_late(adev); - - amdgpu_dm_smu_write_watermarks_table(adev); - - return 0; -} - -/** - * DOC: DM Lifecycle - * - * DM (and consequently DC) is registered in the amdgpu base driver as a IP - * block. When CONFIG_DRM_AMD_DC is enabled, the DM device IP block is added to - * the base driver's device list to be initialized and torn down accordingly. - * - * The functions to do so are provided as hooks in &struct amd_ip_funcs. - */ - -static const struct amd_ip_funcs amdgpu_dm_funcs = { - .name = "dm", - .early_init = dm_early_init, - .late_init = dm_late_init, - .sw_init = dm_sw_init, - .sw_fini = dm_sw_fini, - .early_fini = amdgpu_dm_early_fini, - .hw_init = dm_hw_init, - .hw_fini = dm_hw_fini, - .suspend = dm_suspend, - .resume = dm_resume, - .is_idle = dm_is_idle, - .wait_for_idle = dm_wait_for_idle, - .check_soft_reset = dm_check_soft_reset, - .soft_reset = dm_soft_reset, - .set_clockgating_state = dm_set_clockgating_state, - .set_powergating_state = dm_set_powergating_state, - .dump_ip_state = NULL, - .print_ip_state = NULL, -}; - -const struct amdgpu_ip_block_version dm_ip_block = { - .type = AMD_IP_BLOCK_TYPE_DCE, - .major = 1, - .minor = 0, - .rev = 0, - .funcs = &amdgpu_dm_funcs, -}; - - -/** - * DOC: atomic - * - * *WIP* - */ - -static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = { - .fb_create = amdgpu_display_user_framebuffer_create, - .get_format_info = amdgpu_dm_plane_get_format_info, - .atomic_check = amdgpu_dm_atomic_check, - .atomic_commit = drm_atomic_helper_commit, -}; - -static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = { - .atomic_commit_tail = amdgpu_dm_atomic_commit_tail, - .atomic_commit_setup = drm_dp_mst_atomic_setup_commit, -}; - -static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) -{ - struct amdgpu_dm_backlight_caps *caps; - struct drm_connector *conn_base; - struct amdgpu_device *adev; - struct drm_luminance_range_info *luminance_range; - - if (aconnector->bl_idx == -1 || - aconnector->dc_link->connector_signal != SIGNAL_TYPE_EDP) - return; - - conn_base = &aconnector->base; - adev = drm_to_adev(conn_base->dev); - - caps = &adev->dm.backlight_caps[aconnector->bl_idx]; - caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps; - caps->aux_support = false; - - if (caps->ext_caps->bits.oled == 1 - /* - * || - * caps->ext_caps->bits.sdr_aux_backlight_control == 1 || - * caps->ext_caps->bits.hdr_aux_backlight_control == 1 - */) - caps->aux_support = true; - - if (amdgpu_backlight == 0) - caps->aux_support = false; - else if (amdgpu_backlight == 1) - caps->aux_support = true; - - luminance_range = &conn_base->display_info.luminance_range; - - if (luminance_range->max_luminance) { - caps->aux_min_input_signal = luminance_range->min_luminance; - caps->aux_max_input_signal = luminance_range->max_luminance; - } else { - caps->aux_min_input_signal = 0; - caps->aux_max_input_signal = 512; - } -} - -void amdgpu_dm_update_connector_after_detect( - struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - struct dc_sink *sink; - - /* MST handled by drm_mst framework */ - if (aconnector->mst_mgr.mst_state == true) - return; - - sink = aconnector->dc_link->local_sink; - if (sink) - dc_sink_retain(sink); - - /* - * Edid mgmt connector gets first update only in mode_valid hook and then - * the connector sink is set to either fake or physical sink depends on link status. - * Skip if already done during boot. - */ - if (aconnector->base.force != DRM_FORCE_UNSPECIFIED - && aconnector->dc_em_sink) { - - /* - * For S3 resume with headless use eml_sink to fake stream - * because on resume connector->sink is set to NULL - */ - mutex_lock(&dev->mode_config.mutex); - - if (sink) { - if (aconnector->dc_sink) { - amdgpu_dm_update_freesync_caps(connector, NULL); - /* - * retain and release below are used to - * bump up refcount for sink because the link doesn't point - * to it anymore after disconnect, so on next crtc to connector - * reshuffle by UMD we will get into unwanted dc_sink release - */ - dc_sink_release(aconnector->dc_sink); - } - aconnector->dc_sink = sink; - dc_sink_retain(aconnector->dc_sink); - amdgpu_dm_update_freesync_caps(connector, - aconnector->edid); - } else { - amdgpu_dm_update_freesync_caps(connector, NULL); - if (!aconnector->dc_sink) { - aconnector->dc_sink = aconnector->dc_em_sink; - dc_sink_retain(aconnector->dc_sink); - } - } - - mutex_unlock(&dev->mode_config.mutex); - - if (sink) - dc_sink_release(sink); - return; - } - - /* - * TODO: temporary guard to look for proper fix - * if this sink is MST sink, we should not do anything - */ - if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - dc_sink_release(sink); - return; - } - - if (aconnector->dc_sink == sink) { - /* - * We got a DP short pulse (Link Loss, DP CTS, etc...). - * Do nothing!! - */ - drm_dbg_kms(dev, "DCHPD: connector_id=%d: dc_sink didn't change.\n", - aconnector->connector_id); - if (sink) - dc_sink_release(sink); - return; - } - - drm_dbg_kms(dev, "DCHPD: connector_id=%d: Old sink=%p New sink=%p\n", - aconnector->connector_id, aconnector->dc_sink, sink); - - mutex_lock(&dev->mode_config.mutex); - - /* - * 1. Update status of the drm connector - * 2. Send an event and let userspace tell us what to do - */ - if (sink) { - /* - * TODO: check if we still need the S3 mode update workaround. - * If yes, put it here. - */ - if (aconnector->dc_sink) { - amdgpu_dm_update_freesync_caps(connector, NULL); - dc_sink_release(aconnector->dc_sink); - } - - aconnector->dc_sink = sink; - dc_sink_retain(aconnector->dc_sink); - if (sink->dc_edid.length == 0) { - aconnector->edid = NULL; - if (aconnector->dc_link->aux_mode) { - drm_dp_cec_unset_edid( - &aconnector->dm_dp_aux.aux); - } - } else { - aconnector->edid = - (struct edid *)sink->dc_edid.raw_edid; - - if (aconnector->dc_link->aux_mode) - drm_dp_cec_set_edid(&aconnector->dm_dp_aux.aux, - aconnector->edid); - } - - if (!aconnector->timing_requested) { - aconnector->timing_requested = - kzalloc(sizeof(struct dc_crtc_timing), GFP_KERNEL); - if (!aconnector->timing_requested) - drm_err(dev, - "failed to create aconnector->requested_timing\n"); - } - - drm_connector_update_edid_property(connector, aconnector->edid); - amdgpu_dm_update_freesync_caps(connector, aconnector->edid); - update_connector_ext_caps(aconnector); - } else { - drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); - amdgpu_dm_update_freesync_caps(connector, NULL); - drm_connector_update_edid_property(connector, NULL); - aconnector->num_modes = 0; - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - aconnector->edid = NULL; - kfree(aconnector->timing_requested); - aconnector->timing_requested = NULL; - /* Set CP to DESIRED if it was ENABLED, so we can re-enable it again on hotplug */ - if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) - connector->state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - } - - mutex_unlock(&dev->mode_config.mutex); - - update_subconnector_property(aconnector); - - if (sink) - dc_sink_release(sink); -} - -static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - enum dc_connection_type new_connection_type = dc_connection_none; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); - struct dc *dc = aconnector->dc_link->ctx->dc; - bool ret = false; - - if (adev->dm.disable_hpd_irq) - return; - - /* - * In case of failure or MST no need to update connector status or notify the OS - * since (for MST case) MST does this in its own context. - */ - mutex_lock(&aconnector->hpd_lock); - - if (adev->dm.hdcp_workqueue) { - hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); - dm_con_state->update_hdcp = true; - } - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - aconnector->timing_changed = false; - - if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(aconnector->dc_link); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED) - drm_kms_helper_connector_hotplug_event(connector); - } else { - mutex_lock(&adev->dm.dc_lock); - dc_exit_ips_for_hw_access(dc); - ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD); - mutex_unlock(&adev->dm.dc_lock); - if (ret) { - amdgpu_dm_update_connector_after_detect(aconnector); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED) - drm_kms_helper_connector_hotplug_event(connector); - } - } - mutex_unlock(&aconnector->hpd_lock); - -} - -static void handle_hpd_irq(void *param) -{ - struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; - - handle_hpd_irq_helper(aconnector); - -} - -static void schedule_hpd_rx_offload_work(struct hpd_rx_irq_offload_work_queue *offload_wq, - union hpd_irq_data hpd_irq_data) -{ - struct hpd_rx_irq_offload_work *offload_work = - kzalloc(sizeof(*offload_work), GFP_KERNEL); - - if (!offload_work) { - DRM_ERROR("Failed to allocate hpd_rx_irq_offload_work.\n"); - return; - } - - INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work); - offload_work->data = hpd_irq_data; - offload_work->offload_wq = offload_wq; - - queue_work(offload_wq->wq, &offload_work->work); - DRM_DEBUG_KMS("queue work to handle hpd_rx offload work"); -} - -static void handle_hpd_rx_irq(void *param) -{ - struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; - struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; - struct dc_link *dc_link = aconnector->dc_link; - bool is_mst_root_connector = aconnector->mst_mgr.mst_state; - bool result = false; - enum dc_connection_type new_connection_type = dc_connection_none; - struct amdgpu_device *adev = drm_to_adev(dev); - union hpd_irq_data hpd_irq_data; - bool link_loss = false; - bool has_left_work = false; - int idx = dc_link->link_index; - struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx]; - struct dc *dc = aconnector->dc_link->ctx->dc; - - memset(&hpd_irq_data, 0, sizeof(hpd_irq_data)); - - if (adev->dm.disable_hpd_irq) - return; - - /* - * TODO:Temporary add mutex to protect hpd interrupt not have a gpio - * conflict, after implement i2c helper, this mutex should be - * retired. - */ - mutex_lock(&aconnector->hpd_lock); - - result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, - &link_loss, true, &has_left_work); - - if (!has_left_work) - goto out; - - if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) { - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - goto out; - } - - if (dc_link_dp_allow_hpd_rx_irq(dc_link)) { - if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || - hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { - bool skip = false; - - /* - * DOWN_REP_MSG_RDY is also handled by polling method - * mgr->cbs->poll_hpd_irq() - */ - spin_lock(&offload_wq->offload_lock); - skip = offload_wq->is_handling_mst_msg_rdy_event; - - if (!skip) - offload_wq->is_handling_mst_msg_rdy_event = true; - - spin_unlock(&offload_wq->offload_lock); - - if (!skip) - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - - goto out; - } - - if (link_loss) { - bool skip = false; - - spin_lock(&offload_wq->offload_lock); - skip = offload_wq->is_handling_link_loss; - - if (!skip) - offload_wq->is_handling_link_loss = true; - - spin_unlock(&offload_wq->offload_lock); - - if (!skip) - schedule_hpd_rx_offload_work(offload_wq, hpd_irq_data); - - goto out; - } - } - -out: - if (result && !is_mst_root_connector) { - /* Downstream Port status changed. */ - if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(dc_link); - - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - amdgpu_dm_update_connector_after_detect(aconnector); - - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - drm_kms_helper_connector_hotplug_event(connector); - } else { - bool ret = false; - - mutex_lock(&adev->dm.dc_lock); - dc_exit_ips_for_hw_access(dc); - ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX); - mutex_unlock(&adev->dm.dc_lock); - - if (ret) { - if (aconnector->fake_enable) - aconnector->fake_enable = false; - - amdgpu_dm_update_connector_after_detect(aconnector); - - drm_modeset_lock_all(dev); - dm_restore_drm_connector_state(dev, connector); - drm_modeset_unlock_all(dev); - - drm_kms_helper_connector_hotplug_event(connector); - } - } - } - if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) { - if (adev->dm.hdcp_workqueue) - hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index); - } - - if (dc_link->type != dc_connection_mst_branch) - drm_dp_cec_irq(&aconnector->dm_dp_aux.aux); - - mutex_unlock(&aconnector->hpd_lock); -} - -static int register_hpd_handlers(struct amdgpu_device *adev) -{ - struct drm_device *dev = adev_to_drm(adev); - struct drm_connector *connector; - struct amdgpu_dm_connector *aconnector; - const struct dc_link *dc_link; - struct dc_interrupt_params int_params = {0}; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - if (dc_is_dmub_outbox_supported(adev->dm.dc)) { - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD, - dmub_hpd_callback, true)) { - DRM_ERROR("amdgpu: fail to register dmub hpd callback"); - return -EINVAL; - } - - if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ, - dmub_hpd_callback, true)) { - DRM_ERROR("amdgpu: fail to register dmub hpd callback"); - return -EINVAL; - } - } - - list_for_each_entry(connector, - &dev->mode_config.connector_list, head) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - dc_link = aconnector->dc_link; - - if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = dc_link->irq_source_hpd; - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_HPD1 || - int_params.irq_source > DC_IRQ_SOURCE_HPD6) { - DRM_ERROR("Failed to register hpd irq!\n"); - return -EINVAL; - } - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - handle_hpd_irq, (void *) aconnector)) - return -ENOMEM; - } - - if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { - - /* Also register for DP short pulse (hpd_rx). */ - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = dc_link->irq_source_hpd_rx; - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_HPD1RX || - int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) { - DRM_ERROR("Failed to register hpd rx irq!\n"); - return -EINVAL; - } - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - handle_hpd_rx_irq, (void *) aconnector)) - return -ENOMEM; - } - } - return 0; -} - -#if defined(CONFIG_DRM_AMD_DC_SI) -/* Register IRQ sources and initialize IRQ callbacks */ -static int dce60_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; - unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VBLANK interrupt */ - for (i = 0; i < adev->mode_info.num_crtc; i++) { - r = amdgpu_irq_add_id(adev, client_id, i + 1, &adev->crtc_irq); - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i + 1, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; - i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, client_id, - VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} -#endif - -/* Register IRQ sources and initialize IRQ callbacks */ -static int dce110_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; - unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; - - if (adev->family >= AMDGPU_FAMILY_AI) - client_id = SOC15_IH_CLIENTID_DCE; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VBLANK interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0; i <= VISLANDS30_IV_SRCID_D6_VERTICAL_INTERRUPT0; i++) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->crtc_irq); - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use VUPDATE interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT; i <= VISLANDS30_IV_SRCID_D6_V_UPDATE_INT; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->vupdate_irq); - if (r) { - DRM_ERROR("Failed to add vupdate irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || - int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { - DRM_ERROR("Failed to register vupdate irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_vupdate_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; - i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { - r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, client_id, - VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} - -/* Register IRQ sources and initialize IRQ callbacks */ -static int dcn10_register_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r; - int i; -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - static const unsigned int vrtl_int_srcid[] = { - DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL, - DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL - }; -#endif - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - /* - * Actions of amdgpu_irq_add_id(): - * 1. Register a set() function with base driver. - * Base driver will call set() function to enable/disable an - * interrupt in DC hardware. - * 2. Register amdgpu_dm_irq_handler(). - * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts - * coming from DC hardware. - * amdgpu_dm_irq_handler() will re-direct the interrupt to DC - * for acknowledging and handling. - */ - - /* Use VSTARTUP interrupt */ - for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP; - i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq); - - if (r) { - DRM_ERROR("Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - DRM_ERROR("Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use otg vertical line interrupt */ -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, - vrtl_int_srcid[i], &adev->vline0_irq); - - if (r) { - DRM_ERROR("Failed to add vline0 irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 || - int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) { - DRM_ERROR("Failed to register vline0 irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vline0_params[int_params.irq_source - - DC_IRQ_SOURCE_DC1_VLINE0]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_dcn_vertical_interrupt0_high_irq, - c_irq_params)) - return -ENOMEM; - } -#endif - - /* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to - * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx - * to trigger at end of each vblank, regardless of state of the lock, - * matching DCE behaviour. - */ - for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT; - i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq); - - if (r) { - DRM_ERROR("Failed to add vupdate irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || - int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { - DRM_ERROR("Failed to register vupdate irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_vupdate_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* Use GRPH_PFLIP interrupt */ - for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT; - i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq); - if (r) { - DRM_ERROR("Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - DRM_ERROR("Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - - /* HPD */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, - &adev->hpd_irq); - if (r) { - DRM_ERROR("Failed to add hpd irq id!\n"); - return r; - } - - r = register_hpd_handlers(adev); - - return r; -} -/* Register Outbox IRQ sources and initialize IRQ callbacks */ -static int register_outbox_irq_handlers(struct amdgpu_device *adev) -{ - struct dc *dc = adev->dm.dc; - struct common_irq_params *c_irq_params; - struct dc_interrupt_params int_params = {0}; - int r, i; - - int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; - int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; - - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT, - &adev->dmub_outbox_irq); - if (r) { - DRM_ERROR("Failed to add outbox irq id!\n"); - return r; - } - - if (dc->ctx->dmub_srv) { - i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT; - int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - c_irq_params = &adev->dm.dmub_outbox_params[0]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_dmub_outbox1_low_irq, c_irq_params)) - return -ENOMEM; - } - - return 0; -} - -/* - * Acquires the lock for the atomic state object and returns - * the new atomic state. - * - * This should only be called during atomic check. - */ -int dm_atomic_get_state(struct drm_atomic_state *state, - struct dm_atomic_state **dm_state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_private_state *priv_state; - - if (*dm_state) - return 0; - - priv_state = drm_atomic_get_private_obj_state(state, &dm->atomic_obj); - if (IS_ERR(priv_state)) - return PTR_ERR(priv_state); - - *dm_state = to_dm_atomic_state(priv_state); - - return 0; -} - -static struct dm_atomic_state * -dm_atomic_get_new_state(struct drm_atomic_state *state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_private_obj *obj; - struct drm_private_state *new_obj_state; - int i; - - for_each_new_private_obj_in_state(state, obj, new_obj_state, i) { - if (obj->funcs == dm->atomic_obj.funcs) - return to_dm_atomic_state(new_obj_state); - } - - return NULL; -} - -static struct drm_private_state * -dm_atomic_duplicate_state(struct drm_private_obj *obj) -{ - struct dm_atomic_state *old_state, *new_state; - - new_state = kzalloc(sizeof(*new_state), GFP_KERNEL); - if (!new_state) - return NULL; - - __drm_atomic_helper_private_obj_duplicate_state(obj, &new_state->base); - - old_state = to_dm_atomic_state(obj->state); - - if (old_state && old_state->context) - new_state->context = dc_state_create_copy(old_state->context); - - if (!new_state->context) { - kfree(new_state); - return NULL; - } - - return &new_state->base; -} - -static void dm_atomic_destroy_state(struct drm_private_obj *obj, - struct drm_private_state *state) -{ - struct dm_atomic_state *dm_state = to_dm_atomic_state(state); - - if (dm_state && dm_state->context) - dc_state_release(dm_state->context); - - kfree(dm_state); -} - -static struct drm_private_state_funcs dm_atomic_state_funcs = { - .atomic_duplicate_state = dm_atomic_duplicate_state, - .atomic_destroy_state = dm_atomic_destroy_state, -}; - -static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) -{ - struct dm_atomic_state *state; - int r; - - adev->mode_info.mode_config_initialized = true; - - adev_to_drm(adev)->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs; - adev_to_drm(adev)->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs; - - adev_to_drm(adev)->mode_config.max_width = 16384; - adev_to_drm(adev)->mode_config.max_height = 16384; - - adev_to_drm(adev)->mode_config.preferred_depth = 24; - if (adev->asic_type == CHIP_HAWAII) - /* disable prefer shadow for now due to hibernation issues */ - adev_to_drm(adev)->mode_config.prefer_shadow = 0; - else - adev_to_drm(adev)->mode_config.prefer_shadow = 1; - /* indicates support for immediate flip */ - adev_to_drm(adev)->mode_config.async_page_flip = true; - - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - - state->context = dc_state_create_current_copy(adev->dm.dc); - if (!state->context) { - kfree(state); - return -ENOMEM; - } - - drm_atomic_private_obj_init(adev_to_drm(adev), - &adev->dm.atomic_obj, - &state->base, - &dm_atomic_state_funcs); - - r = amdgpu_display_modeset_create_props(adev); - if (r) { - dc_state_release(state->context); - kfree(state); - return r; - } - -#ifdef AMD_PRIVATE_COLOR - if (amdgpu_dm_create_color_properties(adev)) { - dc_state_release(state->context); - kfree(state); - return -ENOMEM; - } -#endif - - r = amdgpu_dm_audio_init(adev); - if (r) { - dc_state_release(state->context); - kfree(state); - return r; - } - - return 0; -} - -#define AMDGPU_DM_DEFAULT_MIN_BACKLIGHT 12 -#define AMDGPU_DM_DEFAULT_MAX_BACKLIGHT 255 -#define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50 - -static void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm, - int bl_idx) -{ -#if defined(CONFIG_ACPI) - struct amdgpu_dm_backlight_caps caps; - - memset(&caps, 0, sizeof(caps)); - - if (dm->backlight_caps[bl_idx].caps_valid) - return; - - amdgpu_acpi_get_backlight_caps(&caps); - if (caps.caps_valid) { - dm->backlight_caps[bl_idx].caps_valid = true; - if (caps.aux_support) - return; - dm->backlight_caps[bl_idx].min_input_signal = caps.min_input_signal; - dm->backlight_caps[bl_idx].max_input_signal = caps.max_input_signal; - } else { - dm->backlight_caps[bl_idx].min_input_signal = - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT; - dm->backlight_caps[bl_idx].max_input_signal = - AMDGPU_DM_DEFAULT_MAX_BACKLIGHT; - } -#else - if (dm->backlight_caps[bl_idx].aux_support) - return; - - dm->backlight_caps[bl_idx].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT; - dm->backlight_caps[bl_idx].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT; -#endif -} - -static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, - unsigned int *min, unsigned int *max) -{ - if (!caps) - return 0; - - if (caps->aux_support) { - // Firmware limits are in nits, DC API wants millinits. - *max = 1000 * caps->aux_max_input_signal; - *min = 1000 * caps->aux_min_input_signal; - } else { - // Firmware limits are 8-bit, PWM control is 16-bit. - *max = 0x101 * caps->max_input_signal; - *min = 0x101 * caps->min_input_signal; - } - return 1; -} - -static u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps, - uint32_t brightness) -{ - unsigned int min, max; - - if (!get_brightness_range(caps, &min, &max)) - return brightness; - - // Rescale 0..255 to min..max - return min + DIV_ROUND_CLOSEST((max - min) * brightness, - AMDGPU_MAX_BL_LEVEL); -} - -static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps, - uint32_t brightness) -{ - unsigned int min, max; - - if (!get_brightness_range(caps, &min, &max)) - return brightness; - - if (brightness < min) - return 0; - // Rescale min..max to 0..255 - return DIV_ROUND_CLOSEST(AMDGPU_MAX_BL_LEVEL * (brightness - min), - max - min); -} - -static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, - int bl_idx, - u32 user_brightness) -{ - struct amdgpu_dm_backlight_caps caps; - struct dc_link *link; - u32 brightness; - bool rc; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - caps = dm->backlight_caps[bl_idx]; - - dm->brightness[bl_idx] = user_brightness; - /* update scratch register */ - if (bl_idx == 0) - amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]); - brightness = convert_brightness_from_user(&caps, dm->brightness[bl_idx]); - link = (struct dc_link *)dm->backlight_link[bl_idx]; - - /* Change brightness based on AUX property */ - if (caps.aux_support) { - rc = dc_link_set_backlight_level_nits(link, true, brightness, - AUX_BL_DEFAULT_TRANSITION_TIME_MS); - if (!rc) - DRM_DEBUG("DM: Failed to update backlight via AUX on eDP[%d]\n", bl_idx); - } else { - rc = dc_link_set_backlight_level(link, brightness, 0); - if (!rc) - DRM_DEBUG("DM: Failed to update backlight on eDP[%d]\n", bl_idx); - } - - if (rc) - dm->actual_brightness[bl_idx] = user_brightness; -} - -static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) -{ - struct amdgpu_display_manager *dm = bl_get_data(bd); - int i; - - for (i = 0; i < dm->num_of_edps; i++) { - if (bd == dm->backlight_dev[i]) - break; - } - if (i >= AMDGPU_DM_MAX_NUM_EDP) - i = 0; - amdgpu_dm_backlight_set_level(dm, i, bd->props.brightness); - - return 0; -} - -static u32 amdgpu_dm_backlight_get_level(struct amdgpu_display_manager *dm, - int bl_idx) -{ - int ret; - struct amdgpu_dm_backlight_caps caps; - struct dc_link *link = (struct dc_link *)dm->backlight_link[bl_idx]; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - caps = dm->backlight_caps[bl_idx]; - - if (caps.aux_support) { - u32 avg, peak; - bool rc; - - rc = dc_link_get_backlight_level_nits(link, &avg, &peak); - if (!rc) - return dm->brightness[bl_idx]; - return convert_brightness_to_user(&caps, avg); - } - - ret = dc_link_get_backlight_level(link); - - if (ret == DC_ERROR_UNEXPECTED) - return dm->brightness[bl_idx]; - - return convert_brightness_to_user(&caps, ret); -} - -static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) -{ - struct amdgpu_display_manager *dm = bl_get_data(bd); - int i; - - for (i = 0; i < dm->num_of_edps; i++) { - if (bd == dm->backlight_dev[i]) - break; - } - if (i >= AMDGPU_DM_MAX_NUM_EDP) - i = 0; - return amdgpu_dm_backlight_get_level(dm, i); -} - -static const struct backlight_ops amdgpu_dm_backlight_ops = { - .options = BL_CORE_SUSPENDRESUME, - .get_brightness = amdgpu_dm_backlight_get_brightness, - .update_status = amdgpu_dm_backlight_update_status, -}; - -static void -amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) -{ - struct drm_device *drm = aconnector->base.dev; - struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm; - struct backlight_properties props = { 0 }; - struct amdgpu_dm_backlight_caps caps = { 0 }; - char bl_name[16]; - - if (aconnector->bl_idx == -1) - return; - - if (!acpi_video_backlight_use_native()) { - drm_info(drm, "Skipping amdgpu DM backlight registration\n"); - /* Try registering an ACPI video backlight device instead. */ - acpi_video_register_backlight(); - return; - } - - amdgpu_acpi_get_backlight_caps(&caps); - if (caps.caps_valid) { - if (power_supply_is_system_supplied() > 0) - props.brightness = caps.ac_level; - else - props.brightness = caps.dc_level; - } else - props.brightness = AMDGPU_MAX_BL_LEVEL; - - props.max_brightness = AMDGPU_MAX_BL_LEVEL; - props.type = BACKLIGHT_RAW; - - snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d", - drm->primary->index + aconnector->bl_idx); - - dm->backlight_dev[aconnector->bl_idx] = - backlight_device_register(bl_name, aconnector->base.kdev, dm, - &amdgpu_dm_backlight_ops, &props); - - if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) { - DRM_ERROR("DM: Backlight registration failed!\n"); - dm->backlight_dev[aconnector->bl_idx] = NULL; - } else - DRM_DEBUG_DRIVER("DM: Registered Backlight device: %s\n", bl_name); -} - -static int initialize_plane(struct amdgpu_display_manager *dm, - struct amdgpu_mode_info *mode_info, int plane_id, - enum drm_plane_type plane_type, - const struct dc_plane_cap *plane_cap) -{ - struct drm_plane *plane; - unsigned long possible_crtcs; - int ret = 0; - - plane = kzalloc(sizeof(struct drm_plane), GFP_KERNEL); - if (!plane) { - DRM_ERROR("KMS: Failed to allocate plane\n"); - return -ENOMEM; - } - plane->type = plane_type; - - /* - * HACK: IGT tests expect that the primary plane for a CRTC - * can only have one possible CRTC. Only expose support for - * any CRTC if they're not going to be used as a primary plane - * for a CRTC - like overlay or underlay planes. - */ - possible_crtcs = 1 << plane_id; - if (plane_id >= dm->dc->caps.max_streams) - possible_crtcs = 0xff; - - ret = amdgpu_dm_plane_init(dm, plane, possible_crtcs, plane_cap); - - if (ret) { - DRM_ERROR("KMS: Failed to initialize plane\n"); - kfree(plane); - return ret; - } - - if (mode_info) - mode_info->planes[plane_id] = plane; - - return ret; -} - - -static void setup_backlight_device(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = aconnector->dc_link; - int bl_idx = dm->num_of_edps; - - if (!(link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) || - link->type == dc_connection_none) - return; - - if (dm->num_of_edps >= AMDGPU_DM_MAX_NUM_EDP) { - drm_warn(adev_to_drm(dm->adev), "Too much eDP connections, skipping backlight setup for additional eDPs\n"); - return; - } - - aconnector->bl_idx = bl_idx; - - amdgpu_dm_update_backlight_caps(dm, bl_idx); - dm->brightness[bl_idx] = AMDGPU_MAX_BL_LEVEL; - dm->backlight_link[bl_idx] = link; - dm->num_of_edps++; - - update_connector_ext_caps(aconnector); -} - -static void amdgpu_set_panel_orientation(struct drm_connector *connector); - -/* - * In this architecture, the association - * connector -> encoder -> crtc - * id not really requried. The crtc and connector will hold the - * display_index as an abstraction to use with DAL component - * - * Returns 0 on success - */ -static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) -{ - struct amdgpu_display_manager *dm = &adev->dm; - s32 i; - struct amdgpu_dm_connector *aconnector = NULL; - struct amdgpu_encoder *aencoder = NULL; - struct amdgpu_mode_info *mode_info = &adev->mode_info; - u32 link_cnt; - s32 primary_planes; - enum dc_connection_type new_connection_type = dc_connection_none; - const struct dc_plane_cap *plane; - bool psr_feature_enabled = false; - bool replay_feature_enabled = false; - int max_overlay = dm->dc->caps.max_slave_planes; - - dm->display_indexes_num = dm->dc->caps.max_streams; - /* Update the actual used number of crtc */ - adev->mode_info.num_crtc = adev->dm.display_indexes_num; - - amdgpu_dm_set_irq_funcs(adev); - - link_cnt = dm->dc->caps.max_links; - if (amdgpu_dm_mode_config_init(dm->adev)) { - DRM_ERROR("DM: Failed to initialize mode config\n"); - return -EINVAL; - } - - /* There is one primary plane per CRTC */ - primary_planes = dm->dc->caps.max_streams; - if (primary_planes > AMDGPU_MAX_PLANES) { - DRM_ERROR("DM: Plane nums out of 6 planes\n"); - return -EINVAL; - } - - /* - * Initialize primary planes, implicit planes for legacy IOCTLS. - * Order is reversed to match iteration order in atomic check. - */ - for (i = (primary_planes - 1); i >= 0; i--) { - plane = &dm->dc->caps.planes[i]; - - if (initialize_plane(dm, mode_info, i, - DRM_PLANE_TYPE_PRIMARY, plane)) { - DRM_ERROR("KMS: Failed to initialize primary plane\n"); - goto fail; - } - } - - /* - * Initialize overlay planes, index starting after primary planes. - * These planes have a higher DRM index than the primary planes since - * they should be considered as having a higher z-order. - * Order is reversed to match iteration order in atomic check. - * - * Only support DCN for now, and only expose one so we don't encourage - * userspace to use up all the pipes. - */ - for (i = 0; i < dm->dc->caps.max_planes; ++i) { - struct dc_plane_cap *plane = &dm->dc->caps.planes[i]; - - /* Do not create overlay if MPO disabled */ - if (amdgpu_dc_debug_mask & DC_DISABLE_MPO) - break; - - if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL) - continue; - - if (!plane->pixel_format_support.argb8888) - continue; - - if (max_overlay-- == 0) - break; - - if (initialize_plane(dm, NULL, primary_planes + i, - DRM_PLANE_TYPE_OVERLAY, plane)) { - DRM_ERROR("KMS: Failed to initialize overlay plane\n"); - goto fail; - } - } - - for (i = 0; i < dm->dc->caps.max_streams; i++) - if (amdgpu_dm_crtc_init(dm, mode_info->planes[i], i)) { - DRM_ERROR("KMS: Failed to initialize crtc\n"); - goto fail; - } - - /* Use Outbox interrupt */ - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - if (register_outbox_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - DRM_DEBUG_KMS("Unsupported DCN IP version for outbox: 0x%X\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - } - - /* Determine whether to enable PSR support by default. */ - if (!(amdgpu_dc_debug_mask & DC_DISABLE_PSR)) { - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - psr_feature_enabled = true; - break; - default: - psr_feature_enabled = amdgpu_dc_feature_mask & DC_PSR_MASK; - break; - } - } - - /* Determine whether to enable Replay support by default. */ - if (!(amdgpu_dc_debug_mask & DC_DISABLE_REPLAY)) { - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - replay_feature_enabled = true; - break; - - default: - replay_feature_enabled = amdgpu_dc_feature_mask & DC_REPLAY_MASK; - break; - } - } - - if (link_cnt > MAX_LINKS) { - DRM_ERROR( - "KMS: Cannot support more than %d display indexes\n", - MAX_LINKS); - goto fail; - } - - /* loops over all connectors on the board */ - for (i = 0; i < link_cnt; i++) { - struct dc_link *link = NULL; - - link = dc_get_link_at_index(dm->dc, i); - - if (link->connector_signal == SIGNAL_TYPE_VIRTUAL) { - struct amdgpu_dm_wb_connector *wbcon = kzalloc(sizeof(*wbcon), GFP_KERNEL); - - if (!wbcon) { - DRM_ERROR("KMS: Failed to allocate writeback connector\n"); - continue; - } - - if (amdgpu_dm_wb_connector_init(dm, wbcon, i)) { - DRM_ERROR("KMS: Failed to initialize writeback connector\n"); - kfree(wbcon); - continue; - } - - link->psr_settings.psr_feature_enabled = false; - link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; - - continue; - } - - aconnector = kzalloc(sizeof(*aconnector), GFP_KERNEL); - if (!aconnector) - goto fail; - - aencoder = kzalloc(sizeof(*aencoder), GFP_KERNEL); - if (!aencoder) - goto fail; - - if (amdgpu_dm_encoder_init(dm->ddev, aencoder, i)) { - DRM_ERROR("KMS: Failed to initialize encoder\n"); - goto fail; - } - - if (amdgpu_dm_connector_init(dm, aconnector, i, aencoder)) { - DRM_ERROR("KMS: Failed to initialize connector\n"); - goto fail; - } - - if (dm->hpd_rx_offload_wq) - dm->hpd_rx_offload_wq[aconnector->base.index].aconnector = - aconnector; - - if (!dc_link_detect_connection_type(link, &new_connection_type)) - DRM_ERROR("KMS: Failed to detect connector\n"); - - if (aconnector->base.force && new_connection_type == dc_connection_none) { - emulated_link_detect(link); - amdgpu_dm_update_connector_after_detect(aconnector); - } else { - bool ret = false; - - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - ret = dc_link_detect(link, DETECT_REASON_BOOT); - mutex_unlock(&dm->dc_lock); - - if (ret) { - amdgpu_dm_update_connector_after_detect(aconnector); - setup_backlight_device(dm, aconnector); - - /* Disable PSR if Replay can be enabled */ - if (replay_feature_enabled) - if (amdgpu_dm_set_replay_caps(link, aconnector)) - psr_feature_enabled = false; - - if (psr_feature_enabled) - amdgpu_dm_set_psr_caps(link); - } - } - amdgpu_set_panel_orientation(&aconnector->base); - } - - /* Software is initialized. Now we can register interrupt handlers. */ - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - case CHIP_OLAND: - if (dce60_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - case CHIP_TONGA: - case CHIP_FIJI: - case CHIP_CARRIZO: - case CHIP_STONEY: - case CHIP_POLARIS11: - case CHIP_POLARIS10: - case CHIP_POLARIS12: - case CHIP_VEGAM: - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - if (dce110_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(1, 0, 0): - case IP_VERSION(1, 0, 1): - case IP_VERSION(2, 0, 2): - case IP_VERSION(2, 0, 3): - case IP_VERSION(2, 0, 0): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 0, 0): - case IP_VERSION(3, 0, 2): - case IP_VERSION(3, 0, 3): - case IP_VERSION(3, 0, 1): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - if (dcn10_register_irq_handlers(dm->adev)) { - DRM_ERROR("DM: Failed to initialize IRQ\n"); - goto fail; - } - break; - default: - DRM_ERROR("Unsupported DCE IP versions: 0x%X\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - goto fail; - } - break; - } - - return 0; -fail: - kfree(aencoder); - kfree(aconnector); - - return -EINVAL; -} - -static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm) -{ - drm_atomic_private_obj_fini(&dm->atomic_obj); -} - -/****************************************************************************** - * amdgpu_display_funcs functions - *****************************************************************************/ - -/* - * dm_bandwidth_update - program display watermarks - * - * @adev: amdgpu_device pointer - * - * Calculate and program the display watermarks and line buffer allocation. - */ -static void dm_bandwidth_update(struct amdgpu_device *adev) -{ - /* TODO: implement later */ -} - -static const struct amdgpu_display_funcs dm_display_funcs = { - .bandwidth_update = dm_bandwidth_update, /* called unconditionally */ - .vblank_get_counter = dm_vblank_get_counter,/* called unconditionally */ - .backlight_set_level = NULL, /* never called for DC */ - .backlight_get_level = NULL, /* never called for DC */ - .hpd_sense = NULL,/* called unconditionally */ - .hpd_set_polarity = NULL, /* called unconditionally */ - .hpd_get_gpio_reg = NULL, /* VBIOS parsing. DAL does it. */ - .page_flip_get_scanoutpos = - dm_crtc_get_scanoutpos,/* called unconditionally */ - .add_encoder = NULL, /* VBIOS parsing. DAL does it. */ - .add_connector = NULL, /* VBIOS parsing. DAL does it. */ -}; - -#if defined(CONFIG_DEBUG_KERNEL_DC) - -static ssize_t s3_debug_store(struct device *device, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - int ret; - int s3_state; - struct drm_device *drm_dev = dev_get_drvdata(device); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - - ret = kstrtoint(buf, 0, &s3_state); - - if (ret == 0) { - if (s3_state) { - dm_resume(adev); - drm_kms_helper_hotplug_event(adev_to_drm(adev)); - } else - dm_suspend(adev); - } - - return ret == 0 ? count : 0; -} - -DEVICE_ATTR_WO(s3_debug); - -#endif - -static int dm_init_microcode(struct amdgpu_device *adev) -{ - char *fw_name_dmub; - int r; - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 1, 0): - fw_name_dmub = FIRMWARE_RENOIR_DMUB; - if (ASICREV_IS_GREEN_SARDINE(adev->external_rev_id)) - fw_name_dmub = FIRMWARE_GREEN_SARDINE_DMUB; - break; - case IP_VERSION(3, 0, 0): - if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 0)) - fw_name_dmub = FIRMWARE_SIENNA_CICHLID_DMUB; - else - fw_name_dmub = FIRMWARE_NAVY_FLOUNDER_DMUB; - break; - case IP_VERSION(3, 0, 1): - fw_name_dmub = FIRMWARE_VANGOGH_DMUB; - break; - case IP_VERSION(3, 0, 2): - fw_name_dmub = FIRMWARE_DIMGREY_CAVEFISH_DMUB; - break; - case IP_VERSION(3, 0, 3): - fw_name_dmub = FIRMWARE_BEIGE_GOBY_DMUB; - break; - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - fw_name_dmub = FIRMWARE_YELLOW_CARP_DMUB; - break; - case IP_VERSION(3, 1, 4): - fw_name_dmub = FIRMWARE_DCN_314_DMUB; - break; - case IP_VERSION(3, 1, 5): - fw_name_dmub = FIRMWARE_DCN_315_DMUB; - break; - case IP_VERSION(3, 1, 6): - fw_name_dmub = FIRMWARE_DCN316_DMUB; - break; - case IP_VERSION(3, 2, 0): - fw_name_dmub = FIRMWARE_DCN_V3_2_0_DMCUB; - break; - case IP_VERSION(3, 2, 1): - fw_name_dmub = FIRMWARE_DCN_V3_2_1_DMCUB; - break; - case IP_VERSION(3, 5, 0): - fw_name_dmub = FIRMWARE_DCN_35_DMUB; - break; - case IP_VERSION(3, 5, 1): - fw_name_dmub = FIRMWARE_DCN_351_DMUB; - break; - case IP_VERSION(4, 0, 1): - fw_name_dmub = FIRMWARE_DCN_401_DMUB; - break; - default: - /* ASIC doesn't support DMUB. */ - return 0; - } - r = amdgpu_ucode_request(adev, &adev->dm.dmub_fw, "%s", fw_name_dmub); - return r; -} - -static int dm_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_mode_info *mode_info = &adev->mode_info; - struct atom_context *ctx = mode_info->atom_context; - int index = GetIndexIntoMasterTable(DATA, Object_Header); - u16 data_offset; - - /* if there is no object header, skip DM */ - if (!amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) { - adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK; - dev_info(adev->dev, "No object header, skipping DM\n"); - return -ENOENT; - } - - switch (adev->asic_type) { -#if defined(CONFIG_DRM_AMD_DC_SI) - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_OLAND: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 2; - adev->mode_info.num_dig = 2; - break; -#endif - case CHIP_BONAIRE: - case CHIP_HAWAII: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_KAVERI: - adev->mode_info.num_crtc = 4; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 7; - break; - case CHIP_KABINI: - case CHIP_MULLINS: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_FIJI: - case CHIP_TONGA: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 7; - break; - case CHIP_CARRIZO: - adev->mode_info.num_crtc = 3; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 9; - break; - case CHIP_STONEY: - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 9; - break; - case CHIP_POLARIS11: - case CHIP_POLARIS12: - adev->mode_info.num_crtc = 5; - adev->mode_info.num_hpd = 5; - adev->mode_info.num_dig = 5; - break; - case CHIP_POLARIS10: - case CHIP_VEGAM: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - default: - - switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { - case IP_VERSION(2, 0, 2): - case IP_VERSION(3, 0, 0): - adev->mode_info.num_crtc = 6; - adev->mode_info.num_hpd = 6; - adev->mode_info.num_dig = 6; - break; - case IP_VERSION(2, 0, 0): - case IP_VERSION(3, 0, 2): - adev->mode_info.num_crtc = 5; - adev->mode_info.num_hpd = 5; - adev->mode_info.num_dig = 5; - break; - case IP_VERSION(2, 0, 3): - case IP_VERSION(3, 0, 3): - adev->mode_info.num_crtc = 2; - adev->mode_info.num_hpd = 2; - adev->mode_info.num_dig = 2; - break; - case IP_VERSION(1, 0, 0): - case IP_VERSION(1, 0, 1): - case IP_VERSION(3, 0, 1): - case IP_VERSION(2, 1, 0): - case IP_VERSION(3, 1, 2): - case IP_VERSION(3, 1, 3): - case IP_VERSION(3, 1, 4): - case IP_VERSION(3, 1, 5): - case IP_VERSION(3, 1, 6): - case IP_VERSION(3, 2, 0): - case IP_VERSION(3, 2, 1): - case IP_VERSION(3, 5, 0): - case IP_VERSION(3, 5, 1): - case IP_VERSION(4, 0, 1): - adev->mode_info.num_crtc = 4; - adev->mode_info.num_hpd = 4; - adev->mode_info.num_dig = 4; - break; - default: - DRM_ERROR("Unsupported DCE IP versions: 0x%x\n", - amdgpu_ip_version(adev, DCE_HWIP, 0)); - return -EINVAL; - } - break; - } - - if (adev->mode_info.funcs == NULL) - adev->mode_info.funcs = &dm_display_funcs; - - /* - * Note: Do NOT change adev->audio_endpt_rreg and - * adev->audio_endpt_wreg because they are initialised in - * amdgpu_device_init() - */ -#if defined(CONFIG_DEBUG_KERNEL_DC) - device_create_file( - adev_to_drm(adev)->dev, - &dev_attr_s3_debug); -#endif - adev->dc_enabled = true; - - return dm_init_microcode(adev); -} - -static bool modereset_required(struct drm_crtc_state *crtc_state) -{ - return !crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state); -} - -static void amdgpu_dm_encoder_destroy(struct drm_encoder *encoder) -{ - drm_encoder_cleanup(encoder); - kfree(encoder); -} - -static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = { - .destroy = amdgpu_dm_encoder_destroy, -}; - -static int -fill_plane_color_attributes(const struct drm_plane_state *plane_state, - const enum surface_pixel_format format, - enum dc_color_space *color_space) -{ - bool full_range; - - *color_space = COLOR_SPACE_SRGB; - - /* DRM color properties only affect non-RGB formats. */ - if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) - return 0; - - full_range = (plane_state->color_range == DRM_COLOR_YCBCR_FULL_RANGE); - - switch (plane_state->color_encoding) { - case DRM_COLOR_YCBCR_BT601: - if (full_range) - *color_space = COLOR_SPACE_YCBCR601; - else - *color_space = COLOR_SPACE_YCBCR601_LIMITED; - break; - - case DRM_COLOR_YCBCR_BT709: - if (full_range) - *color_space = COLOR_SPACE_YCBCR709; - else - *color_space = COLOR_SPACE_YCBCR709_LIMITED; - break; - - case DRM_COLOR_YCBCR_BT2020: - if (full_range) - *color_space = COLOR_SPACE_2020_YCBCR; - else - return -EINVAL; - break; - - default: - return -EINVAL; - } - - return 0; -} - -static int -fill_dc_plane_info_and_addr(struct amdgpu_device *adev, - const struct drm_plane_state *plane_state, - const u64 tiling_flags, - struct dc_plane_info *plane_info, - struct dc_plane_address *address, - bool tmz_surface, - bool force_disable_dcc) -{ - const struct drm_framebuffer *fb = plane_state->fb; - const struct amdgpu_framebuffer *afb = - to_amdgpu_framebuffer(plane_state->fb); - int ret; - - memset(plane_info, 0, sizeof(*plane_info)); - - switch (fb->format->format) { - case DRM_FORMAT_C8: - plane_info->format = - SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS; - break; - case DRM_FORMAT_RGB565: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565; - break; - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888; - break; - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_ARGB2101010: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010; - break; - case DRM_FORMAT_XBGR2101010: - case DRM_FORMAT_ABGR2101010: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010; - break; - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888; - break; - case DRM_FORMAT_NV21: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr; - break; - case DRM_FORMAT_NV12: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb; - break; - case DRM_FORMAT_P010: - plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb; - break; - case DRM_FORMAT_XRGB16161616F: - case DRM_FORMAT_ARGB16161616F: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F; - break; - case DRM_FORMAT_XBGR16161616F: - case DRM_FORMAT_ABGR16161616F: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F; - break; - case DRM_FORMAT_XRGB16161616: - case DRM_FORMAT_ARGB16161616: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616; - break; - case DRM_FORMAT_XBGR16161616: - case DRM_FORMAT_ABGR16161616: - plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616; - break; - default: - DRM_ERROR( - "Unsupported screen format %p4cc\n", - &fb->format->format); - return -EINVAL; - } - - switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) { - case DRM_MODE_ROTATE_0: - plane_info->rotation = ROTATION_ANGLE_0; - break; - case DRM_MODE_ROTATE_90: - plane_info->rotation = ROTATION_ANGLE_90; - break; - case DRM_MODE_ROTATE_180: - plane_info->rotation = ROTATION_ANGLE_180; - break; - case DRM_MODE_ROTATE_270: - plane_info->rotation = ROTATION_ANGLE_270; - break; - default: - plane_info->rotation = ROTATION_ANGLE_0; - break; - } - - - plane_info->visible = true; - plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE; - - plane_info->layer_index = plane_state->normalized_zpos; - - ret = fill_plane_color_attributes(plane_state, plane_info->format, - &plane_info->color_space); - if (ret) - return ret; - - ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, plane_info->format, - plane_info->rotation, tiling_flags, - &plane_info->tiling_info, - &plane_info->plane_size, - &plane_info->dcc, address, - tmz_surface, force_disable_dcc); - if (ret) - return ret; - - amdgpu_dm_plane_fill_blending_from_plane_state( - plane_state, &plane_info->per_pixel_alpha, &plane_info->pre_multiplied_alpha, - &plane_info->global_alpha, &plane_info->global_alpha_value); - - return 0; -} - -static int fill_dc_plane_attributes(struct amdgpu_device *adev, - struct dc_plane_state *dc_plane_state, - struct drm_plane_state *plane_state, - struct drm_crtc_state *crtc_state) -{ - struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); - struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)plane_state->fb; - struct dc_scaling_info scaling_info; - struct dc_plane_info plane_info; - int ret; - bool force_disable_dcc = false; - - ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, plane_state, &scaling_info); - if (ret) - return ret; - - dc_plane_state->src_rect = scaling_info.src_rect; - dc_plane_state->dst_rect = scaling_info.dst_rect; - dc_plane_state->clip_rect = scaling_info.clip_rect; - dc_plane_state->scaling_quality = scaling_info.scaling_quality; - - force_disable_dcc = adev->asic_type == CHIP_RAVEN && adev->in_suspend; - ret = fill_dc_plane_info_and_addr(adev, plane_state, - afb->tiling_flags, - &plane_info, - &dc_plane_state->address, - afb->tmz_surface, - force_disable_dcc); - if (ret) - return ret; - - dc_plane_state->format = plane_info.format; - dc_plane_state->color_space = plane_info.color_space; - dc_plane_state->format = plane_info.format; - dc_plane_state->plane_size = plane_info.plane_size; - dc_plane_state->rotation = plane_info.rotation; - dc_plane_state->horizontal_mirror = plane_info.horizontal_mirror; - dc_plane_state->stereo_format = plane_info.stereo_format; - dc_plane_state->tiling_info = plane_info.tiling_info; - dc_plane_state->visible = plane_info.visible; - dc_plane_state->per_pixel_alpha = plane_info.per_pixel_alpha; - dc_plane_state->pre_multiplied_alpha = plane_info.pre_multiplied_alpha; - dc_plane_state->global_alpha = plane_info.global_alpha; - dc_plane_state->global_alpha_value = plane_info.global_alpha_value; - dc_plane_state->dcc = plane_info.dcc; - dc_plane_state->layer_index = plane_info.layer_index; - dc_plane_state->flip_int_enabled = true; - - /* - * Always set input transfer function, since plane state is refreshed - * every time. - */ - ret = amdgpu_dm_update_plane_color_mgmt(dm_crtc_state, - plane_state, - dc_plane_state); - if (ret) - return ret; - - return 0; -} - -static inline void fill_dc_dirty_rect(struct drm_plane *plane, - struct rect *dirty_rect, int32_t x, - s32 y, s32 width, s32 height, - int *i, bool ffu) -{ - WARN_ON(*i >= DC_MAX_DIRTY_RECTS); - - dirty_rect->x = x; - dirty_rect->y = y; - dirty_rect->width = width; - dirty_rect->height = height; - - if (ffu) - drm_dbg(plane->dev, - "[PLANE:%d] PSR FFU dirty rect size (%d, %d)\n", - plane->base.id, width, height); - else - drm_dbg(plane->dev, - "[PLANE:%d] PSR SU dirty rect at (%d, %d) size (%d, %d)", - plane->base.id, x, y, width, height); - - (*i)++; -} - -/** - * fill_dc_dirty_rects() - Fill DC dirty regions for PSR selective updates - * - * @plane: DRM plane containing dirty regions that need to be flushed to the eDP - * remote fb - * @old_plane_state: Old state of @plane - * @new_plane_state: New state of @plane - * @crtc_state: New state of CRTC connected to the @plane - * @flip_addrs: DC flip tracking struct, which also tracts dirty rects - * @is_psr_su: Flag indicating whether Panel Self Refresh Selective Update (PSR SU) is enabled. - * If PSR SU is enabled and damage clips are available, only the regions of the screen - * that have changed will be updated. If PSR SU is not enabled, - * or if damage clips are not available, the entire screen will be updated. - * @dirty_regions_changed: dirty regions changed - * - * For PSR SU, DC informs the DMUB uController of dirty rectangle regions - * (referred to as "damage clips" in DRM nomenclature) that require updating on - * the eDP remote buffer. The responsibility of specifying the dirty regions is - * amdgpu_dm's. - * - * A damage-aware DRM client should fill the FB_DAMAGE_CLIPS property on the - * plane with regions that require flushing to the eDP remote buffer. In - * addition, certain use cases - such as cursor and multi-plane overlay (MPO) - - * implicitly provide damage clips without any client support via the plane - * bounds. - */ -static void fill_dc_dirty_rects(struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state, - struct drm_crtc_state *crtc_state, - struct dc_flip_addrs *flip_addrs, - bool is_psr_su, - bool *dirty_regions_changed) -{ - struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); - struct rect *dirty_rects = flip_addrs->dirty_rects; - u32 num_clips; - struct drm_mode_rect *clips; - bool bb_changed; - bool fb_changed; - u32 i = 0; - *dirty_regions_changed = false; - - /* - * Cursor plane has it's own dirty rect update interface. See - * dcn10_dmub_update_cursor_data and dmub_cmd_update_cursor_info_data - */ - if (plane->type == DRM_PLANE_TYPE_CURSOR) - return; - - if (new_plane_state->rotation != DRM_MODE_ROTATE_0) - goto ffu; - - num_clips = drm_plane_get_damage_clips_count(new_plane_state); - clips = drm_plane_get_damage_clips(new_plane_state); - - if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 && - is_psr_su))) - goto ffu; - - if (!dm_crtc_state->mpo_requested) { - if (!num_clips || num_clips > DC_MAX_DIRTY_RECTS) - goto ffu; - - for (; flip_addrs->dirty_rect_count < num_clips; clips++) - fill_dc_dirty_rect(new_plane_state->plane, - &dirty_rects[flip_addrs->dirty_rect_count], - clips->x1, clips->y1, - clips->x2 - clips->x1, clips->y2 - clips->y1, - &flip_addrs->dirty_rect_count, - false); - return; - } - - /* - * MPO is requested. Add entire plane bounding box to dirty rects if - * flipped to or damaged. - * - * If plane is moved or resized, also add old bounding box to dirty - * rects. - */ - fb_changed = old_plane_state->fb->base.id != - new_plane_state->fb->base.id; - bb_changed = (old_plane_state->crtc_x != new_plane_state->crtc_x || - old_plane_state->crtc_y != new_plane_state->crtc_y || - old_plane_state->crtc_w != new_plane_state->crtc_w || - old_plane_state->crtc_h != new_plane_state->crtc_h); - - drm_dbg(plane->dev, - "[PLANE:%d] PSR bb_changed:%d fb_changed:%d num_clips:%d\n", - new_plane_state->plane->base.id, - bb_changed, fb_changed, num_clips); - - *dirty_regions_changed = bb_changed; - - if ((num_clips + (bb_changed ? 2 : 0)) > DC_MAX_DIRTY_RECTS) - goto ffu; - - if (bb_changed) { - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - new_plane_state->crtc_x, - new_plane_state->crtc_y, - new_plane_state->crtc_w, - new_plane_state->crtc_h, &i, false); - - /* Add old plane bounding-box if plane is moved or resized */ - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - old_plane_state->crtc_x, - old_plane_state->crtc_y, - old_plane_state->crtc_w, - old_plane_state->crtc_h, &i, false); - } - - if (num_clips) { - for (; i < num_clips; clips++) - fill_dc_dirty_rect(new_plane_state->plane, - &dirty_rects[i], clips->x1, - clips->y1, clips->x2 - clips->x1, - clips->y2 - clips->y1, &i, false); - } else if (fb_changed && !bb_changed) { - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i], - new_plane_state->crtc_x, - new_plane_state->crtc_y, - new_plane_state->crtc_w, - new_plane_state->crtc_h, &i, false); - } - - flip_addrs->dirty_rect_count = i; - return; - -ffu: - fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[0], 0, 0, - dm_crtc_state->base.mode.crtc_hdisplay, - dm_crtc_state->base.mode.crtc_vdisplay, - &flip_addrs->dirty_rect_count, true); -} - -static void update_stream_scaling_settings(const struct drm_display_mode *mode, - const struct dm_connector_state *dm_state, - struct dc_stream_state *stream) -{ - enum amdgpu_rmx_type rmx_type; - - struct rect src = { 0 }; /* viewport in composition space*/ - struct rect dst = { 0 }; /* stream addressable area */ - - /* no mode. nothing to be done */ - if (!mode) - return; - - /* Full screen scaling by default */ - src.width = mode->hdisplay; - src.height = mode->vdisplay; - dst.width = stream->timing.h_addressable; - dst.height = stream->timing.v_addressable; - - if (dm_state) { - rmx_type = dm_state->scaling; - if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) { - if (src.width * dst.height < - src.height * dst.width) { - /* height needs less upscaling/more downscaling */ - dst.width = src.width * - dst.height / src.height; - } else { - /* width needs less upscaling/more downscaling */ - dst.height = src.height * - dst.width / src.width; - } - } else if (rmx_type == RMX_CENTER) { - dst = src; - } - - dst.x = (stream->timing.h_addressable - dst.width) / 2; - dst.y = (stream->timing.v_addressable - dst.height) / 2; - - if (dm_state->underscan_enable) { - dst.x += dm_state->underscan_hborder / 2; - dst.y += dm_state->underscan_vborder / 2; - dst.width -= dm_state->underscan_hborder; - dst.height -= dm_state->underscan_vborder; - } - } - - stream->src = src; - stream->dst = dst; - - DRM_DEBUG_KMS("Destination Rectangle x:%d y:%d width:%d height:%d\n", - dst.x, dst.y, dst.width, dst.height); - -} - -static enum dc_color_depth -convert_color_depth_from_display_info(const struct drm_connector *connector, - bool is_y420, int requested_bpc) -{ - u8 bpc; - - if (is_y420) { - bpc = 8; - - /* Cap display bpc based on HDMI 2.0 HF-VSDB */ - if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48) - bpc = 16; - else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36) - bpc = 12; - else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30) - bpc = 10; - } else { - bpc = (uint8_t)connector->display_info.bpc; - /* Assume 8 bpc by default if no bpc is specified. */ - bpc = bpc ? bpc : 8; - } - - if (requested_bpc > 0) { - /* - * Cap display bpc based on the user requested value. - * - * The value for state->max_bpc may not correctly updated - * depending on when the connector gets added to the state - * or if this was called outside of atomic check, so it - * can't be used directly. - */ - bpc = min_t(u8, bpc, requested_bpc); - - /* Round down to the nearest even number. */ - bpc = bpc - (bpc & 1); - } - - switch (bpc) { - case 0: - /* - * Temporary Work around, DRM doesn't parse color depth for - * EDID revision before 1.4 - * TODO: Fix edid parsing - */ - return COLOR_DEPTH_888; - case 6: - return COLOR_DEPTH_666; - case 8: - return COLOR_DEPTH_888; - case 10: - return COLOR_DEPTH_101010; - case 12: - return COLOR_DEPTH_121212; - case 14: - return COLOR_DEPTH_141414; - case 16: - return COLOR_DEPTH_161616; - default: - return COLOR_DEPTH_UNDEFINED; - } -} - -static enum dc_aspect_ratio -get_aspect_ratio(const struct drm_display_mode *mode_in) -{ - /* 1-1 mapping, since both enums follow the HDMI spec. */ - return (enum dc_aspect_ratio) mode_in->picture_aspect_ratio; -} - -static enum dc_color_space -get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, - const struct drm_connector_state *connector_state) -{ - enum dc_color_space color_space = COLOR_SPACE_SRGB; - - switch (connector_state->colorspace) { - case DRM_MODE_COLORIMETRY_BT601_YCC: - if (dc_crtc_timing->flags.Y_ONLY) - color_space = COLOR_SPACE_YCBCR601_LIMITED; - else - color_space = COLOR_SPACE_YCBCR601; - break; - case DRM_MODE_COLORIMETRY_BT709_YCC: - if (dc_crtc_timing->flags.Y_ONLY) - color_space = COLOR_SPACE_YCBCR709_LIMITED; - else - color_space = COLOR_SPACE_YCBCR709; - break; - case DRM_MODE_COLORIMETRY_OPRGB: - color_space = COLOR_SPACE_ADOBERGB; - break; - case DRM_MODE_COLORIMETRY_BT2020_RGB: - case DRM_MODE_COLORIMETRY_BT2020_YCC: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) - color_space = COLOR_SPACE_2020_RGB_FULLRANGE; - else - color_space = COLOR_SPACE_2020_YCBCR; - break; - case DRM_MODE_COLORIMETRY_DEFAULT: // ITU601 - default: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { - color_space = COLOR_SPACE_SRGB; - /* - * 27030khz is the separation point between HDTV and SDTV - * according to HDMI spec, we use YCbCr709 and YCbCr601 - * respectively - */ - } else if (dc_crtc_timing->pix_clk_100hz > 270300) { - if (dc_crtc_timing->flags.Y_ONLY) - color_space = - COLOR_SPACE_YCBCR709_LIMITED; - else - color_space = COLOR_SPACE_YCBCR709; - } else { - if (dc_crtc_timing->flags.Y_ONLY) - color_space = - COLOR_SPACE_YCBCR601_LIMITED; - else - color_space = COLOR_SPACE_YCBCR601; - } - break; - } - - return color_space; -} - -static enum display_content_type -get_output_content_type(const struct drm_connector_state *connector_state) -{ - switch (connector_state->content_type) { - default: - case DRM_MODE_CONTENT_TYPE_NO_DATA: - return DISPLAY_CONTENT_TYPE_NO_DATA; - case DRM_MODE_CONTENT_TYPE_GRAPHICS: - return DISPLAY_CONTENT_TYPE_GRAPHICS; - case DRM_MODE_CONTENT_TYPE_PHOTO: - return DISPLAY_CONTENT_TYPE_PHOTO; - case DRM_MODE_CONTENT_TYPE_CINEMA: - return DISPLAY_CONTENT_TYPE_CINEMA; - case DRM_MODE_CONTENT_TYPE_GAME: - return DISPLAY_CONTENT_TYPE_GAME; - } -} - -static bool adjust_colour_depth_from_display_info( - struct dc_crtc_timing *timing_out, - const struct drm_display_info *info) -{ - enum dc_color_depth depth = timing_out->display_color_depth; - int normalized_clk; - - do { - normalized_clk = timing_out->pix_clk_100hz / 10; - /* YCbCr 4:2:0 requires additional adjustment of 1/2 */ - if (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420) - normalized_clk /= 2; - /* Adjusting pix clock following on HDMI spec based on colour depth */ - switch (depth) { - case COLOR_DEPTH_888: - break; - case COLOR_DEPTH_101010: - normalized_clk = (normalized_clk * 30) / 24; - break; - case COLOR_DEPTH_121212: - normalized_clk = (normalized_clk * 36) / 24; - break; - case COLOR_DEPTH_161616: - normalized_clk = (normalized_clk * 48) / 24; - break; - default: - /* The above depths are the only ones valid for HDMI. */ - return false; - } - if (normalized_clk <= info->max_tmds_clock) { - timing_out->display_color_depth = depth; - return true; - } - } while (--depth > COLOR_DEPTH_666); - return false; -} - -static void fill_stream_properties_from_drm_display_mode( - struct dc_stream_state *stream, - const struct drm_display_mode *mode_in, - const struct drm_connector *connector, - const struct drm_connector_state *connector_state, - const struct dc_stream_state *old_stream, - int requested_bpc) -{ - struct dc_crtc_timing *timing_out = &stream->timing; - const struct drm_display_info *info = &connector->display_info; - struct amdgpu_dm_connector *aconnector = NULL; - struct hdmi_vendor_infoframe hv_frame; - struct hdmi_avi_infoframe avi_frame; - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - aconnector = to_amdgpu_dm_connector(connector); - - memset(&hv_frame, 0, sizeof(hv_frame)); - memset(&avi_frame, 0, sizeof(avi_frame)); - - timing_out->h_border_left = 0; - timing_out->h_border_right = 0; - timing_out->v_border_top = 0; - timing_out->v_border_bottom = 0; - /* TODO: un-hardcode */ - if (drm_mode_is_420_only(info, mode_in) - && stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - else if (drm_mode_is_420_also(info, mode_in) - && aconnector - && aconnector->force_yuv420_output) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - else if ((connector->display_info.color_formats & DRM_COLOR_FORMAT_YCBCR444) - && stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR444; - else - timing_out->pixel_encoding = PIXEL_ENCODING_RGB; - - timing_out->timing_3d_format = TIMING_3D_FORMAT_NONE; - timing_out->display_color_depth = convert_color_depth_from_display_info( - connector, - (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420), - requested_bpc); - timing_out->scan_type = SCANNING_TYPE_NODATA; - timing_out->hdmi_vic = 0; - - if (old_stream) { - timing_out->vic = old_stream->timing.vic; - timing_out->flags.HSYNC_POSITIVE_POLARITY = old_stream->timing.flags.HSYNC_POSITIVE_POLARITY; - timing_out->flags.VSYNC_POSITIVE_POLARITY = old_stream->timing.flags.VSYNC_POSITIVE_POLARITY; - } else { - timing_out->vic = drm_match_cea_mode(mode_in); - if (mode_in->flags & DRM_MODE_FLAG_PHSYNC) - timing_out->flags.HSYNC_POSITIVE_POLARITY = 1; - if (mode_in->flags & DRM_MODE_FLAG_PVSYNC) - timing_out->flags.VSYNC_POSITIVE_POLARITY = 1; - } - - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) { - drm_hdmi_avi_infoframe_from_display_mode(&avi_frame, (struct drm_connector *)connector, mode_in); - timing_out->vic = avi_frame.video_code; - drm_hdmi_vendor_infoframe_from_display_mode(&hv_frame, (struct drm_connector *)connector, mode_in); - timing_out->hdmi_vic = hv_frame.vic; - } - - if (aconnector && is_freesync_video_mode(mode_in, aconnector)) { - timing_out->h_addressable = mode_in->hdisplay; - timing_out->h_total = mode_in->htotal; - timing_out->h_sync_width = mode_in->hsync_end - mode_in->hsync_start; - timing_out->h_front_porch = mode_in->hsync_start - mode_in->hdisplay; - timing_out->v_total = mode_in->vtotal; - timing_out->v_addressable = mode_in->vdisplay; - timing_out->v_front_porch = mode_in->vsync_start - mode_in->vdisplay; - timing_out->v_sync_width = mode_in->vsync_end - mode_in->vsync_start; - timing_out->pix_clk_100hz = mode_in->clock * 10; - } else { - timing_out->h_addressable = mode_in->crtc_hdisplay; - timing_out->h_total = mode_in->crtc_htotal; - timing_out->h_sync_width = mode_in->crtc_hsync_end - mode_in->crtc_hsync_start; - timing_out->h_front_porch = mode_in->crtc_hsync_start - mode_in->crtc_hdisplay; - timing_out->v_total = mode_in->crtc_vtotal; - timing_out->v_addressable = mode_in->crtc_vdisplay; - timing_out->v_front_porch = mode_in->crtc_vsync_start - mode_in->crtc_vdisplay; - timing_out->v_sync_width = mode_in->crtc_vsync_end - mode_in->crtc_vsync_start; - timing_out->pix_clk_100hz = mode_in->crtc_clock * 10; - } - - timing_out->aspect_ratio = get_aspect_ratio(mode_in); - - stream->out_transfer_func.type = TF_TYPE_PREDEFINED; - stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB; - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) { - if (!adjust_colour_depth_from_display_info(timing_out, info) && - drm_mode_is_420_also(info, mode_in) && - timing_out->pixel_encoding != PIXEL_ENCODING_YCBCR420) { - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - adjust_colour_depth_from_display_info(timing_out, info); - } - } - - stream->output_color_space = get_output_color_space(timing_out, connector_state); - stream->content_type = get_output_content_type(connector_state); -} - -static void fill_audio_info(struct audio_info *audio_info, - const struct drm_connector *drm_connector, - const struct dc_sink *dc_sink) -{ - int i = 0; - int cea_revision = 0; - const struct dc_edid_caps *edid_caps = &dc_sink->edid_caps; - - audio_info->manufacture_id = edid_caps->manufacturer_id; - audio_info->product_id = edid_caps->product_id; - - cea_revision = drm_connector->display_info.cea_rev; - - strscpy(audio_info->display_name, - edid_caps->display_name, - AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS); - - if (cea_revision >= 3) { - audio_info->mode_count = edid_caps->audio_mode_count; - - for (i = 0; i < audio_info->mode_count; ++i) { - audio_info->modes[i].format_code = - (enum audio_format_code) - (edid_caps->audio_modes[i].format_code); - audio_info->modes[i].channel_count = - edid_caps->audio_modes[i].channel_count; - audio_info->modes[i].sample_rates.all = - edid_caps->audio_modes[i].sample_rate; - audio_info->modes[i].sample_size = - edid_caps->audio_modes[i].sample_size; - } - } - - audio_info->flags.all = edid_caps->speaker_flags; - - /* TODO: We only check for the progressive mode, check for interlace mode too */ - if (drm_connector->latency_present[0]) { - audio_info->video_latency = drm_connector->video_latency[0]; - audio_info->audio_latency = drm_connector->audio_latency[0]; - } - - /* TODO: For DP, video and audio latency should be calculated from DPCD caps */ - -} - -static void -copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode, - struct drm_display_mode *dst_mode) -{ - dst_mode->crtc_hdisplay = src_mode->crtc_hdisplay; - dst_mode->crtc_vdisplay = src_mode->crtc_vdisplay; - dst_mode->crtc_clock = src_mode->crtc_clock; - dst_mode->crtc_hblank_start = src_mode->crtc_hblank_start; - dst_mode->crtc_hblank_end = src_mode->crtc_hblank_end; - dst_mode->crtc_hsync_start = src_mode->crtc_hsync_start; - dst_mode->crtc_hsync_end = src_mode->crtc_hsync_end; - dst_mode->crtc_htotal = src_mode->crtc_htotal; - dst_mode->crtc_hskew = src_mode->crtc_hskew; - dst_mode->crtc_vblank_start = src_mode->crtc_vblank_start; - dst_mode->crtc_vblank_end = src_mode->crtc_vblank_end; - dst_mode->crtc_vsync_start = src_mode->crtc_vsync_start; - dst_mode->crtc_vsync_end = src_mode->crtc_vsync_end; - dst_mode->crtc_vtotal = src_mode->crtc_vtotal; -} - -static void -decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode, - const struct drm_display_mode *native_mode, - bool scale_enabled) -{ - if (scale_enabled) { - copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode); - } else if (native_mode->clock == drm_mode->clock && - native_mode->htotal == drm_mode->htotal && - native_mode->vtotal == drm_mode->vtotal) { - copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode); - } else { - /* no scaling nor amdgpu inserted, no need to patch */ - } -} - -static struct dc_sink * -create_fake_sink(struct dc_link *link) -{ - struct dc_sink_init_data sink_init_data = { 0 }; - struct dc_sink *sink = NULL; - - sink_init_data.link = link; - sink_init_data.sink_signal = link->connector_signal; - - sink = dc_sink_create(&sink_init_data); - if (!sink) { - DRM_ERROR("Failed to create sink!\n"); - return NULL; - } - sink->sink_signal = SIGNAL_TYPE_VIRTUAL; - - return sink; -} - -static void set_multisync_trigger_params( - struct dc_stream_state *stream) -{ - struct dc_stream_state *master = NULL; - - if (stream->triggered_crtc_reset.enabled) { - master = stream->triggered_crtc_reset.event_source; - stream->triggered_crtc_reset.event = - master->timing.flags.VSYNC_POSITIVE_POLARITY ? - CRTC_EVENT_VSYNC_RISING : CRTC_EVENT_VSYNC_FALLING; - stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_PIXEL; - } -} - -static void set_master_stream(struct dc_stream_state *stream_set[], - int stream_count) -{ - int j, highest_rfr = 0, master_stream = 0; - - for (j = 0; j < stream_count; j++) { - if (stream_set[j] && stream_set[j]->triggered_crtc_reset.enabled) { - int refresh_rate = 0; - - refresh_rate = (stream_set[j]->timing.pix_clk_100hz*100)/ - (stream_set[j]->timing.h_total*stream_set[j]->timing.v_total); - if (refresh_rate > highest_rfr) { - highest_rfr = refresh_rate; - master_stream = j; - } - } - } - for (j = 0; j < stream_count; j++) { - if (stream_set[j]) - stream_set[j]->triggered_crtc_reset.event_source = stream_set[master_stream]; - } -} - -static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context) -{ - int i = 0; - struct dc_stream_state *stream; - - if (context->stream_count < 2) - return; - for (i = 0; i < context->stream_count ; i++) { - if (!context->streams[i]) - continue; - /* - * TODO: add a function to read AMD VSDB bits and set - * crtc_sync_master.multi_sync_enabled flag - * For now it's set to false - */ - } - - set_master_stream(context->streams, context->stream_count); - - for (i = 0; i < context->stream_count ; i++) { - stream = context->streams[i]; - - if (!stream) - continue; - - set_multisync_trigger_params(stream); - } -} - -/** - * DOC: FreeSync Video - * - * When a userspace application wants to play a video, the content follows a - * standard format definition that usually specifies the FPS for that format. - * The below list illustrates some video format and the expected FPS, - * respectively: - * - * - TV/NTSC (23.976 FPS) - * - Cinema (24 FPS) - * - TV/PAL (25 FPS) - * - TV/NTSC (29.97 FPS) - * - TV/NTSC (30 FPS) - * - Cinema HFR (48 FPS) - * - TV/PAL (50 FPS) - * - Commonly used (60 FPS) - * - Multiples of 24 (48,72,96 FPS) - * - * The list of standards video format is not huge and can be added to the - * connector modeset list beforehand. With that, userspace can leverage - * FreeSync to extends the front porch in order to attain the target refresh - * rate. Such a switch will happen seamlessly, without screen blanking or - * reprogramming of the output in any other way. If the userspace requests a - * modesetting change compatible with FreeSync modes that only differ in the - * refresh rate, DC will skip the full update and avoid blink during the - * transition. For example, the video player can change the modesetting from - * 60Hz to 30Hz for playing TV/NTSC content when it goes full screen without - * causing any display blink. This same concept can be applied to a mode - * setting change. - */ -static struct drm_display_mode * -get_highest_refresh_rate_mode(struct amdgpu_dm_connector *aconnector, - bool use_probed_modes) -{ - struct drm_display_mode *m, *m_pref = NULL; - u16 current_refresh, highest_refresh; - struct list_head *list_head = use_probed_modes ? - &aconnector->base.probed_modes : - &aconnector->base.modes; - - if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return NULL; - - if (aconnector->freesync_vid_base.clock != 0) - return &aconnector->freesync_vid_base; - - /* Find the preferred mode */ - list_for_each_entry(m, list_head, head) { - if (m->type & DRM_MODE_TYPE_PREFERRED) { - m_pref = m; - break; - } - } - - if (!m_pref) { - /* Probably an EDID with no preferred mode. Fallback to first entry */ - m_pref = list_first_entry_or_null( - &aconnector->base.modes, struct drm_display_mode, head); - if (!m_pref) { - DRM_DEBUG_DRIVER("No preferred mode found in EDID\n"); - return NULL; - } - } - - highest_refresh = drm_mode_vrefresh(m_pref); - - /* - * Find the mode with highest refresh rate with same resolution. - * For some monitors, preferred mode is not the mode with highest - * supported refresh rate. - */ - list_for_each_entry(m, list_head, head) { - current_refresh = drm_mode_vrefresh(m); - - if (m->hdisplay == m_pref->hdisplay && - m->vdisplay == m_pref->vdisplay && - highest_refresh < current_refresh) { - highest_refresh = current_refresh; - m_pref = m; - } - } - - drm_mode_copy(&aconnector->freesync_vid_base, m_pref); - return m_pref; -} - -static bool is_freesync_video_mode(const struct drm_display_mode *mode, - struct amdgpu_dm_connector *aconnector) -{ - struct drm_display_mode *high_mode; - int timing_diff; - - high_mode = get_highest_refresh_rate_mode(aconnector, false); - if (!high_mode || !mode) - return false; - - timing_diff = high_mode->vtotal - mode->vtotal; - - if (high_mode->clock == 0 || high_mode->clock != mode->clock || - high_mode->hdisplay != mode->hdisplay || - high_mode->vdisplay != mode->vdisplay || - high_mode->hsync_start != mode->hsync_start || - high_mode->hsync_end != mode->hsync_end || - high_mode->htotal != mode->htotal || - high_mode->hskew != mode->hskew || - high_mode->vscan != mode->vscan || - high_mode->vsync_start - mode->vsync_start != timing_diff || - high_mode->vsync_end - mode->vsync_end != timing_diff) - return false; - else - return true; -} - -#if defined(CONFIG_DRM_AMD_DC_FP) -static void update_dsc_caps(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps) -{ - stream->timing.flags.DSC = 0; - dsc_caps->is_dsc_supported = false; - - if (aconnector->dc_link && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { - if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE || - sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) - dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc, - aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw, - aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw, - dsc_caps); - } -} - -static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps, - uint32_t max_dsc_target_bpp_limit_override) -{ - const struct dc_link_settings *verified_link_cap = NULL; - u32 link_bw_in_kbps; - u32 edp_min_bpp_x16, edp_max_bpp_x16; - struct dc *dc = sink->ctx->dc; - struct dc_dsc_bw_range bw_range = {0}; - struct dc_dsc_config dsc_cfg = {0}; - struct dc_dsc_config_options dsc_options = {0}; - - dc_dsc_get_default_config_option(dc, &dsc_options); - dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16; - - verified_link_cap = dc_link_get_link_cap(stream->link); - link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap); - edp_min_bpp_x16 = 8 * 16; - edp_max_bpp_x16 = 8 * 16; - - if (edp_max_bpp_x16 > dsc_caps->edp_max_bits_per_pixel) - edp_max_bpp_x16 = dsc_caps->edp_max_bits_per_pixel; - - if (edp_max_bpp_x16 < edp_min_bpp_x16) - edp_min_bpp_x16 = edp_max_bpp_x16; - - if (dc_dsc_compute_bandwidth_range(dc->res_pool->dscs[0], - dc->debug.dsc_min_slice_height_override, - edp_min_bpp_x16, edp_max_bpp_x16, - dsc_caps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &bw_range)) { - - if (bw_range.max_kbps < link_bw_in_kbps) { - if (dc_dsc_compute_config(dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - 0, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &dsc_cfg)) { - stream->timing.dsc_cfg = dsc_cfg; - stream->timing.flags.DSC = 1; - stream->timing.dsc_cfg.bits_per_pixel = edp_max_bpp_x16; - } - return; - } - } - - if (dc_dsc_compute_config(dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - link_bw_in_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &dsc_cfg)) { - stream->timing.dsc_cfg = dsc_cfg; - stream->timing.flags.DSC = 1; - } -} - -static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector, - struct dc_sink *sink, struct dc_stream_state *stream, - struct dsc_dec_dpcd_caps *dsc_caps) -{ - struct drm_connector *drm_connector = &aconnector->base; - u32 link_bandwidth_kbps; - struct dc *dc = sink->ctx->dc; - u32 max_supported_bw_in_kbps, timing_bw_in_kbps; - u32 dsc_max_supported_bw_in_kbps; - u32 max_dsc_target_bpp_limit_override = - drm_connector->display_info.max_dsc_bpp; - struct dc_dsc_config_options dsc_options = {0}; - - dc_dsc_get_default_config_option(dc, &dsc_options); - dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16; - - link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link, - dc_link_get_link_cap(aconnector->dc_link)); - - /* Set DSC policy according to dsc_clock_en */ - dc_dsc_policy_set_enable_dsc_when_not_needed( - aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE); - - if (sink->sink_signal == SIGNAL_TYPE_EDP && - !aconnector->dc_link->panel_config.dsc.disable_dsc_edp && - dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) { - - apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override); - - } else if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) { - if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) { - if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - link_bandwidth_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &stream->timing.dsc_cfg)) { - stream->timing.flags.DSC = 1; - DRM_DEBUG_DRIVER("%s: [%s] DSC is selected from SST RX\n", __func__, drm_connector->name); - } - } else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) { - timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link)); - max_supported_bw_in_kbps = link_bandwidth_kbps; - dsc_max_supported_bw_in_kbps = link_bandwidth_kbps; - - if (timing_bw_in_kbps > max_supported_bw_in_kbps && - max_supported_bw_in_kbps > 0 && - dsc_max_supported_bw_in_kbps > 0) - if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0], - dsc_caps, - &dsc_options, - dsc_max_supported_bw_in_kbps, - &stream->timing, - dc_link_get_highest_encoding_format(aconnector->dc_link), - &stream->timing.dsc_cfg)) { - stream->timing.flags.DSC = 1; - DRM_DEBUG_DRIVER("%s: [%s] DSC is selected from DP-HDMI PCON\n", - __func__, drm_connector->name); - } - } - } - - /* Overwrite the stream flag if DSC is enabled through debugfs */ - if (aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE) - stream->timing.flags.DSC = 1; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_h) - stream->timing.dsc_cfg.num_slices_h = aconnector->dsc_settings.dsc_num_slices_h; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_v) - stream->timing.dsc_cfg.num_slices_v = aconnector->dsc_settings.dsc_num_slices_v; - - if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel) - stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel; -} -#endif - -static struct dc_stream_state * -create_stream_for_sink(struct drm_connector *connector, - const struct drm_display_mode *drm_mode, - const struct dm_connector_state *dm_state, - const struct dc_stream_state *old_stream, - int requested_bpc) -{ - struct amdgpu_dm_connector *aconnector = NULL; - struct drm_display_mode *preferred_mode = NULL; - const struct drm_connector_state *con_state = &dm_state->base; - struct dc_stream_state *stream = NULL; - struct drm_display_mode mode; - struct drm_display_mode saved_mode; - struct drm_display_mode *freesync_mode = NULL; - bool native_mode_found = false; - bool recalculate_timing = false; - bool scale = dm_state->scaling != RMX_OFF; - int mode_refresh; - int preferred_refresh = 0; - enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN; -#if defined(CONFIG_DRM_AMD_DC_FP) - struct dsc_dec_dpcd_caps dsc_caps; -#endif - struct dc_link *link = NULL; - struct dc_sink *sink = NULL; - - drm_mode_init(&mode, drm_mode); - memset(&saved_mode, 0, sizeof(saved_mode)); - - if (connector == NULL) { - DRM_ERROR("connector is NULL!\n"); - return stream; - } - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) { - aconnector = NULL; - aconnector = to_amdgpu_dm_connector(connector); - link = aconnector->dc_link; - } else { - struct drm_writeback_connector *wbcon = NULL; - struct amdgpu_dm_wb_connector *dm_wbcon = NULL; - - wbcon = drm_connector_to_writeback(connector); - dm_wbcon = to_amdgpu_dm_wb_connector(wbcon); - link = dm_wbcon->link; - } - - if (!aconnector || !aconnector->dc_sink) { - sink = create_fake_sink(link); - if (!sink) - return stream; - - } else { - sink = aconnector->dc_sink; - dc_sink_retain(sink); - } - - stream = dc_create_stream_for_sink(sink); - - if (stream == NULL) { - DRM_ERROR("Failed to create stream for sink!\n"); - goto finish; - } - - /* We leave this NULL for writeback connectors */ - stream->dm_stream_context = aconnector; - - stream->timing.flags.LTE_340MCSC_SCRAMBLE = - connector->display_info.hdmi.scdc.scrambling.low_rates; - - list_for_each_entry(preferred_mode, &connector->modes, head) { - /* Search for preferred mode */ - if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) { - native_mode_found = true; - break; - } - } - if (!native_mode_found) - preferred_mode = list_first_entry_or_null( - &connector->modes, - struct drm_display_mode, - head); - - mode_refresh = drm_mode_vrefresh(&mode); - - if (preferred_mode == NULL) { - /* - * This may not be an error, the use case is when we have no - * usermode calls to reset and set mode upon hotplug. In this - * case, we call set mode ourselves to restore the previous mode - * and the modelist may not be filled in time. - */ - DRM_DEBUG_DRIVER("No preferred mode found\n"); - } else if (aconnector) { - recalculate_timing = amdgpu_freesync_vid_mode && - is_freesync_video_mode(&mode, aconnector); - if (recalculate_timing) { - freesync_mode = get_highest_refresh_rate_mode(aconnector, false); - drm_mode_copy(&saved_mode, &mode); - saved_mode.picture_aspect_ratio = mode.picture_aspect_ratio; - drm_mode_copy(&mode, freesync_mode); - mode.picture_aspect_ratio = saved_mode.picture_aspect_ratio; - } else { - decide_crtc_timing_for_drm_display_mode( - &mode, preferred_mode, scale); - - preferred_refresh = drm_mode_vrefresh(preferred_mode); - } - } - - if (recalculate_timing) - drm_mode_set_crtcinfo(&saved_mode, 0); - - /* - * If scaling is enabled and refresh rate didn't change - * we copy the vic and polarities of the old timings - */ - if (!scale || mode_refresh != preferred_refresh) - fill_stream_properties_from_drm_display_mode( - stream, &mode, connector, con_state, NULL, - requested_bpc); - else - fill_stream_properties_from_drm_display_mode( - stream, &mode, connector, con_state, old_stream, - requested_bpc); - - /* The rest isn't needed for writeback connectors */ - if (!aconnector) - goto finish; - - if (aconnector->timing_changed) { - drm_dbg(aconnector->base.dev, - "overriding timing for automated test, bpc %d, changing to %d\n", - stream->timing.display_color_depth, - aconnector->timing_requested->display_color_depth); - stream->timing = *aconnector->timing_requested; - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - /* SST DSC determination policy */ - update_dsc_caps(aconnector, sink, stream, &dsc_caps); - if (aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE && dsc_caps.is_dsc_supported) - apply_dsc_policy_for_stream(aconnector, sink, stream, &dsc_caps); -#endif - - update_stream_scaling_settings(&mode, dm_state, stream); - - fill_audio_info( - &stream->audio_info, - connector, - sink); - - update_stream_signal(stream, sink); - - if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket); - - if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || - stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST || - stream->signal == SIGNAL_TYPE_EDP) { - // - // should decide stream support vsc sdp colorimetry capability - // before building vsc info packet - // - stream->use_vsc_sdp_for_colorimetry = stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 && - stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED; - - if (stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) - tf = TRANSFER_FUNC_GAMMA_22; - mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf); - aconnector->psr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY; - - } -finish: - dc_sink_release(sink); - - return stream; -} - -static enum drm_connector_status -amdgpu_dm_connector_detect(struct drm_connector *connector, bool force) -{ - bool connected; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - /* - * Notes: - * 1. This interface is NOT called in context of HPD irq. - * 2. This interface *is called* in context of user-mode ioctl. Which - * makes it a bad place for *any* MST-related activity. - */ - - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED && - !aconnector->fake_enable) - connected = (aconnector->dc_sink != NULL); - else - connected = (aconnector->base.force == DRM_FORCE_ON || - aconnector->base.force == DRM_FORCE_ON_DIGITAL); - - update_subconnector_property(aconnector); - - return (connected ? connector_status_connected : - connector_status_disconnected); -} - -int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector, - struct drm_connector_state *connector_state, - struct drm_property *property, - uint64_t val) -{ - struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_old_state = - to_dm_connector_state(connector->state); - struct dm_connector_state *dm_new_state = - to_dm_connector_state(connector_state); - - int ret = -EINVAL; - - if (property == dev->mode_config.scaling_mode_property) { - enum amdgpu_rmx_type rmx_type; - - switch (val) { - case DRM_MODE_SCALE_CENTER: - rmx_type = RMX_CENTER; - break; - case DRM_MODE_SCALE_ASPECT: - rmx_type = RMX_ASPECT; - break; - case DRM_MODE_SCALE_FULLSCREEN: - rmx_type = RMX_FULL; - break; - case DRM_MODE_SCALE_NONE: - default: - rmx_type = RMX_OFF; - break; - } - - if (dm_old_state->scaling == rmx_type) - return 0; - - dm_new_state->scaling = rmx_type; - ret = 0; - } else if (property == adev->mode_info.underscan_hborder_property) { - dm_new_state->underscan_hborder = val; - ret = 0; - } else if (property == adev->mode_info.underscan_vborder_property) { - dm_new_state->underscan_vborder = val; - ret = 0; - } else if (property == adev->mode_info.underscan_property) { - dm_new_state->underscan_enable = val; - ret = 0; - } - - return ret; -} - -int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector, - const struct drm_connector_state *state, - struct drm_property *property, - uint64_t *val) -{ - struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_connector_state *dm_state = - to_dm_connector_state(state); - int ret = -EINVAL; - - if (property == dev->mode_config.scaling_mode_property) { - switch (dm_state->scaling) { - case RMX_CENTER: - *val = DRM_MODE_SCALE_CENTER; - break; - case RMX_ASPECT: - *val = DRM_MODE_SCALE_ASPECT; - break; - case RMX_FULL: - *val = DRM_MODE_SCALE_FULLSCREEN; - break; - case RMX_OFF: - default: - *val = DRM_MODE_SCALE_NONE; - break; - } - ret = 0; - } else if (property == adev->mode_info.underscan_hborder_property) { - *val = dm_state->underscan_hborder; - ret = 0; - } else if (property == adev->mode_info.underscan_vborder_property) { - *val = dm_state->underscan_vborder; - ret = 0; - } else if (property == adev->mode_info.underscan_property) { - *val = dm_state->underscan_enable; - ret = 0; - } - - return ret; -} - -/** - * DOC: panel power savings - * - * The display manager allows you to set your desired **panel power savings** - * level (between 0-4, with 0 representing off), e.g. using the following:: - * - * # echo 3 > /sys/class/drm/card0-eDP-1/amdgpu/panel_power_savings - * - * Modifying this value can have implications on color accuracy, so tread - * carefully. - */ - -static ssize_t panel_power_savings_show(struct device *device, - struct device_attribute *attr, - char *buf) -{ - struct drm_connector *connector = dev_get_drvdata(device); - struct drm_device *dev = connector->dev; - u8 val; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - val = to_dm_connector_state(connector->state)->abm_level == - ABM_LEVEL_IMMEDIATE_DISABLE ? 0 : - to_dm_connector_state(connector->state)->abm_level; - drm_modeset_unlock(&dev->mode_config.connection_mutex); - - return sysfs_emit(buf, "%u\n", val); -} - -static ssize_t panel_power_savings_store(struct device *device, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct drm_connector *connector = dev_get_drvdata(device); - struct drm_device *dev = connector->dev; - long val; - int ret; - - ret = kstrtol(buf, 0, &val); - - if (ret) - return ret; - - if (val < 0 || val > 4) - return -EINVAL; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - to_dm_connector_state(connector->state)->abm_level = val ?: - ABM_LEVEL_IMMEDIATE_DISABLE; - drm_modeset_unlock(&dev->mode_config.connection_mutex); - - drm_kms_helper_hotplug_event(dev); - - return count; -} - -static DEVICE_ATTR_RW(panel_power_savings); - -static struct attribute *amdgpu_attrs[] = { - &dev_attr_panel_power_savings.attr, - NULL -}; - -static const struct attribute_group amdgpu_group = { - .name = "amdgpu", - .attrs = amdgpu_attrs -}; - -static bool -amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *amdgpu_dm_connector) -{ - if (amdgpu_dm_abm_level >= 0) - return false; - - if (amdgpu_dm_connector->base.connector_type != DRM_MODE_CONNECTOR_eDP) - return false; - - /* check for OLED panels */ - if (amdgpu_dm_connector->bl_idx >= 0) { - struct drm_device *drm = amdgpu_dm_connector->base.dev; - struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm; - struct amdgpu_dm_backlight_caps *caps; - - caps = &dm->backlight_caps[amdgpu_dm_connector->bl_idx]; - if (caps->aux_support) - return false; - } - - return true; -} - -static void amdgpu_dm_connector_unregister(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector); - - if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) - sysfs_remove_group(&connector->kdev->kobj, &amdgpu_group); - - drm_dp_aux_unregister(&amdgpu_dm_connector->dm_dp_aux.aux); -} - -static void amdgpu_dm_connector_destroy(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct amdgpu_display_manager *dm = &adev->dm; - - /* - * Call only if mst_mgr was initialized before since it's not done - * for all connector types. - */ - if (aconnector->mst_mgr.dev) - drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr); - - if (aconnector->bl_idx != -1) { - backlight_device_unregister(dm->backlight_dev[aconnector->bl_idx]); - dm->backlight_dev[aconnector->bl_idx] = NULL; - } - - if (aconnector->dc_em_sink) - dc_sink_release(aconnector->dc_em_sink); - aconnector->dc_em_sink = NULL; - if (aconnector->dc_sink) - dc_sink_release(aconnector->dc_sink); - aconnector->dc_sink = NULL; - - drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux); - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - if (aconnector->i2c) { - i2c_del_adapter(&aconnector->i2c->base); - kfree(aconnector->i2c); - } - kfree(aconnector->dm_dp_aux.aux.name); - - kfree(connector); -} - -void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector) -{ - struct dm_connector_state *state = - to_dm_connector_state(connector->state); - - if (connector->state) - __drm_atomic_helper_connector_destroy_state(connector->state); - - kfree(state); - - state = kzalloc(sizeof(*state), GFP_KERNEL); - - if (state) { - state->scaling = RMX_OFF; - state->underscan_enable = false; - state->underscan_hborder = 0; - state->underscan_vborder = 0; - state->base.max_requested_bpc = 8; - state->vcpi_slots = 0; - state->pbn = 0; - - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { - if (amdgpu_dm_abm_level <= 0) - state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE; - else - state->abm_level = amdgpu_dm_abm_level; - } - - __drm_atomic_helper_connector_reset(connector, &state->base); - } -} - -struct drm_connector_state * -amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector) -{ - struct dm_connector_state *state = - to_dm_connector_state(connector->state); - - struct dm_connector_state *new_state = - kmemdup(state, sizeof(*state), GFP_KERNEL); - - if (!new_state) - return NULL; - - __drm_atomic_helper_connector_duplicate_state(connector, &new_state->base); - - new_state->freesync_capable = state->freesync_capable; - new_state->abm_level = state->abm_level; - new_state->scaling = state->scaling; - new_state->underscan_enable = state->underscan_enable; - new_state->underscan_hborder = state->underscan_hborder; - new_state->underscan_vborder = state->underscan_vborder; - new_state->vcpi_slots = state->vcpi_slots; - new_state->pbn = state->pbn; - return &new_state->base; -} - -static int -amdgpu_dm_connector_late_register(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - int r; - - if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) { - r = sysfs_create_group(&connector->kdev->kobj, - &amdgpu_group); - if (r) - return r; - } - - amdgpu_dm_register_backlight_device(amdgpu_dm_connector); - - if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) || - (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) { - amdgpu_dm_connector->dm_dp_aux.aux.dev = connector->kdev; - r = drm_dp_aux_register(&amdgpu_dm_connector->dm_dp_aux.aux); - if (r) - return r; - } - -#if defined(CONFIG_DEBUG_FS) - connector_debugfs_init(amdgpu_dm_connector); -#endif - - return 0; -} - -static void amdgpu_dm_connector_funcs_force(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dc_link *dc_link = aconnector->dc_link; - struct dc_sink *dc_em_sink = aconnector->dc_em_sink; - struct edid *edid; - struct i2c_adapter *ddc; - - if (dc_link && dc_link->aux_mode) - ddc = &aconnector->dm_dp_aux.aux.ddc; - else - ddc = &aconnector->i2c->base; - - /* - * Note: drm_get_edid gets edid in the following order: - * 1) override EDID if set via edid_override debugfs, - * 2) firmware EDID if set via edid_firmware module parameter - * 3) regular DDC read. - */ - edid = drm_get_edid(connector, ddc); - if (!edid) { - DRM_ERROR("No EDID found on connector: %s.\n", connector->name); - return; - } - - aconnector->edid = edid; - - /* Update emulated (virtual) sink's EDID */ - if (dc_em_sink && dc_link) { - memset(&dc_em_sink->edid_caps, 0, sizeof(struct dc_edid_caps)); - memmove(dc_em_sink->dc_edid.raw_edid, edid, (edid->extensions + 1) * EDID_LENGTH); - dm_helpers_parse_edid_caps( - dc_link, - &dc_em_sink->dc_edid, - &dc_em_sink->edid_caps); - } -} - -static const struct drm_connector_funcs amdgpu_dm_connector_funcs = { - .reset = amdgpu_dm_connector_funcs_reset, - .detect = amdgpu_dm_connector_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = amdgpu_dm_connector_destroy, - .atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_set_property = amdgpu_dm_connector_atomic_set_property, - .atomic_get_property = amdgpu_dm_connector_atomic_get_property, - .late_register = amdgpu_dm_connector_late_register, - .early_unregister = amdgpu_dm_connector_unregister, - .force = amdgpu_dm_connector_funcs_force -}; - -static int get_modes(struct drm_connector *connector) -{ - return amdgpu_dm_connector_get_modes(connector); -} - -static void create_eml_sink(struct amdgpu_dm_connector *aconnector) -{ - struct drm_connector *connector = &aconnector->base; - struct dc_link *dc_link = aconnector->dc_link; - struct dc_sink_init_data init_params = { - .link = aconnector->dc_link, - .sink_signal = SIGNAL_TYPE_VIRTUAL - }; - struct edid *edid; - struct i2c_adapter *ddc; - - if (dc_link->aux_mode) - ddc = &aconnector->dm_dp_aux.aux.ddc; - else - ddc = &aconnector->i2c->base; - - /* - * Note: drm_get_edid gets edid in the following order: - * 1) override EDID if set via edid_override debugfs, - * 2) firmware EDID if set via edid_firmware module parameter - * 3) regular DDC read. - */ - edid = drm_get_edid(connector, ddc); - if (!edid) { - DRM_ERROR("No EDID found on connector: %s.\n", connector->name); - return; - } - - if (drm_detect_hdmi_monitor(edid)) - init_params.sink_signal = SIGNAL_TYPE_HDMI_TYPE_A; - - aconnector->edid = edid; - - aconnector->dc_em_sink = dc_link_add_remote_sink( - aconnector->dc_link, - (uint8_t *)edid, - (edid->extensions + 1) * EDID_LENGTH, - &init_params); - - if (aconnector->base.force == DRM_FORCE_ON) { - aconnector->dc_sink = aconnector->dc_link->local_sink ? - aconnector->dc_link->local_sink : - aconnector->dc_em_sink; - if (aconnector->dc_sink) - dc_sink_retain(aconnector->dc_sink); - } -} - -static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector) -{ - struct dc_link *link = (struct dc_link *)aconnector->dc_link; - - /* - * In case of headless boot with force on for DP managed connector - * Those settings have to be != 0 to get initial modeset - */ - if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) { - link->verified_link_cap.lane_count = LANE_COUNT_FOUR; - link->verified_link_cap.link_rate = LINK_RATE_HIGH2; - } - - create_eml_sink(aconnector); -} - -static enum dc_status dm_validate_stream_and_context(struct dc *dc, - struct dc_stream_state *stream) -{ - enum dc_status dc_result = DC_ERROR_UNEXPECTED; - struct dc_plane_state *dc_plane_state = NULL; - struct dc_state *dc_state = NULL; - - if (!stream) - goto cleanup; - - dc_plane_state = dc_create_plane_state(dc); - if (!dc_plane_state) - goto cleanup; - - dc_state = dc_state_create(dc, NULL); - if (!dc_state) - goto cleanup; - - /* populate stream to plane */ - dc_plane_state->src_rect.height = stream->src.height; - dc_plane_state->src_rect.width = stream->src.width; - dc_plane_state->dst_rect.height = stream->src.height; - dc_plane_state->dst_rect.width = stream->src.width; - dc_plane_state->clip_rect.height = stream->src.height; - dc_plane_state->clip_rect.width = stream->src.width; - dc_plane_state->plane_size.surface_pitch = ((stream->src.width + 255) / 256) * 256; - dc_plane_state->plane_size.surface_size.height = stream->src.height; - dc_plane_state->plane_size.surface_size.width = stream->src.width; - dc_plane_state->plane_size.chroma_size.height = stream->src.height; - dc_plane_state->plane_size.chroma_size.width = stream->src.width; - dc_plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888; - dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN; - dc_plane_state->rotation = ROTATION_ANGLE_0; - dc_plane_state->is_tiling_rotated = false; - dc_plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_LINEAR_GENERAL; - - dc_result = dc_validate_stream(dc, stream); - if (dc_result == DC_OK) - dc_result = dc_validate_plane(dc, dc_plane_state); - - if (dc_result == DC_OK) - dc_result = dc_state_add_stream(dc, dc_state, stream); - - if (dc_result == DC_OK && !dc_state_add_plane( - dc, - stream, - dc_plane_state, - dc_state)) - dc_result = DC_FAIL_ATTACH_SURFACES; - - if (dc_result == DC_OK) - dc_result = dc_validate_global_state(dc, dc_state, true); - -cleanup: - if (dc_state) - dc_state_release(dc_state); - - if (dc_plane_state) - dc_plane_state_release(dc_plane_state); - - return dc_result; -} - -struct dc_stream_state * -create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, - const struct drm_display_mode *drm_mode, - const struct dm_connector_state *dm_state, - const struct dc_stream_state *old_stream) -{ - struct drm_connector *connector = &aconnector->base; - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct dc_stream_state *stream; - const struct drm_connector_state *drm_state = dm_state ? &dm_state->base : NULL; - int requested_bpc = drm_state ? drm_state->max_requested_bpc : 8; - enum dc_status dc_result = DC_OK; - - if (!dm_state) - return NULL; - - do { - stream = create_stream_for_sink(connector, drm_mode, - dm_state, old_stream, - requested_bpc); - if (stream == NULL) { - DRM_ERROR("Failed to create stream for sink!\n"); - break; - } - - if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return stream; - - dc_result = dc_validate_stream(adev->dm.dc, stream); - if (dc_result == DC_OK && stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) - dc_result = dm_dp_mst_is_port_support_mode(aconnector, stream); - - if (dc_result == DC_OK) - dc_result = dm_validate_stream_and_context(adev->dm.dc, stream); - - if (dc_result != DC_OK) { - DRM_DEBUG_KMS("Mode %dx%d (clk %d) failed DC validation with error %d (%s)\n", - drm_mode->hdisplay, - drm_mode->vdisplay, - drm_mode->clock, - dc_result, - dc_status_to_str(dc_result)); - - dc_stream_release(stream); - stream = NULL; - requested_bpc -= 2; /* lower bpc to retry validation */ - } - - } while (stream == NULL && requested_bpc >= 6); - - if (dc_result == DC_FAIL_ENC_VALIDATE && !aconnector->force_yuv420_output) { - DRM_DEBUG_KMS("Retry forcing YCbCr420 encoding\n"); - - aconnector->force_yuv420_output = true; - stream = create_validate_stream_for_sink(aconnector, drm_mode, - dm_state, old_stream); - aconnector->force_yuv420_output = false; - } - - return stream; -} - -enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - int result = MODE_ERROR; - struct dc_sink *dc_sink; - /* TODO: Unhardcode stream count */ - struct dc_stream_state *stream; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - if ((mode->flags & DRM_MODE_FLAG_INTERLACE) || - (mode->flags & DRM_MODE_FLAG_DBLSCAN)) - return result; - - /* - * Only run this the first time mode_valid is called to initilialize - * EDID mgmt - */ - if (aconnector->base.force != DRM_FORCE_UNSPECIFIED && - !aconnector->dc_em_sink) - handle_edid_mgmt(aconnector); - - dc_sink = to_amdgpu_dm_connector(connector)->dc_sink; - - if (dc_sink == NULL && aconnector->base.force != DRM_FORCE_ON_DIGITAL && - aconnector->base.force != DRM_FORCE_ON) { - DRM_ERROR("dc_sink is NULL!\n"); - goto fail; - } - - drm_mode_set_crtcinfo(mode, 0); - - stream = create_validate_stream_for_sink(aconnector, mode, - to_dm_connector_state(connector->state), - NULL); - if (stream) { - dc_stream_release(stream); - result = MODE_OK; - } - -fail: - /* TODO: error handling*/ - return result; -} - -static int fill_hdr_info_packet(const struct drm_connector_state *state, - struct dc_info_packet *out) -{ - struct hdmi_drm_infoframe frame; - unsigned char buf[30]; /* 26 + 4 */ - ssize_t len; - int ret, i; - - memset(out, 0, sizeof(*out)); - - if (!state->hdr_output_metadata) - return 0; - - ret = drm_hdmi_infoframe_set_hdr_metadata(&frame, state); - if (ret) - return ret; - - len = hdmi_drm_infoframe_pack_only(&frame, buf, sizeof(buf)); - if (len < 0) - return (int)len; - - /* Static metadata is a fixed 26 bytes + 4 byte header. */ - if (len != 30) - return -EINVAL; - - /* Prepare the infopacket for DC. */ - switch (state->connector->connector_type) { - case DRM_MODE_CONNECTOR_HDMIA: - out->hb0 = 0x87; /* type */ - out->hb1 = 0x01; /* version */ - out->hb2 = 0x1A; /* length */ - out->sb[0] = buf[3]; /* checksum */ - i = 1; - break; - - case DRM_MODE_CONNECTOR_DisplayPort: - case DRM_MODE_CONNECTOR_eDP: - out->hb0 = 0x00; /* sdp id, zero */ - out->hb1 = 0x87; /* type */ - out->hb2 = 0x1D; /* payload len - 1 */ - out->hb3 = (0x13 << 2); /* sdp version */ - out->sb[0] = 0x01; /* version */ - out->sb[1] = 0x1A; /* length */ - i = 2; - break; - - default: - return -EINVAL; - } - - memcpy(&out->sb[i], &buf[4], 26); - out->valid = true; - - print_hex_dump(KERN_DEBUG, "HDR SB:", DUMP_PREFIX_NONE, 16, 1, out->sb, - sizeof(out->sb), false); - - return 0; -} - -static int -amdgpu_dm_connector_atomic_check(struct drm_connector *conn, - struct drm_atomic_state *state) -{ - struct drm_connector_state *new_con_state = - drm_atomic_get_new_connector_state(state, conn); - struct drm_connector_state *old_con_state = - drm_atomic_get_old_connector_state(state, conn); - struct drm_crtc *crtc = new_con_state->crtc; - struct drm_crtc_state *new_crtc_state; - struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn); - int ret; - - trace_amdgpu_dm_connector_atomic_check(new_con_state); - - if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { - ret = drm_dp_mst_root_conn_atomic_check(new_con_state, &aconn->mst_mgr); - if (ret < 0) - return ret; - } - - if (!crtc) - return 0; - - if (new_con_state->colorspace != old_con_state->colorspace) { - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - new_crtc_state->mode_changed = true; - } - - if (new_con_state->content_type != old_con_state->content_type) { - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - new_crtc_state->mode_changed = true; - } - - if (!drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state)) { - struct dc_info_packet hdr_infopacket; - - ret = fill_hdr_info_packet(new_con_state, &hdr_infopacket); - if (ret) - return ret; - - new_crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(new_crtc_state)) - return PTR_ERR(new_crtc_state); - - /* - * DC considers the stream backends changed if the - * static metadata changes. Forcing the modeset also - * gives a simple way for userspace to switch from - * 8bpc to 10bpc when setting the metadata to enter - * or exit HDR. - * - * Changing the static metadata after it's been - * set is permissible, however. So only force a - * modeset if we're entering or exiting HDR. - */ - new_crtc_state->mode_changed = new_crtc_state->mode_changed || - !old_con_state->hdr_output_metadata || - !new_con_state->hdr_output_metadata; - } - - return 0; -} - -static const struct drm_connector_helper_funcs -amdgpu_dm_connector_helper_funcs = { - /* - * If hotplugging a second bigger display in FB Con mode, bigger resolution - * modes will be filtered by drm_mode_validate_size(), and those modes - * are missing after user start lightdm. So we need to renew modes list. - * in get_modes call back, not just return the modes count - */ - .get_modes = get_modes, - .mode_valid = amdgpu_dm_connector_mode_valid, - .atomic_check = amdgpu_dm_connector_atomic_check, -}; - -static void dm_encoder_helper_disable(struct drm_encoder *encoder) -{ - -} - -int convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth) -{ - switch (display_color_depth) { - case COLOR_DEPTH_666: - return 6; - case COLOR_DEPTH_888: - return 8; - case COLOR_DEPTH_101010: - return 10; - case COLOR_DEPTH_121212: - return 12; - case COLOR_DEPTH_141414: - return 14; - case COLOR_DEPTH_161616: - return 16; - default: - break; - } - return 0; -} - -static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder, - struct drm_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct drm_atomic_state *state = crtc_state->state; - struct drm_connector *connector = conn_state->connector; - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_new_connector_state = to_dm_connector_state(conn_state); - const struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode; - struct drm_dp_mst_topology_mgr *mst_mgr; - struct drm_dp_mst_port *mst_port; - struct drm_dp_mst_topology_state *mst_state; - enum dc_color_depth color_depth; - int clock, bpp = 0; - bool is_y420 = false; - - if (!aconnector->mst_output_port) - return 0; - - mst_port = aconnector->mst_output_port; - mst_mgr = &aconnector->mst_root->mst_mgr; - - if (!crtc_state->connectors_changed && !crtc_state->mode_changed) - return 0; - - mst_state = drm_atomic_get_mst_topology_state(state, mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - mst_state->pbn_div.full = dfixed_const(dm_mst_get_pbn_divider(aconnector->mst_root->dc_link)); - - if (!state->duplicated) { - int max_bpc = conn_state->max_requested_bpc; - - is_y420 = drm_mode_is_420_also(&connector->display_info, adjusted_mode) && - aconnector->force_yuv420_output; - color_depth = convert_color_depth_from_display_info(connector, - is_y420, - max_bpc); - bpp = convert_dc_color_depth_into_bpc(color_depth) * 3; - clock = adjusted_mode->clock; - dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp << 4); - } - - dm_new_connector_state->vcpi_slots = - drm_dp_atomic_find_time_slots(state, mst_mgr, mst_port, - dm_new_connector_state->pbn); - if (dm_new_connector_state->vcpi_slots < 0) { - DRM_DEBUG_ATOMIC("failed finding vcpi slots: %d\n", (int)dm_new_connector_state->vcpi_slots); - return dm_new_connector_state->vcpi_slots; - } - return 0; -} - -const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = { - .disable = dm_encoder_helper_disable, - .atomic_check = dm_encoder_helper_atomic_check -}; - -static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state, - struct dc_state *dc_state, - struct dsc_mst_fairness_vars *vars) -{ - struct dc_stream_state *stream = NULL; - struct drm_connector *connector; - struct drm_connector_state *new_con_state; - struct amdgpu_dm_connector *aconnector; - struct dm_connector_state *dm_conn_state; - int i, j, ret; - int vcpi, pbn_div, pbn = 0, slot_num = 0; - - for_each_new_connector_in_state(state, connector, new_con_state, i) { - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->mst_output_port) - continue; - - if (!new_con_state || !new_con_state->crtc) - continue; - - dm_conn_state = to_dm_connector_state(new_con_state); - - for (j = 0; j < dc_state->stream_count; j++) { - stream = dc_state->streams[j]; - if (!stream) - continue; - - if ((struct amdgpu_dm_connector *)stream->dm_stream_context == aconnector) - break; - - stream = NULL; - } - - if (!stream) - continue; - - pbn_div = dm_mst_get_pbn_divider(stream->link); - /* pbn is calculated by compute_mst_dsc_configs_for_state*/ - for (j = 0; j < dc_state->stream_count; j++) { - if (vars[j].aconnector == aconnector) { - pbn = vars[j].pbn; - break; - } - } - - if (j == dc_state->stream_count || pbn_div == 0) - continue; - - slot_num = DIV_ROUND_UP(pbn, pbn_div); - - if (stream->timing.flags.DSC != 1) { - dm_conn_state->pbn = pbn; - dm_conn_state->vcpi_slots = slot_num; - - ret = drm_dp_mst_atomic_enable_dsc(state, aconnector->mst_output_port, - dm_conn_state->pbn, false); - if (ret < 0) - return ret; - - continue; - } - - vcpi = drm_dp_mst_atomic_enable_dsc(state, aconnector->mst_output_port, pbn, true); - if (vcpi < 0) - return vcpi; - - dm_conn_state->pbn = pbn; - dm_conn_state->vcpi_slots = vcpi; - } - return 0; -} - -static int to_drm_connector_type(enum signal_type st) -{ - switch (st) { - case SIGNAL_TYPE_HDMI_TYPE_A: - return DRM_MODE_CONNECTOR_HDMIA; - case SIGNAL_TYPE_EDP: - return DRM_MODE_CONNECTOR_eDP; - case SIGNAL_TYPE_LVDS: - return DRM_MODE_CONNECTOR_LVDS; - case SIGNAL_TYPE_RGB: - return DRM_MODE_CONNECTOR_VGA; - case SIGNAL_TYPE_DISPLAY_PORT: - case SIGNAL_TYPE_DISPLAY_PORT_MST: - return DRM_MODE_CONNECTOR_DisplayPort; - case SIGNAL_TYPE_DVI_DUAL_LINK: - case SIGNAL_TYPE_DVI_SINGLE_LINK: - return DRM_MODE_CONNECTOR_DVID; - case SIGNAL_TYPE_VIRTUAL: - return DRM_MODE_CONNECTOR_VIRTUAL; - - default: - return DRM_MODE_CONNECTOR_Unknown; - } -} - -static struct drm_encoder *amdgpu_dm_connector_to_encoder(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - - /* There is only one encoder per connector */ - drm_connector_for_each_possible_encoder(connector, encoder) - return encoder; - - return NULL; -} - -static void amdgpu_dm_get_native_mode(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - struct amdgpu_encoder *amdgpu_encoder; - - encoder = amdgpu_dm_connector_to_encoder(connector); - - if (encoder == NULL) - return; - - amdgpu_encoder = to_amdgpu_encoder(encoder); - - amdgpu_encoder->native_mode.clock = 0; - - if (!list_empty(&connector->probed_modes)) { - struct drm_display_mode *preferred_mode = NULL; - - list_for_each_entry(preferred_mode, - &connector->probed_modes, - head) { - if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) - amdgpu_encoder->native_mode = *preferred_mode; - - break; - } - - } -} - -static struct drm_display_mode * -amdgpu_dm_create_common_mode(struct drm_encoder *encoder, - char *name, - int hdisplay, int vdisplay) -{ - struct drm_device *dev = encoder->dev; - struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); - struct drm_display_mode *mode = NULL; - struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; - - mode = drm_mode_duplicate(dev, native_mode); - - if (mode == NULL) - return NULL; - - mode->hdisplay = hdisplay; - mode->vdisplay = vdisplay; - mode->type &= ~DRM_MODE_TYPE_PREFERRED; - strscpy(mode->name, name, DRM_DISPLAY_MODE_LEN); - - return mode; - -} - -static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder, - struct drm_connector *connector) -{ - struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); - struct drm_display_mode *mode = NULL; - struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - int i; - int n; - struct mode_size { - char name[DRM_DISPLAY_MODE_LEN]; - int w; - int h; - } common_modes[] = { - { "640x480", 640, 480}, - { "800x600", 800, 600}, - { "1024x768", 1024, 768}, - { "1280x720", 1280, 720}, - { "1280x800", 1280, 800}, - {"1280x1024", 1280, 1024}, - { "1440x900", 1440, 900}, - {"1680x1050", 1680, 1050}, - {"1600x1200", 1600, 1200}, - {"1920x1080", 1920, 1080}, - {"1920x1200", 1920, 1200} - }; - - n = ARRAY_SIZE(common_modes); - - for (i = 0; i < n; i++) { - struct drm_display_mode *curmode = NULL; - bool mode_existed = false; - - if (common_modes[i].w > native_mode->hdisplay || - common_modes[i].h > native_mode->vdisplay || - (common_modes[i].w == native_mode->hdisplay && - common_modes[i].h == native_mode->vdisplay)) - continue; - - list_for_each_entry(curmode, &connector->probed_modes, head) { - if (common_modes[i].w == curmode->hdisplay && - common_modes[i].h == curmode->vdisplay) { - mode_existed = true; - break; - } - } - - if (mode_existed) - continue; - - mode = amdgpu_dm_create_common_mode(encoder, - common_modes[i].name, common_modes[i].w, - common_modes[i].h); - if (!mode) - continue; - - drm_mode_probed_add(connector, mode); - amdgpu_dm_connector->num_modes++; - } -} - -static void amdgpu_set_panel_orientation(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - struct amdgpu_encoder *amdgpu_encoder; - const struct drm_display_mode *native_mode; - - if (connector->connector_type != DRM_MODE_CONNECTOR_eDP && - connector->connector_type != DRM_MODE_CONNECTOR_LVDS) - return; - - mutex_lock(&connector->dev->mode_config.mutex); - amdgpu_dm_connector_get_modes(connector); - mutex_unlock(&connector->dev->mode_config.mutex); - - encoder = amdgpu_dm_connector_to_encoder(connector); - if (!encoder) - return; - - amdgpu_encoder = to_amdgpu_encoder(encoder); - - native_mode = &amdgpu_encoder->native_mode; - if (native_mode->hdisplay == 0 || native_mode->vdisplay == 0) - return; - - drm_connector_set_panel_orientation_with_quirk(connector, - DRM_MODE_PANEL_ORIENTATION_UNKNOWN, - native_mode->hdisplay, - native_mode->vdisplay); -} - -static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector, - struct edid *edid) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - - if (edid) { - /* empty probed_modes */ - INIT_LIST_HEAD(&connector->probed_modes); - amdgpu_dm_connector->num_modes = - drm_add_edid_modes(connector, edid); - - /* sorting the probed modes before calling function - * amdgpu_dm_get_native_mode() since EDID can have - * more than one preferred mode. The modes that are - * later in the probed mode list could be of higher - * and preferred resolution. For example, 3840x2160 - * resolution in base EDID preferred timing and 4096x2160 - * preferred resolution in DID extension block later. - */ - drm_mode_sort(&connector->probed_modes); - amdgpu_dm_get_native_mode(connector); - - /* Freesync capabilities are reset by calling - * drm_add_edid_modes() and need to be - * restored here. - */ - amdgpu_dm_update_freesync_caps(connector, edid); - } else { - amdgpu_dm_connector->num_modes = 0; - } -} - -static bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector, - struct drm_display_mode *mode) -{ - struct drm_display_mode *m; - - list_for_each_entry(m, &aconnector->base.probed_modes, head) { - if (drm_mode_equal(m, mode)) - return true; - } - - return false; -} - -static uint add_fs_modes(struct amdgpu_dm_connector *aconnector) -{ - const struct drm_display_mode *m; - struct drm_display_mode *new_mode; - uint i; - u32 new_modes_count = 0; - - /* Standard FPS values - * - * 23.976 - TV/NTSC - * 24 - Cinema - * 25 - TV/PAL - * 29.97 - TV/NTSC - * 30 - TV/NTSC - * 48 - Cinema HFR - * 50 - TV/PAL - * 60 - Commonly used - * 48,72,96,120 - Multiples of 24 - */ - static const u32 common_rates[] = { - 23976, 24000, 25000, 29970, 30000, - 48000, 50000, 60000, 72000, 96000, 120000 - }; - - /* - * Find mode with highest refresh rate with the same resolution - * as the preferred mode. Some monitors report a preferred mode - * with lower resolution than the highest refresh rate supported. - */ - - m = get_highest_refresh_rate_mode(aconnector, true); - if (!m) - return 0; - - for (i = 0; i < ARRAY_SIZE(common_rates); i++) { - u64 target_vtotal, target_vtotal_diff; - u64 num, den; - - if (drm_mode_vrefresh(m) * 1000 < common_rates[i]) - continue; - - if (common_rates[i] < aconnector->min_vfreq * 1000 || - common_rates[i] > aconnector->max_vfreq * 1000) - continue; - - num = (unsigned long long)m->clock * 1000 * 1000; - den = common_rates[i] * (unsigned long long)m->htotal; - target_vtotal = div_u64(num, den); - target_vtotal_diff = target_vtotal - m->vtotal; - - /* Check for illegal modes */ - if (m->vsync_start + target_vtotal_diff < m->vdisplay || - m->vsync_end + target_vtotal_diff < m->vsync_start || - m->vtotal + target_vtotal_diff < m->vsync_end) - continue; - - new_mode = drm_mode_duplicate(aconnector->base.dev, m); - if (!new_mode) - goto out; - - new_mode->vtotal += (u16)target_vtotal_diff; - new_mode->vsync_start += (u16)target_vtotal_diff; - new_mode->vsync_end += (u16)target_vtotal_diff; - new_mode->type &= ~DRM_MODE_TYPE_PREFERRED; - new_mode->type |= DRM_MODE_TYPE_DRIVER; - - if (!is_duplicate_mode(aconnector, new_mode)) { - drm_mode_probed_add(&aconnector->base, new_mode); - new_modes_count += 1; - } else - drm_mode_destroy(aconnector->base.dev, new_mode); - } - out: - return new_modes_count; -} - -static void amdgpu_dm_connector_add_freesync_modes(struct drm_connector *connector, - struct edid *edid) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - - if (!(amdgpu_freesync_vid_mode && edid)) - return; - - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - amdgpu_dm_connector->num_modes += - add_fs_modes(amdgpu_dm_connector); -} - -static int amdgpu_dm_connector_get_modes(struct drm_connector *connector) -{ - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - struct drm_encoder *encoder; - struct edid *edid = amdgpu_dm_connector->edid; - struct dc_link_settings *verified_link_cap = - &amdgpu_dm_connector->dc_link->verified_link_cap; - const struct dc *dc = amdgpu_dm_connector->dc_link->dc; - - encoder = amdgpu_dm_connector_to_encoder(connector); - - if (!drm_edid_is_valid(edid)) { - amdgpu_dm_connector->num_modes = - drm_add_modes_noedid(connector, 640, 480); - if (dc->link_srv->dp_get_encoding_format(verified_link_cap) == DP_128b_132b_ENCODING) - amdgpu_dm_connector->num_modes += - drm_add_modes_noedid(connector, 1920, 1080); - } else { - amdgpu_dm_connector_ddc_get_modes(connector, edid); - if (encoder) - amdgpu_dm_connector_add_common_modes(encoder, connector); - amdgpu_dm_connector_add_freesync_modes(connector, edid); - } - amdgpu_dm_fbc_init(connector); - - return amdgpu_dm_connector->num_modes; -} - -static const u32 supported_colorspaces = - BIT(DRM_MODE_COLORIMETRY_BT709_YCC) | - BIT(DRM_MODE_COLORIMETRY_OPRGB) | - BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) | - BIT(DRM_MODE_COLORIMETRY_BT2020_YCC); - -void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector, - int connector_type, - struct dc_link *link, - int link_index) -{ - struct amdgpu_device *adev = drm_to_adev(dm->ddev); - - /* - * Some of the properties below require access to state, like bpc. - * Allocate some default initial connector state with our reset helper. - */ - if (aconnector->base.funcs->reset) - aconnector->base.funcs->reset(&aconnector->base); - - aconnector->connector_id = link_index; - aconnector->bl_idx = -1; - aconnector->dc_link = link; - aconnector->base.interlace_allowed = false; - aconnector->base.doublescan_allowed = false; - aconnector->base.stereo_allowed = false; - aconnector->base.dpms = DRM_MODE_DPMS_OFF; - aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */ - aconnector->audio_inst = -1; - aconnector->pack_sdp_v1_3 = false; - aconnector->as_type = ADAPTIVE_SYNC_TYPE_NONE; - memset(&aconnector->vsdb_info, 0, sizeof(aconnector->vsdb_info)); - mutex_init(&aconnector->hpd_lock); - mutex_init(&aconnector->handle_mst_msg_ready); - - /* - * configure support HPD hot plug connector_>polled default value is 0 - * which means HPD hot plug not supported - */ - switch (connector_type) { - case DRM_MODE_CONNECTOR_HDMIA: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - aconnector->base.ycbcr_420_allowed = - link->link_enc->features.hdmi_ycbcr420_supported ? true : false; - break; - case DRM_MODE_CONNECTOR_DisplayPort: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - link->link_enc = link_enc_cfg_get_link_enc(link); - ASSERT(link->link_enc); - if (link->link_enc) - aconnector->base.ycbcr_420_allowed = - link->link_enc->features.dp_ycbcr420_supported ? true : false; - break; - case DRM_MODE_CONNECTOR_DVID: - aconnector->base.polled = DRM_CONNECTOR_POLL_HPD; - break; - default: - break; - } - - drm_object_attach_property(&aconnector->base.base, - dm->ddev->mode_config.scaling_mode_property, - DRM_MODE_SCALE_NONE); - - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_property, - UNDERSCAN_OFF); - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_hborder_property, - 0); - drm_object_attach_property(&aconnector->base.base, - adev->mode_info.underscan_vborder_property, - 0); - - if (!aconnector->mst_root) - drm_connector_attach_max_bpc_property(&aconnector->base, 8, 16); - - aconnector->base.state->max_bpc = 16; - aconnector->base.state->max_requested_bpc = aconnector->base.state->max_bpc; - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA) { - /* Content Type is currently only implemented for HDMI. */ - drm_connector_attach_content_type_property(&aconnector->base); - } - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA) { - if (!drm_mode_create_hdmi_colorspace_property(&aconnector->base, supported_colorspaces)) - drm_connector_attach_colorspace_property(&aconnector->base); - } else if ((connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root) || - connector_type == DRM_MODE_CONNECTOR_eDP) { - if (!drm_mode_create_dp_colorspace_property(&aconnector->base, supported_colorspaces)) - drm_connector_attach_colorspace_property(&aconnector->base); - } - - if (connector_type == DRM_MODE_CONNECTOR_HDMIA || - connector_type == DRM_MODE_CONNECTOR_DisplayPort || - connector_type == DRM_MODE_CONNECTOR_eDP) { - drm_connector_attach_hdr_output_metadata_property(&aconnector->base); - - if (!aconnector->mst_root) - drm_connector_attach_vrr_capable_property(&aconnector->base); - - if (adev->dm.hdcp_workqueue) - drm_connector_attach_content_protection_property(&aconnector->base, true); - } -} - -static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap, - struct i2c_msg *msgs, int num) -{ - struct amdgpu_i2c_adapter *i2c = i2c_get_adapdata(i2c_adap); - struct ddc_service *ddc_service = i2c->ddc_service; - struct i2c_command cmd; - int i; - int result = -EIO; - - if (!ddc_service->ddc_pin || !ddc_service->ddc_pin->hw_info.hw_supported) - return result; - - cmd.payloads = kcalloc(num, sizeof(struct i2c_payload), GFP_KERNEL); - - if (!cmd.payloads) - return result; - - cmd.number_of_payloads = num; - cmd.engine = I2C_COMMAND_ENGINE_DEFAULT; - cmd.speed = 100; - - for (i = 0; i < num; i++) { - cmd.payloads[i].write = !(msgs[i].flags & I2C_M_RD); - cmd.payloads[i].address = msgs[i].addr; - cmd.payloads[i].length = msgs[i].len; - cmd.payloads[i].data = msgs[i].buf; - } - - if (dc_submit_i2c( - ddc_service->ctx->dc, - ddc_service->link->link_index, - &cmd)) - result = num; - - kfree(cmd.payloads); - return result; -} - -static u32 amdgpu_dm_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -} - -static const struct i2c_algorithm amdgpu_dm_i2c_algo = { - .master_xfer = amdgpu_dm_i2c_xfer, - .functionality = amdgpu_dm_i2c_func, -}; - -static struct amdgpu_i2c_adapter * -create_i2c(struct ddc_service *ddc_service, - int link_index, - int *res) -{ - struct amdgpu_device *adev = ddc_service->ctx->driver_context; - struct amdgpu_i2c_adapter *i2c; - - i2c = kzalloc(sizeof(struct amdgpu_i2c_adapter), GFP_KERNEL); - if (!i2c) - return NULL; - i2c->base.owner = THIS_MODULE; - i2c->base.dev.parent = &adev->pdev->dev; - i2c->base.algo = &amdgpu_dm_i2c_algo; - snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c hw bus %d", link_index); - i2c_set_adapdata(&i2c->base, i2c); - i2c->ddc_service = ddc_service; - - return i2c; -} - - -/* - * Note: this function assumes that dc_link_detect() was called for the - * dc_link which will be represented by this aconnector. - */ -static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, - struct amdgpu_dm_connector *aconnector, - u32 link_index, - struct amdgpu_encoder *aencoder) -{ - int res = 0; - int connector_type; - struct dc *dc = dm->dc; - struct dc_link *link = dc_get_link_at_index(dc, link_index); - struct amdgpu_i2c_adapter *i2c; - - /* Not needed for writeback connector */ - link->priv = aconnector; - - - i2c = create_i2c(link->ddc, link->link_index, &res); - if (!i2c) { - DRM_ERROR("Failed to create i2c adapter data\n"); - return -ENOMEM; - } - - aconnector->i2c = i2c; - res = i2c_add_adapter(&i2c->base); - - if (res) { - DRM_ERROR("Failed to register hw i2c %d\n", link->link_index); - goto out_free; - } - - connector_type = to_drm_connector_type(link->connector_signal); - - res = drm_connector_init_with_ddc( - dm->ddev, - &aconnector->base, - &amdgpu_dm_connector_funcs, - connector_type, - &i2c->base); - - if (res) { - DRM_ERROR("connector_init failed\n"); - aconnector->connector_id = -1; - goto out_free; - } - - drm_connector_helper_add( - &aconnector->base, - &amdgpu_dm_connector_helper_funcs); - - amdgpu_dm_connector_init_helper( - dm, - aconnector, - connector_type, - link, - link_index); - - drm_connector_attach_encoder( - &aconnector->base, &aencoder->base); - - if (connector_type == DRM_MODE_CONNECTOR_DisplayPort - || connector_type == DRM_MODE_CONNECTOR_eDP) - amdgpu_dm_initialize_dp_connector(dm, aconnector, link->link_index); - -out_free: - if (res) { - kfree(i2c); - aconnector->i2c = NULL; - } - return res; -} - -int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev) -{ - switch (adev->mode_info.num_crtc) { - case 1: - return 0x1; - case 2: - return 0x3; - case 3: - return 0x7; - case 4: - return 0xf; - case 5: - return 0x1f; - case 6: - default: - return 0x3f; - } -} - -static int amdgpu_dm_encoder_init(struct drm_device *dev, - struct amdgpu_encoder *aencoder, - uint32_t link_index) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - - int res = drm_encoder_init(dev, - &aencoder->base, - &amdgpu_dm_encoder_funcs, - DRM_MODE_ENCODER_TMDS, - NULL); - - aencoder->base.possible_crtcs = amdgpu_dm_get_encoder_crtc_mask(adev); - - if (!res) - aencoder->encoder_id = link_index; - else - aencoder->encoder_id = -1; - - drm_encoder_helper_add(&aencoder->base, &amdgpu_dm_encoder_helper_funcs); - - return res; -} - -static void manage_dm_interrupts(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc, - struct dm_crtc_state *acrtc_state) -{ -<<<<<<< - /* - * We have no guarantee that the frontend index maps to the same - * backend index - some even map to more than one. - * - * TODO: Use a different interrupt or check DC itself for the mapping. - */ - int irq_type = - amdgpu_display_crtc_idx_to_irq_type( - adev, - acrtc->crtc_id); - struct drm_vblank_crtc_config config = {0}; - struct dc_crtc_timing *timing; - int offdelay; - - if (acrtc_state) { - if (amdgpu_ip_version(adev, DCE_HWIP, 0) < - IP_VERSION(3, 5, 0) || - acrtc_state->stream->link->psr_settings.psr_version < - DC_PSR_VERSION_UNSUPPORTED) { - timing = &acrtc_state->stream->timing; - - /* at least 2 frames */ - offdelay = DIV64_U64_ROUND_UP((u64)20 * - timing->v_total * - timing->h_total, - timing->pix_clk_100hz); - - config.offdelay_ms = offdelay ?: 30; - } else { - config.disable_immediate = true; - } - - drm_crtc_vblank_on_config(&acrtc->base, - &config); - - amdgpu_irq_get( - adev, - &adev->pageflip_irq, - irq_type); -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - amdgpu_irq_get( - adev, - &adev->vline0_irq, - irq_type); -#endif - } else { -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - amdgpu_irq_put( - adev, - &adev->vline0_irq, - irq_type); -#endif - amdgpu_irq_put( - adev, - &adev->pageflip_irq, - irq_type); -======= - if (enable) - drm_crtc_vblank_on(&acrtc->base); - else ->>>>>>> - drm_crtc_vblank_off(&acrtc->base); -} - -static void dm_update_pflip_irq_state(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc) -{ - int irq_type = - amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); - - /** - * This reads the current state for the IRQ and force reapplies - * the setting to hardware. - */ - amdgpu_irq_update(adev, &adev->pageflip_irq, irq_type); -} - -static bool -is_scaling_state_different(const struct dm_connector_state *dm_state, - const struct dm_connector_state *old_dm_state) -{ - if (dm_state->scaling != old_dm_state->scaling) - return true; - if (!dm_state->underscan_enable && old_dm_state->underscan_enable) { - if (old_dm_state->underscan_hborder != 0 && old_dm_state->underscan_vborder != 0) - return true; - } else if (dm_state->underscan_enable && !old_dm_state->underscan_enable) { - if (dm_state->underscan_hborder != 0 && dm_state->underscan_vborder != 0) - return true; - } else if (dm_state->underscan_hborder != old_dm_state->underscan_hborder || - dm_state->underscan_vborder != old_dm_state->underscan_vborder) - return true; - return false; -} - -static bool is_content_protection_different(struct drm_crtc_state *new_crtc_state, - struct drm_crtc_state *old_crtc_state, - struct drm_connector_state *new_conn_state, - struct drm_connector_state *old_conn_state, - const struct drm_connector *connector, - struct hdcp_workqueue *hdcp_w) -{ - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); - - pr_debug("[HDCP_DM] connector->index: %x connect_status: %x dpms: %x\n", - connector->index, connector->status, connector->dpms); - pr_debug("[HDCP_DM] state protection old: %x new: %x\n", - old_conn_state->content_protection, new_conn_state->content_protection); - - if (old_crtc_state) - pr_debug("[HDCP_DM] old crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - old_crtc_state->enable, - old_crtc_state->active, - old_crtc_state->mode_changed, - old_crtc_state->active_changed, - old_crtc_state->connectors_changed); - - if (new_crtc_state) - pr_debug("[HDCP_DM] NEW crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* hdcp content type change */ - if (old_conn_state->hdcp_content_type != new_conn_state->hdcp_content_type && - new_conn_state->content_protection != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - pr_debug("[HDCP_DM] Type0/1 change %s :true\n", __func__); - return true; - } - - /* CP is being re enabled, ignore this */ - if (old_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) { - if (new_crtc_state && new_crtc_state->mode_changed) { - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - pr_debug("[HDCP_DM] ENABLED->DESIRED & mode_changed %s :true\n", __func__); - return true; - } - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_ENABLED; - pr_debug("[HDCP_DM] ENABLED -> DESIRED %s :false\n", __func__); - return false; - } - - /* S3 resume case, since old state will always be 0 (UNDESIRED) and the restored state will be ENABLED - * - * Handles: UNDESIRED -> ENABLED - */ - if (old_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_UNDESIRED && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) - new_conn_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - - /* Stream removed and re-enabled - * - * Can sometimes overlap with the HPD case, - * thus set update_hdcp to false to avoid - * setting HDCP multiple times. - * - * Handles: DESIRED -> DESIRED (Special case) - */ - if (!(old_conn_state->crtc && old_conn_state->crtc->enabled) && - new_conn_state->crtc && new_conn_state->crtc->enabled && - connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) { - dm_con_state->update_hdcp = false; - pr_debug("[HDCP_DM] DESIRED->DESIRED (Stream removed and re-enabled) %s :true\n", - __func__); - return true; - } - - /* Hot-plug, headless s3, dpms - * - * Only start HDCP if the display is connected/enabled. - * update_hdcp flag will be set to false until the next - * HPD comes in. - * - * Handles: DESIRED -> DESIRED (Special case) - */ - if (dm_con_state->update_hdcp && - new_conn_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED && - connector->dpms == DRM_MODE_DPMS_ON && aconnector->dc_sink != NULL) { - dm_con_state->update_hdcp = false; - pr_debug("[HDCP_DM] DESIRED->DESIRED (Hot-plug, headless s3, dpms) %s :true\n", - __func__); - return true; - } - - if (old_conn_state->content_protection == new_conn_state->content_protection) { - if (new_conn_state->content_protection >= DRM_MODE_CONTENT_PROTECTION_DESIRED) { - if (new_crtc_state && new_crtc_state->mode_changed) { - pr_debug("[HDCP_DM] DESIRED->DESIRED or ENABLE->ENABLE mode_change %s :true\n", - __func__); - return true; - } - pr_debug("[HDCP_DM] DESIRED->DESIRED & ENABLE->ENABLE %s :false\n", - __func__); - return false; - } - - pr_debug("[HDCP_DM] UNDESIRED->UNDESIRED %s :false\n", __func__); - return false; - } - - if (new_conn_state->content_protection != DRM_MODE_CONTENT_PROTECTION_ENABLED) { - pr_debug("[HDCP_DM] UNDESIRED->DESIRED or DESIRED->UNDESIRED or ENABLED->UNDESIRED %s :true\n", - __func__); - return true; - } - - pr_debug("[HDCP_DM] DESIRED->ENABLED %s :false\n", __func__); - return false; -} - -static void remove_stream(struct amdgpu_device *adev, - struct amdgpu_crtc *acrtc, - struct dc_stream_state *stream) -{ - /* this is the update mode case */ - - acrtc->otg_inst = -1; - acrtc->enabled = false; -} - -static void prepare_flip_isr(struct amdgpu_crtc *acrtc) -{ - - assert_spin_locked(&acrtc->base.dev->event_lock); - WARN_ON(acrtc->event); - - acrtc->event = acrtc->base.state->event; - - /* Set the flip status */ - acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED; - - /* Mark this event as consumed */ - acrtc->base.state->event = NULL; - - drm_dbg_state(acrtc->base.dev, - "crtc:%d, pflip_stat:AMDGPU_FLIP_SUBMITTED\n", - acrtc->crtc_id); -} - -static void update_freesync_state_on_stream( - struct amdgpu_display_manager *dm, - struct dm_crtc_state *new_crtc_state, - struct dc_stream_state *new_stream, - struct dc_plane_state *surface, - u32 flip_timestamp_in_us) -{ - struct mod_vrr_params vrr_params; - struct dc_info_packet vrr_infopacket = {0}; - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(new_crtc_state->base.crtc); - unsigned long flags; - bool pack_sdp_v1_3 = false; - struct amdgpu_dm_connector *aconn; - enum vrr_packet_type packet_type = PACKET_TYPE_VRR; - - if (!new_stream) - return; - - /* - * TODO: Determine why min/max totals and vrefresh can be 0 here. - * For now it's sufficient to just guard against these conditions. - */ - - if (!new_stream->timing.h_total || !new_stream->timing.v_total) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - vrr_params = acrtc->dm_irq_params.vrr_params; - - if (surface) { - mod_freesync_handle_preflip( - dm->freesync_module, - surface, - new_stream, - flip_timestamp_in_us, - &vrr_params); - - if (adev->family < AMDGPU_FAMILY_AI && - amdgpu_dm_crtc_vrr_active(new_crtc_state)) { - mod_freesync_handle_v_update(dm->freesync_module, - new_stream, &vrr_params); - - /* Need to call this before the frame ends. */ - dc_stream_adjust_vmin_vmax(dm->dc, - new_crtc_state->stream, - &vrr_params.adjust); - } - } - - aconn = (struct amdgpu_dm_connector *)new_stream->dm_stream_context; - - if (aconn && (aconn->as_type == FREESYNC_TYPE_PCON_IN_WHITELIST || aconn->vsdb_info.replay_mode)) { - pack_sdp_v1_3 = aconn->pack_sdp_v1_3; - - if (aconn->vsdb_info.amd_vsdb_version == 1) - packet_type = PACKET_TYPE_FS_V1; - else if (aconn->vsdb_info.amd_vsdb_version == 2) - packet_type = PACKET_TYPE_FS_V2; - else if (aconn->vsdb_info.amd_vsdb_version == 3) - packet_type = PACKET_TYPE_FS_V3; - - mod_build_adaptive_sync_infopacket(new_stream, aconn->as_type, NULL, - &new_stream->adaptive_sync_infopacket); - } - - mod_freesync_build_vrr_infopacket( - dm->freesync_module, - new_stream, - &vrr_params, - packet_type, - TRANSFER_FUNC_UNKNOWN, - &vrr_infopacket, - pack_sdp_v1_3); - - new_crtc_state->freesync_vrr_info_changed |= - (memcmp(&new_crtc_state->vrr_infopacket, - &vrr_infopacket, - sizeof(vrr_infopacket)) != 0); - - acrtc->dm_irq_params.vrr_params = vrr_params; - new_crtc_state->vrr_infopacket = vrr_infopacket; - - new_stream->vrr_infopacket = vrr_infopacket; - new_stream->allow_freesync = mod_freesync_get_freesync_enabled(&vrr_params); - - if (new_crtc_state->freesync_vrr_info_changed) - DRM_DEBUG_KMS("VRR packet update: crtc=%u enabled=%d state=%d", - new_crtc_state->base.crtc->base.id, - (int)new_crtc_state->base.vrr_enabled, - (int)vrr_params.state); - - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -static void update_stream_irq_parameters( - struct amdgpu_display_manager *dm, - struct dm_crtc_state *new_crtc_state) -{ - struct dc_stream_state *new_stream = new_crtc_state->stream; - struct mod_vrr_params vrr_params; - struct mod_freesync_config config = new_crtc_state->freesync_config; - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(new_crtc_state->base.crtc); - unsigned long flags; - - if (!new_stream) - return; - - /* - * TODO: Determine why min/max totals and vrefresh can be 0 here. - * For now it's sufficient to just guard against these conditions. - */ - if (!new_stream->timing.h_total || !new_stream->timing.v_total) - return; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - vrr_params = acrtc->dm_irq_params.vrr_params; - - if (new_crtc_state->vrr_supported && - config.min_refresh_in_uhz && - config.max_refresh_in_uhz) { - /* - * if freesync compatible mode was set, config.state will be set - * in atomic check - */ - if (config.state == VRR_STATE_ACTIVE_FIXED && config.fixed_refresh_in_uhz && - (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) || - new_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED)) { - vrr_params.max_refresh_in_uhz = config.max_refresh_in_uhz; - vrr_params.min_refresh_in_uhz = config.min_refresh_in_uhz; - vrr_params.fixed_refresh_in_uhz = config.fixed_refresh_in_uhz; - vrr_params.state = VRR_STATE_ACTIVE_FIXED; - } else { - config.state = new_crtc_state->base.vrr_enabled ? - VRR_STATE_ACTIVE_VARIABLE : - VRR_STATE_INACTIVE; - } - } else { - config.state = VRR_STATE_UNSUPPORTED; - } - - mod_freesync_build_vrr_params(dm->freesync_module, - new_stream, - &config, &vrr_params); - - new_crtc_state->freesync_config = config; - /* Copy state for access from DM IRQ handler */ - acrtc->dm_irq_params.freesync_config = config; - acrtc->dm_irq_params.active_planes = new_crtc_state->active_planes; - acrtc->dm_irq_params.vrr_params = vrr_params; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -} - -static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, - struct dm_crtc_state *new_state) -{ - bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state); - bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state); - - if (!old_vrr_active && new_vrr_active) { - /* Transition VRR inactive -> active: - * While VRR is active, we must not disable vblank irq, as a - * reenable after disable would compute bogus vblank/pflip - * timestamps if it likely happened inside display front-porch. - * - * We also need vupdate irq for the actual core vblank handling - * at end of vblank. - */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); - WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0); - DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n", - __func__, new_state->base.crtc->base.id); - } else if (old_vrr_active && !new_vrr_active) { - /* Transition VRR active -> inactive: - * Allow vblank irq disable again for fixed refresh rate. - */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); - drm_crtc_vblank_put(new_state->base.crtc); - DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n", - __func__, new_state->base.crtc->base.id); - } -} - -static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state) -{ - struct drm_plane *plane; - struct drm_plane_state *old_plane_state; - int i; - - /* - * TODO: Make this per-stream so we don't issue redundant updates for - * commits with multiple streams. - */ - for_each_old_plane_in_state(state, plane, old_plane_state, i) - if (plane->type == DRM_PLANE_TYPE_CURSOR) - amdgpu_dm_plane_handle_cursor_update(plane, old_plane_state); -} - -static inline uint32_t get_mem_type(struct drm_framebuffer *fb) -{ - struct amdgpu_bo *abo = gem_to_amdgpu_bo(fb->obj[0]); - - return abo->tbo.resource ? abo->tbo.resource->mem_type : 0; -} - -static void amdgpu_dm_update_cursor(struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct dc_stream_update *update) -{ - struct amdgpu_device *adev = drm_to_adev(plane->dev); - struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(plane->state->fb); - struct drm_crtc *crtc = afb ? plane->state->crtc : old_plane_state->crtc; - struct dm_crtc_state *crtc_state = crtc ? to_dm_crtc_state(crtc->state) : NULL; - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - uint64_t address = afb ? afb->address : 0; - struct dc_cursor_position position = {0}; - struct dc_cursor_attributes attributes; - int ret; - - if (!plane->state->fb && !old_plane_state->fb) - return; - - drm_dbg_atomic(plane->dev, "crtc_id=%d with size %d to %d\n", - amdgpu_crtc->crtc_id, plane->state->crtc_w, - plane->state->crtc_h); - - ret = amdgpu_dm_plane_get_cursor_position(plane, crtc, &position); - if (ret) - return; - - if (!position.enable) { - /* turn off cursor */ - if (crtc_state && crtc_state->stream) { - dc_stream_set_cursor_position(crtc_state->stream, - &position); - update->cursor_position = &crtc_state->stream->cursor_position; - } - return; - } - - amdgpu_crtc->cursor_width = plane->state->crtc_w; - amdgpu_crtc->cursor_height = plane->state->crtc_h; - - memset(&attributes, 0, sizeof(attributes)); - attributes.address.high_part = upper_32_bits(address); - attributes.address.low_part = lower_32_bits(address); - attributes.width = plane->state->crtc_w; - attributes.height = plane->state->crtc_h; - attributes.color_format = CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA; - attributes.rotation_angle = 0; - attributes.attribute_flags.value = 0; - - /* Enable cursor degamma ROM on DCN3+ for implicit sRGB degamma in DRM - * legacy gamma setup. - */ - if (crtc_state->cm_is_degamma_srgb && - adev->dm.dc->caps.color.dpp.gamma_corr) - attributes.attribute_flags.bits.ENABLE_CURSOR_DEGAMMA = 1; - - if (afb) - attributes.pitch = afb->base.pitches[0] / afb->base.format->cpp[0]; - - if (crtc_state->stream) { - if (!dc_stream_set_cursor_attributes(crtc_state->stream, - &attributes)) - DRM_ERROR("DC failed to set cursor attributes\n"); - - update->cursor_attributes = &crtc_state->stream->cursor_attributes; - - if (!dc_stream_set_cursor_position(crtc_state->stream, - &position)) - DRM_ERROR("DC failed to set cursor position\n"); - - update->cursor_position = &crtc_state->stream->cursor_position; - } -} - -static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, - struct drm_device *dev, - struct amdgpu_display_manager *dm, - struct drm_crtc *pcrtc, - bool wait_for_vblank) -{ - u32 i; - u64 timestamp_ns = ktime_get_ns(); - struct drm_plane *plane; - struct drm_plane_state *old_plane_state, *new_plane_state; - struct amdgpu_crtc *acrtc_attach = to_amdgpu_crtc(pcrtc); - struct drm_crtc_state *new_pcrtc_state = - drm_atomic_get_new_crtc_state(state, pcrtc); - struct dm_crtc_state *acrtc_state = to_dm_crtc_state(new_pcrtc_state); - struct dm_crtc_state *dm_old_crtc_state = - to_dm_crtc_state(drm_atomic_get_old_crtc_state(state, pcrtc)); - int planes_count = 0, vpos, hpos; - unsigned long flags; - u32 target_vblank, last_flip_vblank; - bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state); - bool cursor_update = false; - bool pflip_present = false; - bool dirty_rects_changed = false; - bool updated_planes_and_streams = false; - struct { - struct dc_surface_update surface_updates[MAX_SURFACES]; - struct dc_plane_info plane_infos[MAX_SURFACES]; - struct dc_scaling_info scaling_infos[MAX_SURFACES]; - struct dc_flip_addrs flip_addrs[MAX_SURFACES]; - struct dc_stream_update stream_update; - } *bundle; - - bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - - if (!bundle) { - drm_err(dev, "Failed to allocate update bundle\n"); - goto cleanup; - } - - /* - * Disable the cursor first if we're disabling all the planes. - * It'll remain on the screen after the planes are re-enabled - * if we don't. - * - * If the cursor is transitioning from native to overlay mode, the - * native cursor needs to be disabled first. - */ - if (acrtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE && - dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) { - struct dc_cursor_position cursor_position = {0}; - - if (!dc_stream_set_cursor_position(acrtc_state->stream, - &cursor_position)) - drm_err(dev, "DC failed to disable native cursor\n"); - - bundle->stream_update.cursor_position = - &acrtc_state->stream->cursor_position; - } - - if (acrtc_state->active_planes == 0 && - dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) - amdgpu_dm_commit_cursors(state); - - /* update planes when needed */ - for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { - struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *new_crtc_state; - struct drm_framebuffer *fb = new_plane_state->fb; - struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)fb; - bool plane_needs_flip; - struct dc_plane_state *dc_plane; - struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state); - - /* Cursor plane is handled after stream updates */ - if (plane->type == DRM_PLANE_TYPE_CURSOR && - acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) { - if ((fb && crtc == pcrtc) || - (old_plane_state->fb && old_plane_state->crtc == pcrtc)) { - cursor_update = true; - if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) - amdgpu_dm_update_cursor(plane, old_plane_state, &bundle->stream_update); - } - - continue; - } - - if (!fb || !crtc || pcrtc != crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - if (!new_crtc_state->active) - continue; - - dc_plane = dm_new_plane_state->dc_state; - if (!dc_plane) - continue; - - bundle->surface_updates[planes_count].surface = dc_plane; - if (new_pcrtc_state->color_mgmt_changed) { - bundle->surface_updates[planes_count].gamma = &dc_plane->gamma_correction; - bundle->surface_updates[planes_count].in_transfer_func = &dc_plane->in_transfer_func; - bundle->surface_updates[planes_count].gamut_remap_matrix = &dc_plane->gamut_remap_matrix; - bundle->surface_updates[planes_count].hdr_mult = dc_plane->hdr_mult; - bundle->surface_updates[planes_count].func_shaper = &dc_plane->in_shaper_func; - bundle->surface_updates[planes_count].lut3d_func = &dc_plane->lut3d_func; - bundle->surface_updates[planes_count].blend_tf = &dc_plane->blend_tf; - } - - amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state, - &bundle->scaling_infos[planes_count]); - - bundle->surface_updates[planes_count].scaling_info = - &bundle->scaling_infos[planes_count]; - - plane_needs_flip = old_plane_state->fb && new_plane_state->fb; - - pflip_present = pflip_present || plane_needs_flip; - - if (!plane_needs_flip) { - planes_count += 1; - continue; - } - - fill_dc_plane_info_and_addr( - dm->adev, new_plane_state, - afb->tiling_flags, - &bundle->plane_infos[planes_count], - &bundle->flip_addrs[planes_count].address, - afb->tmz_surface, false); - - drm_dbg_state(state->dev, "plane: id=%d dcc_en=%d\n", - new_plane_state->plane->index, - bundle->plane_infos[planes_count].dcc.enable); - - bundle->surface_updates[planes_count].plane_info = - &bundle->plane_infos[planes_count]; - - if (acrtc_state->stream->link->psr_settings.psr_feature_enabled || - acrtc_state->stream->link->replay_settings.replay_feature_enabled) { - fill_dc_dirty_rects(plane, old_plane_state, - new_plane_state, new_crtc_state, - &bundle->flip_addrs[planes_count], - acrtc_state->stream->link->psr_settings.psr_version == - DC_PSR_VERSION_SU_1, - &dirty_rects_changed); - - /* - * If the dirty regions changed, PSR-SU need to be disabled temporarily - * and enabled it again after dirty regions are stable to avoid video glitch. - * PSR-SU will be enabled in vblank_control_worker() if user pause the video - * during the PSR-SU was disabled. - */ - if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 && - acrtc_attach->dm_irq_params.allow_psr_entry && -#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY - !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) && -#endif - dirty_rects_changed) { - mutex_lock(&dm->dc_lock); - acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns = - timestamp_ns; - if (acrtc_state->stream->link->psr_settings.psr_allow_active) - amdgpu_dm_psr_disable(acrtc_state->stream); - mutex_unlock(&dm->dc_lock); - } - } - - /* - * Only allow immediate flips for fast updates that don't - * change memory domain, FB pitch, DCC state, rotation or - * mirroring. - * - * dm_crtc_helper_atomic_check() only accepts async flips with - * fast updates. - */ - if (crtc->state->async_flip && - (acrtc_state->update_type != UPDATE_TYPE_FAST || - get_mem_type(old_plane_state->fb) != get_mem_type(fb))) - drm_warn_once(state->dev, - "[PLANE:%d:%s] async flip with non-fast update\n", - plane->base.id, plane->name); - - bundle->flip_addrs[planes_count].flip_immediate = - crtc->state->async_flip && - acrtc_state->update_type == UPDATE_TYPE_FAST && - get_mem_type(old_plane_state->fb) == get_mem_type(fb); - - timestamp_ns = ktime_get_ns(); - bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000); - bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count]; - bundle->surface_updates[planes_count].surface = dc_plane; - - if (!bundle->surface_updates[planes_count].surface) { - DRM_ERROR("No surface for CRTC: id=%d\n", - acrtc_attach->crtc_id); - continue; - } - - if (plane == pcrtc->primary) - update_freesync_state_on_stream( - dm, - acrtc_state, - acrtc_state->stream, - dc_plane, - bundle->flip_addrs[planes_count].flip_timestamp_in_us); - - drm_dbg_state(state->dev, "%s Flipping to hi: 0x%x, low: 0x%x\n", - __func__, - bundle->flip_addrs[planes_count].address.grph.addr.high_part, - bundle->flip_addrs[planes_count].address.grph.addr.low_part); - - planes_count += 1; - - } - - if (pflip_present) { - if (!vrr_active) { - /* Use old throttling in non-vrr fixed refresh rate mode - * to keep flip scheduling based on target vblank counts - * working in a backwards compatible way, e.g., for - * clients using the GLX_OML_sync_control extension or - * DRI3/Present extension with defined target_msc. - */ - last_flip_vblank = amdgpu_get_vblank_counter_kms(pcrtc); - } else { - /* For variable refresh rate mode only: - * Get vblank of last completed flip to avoid > 1 vrr - * flips per video frame by use of throttling, but allow - * flip programming anywhere in the possibly large - * variable vrr vblank interval for fine-grained flip - * timing control and more opportunity to avoid stutter - * on late submission of flips. - */ - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - last_flip_vblank = acrtc_attach->dm_irq_params.last_flip_vblank; - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - target_vblank = last_flip_vblank + wait_for_vblank; - - /* - * Wait until we're out of the vertical blank period before the one - * targeted by the flip - */ - while ((acrtc_attach->enabled && - (amdgpu_display_get_crtc_scanoutpos(dm->ddev, acrtc_attach->crtc_id, - 0, &vpos, &hpos, NULL, - NULL, &pcrtc->hwmode) - & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) == - (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) && - (int)(target_vblank - - amdgpu_get_vblank_counter_kms(pcrtc)) > 0)) { - usleep_range(1000, 1100); - } - - /** - * Prepare the flip event for the pageflip interrupt to handle. - * - * This only works in the case where we've already turned on the - * appropriate hardware blocks (eg. HUBP) so in the transition case - * from 0 -> n planes we have to skip a hardware generated event - * and rely on sending it from software. - */ - if (acrtc_attach->base.state->event && - acrtc_state->active_planes > 0) { - drm_crtc_vblank_get(pcrtc); - - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - - WARN_ON(acrtc_attach->pflip_status != AMDGPU_FLIP_NONE); - prepare_flip_isr(acrtc_attach); - - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - if (acrtc_state->stream) { - if (acrtc_state->freesync_vrr_info_changed) - bundle->stream_update.vrr_infopacket = - &acrtc_state->stream->vrr_infopacket; - } - } else if (cursor_update && acrtc_state->active_planes > 0) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - if (acrtc_attach->base.state->event) { - drm_crtc_vblank_get(pcrtc); - acrtc_attach->event = acrtc_attach->base.state->event; - acrtc_attach->base.state->event = NULL; - } - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - - /* Update the planes if changed or disable if we don't have any. */ - if ((planes_count || acrtc_state->active_planes == 0) && - acrtc_state->stream) { - /* - * If PSR or idle optimizations are enabled then flush out - * any pending work before hardware programming. - */ - if (dm->vblank_control_workqueue) - flush_workqueue(dm->vblank_control_workqueue); - - bundle->stream_update.stream = acrtc_state->stream; - if (new_pcrtc_state->mode_changed) { - bundle->stream_update.src = acrtc_state->stream->src; - bundle->stream_update.dst = acrtc_state->stream->dst; - } - - if (new_pcrtc_state->color_mgmt_changed) { - /* - * TODO: This isn't fully correct since we've actually - * already modified the stream in place. - */ - bundle->stream_update.gamut_remap = - &acrtc_state->stream->gamut_remap_matrix; - bundle->stream_update.output_csc_transform = - &acrtc_state->stream->csc_color_matrix; - bundle->stream_update.out_transfer_func = - &acrtc_state->stream->out_transfer_func; - bundle->stream_update.lut3d_func = - (struct dc_3dlut *) acrtc_state->stream->lut3d_func; - bundle->stream_update.func_shaper = - (struct dc_transfer_func *) acrtc_state->stream->func_shaper; - } - - acrtc_state->stream->abm_level = acrtc_state->abm_level; - if (acrtc_state->abm_level != dm_old_crtc_state->abm_level) - bundle->stream_update.abm_level = &acrtc_state->abm_level; - - mutex_lock(&dm->dc_lock); - if ((acrtc_state->update_type > UPDATE_TYPE_FAST) && - acrtc_state->stream->link->psr_settings.psr_allow_active) - amdgpu_dm_psr_disable(acrtc_state->stream); - mutex_unlock(&dm->dc_lock); - - /* - * If FreeSync state on the stream has changed then we need to - * re-adjust the min/max bounds now that DC doesn't handle this - * as part of commit. - */ - if (is_dc_timing_adjust_needed(dm_old_crtc_state, acrtc_state)) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - dc_stream_adjust_vmin_vmax( - dm->dc, acrtc_state->stream, - &acrtc_attach->dm_irq_params.vrr_params.adjust); - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - mutex_lock(&dm->dc_lock); - update_planes_and_stream_adapter(dm->dc, - acrtc_state->update_type, - planes_count, - acrtc_state->stream, - &bundle->stream_update, - bundle->surface_updates); - updated_planes_and_streams = true; - - /** - * Enable or disable the interrupts on the backend. - * - * Most pipes are put into power gating when unused. - * - * When power gating is enabled on a pipe we lose the - * interrupt enablement state when power gating is disabled. - * - * So we need to update the IRQ control state in hardware - * whenever the pipe turns on (since it could be previously - * power gated) or off (since some pipes can't be power gated - * on some ASICs). - */ - if (dm_old_crtc_state->active_planes != acrtc_state->active_planes) - dm_update_pflip_irq_state(drm_to_adev(dev), - acrtc_attach); - - if (acrtc_state->update_type > UPDATE_TYPE_FAST) { - if (acrtc_state->stream->link->replay_settings.config.replay_supported && - !acrtc_state->stream->link->replay_settings.replay_feature_enabled) { - struct amdgpu_dm_connector *aconn = - (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; - amdgpu_dm_link_setup_replay(acrtc_state->stream->link, aconn); - } else if (acrtc_state->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED && - !acrtc_state->stream->link->psr_settings.psr_feature_enabled) { - - struct amdgpu_dm_connector *aconn = (struct amdgpu_dm_connector *) - acrtc_state->stream->dm_stream_context; - - if (!aconn->disallow_edp_enter_psr) - amdgpu_dm_link_setup_psr(acrtc_state->stream); - } - } - - /* Decrement skip count when PSR is enabled and we're doing fast updates. */ - if (acrtc_state->update_type == UPDATE_TYPE_FAST && - acrtc_state->stream->link->psr_settings.psr_feature_enabled) { - struct amdgpu_dm_connector *aconn = - (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; - - if (aconn->psr_skip_count > 0) - aconn->psr_skip_count--; - - /* Allow PSR when skip count is 0. */ - acrtc_attach->dm_irq_params.allow_psr_entry = !aconn->psr_skip_count; - - /* - * If sink supports PSR SU, there is no need to rely on - * a vblank event disable request to enable PSR. PSR SU - * can be enabled immediately once OS demonstrates an - * adequate number of fast atomic commits to notify KMD - * of update events. See `vblank_control_worker()`. - */ - if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 && - acrtc_attach->dm_irq_params.allow_psr_entry && -#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY - !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) && -#endif - !acrtc_state->stream->link->psr_settings.psr_allow_active && - !aconn->disallow_edp_enter_psr && - (timestamp_ns - - acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns) > - 500000000) - amdgpu_dm_psr_enable(acrtc_state->stream); - } else { - acrtc_attach->dm_irq_params.allow_psr_entry = false; - } - - mutex_unlock(&dm->dc_lock); - } - - /* - * Update cursor state *after* programming all the planes. - * This avoids redundant programming in the case where we're going - * to be disabling a single plane - those pipes are being disabled. - */ - if (acrtc_state->active_planes && - (!updated_planes_and_streams || amdgpu_ip_version(dm->adev, DCE_HWIP, 0) == 0) && - acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) - amdgpu_dm_commit_cursors(state); - -cleanup: - kfree(bundle); -} - -static void amdgpu_dm_commit_audio(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct drm_crtc_state *new_crtc_state; - struct dm_crtc_state *new_dm_crtc_state; - const struct dc_stream_status *status; - int i, inst; - - /* Notify device removals. */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - if (old_con_state->crtc != new_con_state->crtc) { - /* CRTC changes require notification. */ - goto notify; - } - - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_con_state->crtc); - - if (!new_crtc_state) - continue; - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - -notify: - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - mutex_lock(&adev->dm.audio_lock); - inst = aconnector->audio_inst; - aconnector->audio_inst = -1; - mutex_unlock(&adev->dm.audio_lock); - - amdgpu_dm_audio_eld_notify(adev, inst); - } - - /* Notify audio device additions. */ - for_each_new_connector_in_state(state, connector, new_con_state, i) { - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_con_state->crtc); - - if (!new_crtc_state) - continue; - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - - new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); - if (!new_dm_crtc_state->stream) - continue; - - status = dc_stream_get_status(new_dm_crtc_state->stream); - if (!status) - continue; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - mutex_lock(&adev->dm.audio_lock); - inst = status->audio_inst; - aconnector->audio_inst = inst; - mutex_unlock(&adev->dm.audio_lock); - - amdgpu_dm_audio_eld_notify(adev, inst); - } -} - -/* - * amdgpu_dm_crtc_copy_transient_flags - copy mirrored flags from DRM to DC - * @crtc_state: the DRM CRTC state - * @stream_state: the DC stream state. - * - * Copy the mirrored transient state flags from DRM, to DC. It is used to bring - * a dc_stream_state's flags in sync with a drm_crtc_state's flags. - */ -static void amdgpu_dm_crtc_copy_transient_flags(struct drm_crtc_state *crtc_state, - struct dc_stream_state *stream_state) -{ - stream_state->mode_changed = drm_atomic_crtc_needs_modeset(crtc_state); -} - -static void dm_clear_writeback(struct amdgpu_display_manager *dm, - struct dm_crtc_state *crtc_state) -{ - dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0); -} - -static void amdgpu_dm_commit_streams(struct drm_atomic_state *state, - struct dc_state *dc_state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct drm_connector_state *old_con_state; - struct drm_connector *connector; - bool mode_set_reset_required = false; - u32 i; - struct dc_commit_streams_params params = {dc_state->streams, dc_state->stream_count}; - - /* Disable writeback */ - for_each_old_connector_in_state(state, connector, old_con_state, i) { - struct dm_connector_state *dm_old_con_state; - struct amdgpu_crtc *acrtc; - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - old_crtc_state = NULL; - - dm_old_con_state = to_dm_connector_state(old_con_state); - if (!dm_old_con_state->base.crtc) - continue; - - acrtc = to_amdgpu_crtc(dm_old_con_state->base.crtc); - if (acrtc) - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - - if (!acrtc || !acrtc->wb_enabled) - continue; - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - dm_clear_writeback(dm, dm_old_crtc_state); - acrtc->wb_enabled = false; - } - - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (old_crtc_state->active && - (!new_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - manage_dm_interrupts(adev, acrtc, NULL); - dc_stream_release(dm_old_crtc_state->stream); - } - } - - drm_atomic_helper_calc_timestamping_constants(state); - - /* update changed items */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - drm_dbg_state(state->dev, - "amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, planes_changed:%d, mode_changed:%d,active_changed:%d,connectors_changed:%d\n", - acrtc->crtc_id, - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->planes_changed, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* Disable cursor if disabling crtc */ - if (old_crtc_state->active && !new_crtc_state->active) { - struct dc_cursor_position position; - - memset(&position, 0, sizeof(position)); - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_stream_program_cursor_position(dm_old_crtc_state->stream, &position); - mutex_unlock(&dm->dc_lock); - } - - /* Copy all transient state flags into dc state */ - if (dm_new_crtc_state->stream) { - amdgpu_dm_crtc_copy_transient_flags(&dm_new_crtc_state->base, - dm_new_crtc_state->stream); - } - - /* handles headless hotplug case, updating new_state and - * aconnector as needed - */ - - if (amdgpu_dm_crtc_modeset_required(new_crtc_state, dm_new_crtc_state->stream, dm_old_crtc_state->stream)) { - - drm_dbg_atomic(dev, - "Atomic commit: SET crtc id %d: [%p]\n", - acrtc->crtc_id, acrtc); - - if (!dm_new_crtc_state->stream) { - /* - * this could happen because of issues with - * userspace notifications delivery. - * In this case userspace tries to set mode on - * display which is disconnected in fact. - * dc_sink is NULL in this case on aconnector. - * We expect reset mode will come soon. - * - * This can also happen when unplug is done - * during resume sequence ended - * - * In this case, we want to pretend we still - * have a sink to keep the pipe running so that - * hw state is consistent with the sw state - */ - drm_dbg_atomic(dev, - "Failed to create new stream for crtc %d\n", - acrtc->base.base.id); - continue; - } - - if (dm_old_crtc_state->stream) - remove_stream(adev, acrtc, dm_old_crtc_state->stream); - - pm_runtime_get_noresume(dev->dev); - - acrtc->enabled = true; - acrtc->hw_mode = new_crtc_state->mode; - crtc->hwmode = new_crtc_state->mode; - mode_set_reset_required = true; - } else if (modereset_required(new_crtc_state)) { - drm_dbg_atomic(dev, - "Atomic commit: RESET. crtc id %d:[%p]\n", - acrtc->crtc_id, acrtc); - /* i.e. reset mode */ - if (dm_old_crtc_state->stream) - remove_stream(adev, acrtc, dm_old_crtc_state->stream); - - mode_set_reset_required = true; - } - } /* for_each_crtc_in_state() */ - - /* if there mode set or reset, disable eDP PSR, Replay */ - if (mode_set_reset_required) { - if (dm->vblank_control_workqueue) - flush_workqueue(dm->vblank_control_workqueue); - - amdgpu_dm_replay_disable_all(dm); - amdgpu_dm_psr_disable_all(dm); - } - - dm_enable_per_frame_crtc_master_sync(dc_state); - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - WARN_ON(!dc_commit_streams(dm->dc, ¶ms)); - - /* Allow idle optimization when vblank count is 0 for display off */ - if (dm->active_vblank_irq_count == 0) - dc_allow_idle_optimizations(dm->dc, true); - mutex_unlock(&dm->dc_lock); - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state->stream != NULL) { - const struct dc_stream_status *status = - dc_stream_get_status(dm_new_crtc_state->stream); - - if (!status) - status = dc_state_get_stream_status(dc_state, - dm_new_crtc_state->stream); - if (!status) - drm_err(dev, - "got no status for stream %p on acrtc%p\n", - dm_new_crtc_state->stream, acrtc); - else - acrtc->otg_inst = status->primary_otg_inst; - } - } -} - -static void dm_set_writeback(struct amdgpu_display_manager *dm, - struct dm_crtc_state *crtc_state, - struct drm_connector *connector, - struct drm_connector_state *new_con_state) -{ - struct drm_writeback_connector *wb_conn = drm_connector_to_writeback(connector); - struct amdgpu_device *adev = dm->adev; - struct amdgpu_crtc *acrtc; - struct dc_writeback_info *wb_info; - struct pipe_ctx *pipe = NULL; - struct amdgpu_framebuffer *afb; - int i = 0; - - wb_info = kzalloc(sizeof(*wb_info), GFP_KERNEL); - if (!wb_info) { - DRM_ERROR("Failed to allocate wb_info\n"); - return; - } - - acrtc = to_amdgpu_crtc(wb_conn->encoder.crtc); - if (!acrtc) { - DRM_ERROR("no amdgpu_crtc found\n"); - kfree(wb_info); - return; - } - - afb = to_amdgpu_framebuffer(new_con_state->writeback_job->fb); - if (!afb) { - DRM_ERROR("No amdgpu_framebuffer found\n"); - kfree(wb_info); - return; - } - - for (i = 0; i < MAX_PIPES; i++) { - if (dm->dc->current_state->res_ctx.pipe_ctx[i].stream == crtc_state->stream) { - pipe = &dm->dc->current_state->res_ctx.pipe_ctx[i]; - break; - } - } - - /* fill in wb_info */ - wb_info->wb_enabled = true; - - wb_info->dwb_pipe_inst = 0; - wb_info->dwb_params.dwbscl_black_color = 0; - wb_info->dwb_params.hdr_mult = 0x1F000; - wb_info->dwb_params.csc_params.gamut_adjust_type = CM_GAMUT_ADJUST_TYPE_BYPASS; - wb_info->dwb_params.csc_params.gamut_coef_format = CM_GAMUT_REMAP_COEF_FORMAT_S2_13; - wb_info->dwb_params.output_depth = DWB_OUTPUT_PIXEL_DEPTH_10BPC; - wb_info->dwb_params.cnv_params.cnv_out_bpc = DWB_CNV_OUT_BPC_10BPC; - - /* width & height from crtc */ - wb_info->dwb_params.cnv_params.src_width = acrtc->base.mode.crtc_hdisplay; - wb_info->dwb_params.cnv_params.src_height = acrtc->base.mode.crtc_vdisplay; - wb_info->dwb_params.dest_width = acrtc->base.mode.crtc_hdisplay; - wb_info->dwb_params.dest_height = acrtc->base.mode.crtc_vdisplay; - - wb_info->dwb_params.cnv_params.crop_en = false; - wb_info->dwb_params.stereo_params.stereo_enabled = false; - - wb_info->dwb_params.cnv_params.out_max_pix_val = 0x3ff; // 10 bits - wb_info->dwb_params.cnv_params.out_min_pix_val = 0; - wb_info->dwb_params.cnv_params.fc_out_format = DWB_OUT_FORMAT_32BPP_ARGB; - wb_info->dwb_params.cnv_params.out_denorm_mode = DWB_OUT_DENORM_BYPASS; - - wb_info->dwb_params.out_format = dwb_scaler_mode_bypass444; - - wb_info->dwb_params.capture_rate = dwb_capture_rate_0; - - wb_info->dwb_params.scaler_taps.h_taps = 4; - wb_info->dwb_params.scaler_taps.v_taps = 4; - wb_info->dwb_params.scaler_taps.h_taps_c = 2; - wb_info->dwb_params.scaler_taps.v_taps_c = 2; - wb_info->dwb_params.subsample_position = DWB_INTERSTITIAL_SUBSAMPLING; - - wb_info->mcif_buf_params.luma_pitch = afb->base.pitches[0]; - wb_info->mcif_buf_params.chroma_pitch = afb->base.pitches[1]; - - for (i = 0; i < DWB_MCIF_BUF_COUNT; i++) { - wb_info->mcif_buf_params.luma_address[i] = afb->address; - wb_info->mcif_buf_params.chroma_address[i] = 0; - } - - wb_info->mcif_buf_params.p_vmid = 1; - if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0)) { - wb_info->mcif_warmup_params.start_address.quad_part = afb->address; - wb_info->mcif_warmup_params.region_size = - wb_info->mcif_buf_params.luma_pitch * wb_info->dwb_params.dest_height; - } - wb_info->mcif_warmup_params.p_vmid = 1; - wb_info->writeback_source_plane = pipe->plane_state; - - dc_stream_add_writeback(dm->dc, crtc_state->stream, wb_info); - - acrtc->wb_pending = true; - acrtc->wb_conn = wb_conn; - drm_writeback_queue_job(wb_conn, new_con_state); -} - -/** - * amdgpu_dm_atomic_commit_tail() - AMDgpu DM's commit tail implementation. - * @state: The atomic state to commit - * - * This will tell DC to commit the constructed DC state from atomic_check, - * programming the hardware. Any failures here implies a hardware failure, since - * atomic check should have filtered anything non-kosher. - */ -static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_display_manager *dm = &adev->dm; - struct dm_atomic_state *dm_state; - struct dc_state *dc_state = NULL; - u32 i, j; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - unsigned long flags; - bool wait_for_vblank = true; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - int crtc_disable_count = 0; - - trace_amdgpu_dm_atomic_commit_tail_begin(state); - - drm_atomic_helper_update_legacy_modeset_state(dev, state); - drm_dp_mst_atomic_wait_for_dependencies(state); - - dm_state = dm_atomic_get_new_state(state); - if (dm_state && dm_state->context) { - dc_state = dm_state->context; - amdgpu_dm_commit_streams(state, dc_state); - } - - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct amdgpu_dm_connector *aconnector; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!adev->dm.hdcp_workqueue) - continue; - - pr_debug("[HDCP_DM] -------------- i : %x ----------\n", i); - - if (!connector) - continue; - - pr_debug("[HDCP_DM] connector->index: %x connect_status: %x dpms: %x\n", - connector->index, connector->status, connector->dpms); - pr_debug("[HDCP_DM] state protection old: %x new: %x\n", - old_con_state->content_protection, new_con_state->content_protection); - - if (aconnector->dc_sink) { - if (aconnector->dc_sink->sink_signal != SIGNAL_TYPE_VIRTUAL && - aconnector->dc_sink->sink_signal != SIGNAL_TYPE_NONE) { - pr_debug("[HDCP_DM] pipe_ctx dispname=%s\n", - aconnector->dc_sink->edid_caps.display_name); - } - } - - new_crtc_state = NULL; - old_crtc_state = NULL; - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - if (old_crtc_state) - pr_debug("old crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - old_crtc_state->enable, - old_crtc_state->active, - old_crtc_state->mode_changed, - old_crtc_state->active_changed, - old_crtc_state->connectors_changed); - - if (new_crtc_state) - pr_debug("NEW crtc en: %x a: %x m: %x a-chg: %x c-chg: %x\n", - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - } - - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - - if (!adev->dm.hdcp_workqueue) - continue; - - new_crtc_state = NULL; - old_crtc_state = NULL; - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state && dm_new_crtc_state->stream == NULL && - connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) { - hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); - new_con_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; - dm_new_con_state->update_hdcp = true; - continue; - } - - if (is_content_protection_different(new_crtc_state, old_crtc_state, new_con_state, - old_con_state, connector, adev->dm.hdcp_workqueue)) { - /* when display is unplugged from mst hub, connctor will - * be destroyed within dm_dp_mst_connector_destroy. connector - * hdcp perperties, like type, undesired, desired, enabled, - * will be lost. So, save hdcp properties into hdcp_work within - * amdgpu_dm_atomic_commit_tail. if the same display is - * plugged back with same display index, its hdcp properties - * will be retrieved from hdcp_work within dm_dp_mst_get_modes - */ - - bool enable_encryption = false; - - if (new_con_state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) - enable_encryption = true; - - if (aconnector->dc_link && aconnector->dc_sink && - aconnector->dc_link->type == dc_connection_mst_branch) { - struct hdcp_workqueue *hdcp_work = adev->dm.hdcp_workqueue; - struct hdcp_workqueue *hdcp_w = - &hdcp_work[aconnector->dc_link->link_index]; - - hdcp_w->hdcp_content_type[connector->index] = - new_con_state->hdcp_content_type; - hdcp_w->content_protection[connector->index] = - new_con_state->content_protection; - } - - if (new_crtc_state && new_crtc_state->mode_changed && - new_con_state->content_protection >= DRM_MODE_CONTENT_PROTECTION_DESIRED) - enable_encryption = true; - - DRM_INFO("[HDCP_DM] hdcp_update_display enable_encryption = %x\n", enable_encryption); - - if (aconnector->dc_link) - hdcp_update_display( - adev->dm.hdcp_workqueue, aconnector->dc_link->link_index, aconnector, - new_con_state->hdcp_content_type, enable_encryption); - } - } - - /* Handle connector state changes */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct dc_surface_update *dummy_updates; - struct dc_stream_update stream_update; - struct dc_info_packet hdr_packet; - struct dc_stream_status *status = NULL; - bool abm_changed, hdr_changed, scaling_changed; - - memset(&stream_update, 0, sizeof(stream_update)); - - if (acrtc) { - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base); - } - - /* Skip any modesets/resets */ - if (!acrtc || drm_atomic_crtc_needs_modeset(new_crtc_state)) - continue; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - scaling_changed = is_scaling_state_different(dm_new_con_state, - dm_old_con_state); - - abm_changed = dm_new_crtc_state->abm_level != - dm_old_crtc_state->abm_level; - - hdr_changed = - !drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state); - - if (!scaling_changed && !abm_changed && !hdr_changed) - continue; - - stream_update.stream = dm_new_crtc_state->stream; - if (scaling_changed) { - update_stream_scaling_settings(&dm_new_con_state->base.crtc->mode, - dm_new_con_state, dm_new_crtc_state->stream); - - stream_update.src = dm_new_crtc_state->stream->src; - stream_update.dst = dm_new_crtc_state->stream->dst; - } - - if (abm_changed) { - dm_new_crtc_state->stream->abm_level = dm_new_crtc_state->abm_level; - - stream_update.abm_level = &dm_new_crtc_state->abm_level; - } - - if (hdr_changed) { - fill_hdr_info_packet(new_con_state, &hdr_packet); - stream_update.hdr_static_metadata = &hdr_packet; - } - - status = dc_stream_get_status(dm_new_crtc_state->stream); - - if (WARN_ON(!status)) - continue; - - WARN_ON(!status->plane_count); - - /* - * TODO: DC refuses to perform stream updates without a dc_surface_update. - * Here we create an empty update on each plane. - * To fix this, DC should permit updating only stream properties. - */ - dummy_updates = kzalloc(sizeof(struct dc_surface_update) * MAX_SURFACES, GFP_ATOMIC); - if (!dummy_updates) { - DRM_ERROR("Failed to allocate memory for dummy_updates.\n"); - continue; - } - for (j = 0; j < status->plane_count; j++) - dummy_updates[j].surface = status->plane_states[0]; - - sort(dummy_updates, status->plane_count, - sizeof(*dummy_updates), dm_plane_layer_index_cmp, NULL); - - mutex_lock(&dm->dc_lock); - dc_exit_ips_for_hw_access(dm->dc); - dc_update_planes_and_stream(dm->dc, - dummy_updates, - status->plane_count, - dm_new_crtc_state->stream, - &stream_update); - mutex_unlock(&dm->dc_lock); - kfree(dummy_updates); - } - - /** - * Enable interrupts for CRTCs that are newly enabled or went through - * a modeset. It was intentionally deferred until after the front end - * state was modified to wait until the OTG was on and so the IRQ - * handlers didn't access stale or invalid state. - */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); -#ifdef CONFIG_DEBUG_FS - enum amdgpu_dm_pipe_crc_source cur_crc_src; -#endif - /* Count number of newly disabled CRTCs for dropping PM refs later. */ - if (old_crtc_state->active && !new_crtc_state->active) - crtc_disable_count++; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - /* For freesync config update on crtc state and params for irq */ - update_stream_irq_parameters(dm, dm_new_crtc_state); - -#ifdef CONFIG_DEBUG_FS - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - cur_crc_src = acrtc->dm_irq_params.crc_src; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); -#endif - - if (new_crtc_state->active && - (!old_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - dc_stream_retain(dm_new_crtc_state->stream); - acrtc->dm_irq_params.stream = dm_new_crtc_state->stream; - manage_dm_interrupts(adev, acrtc, dm_new_crtc_state); - } - /* Handle vrr on->off / off->on transitions */ - amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state); - -#ifdef CONFIG_DEBUG_FS - if (new_crtc_state->active && - (!old_crtc_state->active || - drm_atomic_crtc_needs_modeset(new_crtc_state))) { - /** - * Frontend may have changed so reapply the CRC capture - * settings for the stream. - */ - if (amdgpu_dm_is_valid_crc_source(cur_crc_src)) { -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - if (amdgpu_dm_crc_window_is_activated(crtc)) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - acrtc->dm_irq_params.window_param.update_win = true; - - /** - * It takes 2 frames for HW to stably generate CRC when - * resuming from suspend, so we set skip_frame_cnt 2. - */ - acrtc->dm_irq_params.window_param.skip_frame_cnt = 2; - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } -#endif - if (amdgpu_dm_crtc_configure_crc_source( - crtc, dm_new_crtc_state, cur_crc_src)) - drm_dbg_atomic(dev, "Failed to configure crc source"); - } - } -#endif - } - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) - if (new_crtc_state->async_flip) - wait_for_vblank = false; - - /* update planes when needed per crtc*/ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (dm_new_crtc_state->stream) - amdgpu_dm_commit_planes(state, dev, dm, crtc, wait_for_vblank); - } - - /* Enable writeback */ - for_each_new_connector_in_state(state, connector, new_con_state, i) { - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - - if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - if (!new_con_state->writeback_job) - continue; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base); - - if (!new_crtc_state) - continue; - - if (acrtc->wb_enabled) - continue; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - dm_set_writeback(dm, dm_new_crtc_state, connector, new_con_state); - acrtc->wb_enabled = true; - } - - /* Update audio instances for each connector. */ - amdgpu_dm_commit_audio(dev, state); - - /* restore the backlight level */ - for (i = 0; i < dm->num_of_edps; i++) { - if (dm->backlight_dev[i] && - (dm->actual_brightness[i] != dm->brightness[i])) - amdgpu_dm_backlight_set_level(dm, i, dm->brightness[i]); - } - - /* - * send vblank event on all events not handled in flip and - * mark consumed event for drm_atomic_helper_commit_hw_done - */ - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - - if (new_crtc_state->event) - drm_send_event_locked(dev, &new_crtc_state->event->base); - - new_crtc_state->event = NULL; - } - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - - /* Signal HW programming completion */ - drm_atomic_helper_commit_hw_done(state); - - if (wait_for_vblank) - drm_atomic_helper_wait_for_flip_done(dev, state); - - drm_atomic_helper_cleanup_planes(dev, state); - - /* Don't free the memory if we are hitting this as part of suspend. - * This way we don't free any memory during suspend; see - * amdgpu_bo_free_kernel(). The memory will be freed in the first - * non-suspend modeset or when the driver is torn down. - */ - if (!adev->in_suspend) { - /* return the stolen vga memory back to VRAM */ - if (!adev->mman.keep_stolen_vga_memory) - amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); - amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); - } - - /* - * Finally, drop a runtime PM reference for each newly disabled CRTC, - * so we can put the GPU into runtime suspend if we're not driving any - * displays anymore - */ - for (i = 0; i < crtc_disable_count; i++) - pm_runtime_put_autosuspend(dev->dev); - pm_runtime_mark_last_busy(dev->dev); -} - -static int dm_force_atomic_commit(struct drm_connector *connector) -{ - int ret = 0; - struct drm_device *ddev = connector->dev; - struct drm_atomic_state *state = drm_atomic_state_alloc(ddev); - struct amdgpu_crtc *disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc); - struct drm_plane *plane = disconnected_acrtc->base.primary; - struct drm_connector_state *conn_state; - struct drm_crtc_state *crtc_state; - struct drm_plane_state *plane_state; - - if (!state) - return -ENOMEM; - - state->acquire_ctx = ddev->mode_config.acquire_ctx; - - /* Construct an atomic state to restore previous display setting */ - - /* - * Attach connectors to drm_atomic_state - */ - conn_state = drm_atomic_get_connector_state(state, connector); - - ret = PTR_ERR_OR_ZERO(conn_state); - if (ret) - goto out; - - /* Attach crtc to drm_atomic_state*/ - crtc_state = drm_atomic_get_crtc_state(state, &disconnected_acrtc->base); - - ret = PTR_ERR_OR_ZERO(crtc_state); - if (ret) - goto out; - - /* force a restore */ - crtc_state->mode_changed = true; - - /* Attach plane to drm_atomic_state */ - plane_state = drm_atomic_get_plane_state(state, plane); - - ret = PTR_ERR_OR_ZERO(plane_state); - if (ret) - goto out; - - /* Call commit internally with the state we just constructed */ - ret = drm_atomic_commit(state); - -out: - drm_atomic_state_put(state); - if (ret) - DRM_ERROR("Restoring old state failed with %i\n", ret); - - return ret; -} - -/* - * This function handles all cases when set mode does not come upon hotplug. - * This includes when a display is unplugged then plugged back into the - * same port and when running without usermode desktop manager supprot - */ -void dm_restore_drm_connector_state(struct drm_device *dev, - struct drm_connector *connector) -{ - struct amdgpu_dm_connector *aconnector; - struct amdgpu_crtc *disconnected_acrtc; - struct dm_crtc_state *acrtc_state; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return; - - aconnector = to_amdgpu_dm_connector(connector); - - if (!aconnector->dc_sink || !connector->state || !connector->encoder) - return; - - disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc); - if (!disconnected_acrtc) - return; - - acrtc_state = to_dm_crtc_state(disconnected_acrtc->base.state); - if (!acrtc_state->stream) - return; - - /* - * If the previous sink is not released and different from the current, - * we deduce we are in a state where we can not rely on usermode call - * to turn on the display, so we do it here - */ - if (acrtc_state->stream->sink != aconnector->dc_sink) - dm_force_atomic_commit(&aconnector->base); -} - -/* - * Grabs all modesetting locks to serialize against any blocking commits, - * Waits for completion of all non blocking commits. - */ -static int do_aquire_global_lock(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct drm_crtc *crtc; - struct drm_crtc_commit *commit; - long ret; - - /* - * Adding all modeset locks to aquire_ctx will - * ensure that when the framework release it the - * extra locks we are locking here will get released to - */ - ret = drm_modeset_lock_all_ctx(dev, state->acquire_ctx); - if (ret) - return ret; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - spin_lock(&crtc->commit_lock); - commit = list_first_entry_or_null(&crtc->commit_list, - struct drm_crtc_commit, commit_entry); - if (commit) - drm_crtc_commit_get(commit); - spin_unlock(&crtc->commit_lock); - - if (!commit) - continue; - - /* - * Make sure all pending HW programming completed and - * page flips done - */ - ret = wait_for_completion_interruptible_timeout(&commit->hw_done, 10*HZ); - - if (ret > 0) - ret = wait_for_completion_interruptible_timeout( - &commit->flip_done, 10*HZ); - - if (ret == 0) - DRM_ERROR("[CRTC:%d:%s] hw_done or flip_done timed out\n", - crtc->base.id, crtc->name); - - drm_crtc_commit_put(commit); - } - - return ret < 0 ? ret : 0; -} - -static void get_freesync_config_for_crtc( - struct dm_crtc_state *new_crtc_state, - struct dm_connector_state *new_con_state) -{ - struct mod_freesync_config config = {0}; - struct amdgpu_dm_connector *aconnector; - struct drm_display_mode *mode = &new_crtc_state->base.mode; - int vrefresh = drm_mode_vrefresh(mode); - bool fs_vid_mode = false; - - if (new_con_state->base.connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - return; - - aconnector = to_amdgpu_dm_connector(new_con_state->base.connector); - - new_crtc_state->vrr_supported = new_con_state->freesync_capable && - vrefresh >= aconnector->min_vfreq && - vrefresh <= aconnector->max_vfreq; - - if (new_crtc_state->vrr_supported) { - new_crtc_state->stream->ignore_msa_timing_param = true; - fs_vid_mode = new_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED; - - config.min_refresh_in_uhz = aconnector->min_vfreq * 1000000; - config.max_refresh_in_uhz = aconnector->max_vfreq * 1000000; - config.vsif_supported = true; - config.btr = true; - - if (fs_vid_mode) { - config.state = VRR_STATE_ACTIVE_FIXED; - config.fixed_refresh_in_uhz = new_crtc_state->freesync_config.fixed_refresh_in_uhz; - goto out; - } else if (new_crtc_state->base.vrr_enabled) { - config.state = VRR_STATE_ACTIVE_VARIABLE; - } else { - config.state = VRR_STATE_INACTIVE; - } - } -out: - new_crtc_state->freesync_config = config; -} - -static void reset_freesync_config_for_crtc( - struct dm_crtc_state *new_crtc_state) -{ - new_crtc_state->vrr_supported = false; - - memset(&new_crtc_state->vrr_infopacket, 0, - sizeof(new_crtc_state->vrr_infopacket)); -} - -static bool -is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state) -{ - const struct drm_display_mode *old_mode, *new_mode; - - if (!old_crtc_state || !new_crtc_state) - return false; - - old_mode = &old_crtc_state->mode; - new_mode = &new_crtc_state->mode; - - if (old_mode->clock == new_mode->clock && - old_mode->hdisplay == new_mode->hdisplay && - old_mode->vdisplay == new_mode->vdisplay && - old_mode->htotal == new_mode->htotal && - old_mode->vtotal != new_mode->vtotal && - old_mode->hsync_start == new_mode->hsync_start && - old_mode->vsync_start != new_mode->vsync_start && - old_mode->hsync_end == new_mode->hsync_end && - old_mode->vsync_end != new_mode->vsync_end && - old_mode->hskew == new_mode->hskew && - old_mode->vscan == new_mode->vscan && - (old_mode->vsync_end - old_mode->vsync_start) == - (new_mode->vsync_end - new_mode->vsync_start)) - return true; - - return false; -} - -static void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state) -{ - u64 num, den, res; - struct drm_crtc_state *new_crtc_state = &dm_new_crtc_state->base; - - dm_new_crtc_state->freesync_config.state = VRR_STATE_ACTIVE_FIXED; - - num = (unsigned long long)new_crtc_state->mode.clock * 1000 * 1000000; - den = (unsigned long long)new_crtc_state->mode.htotal * - (unsigned long long)new_crtc_state->mode.vtotal; - - res = div_u64(num, den); - dm_new_crtc_state->freesync_config.fixed_refresh_in_uhz = res; -} - -static int dm_update_crtc_state(struct amdgpu_display_manager *dm, - struct drm_atomic_state *state, - struct drm_crtc *crtc, - struct drm_crtc_state *old_crtc_state, - struct drm_crtc_state *new_crtc_state, - bool enable, - bool *lock_and_validation_needed) -{ - struct dm_atomic_state *dm_state = NULL; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct dc_stream_state *new_stream; - int ret = 0; - - /* - * TODO Move this code into dm_crtc_atomic_check once we get rid of dc_validation_set - * update changed items - */ - struct amdgpu_crtc *acrtc = NULL; - struct drm_connector *connector = NULL; - struct amdgpu_dm_connector *aconnector = NULL; - struct drm_connector_state *drm_new_conn_state = NULL, *drm_old_conn_state = NULL; - struct dm_connector_state *dm_new_conn_state = NULL, *dm_old_conn_state = NULL; - - new_stream = NULL; - - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - acrtc = to_amdgpu_crtc(crtc); - connector = amdgpu_dm_find_first_crtc_matching_connector(state, crtc); - if (connector) - aconnector = to_amdgpu_dm_connector(connector); - - /* TODO This hack should go away */ - if (connector && enable) { - /* Make sure fake sink is created in plug-in scenario */ - drm_new_conn_state = drm_atomic_get_new_connector_state(state, - connector); - drm_old_conn_state = drm_atomic_get_old_connector_state(state, - connector); - - if (IS_ERR(drm_new_conn_state)) { - ret = PTR_ERR_OR_ZERO(drm_new_conn_state); - goto fail; - } - - dm_new_conn_state = to_dm_connector_state(drm_new_conn_state); - dm_old_conn_state = to_dm_connector_state(drm_old_conn_state); - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - goto skip_modeset; - - new_stream = create_validate_stream_for_sink(aconnector, - &new_crtc_state->mode, - dm_new_conn_state, - dm_old_crtc_state->stream); - - /* - * we can have no stream on ACTION_SET if a display - * was disconnected during S3, in this case it is not an - * error, the OS will be updated after detection, and - * will do the right thing on next atomic commit - */ - - if (!new_stream) { - DRM_DEBUG_DRIVER("%s: Failed to create new stream for crtc %d\n", - __func__, acrtc->base.base.id); - ret = -ENOMEM; - goto fail; - } - - /* - * TODO: Check VSDB bits to decide whether this should - * be enabled or not. - */ - new_stream->triggered_crtc_reset.enabled = - dm->force_timing_sync; - - dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level; - - ret = fill_hdr_info_packet(drm_new_conn_state, - &new_stream->hdr_static_metadata); - if (ret) - goto fail; - - /* - * If we already removed the old stream from the context - * (and set the new stream to NULL) then we can't reuse - * the old stream even if the stream and scaling are unchanged. - * We'll hit the BUG_ON and black screen. - * - * TODO: Refactor this function to allow this check to work - * in all conditions. - */ - if (amdgpu_freesync_vid_mode && - dm_new_crtc_state->stream && - is_timing_unchanged_for_freesync(new_crtc_state, old_crtc_state)) - goto skip_modeset; - - if (dm_new_crtc_state->stream && - dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) && - dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream)) { - new_crtc_state->mode_changed = false; - DRM_DEBUG_DRIVER("Mode change not required, setting mode_changed to %d", - new_crtc_state->mode_changed); - } - } - - /* mode_changed flag may get updated above, need to check again */ - if (!drm_atomic_crtc_needs_modeset(new_crtc_state)) - goto skip_modeset; - - drm_dbg_state(state->dev, - "amdgpu_crtc id:%d crtc_state_flags: enable:%d, active:%d, planes_changed:%d, mode_changed:%d,active_changed:%d,connectors_changed:%d\n", - acrtc->crtc_id, - new_crtc_state->enable, - new_crtc_state->active, - new_crtc_state->planes_changed, - new_crtc_state->mode_changed, - new_crtc_state->active_changed, - new_crtc_state->connectors_changed); - - /* Remove stream for any changed/disabled CRTC */ - if (!enable) { - - if (!dm_old_crtc_state->stream) - goto skip_modeset; - - /* Unset freesync video if it was active before */ - if (dm_old_crtc_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED) { - dm_new_crtc_state->freesync_config.state = VRR_STATE_INACTIVE; - dm_new_crtc_state->freesync_config.fixed_refresh_in_uhz = 0; - } - - /* Now check if we should set freesync video mode */ - if (amdgpu_freesync_vid_mode && dm_new_crtc_state->stream && - dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) && - dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream) && - is_timing_unchanged_for_freesync(new_crtc_state, - old_crtc_state)) { - new_crtc_state->mode_changed = false; - DRM_DEBUG_DRIVER( - "Mode change not required for front porch change, setting mode_changed to %d", - new_crtc_state->mode_changed); - - set_freesync_fixed_config(dm_new_crtc_state); - - goto skip_modeset; - } else if (amdgpu_freesync_vid_mode && aconnector && - is_freesync_video_mode(&new_crtc_state->mode, - aconnector)) { - struct drm_display_mode *high_mode; - - high_mode = get_highest_refresh_rate_mode(aconnector, false); - if (!drm_mode_equal(&new_crtc_state->mode, high_mode)) - set_freesync_fixed_config(dm_new_crtc_state); - } - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - goto fail; - - DRM_DEBUG_DRIVER("Disabling DRM crtc: %d\n", - crtc->base.id); - - /* i.e. reset mode */ - if (dc_state_remove_stream( - dm->dc, - dm_state->context, - dm_old_crtc_state->stream) != DC_OK) { - ret = -EINVAL; - goto fail; - } - - dc_stream_release(dm_old_crtc_state->stream); - dm_new_crtc_state->stream = NULL; - - reset_freesync_config_for_crtc(dm_new_crtc_state); - - *lock_and_validation_needed = true; - - } else {/* Add stream for any updated/enabled CRTC */ - /* - * Quick fix to prevent NULL pointer on new_stream when - * added MST connectors not found in existing crtc_state in the chained mode - * TODO: need to dig out the root cause of that - */ - if (!connector) - goto skip_modeset; - - if (modereset_required(new_crtc_state)) - goto skip_modeset; - - if (amdgpu_dm_crtc_modeset_required(new_crtc_state, new_stream, - dm_old_crtc_state->stream)) { - - WARN_ON(dm_new_crtc_state->stream); - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - goto fail; - - dm_new_crtc_state->stream = new_stream; - - dc_stream_retain(new_stream); - - DRM_DEBUG_ATOMIC("Enabling DRM crtc: %d\n", - crtc->base.id); - - if (dc_state_add_stream( - dm->dc, - dm_state->context, - dm_new_crtc_state->stream) != DC_OK) { - ret = -EINVAL; - goto fail; - } - - *lock_and_validation_needed = true; - } - } - -skip_modeset: - /* Release extra reference */ - if (new_stream) - dc_stream_release(new_stream); - - /* - * We want to do dc stream updates that do not require a - * full modeset below. - */ - if (!(enable && connector && new_crtc_state->active)) - return 0; - /* - * Given above conditions, the dc state cannot be NULL because: - * 1. We're in the process of enabling CRTCs (just been added - * to the dc context, or already is on the context) - * 2. Has a valid connector attached, and - * 3. Is currently active and enabled. - * => The dc stream state currently exists. - */ - BUG_ON(dm_new_crtc_state->stream == NULL); - - /* Scaling or underscan settings */ - if (is_scaling_state_different(dm_old_conn_state, dm_new_conn_state) || - drm_atomic_crtc_needs_modeset(new_crtc_state)) - update_stream_scaling_settings( - &new_crtc_state->mode, dm_new_conn_state, dm_new_crtc_state->stream); - - /* ABM settings */ - dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level; - - /* - * Color management settings. We also update color properties - * when a modeset is needed, to ensure it gets reprogrammed. - */ - if (dm_new_crtc_state->base.color_mgmt_changed || - dm_old_crtc_state->regamma_tf != dm_new_crtc_state->regamma_tf || - drm_atomic_crtc_needs_modeset(new_crtc_state)) { - ret = amdgpu_dm_update_crtc_color_mgmt(dm_new_crtc_state); - if (ret) - goto fail; - } - - /* Update Freesync settings. */ - get_freesync_config_for_crtc(dm_new_crtc_state, - dm_new_conn_state); - - return ret; - -fail: - if (new_stream) - dc_stream_release(new_stream); - return ret; -} - -static bool should_reset_plane(struct drm_atomic_state *state, - struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state) -{ - struct drm_plane *other; - struct drm_plane_state *old_other_state, *new_other_state; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *old_dm_crtc_state, *new_dm_crtc_state; - struct amdgpu_device *adev = drm_to_adev(plane->dev); - int i; - - /* - * TODO: Remove this hack for all asics once it proves that the - * fast updates works fine on DCN3.2+. - */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 2, 0) && - state->allow_modeset) - return true; - - /* Exit early if we know that we're adding or removing the plane. */ - if (old_plane_state->crtc != new_plane_state->crtc) - return true; - - /* old crtc == new_crtc == NULL, plane not in context. */ - if (!new_plane_state->crtc) - return false; - - new_crtc_state = - drm_atomic_get_new_crtc_state(state, new_plane_state->crtc); - old_crtc_state = - drm_atomic_get_old_crtc_state(state, old_plane_state->crtc); - - if (!new_crtc_state) - return true; - - /* - * A change in cursor mode means a new dc pipe needs to be acquired or - * released from the state - */ - old_dm_crtc_state = to_dm_crtc_state(old_crtc_state); - new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); - if (plane->type == DRM_PLANE_TYPE_CURSOR && - old_dm_crtc_state != NULL && - old_dm_crtc_state->cursor_mode != new_dm_crtc_state->cursor_mode) { - return true; - } - - /* CRTC Degamma changes currently require us to recreate planes. */ - if (new_crtc_state->color_mgmt_changed) - return true; - - /* - * On zpos change, planes need to be reordered by removing and re-adding - * them one by one to the dc state, in order of descending zpos. - * - * TODO: We can likely skip bandwidth validation if the only thing that - * changed about the plane was it'z z-ordering. - */ - if (new_crtc_state->zpos_changed) - return true; - - if (drm_atomic_crtc_needs_modeset(new_crtc_state)) - return true; - - /* - * If there are any new primary or overlay planes being added or - * removed then the z-order can potentially change. To ensure - * correct z-order and pipe acquisition the current DC architecture - * requires us to remove and recreate all existing planes. - * - * TODO: Come up with a more elegant solution for this. - */ - for_each_oldnew_plane_in_state(state, other, old_other_state, new_other_state, i) { - struct amdgpu_framebuffer *old_afb, *new_afb; - struct dm_plane_state *dm_new_other_state, *dm_old_other_state; - - dm_new_other_state = to_dm_plane_state(new_other_state); - dm_old_other_state = to_dm_plane_state(old_other_state); - - if (other->type == DRM_PLANE_TYPE_CURSOR) - continue; - - if (old_other_state->crtc != new_plane_state->crtc && - new_other_state->crtc != new_plane_state->crtc) - continue; - - if (old_other_state->crtc != new_other_state->crtc) - return true; - - /* Src/dst size and scaling updates. */ - if (old_other_state->src_w != new_other_state->src_w || - old_other_state->src_h != new_other_state->src_h || - old_other_state->crtc_w != new_other_state->crtc_w || - old_other_state->crtc_h != new_other_state->crtc_h) - return true; - - /* Rotation / mirroring updates. */ - if (old_other_state->rotation != new_other_state->rotation) - return true; - - /* Blending updates. */ - if (old_other_state->pixel_blend_mode != - new_other_state->pixel_blend_mode) - return true; - - /* Alpha updates. */ - if (old_other_state->alpha != new_other_state->alpha) - return true; - - /* Colorspace changes. */ - if (old_other_state->color_range != new_other_state->color_range || - old_other_state->color_encoding != new_other_state->color_encoding) - return true; - - /* HDR/Transfer Function changes. */ - if (dm_old_other_state->degamma_tf != dm_new_other_state->degamma_tf || - dm_old_other_state->degamma_lut != dm_new_other_state->degamma_lut || - dm_old_other_state->hdr_mult != dm_new_other_state->hdr_mult || - dm_old_other_state->ctm != dm_new_other_state->ctm || - dm_old_other_state->shaper_lut != dm_new_other_state->shaper_lut || - dm_old_other_state->shaper_tf != dm_new_other_state->shaper_tf || - dm_old_other_state->lut3d != dm_new_other_state->lut3d || - dm_old_other_state->blend_lut != dm_new_other_state->blend_lut || - dm_old_other_state->blend_tf != dm_new_other_state->blend_tf) - return true; - - /* Framebuffer checks fall at the end. */ - if (!old_other_state->fb || !new_other_state->fb) - continue; - - /* Pixel format changes can require bandwidth updates. */ - if (old_other_state->fb->format != new_other_state->fb->format) - return true; - - old_afb = (struct amdgpu_framebuffer *)old_other_state->fb; - new_afb = (struct amdgpu_framebuffer *)new_other_state->fb; - - /* Tiling and DCC changes also require bandwidth updates. */ - if (old_afb->tiling_flags != new_afb->tiling_flags || - old_afb->base.modifier != new_afb->base.modifier) - return true; - } - - return false; -} - -static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc, - struct drm_plane_state *new_plane_state, - struct drm_framebuffer *fb) -{ - struct amdgpu_device *adev = drm_to_adev(new_acrtc->base.dev); - struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(fb); - unsigned int pitch; - bool linear; - - if (fb->width > new_acrtc->max_cursor_width || - fb->height > new_acrtc->max_cursor_height) { - DRM_DEBUG_ATOMIC("Bad cursor FB size %dx%d\n", - new_plane_state->fb->width, - new_plane_state->fb->height); - return -EINVAL; - } - if (new_plane_state->src_w != fb->width << 16 || - new_plane_state->src_h != fb->height << 16) { - DRM_DEBUG_ATOMIC("Cropping not supported for cursor plane\n"); - return -EINVAL; - } - - /* Pitch in pixels */ - pitch = fb->pitches[0] / fb->format->cpp[0]; - - if (fb->width != pitch) { - DRM_DEBUG_ATOMIC("Cursor FB width %d doesn't match pitch %d", - fb->width, pitch); - return -EINVAL; - } - - switch (pitch) { - case 64: - case 128: - case 256: - /* FB pitch is supported by cursor plane */ - break; - default: - DRM_DEBUG_ATOMIC("Bad cursor FB pitch %d px\n", pitch); - return -EINVAL; - } - - /* Core DRM takes care of checking FB modifiers, so we only need to - * check tiling flags when the FB doesn't have a modifier. - */ - if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) { - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { - linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0; - } else if (adev->family >= AMDGPU_FAMILY_AI) { - linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0; - } else { - linear = AMDGPU_TILING_GET(afb->tiling_flags, ARRAY_MODE) != DC_ARRAY_2D_TILED_THIN1 && - AMDGPU_TILING_GET(afb->tiling_flags, ARRAY_MODE) != DC_ARRAY_1D_TILED_THIN1 && - AMDGPU_TILING_GET(afb->tiling_flags, MICRO_TILE_MODE) == 0; - } - if (!linear) { - DRM_DEBUG_ATOMIC("Cursor FB not linear"); - return -EINVAL; - } - } - - return 0; -} - -/* - * Helper function for checking the cursor in native mode - */ -static int dm_check_native_cursor_state(struct drm_crtc *new_plane_crtc, - struct drm_plane *plane, - struct drm_plane_state *new_plane_state, - bool enable) -{ - - struct amdgpu_crtc *new_acrtc; - int ret; - - if (!enable || !new_plane_crtc || - drm_atomic_plane_disabling(plane->state, new_plane_state)) - return 0; - - new_acrtc = to_amdgpu_crtc(new_plane_crtc); - - if (new_plane_state->src_x != 0 || new_plane_state->src_y != 0) { - DRM_DEBUG_ATOMIC("Cropping not supported for cursor plane\n"); - return -EINVAL; - } - - if (new_plane_state->fb) { - ret = dm_check_cursor_fb(new_acrtc, new_plane_state, - new_plane_state->fb); - if (ret) - return ret; - } - - return 0; -} - -static bool dm_should_update_native_cursor(struct drm_atomic_state *state, - struct drm_crtc *old_plane_crtc, - struct drm_crtc *new_plane_crtc, - bool enable) -{ - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - - if (!enable) { - if (old_plane_crtc == NULL) - return true; - - old_crtc_state = drm_atomic_get_old_crtc_state( - state, old_plane_crtc); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - return dm_old_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE; - } else { - if (new_plane_crtc == NULL) - return true; - - new_crtc_state = drm_atomic_get_new_crtc_state( - state, new_plane_crtc); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - return dm_new_crtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE; - } -} - -static int dm_update_plane_state(struct dc *dc, - struct drm_atomic_state *state, - struct drm_plane *plane, - struct drm_plane_state *old_plane_state, - struct drm_plane_state *new_plane_state, - bool enable, - bool *lock_and_validation_needed, - bool *is_top_most_overlay) -{ - - struct dm_atomic_state *dm_state = NULL; - struct drm_crtc *new_plane_crtc, *old_plane_crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct dm_crtc_state *dm_new_crtc_state, *dm_old_crtc_state; - struct dm_plane_state *dm_new_plane_state, *dm_old_plane_state; - bool needs_reset, update_native_cursor; - int ret = 0; - - - new_plane_crtc = new_plane_state->crtc; - old_plane_crtc = old_plane_state->crtc; - dm_new_plane_state = to_dm_plane_state(new_plane_state); - dm_old_plane_state = to_dm_plane_state(old_plane_state); - - update_native_cursor = dm_should_update_native_cursor(state, - old_plane_crtc, - new_plane_crtc, - enable); - - if (plane->type == DRM_PLANE_TYPE_CURSOR && update_native_cursor) { - ret = dm_check_native_cursor_state(new_plane_crtc, plane, - new_plane_state, enable); - if (ret) - return ret; - - return 0; - } - - needs_reset = should_reset_plane(state, plane, old_plane_state, - new_plane_state); - - /* Remove any changed/removed planes */ - if (!enable) { - if (!needs_reset) - return 0; - - if (!old_plane_crtc) - return 0; - - old_crtc_state = drm_atomic_get_old_crtc_state( - state, old_plane_crtc); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (!dm_old_crtc_state->stream) - return 0; - - DRM_DEBUG_ATOMIC("Disabling DRM plane: %d on DRM crtc %d\n", - plane->base.id, old_plane_crtc->base.id); - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - return ret; - - if (!dc_state_remove_plane( - dc, - dm_old_crtc_state->stream, - dm_old_plane_state->dc_state, - dm_state->context)) { - - return -EINVAL; - } - - if (dm_old_plane_state->dc_state) - dc_plane_state_release(dm_old_plane_state->dc_state); - - dm_new_plane_state->dc_state = NULL; - - *lock_and_validation_needed = true; - - } else { /* Add new planes */ - struct dc_plane_state *dc_new_plane_state; - - if (drm_atomic_plane_disabling(plane->state, new_plane_state)) - return 0; - - if (!new_plane_crtc) - return 0; - - new_crtc_state = drm_atomic_get_new_crtc_state(state, new_plane_crtc); - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - if (!dm_new_crtc_state->stream) - return 0; - - if (!needs_reset) - return 0; - - ret = amdgpu_dm_plane_helper_check_state(new_plane_state, new_crtc_state); - if (ret) - goto out; - - WARN_ON(dm_new_plane_state->dc_state); - - dc_new_plane_state = dc_create_plane_state(dc); - if (!dc_new_plane_state) { - ret = -ENOMEM; - goto out; - } - - DRM_DEBUG_ATOMIC("Enabling DRM plane: %d on DRM crtc %d\n", - plane->base.id, new_plane_crtc->base.id); - - ret = fill_dc_plane_attributes( - drm_to_adev(new_plane_crtc->dev), - dc_new_plane_state, - new_plane_state, - new_crtc_state); - if (ret) { - dc_plane_state_release(dc_new_plane_state); - goto out; - } - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) { - dc_plane_state_release(dc_new_plane_state); - goto out; - } - - /* - * Any atomic check errors that occur after this will - * not need a release. The plane state will be attached - * to the stream, and therefore part of the atomic - * state. It'll be released when the atomic state is - * cleaned. - */ - if (!dc_state_add_plane( - dc, - dm_new_crtc_state->stream, - dc_new_plane_state, - dm_state->context)) { - - dc_plane_state_release(dc_new_plane_state); - ret = -EINVAL; - goto out; - } - - dm_new_plane_state->dc_state = dc_new_plane_state; - - dm_new_crtc_state->mpo_requested |= (plane->type == DRM_PLANE_TYPE_OVERLAY); - - /* Tell DC to do a full surface update every time there - * is a plane change. Inefficient, but works for now. - */ - dm_new_plane_state->dc_state->update_flags.bits.full_update = 1; - - *lock_and_validation_needed = true; - } - -out: - /* If enabling cursor overlay failed, attempt fallback to native mode */ - if (enable && ret == -EINVAL && plane->type == DRM_PLANE_TYPE_CURSOR) { - ret = dm_check_native_cursor_state(new_plane_crtc, plane, - new_plane_state, enable); - if (ret) - return ret; - - dm_new_crtc_state->cursor_mode = DM_CURSOR_NATIVE_MODE; - } - - return ret; -} - -static void dm_get_oriented_plane_size(struct drm_plane_state *plane_state, - int *src_w, int *src_h) -{ - switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) { - case DRM_MODE_ROTATE_90: - case DRM_MODE_ROTATE_270: - *src_w = plane_state->src_h >> 16; - *src_h = plane_state->src_w >> 16; - break; - case DRM_MODE_ROTATE_0: - case DRM_MODE_ROTATE_180: - default: - *src_w = plane_state->src_w >> 16; - *src_h = plane_state->src_h >> 16; - break; - } -} - -static void -dm_get_plane_scale(struct drm_plane_state *plane_state, - int *out_plane_scale_w, int *out_plane_scale_h) -{ - int plane_src_w, plane_src_h; - - dm_get_oriented_plane_size(plane_state, &plane_src_w, &plane_src_h); - *out_plane_scale_w = plane_state->crtc_w * 1000 / plane_src_w; - *out_plane_scale_h = plane_state->crtc_h * 1000 / plane_src_h; -} - -/* - * The normalized_zpos value cannot be used by this iterator directly. It's only - * calculated for enabled planes, potentially causing normalized_zpos collisions - * between enabled/disabled planes in the atomic state. We need a unique value - * so that the iterator will not generate the same object twice, or loop - * indefinitely. - */ -static inline struct __drm_planes_state *__get_next_zpos( - struct drm_atomic_state *state, - struct __drm_planes_state *prev) -{ - unsigned int highest_zpos = 0, prev_zpos = 256; - uint32_t highest_id = 0, prev_id = UINT_MAX; - struct drm_plane_state *new_plane_state; - struct drm_plane *plane; - int i, highest_i = -1; - - if (prev != NULL) { - prev_zpos = prev->new_state->zpos; - prev_id = prev->ptr->base.id; - } - - for_each_new_plane_in_state(state, plane, new_plane_state, i) { - /* Skip planes with higher zpos than the previously returned */ - if (new_plane_state->zpos > prev_zpos || - (new_plane_state->zpos == prev_zpos && - plane->base.id >= prev_id)) - continue; - - /* Save the index of the plane with highest zpos */ - if (new_plane_state->zpos > highest_zpos || - (new_plane_state->zpos == highest_zpos && - plane->base.id > highest_id)) { - highest_zpos = new_plane_state->zpos; - highest_id = plane->base.id; - highest_i = i; - } - } - - if (highest_i < 0) - return NULL; - - return &state->planes[highest_i]; -} - -/* - * Use the uniqueness of the plane's (zpos, drm obj ID) combination to iterate - * by descending zpos, as read from the new plane state. This is the same - * ordering as defined by drm_atomic_normalize_zpos(). - */ -#define for_each_oldnew_plane_in_descending_zpos(__state, plane, old_plane_state, new_plane_state) \ - for (struct __drm_planes_state *__i = __get_next_zpos((__state), NULL); \ - __i != NULL; __i = __get_next_zpos((__state), __i)) \ - for_each_if(((plane) = __i->ptr, \ - (void)(plane) /* Only to avoid unused-but-set-variable warning */, \ - (old_plane_state) = __i->old_state, \ - (new_plane_state) = __i->new_state, 1)) - -static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc) -{ - struct drm_connector *connector; - struct drm_connector_state *conn_state, *old_conn_state; - struct amdgpu_dm_connector *aconnector = NULL; - int i; - - for_each_oldnew_connector_in_state(state, connector, old_conn_state, conn_state, i) { - if (!conn_state->crtc) - conn_state = old_conn_state; - - if (conn_state->crtc != crtc) - continue; - - if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) - continue; - - aconnector = to_amdgpu_dm_connector(connector); - if (!aconnector->mst_output_port || !aconnector->mst_root) - aconnector = NULL; - else - break; - } - - if (!aconnector) - return 0; - - return drm_dp_mst_add_affected_dsc_crtcs(state, &aconnector->mst_root->mst_mgr); -} - -/** - * DOC: Cursor Modes - Native vs Overlay - * - * In native mode, the cursor uses a integrated cursor pipe within each DCN hw - * plane. It does not require a dedicated hw plane to enable, but it is - * subjected to the same z-order and scaling as the hw plane. It also has format - * restrictions, a RGB cursor in native mode cannot be enabled within a non-RGB - * hw plane. - * - * In overlay mode, the cursor uses a separate DCN hw plane, and thus has its - * own scaling and z-pos. It also has no blending restrictions. It lends to a - * cursor behavior more akin to a DRM client's expectations. However, it does - * occupy an extra DCN plane, and therefore will only be used if a DCN plane is - * available. - */ - -/** - * dm_crtc_get_cursor_mode() - Determine the required cursor mode on crtc - * @adev: amdgpu device - * @state: DRM atomic state - * @dm_crtc_state: amdgpu state for the CRTC containing the cursor - * @cursor_mode: Returns the required cursor mode on dm_crtc_state - * - * Get whether the cursor should be enabled in native mode, or overlay mode, on - * the dm_crtc_state. - * - * The cursor should be enabled in overlay mode if there exists an underlying - * plane - on which the cursor may be blended - that is either YUV formatted, or - * scaled differently from the cursor. - * - * Since zpos info is required, drm_atomic_normalize_zpos must be called before - * calling this function. - * - * Return: 0 on success, or an error code if getting the cursor plane state - * failed. - */ -static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, - struct drm_atomic_state *state, - struct dm_crtc_state *dm_crtc_state, - enum amdgpu_dm_cursor_mode *cursor_mode) -{ - struct drm_plane_state *old_plane_state, *plane_state, *cursor_state; - struct drm_crtc_state *crtc_state = &dm_crtc_state->base; - struct drm_plane *plane; - bool consider_mode_change = false; - bool entire_crtc_covered = false; - bool cursor_changed = false; - int underlying_scale_w, underlying_scale_h; - int cursor_scale_w, cursor_scale_h; - int i; - - /* Overlay cursor not supported on HW before DCN - * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE - */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { - *cursor_mode = DM_CURSOR_NATIVE_MODE; - return 0; - } - - /* Init cursor_mode to be the same as current */ - *cursor_mode = dm_crtc_state->cursor_mode; - - /* - * Cursor mode can change if a plane's format changes, scale changes, is - * enabled/disabled, or z-order changes. - */ - for_each_oldnew_plane_in_state(state, plane, old_plane_state, plane_state, i) { - int new_scale_w, new_scale_h, old_scale_w, old_scale_h; - - /* Only care about planes on this CRTC */ - if ((drm_plane_mask(plane) & crtc_state->plane_mask) == 0) - continue; - - if (plane->type == DRM_PLANE_TYPE_CURSOR) - cursor_changed = true; - - if (drm_atomic_plane_enabling(old_plane_state, plane_state) || - drm_atomic_plane_disabling(old_plane_state, plane_state) || - old_plane_state->fb->format != plane_state->fb->format) { - consider_mode_change = true; - break; - } - - dm_get_plane_scale(plane_state, &new_scale_w, &new_scale_h); - dm_get_plane_scale(old_plane_state, &old_scale_w, &old_scale_h); - if (new_scale_w != old_scale_w || new_scale_h != old_scale_h) { - consider_mode_change = true; - break; - } - } - - if (!consider_mode_change && !crtc_state->zpos_changed) - return 0; - - /* - * If no cursor change on this CRTC, and not enabled on this CRTC, then - * no need to set cursor mode. This avoids needlessly locking the cursor - * state. - */ - if (!cursor_changed && - !(drm_plane_mask(crtc_state->crtc->cursor) & crtc_state->plane_mask)) { - return 0; - } - - cursor_state = drm_atomic_get_plane_state(state, - crtc_state->crtc->cursor); - if (IS_ERR(cursor_state)) - return PTR_ERR(cursor_state); - - /* Cursor is disabled */ - if (!cursor_state->fb) - return 0; - - /* For all planes in descending z-order (all of which are below cursor - * as per zpos definitions), check their scaling and format - */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, plane_state) { - - /* Only care about non-cursor planes on this CRTC */ - if ((drm_plane_mask(plane) & crtc_state->plane_mask) == 0 || - plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - /* Underlying plane is YUV format - use overlay cursor */ - if (amdgpu_dm_plane_is_video_format(plane_state->fb->format->format)) { - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - return 0; - } - - dm_get_plane_scale(plane_state, - &underlying_scale_w, &underlying_scale_h); - dm_get_plane_scale(cursor_state, - &cursor_scale_w, &cursor_scale_h); - - /* Underlying plane has different scale - use overlay cursor */ - if (cursor_scale_w != underlying_scale_w && - cursor_scale_h != underlying_scale_h) { - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - return 0; - } - - /* If this plane covers the whole CRTC, no need to check planes underneath */ - if (plane_state->crtc_x <= 0 && plane_state->crtc_y <= 0 && - plane_state->crtc_x + plane_state->crtc_w >= crtc_state->mode.hdisplay && - plane_state->crtc_y + plane_state->crtc_h >= crtc_state->mode.vdisplay) { - entire_crtc_covered = true; - break; - } - } - - /* If planes do not cover the entire CRTC, use overlay mode to enable - * cursor over holes - */ - if (entire_crtc_covered) - *cursor_mode = DM_CURSOR_NATIVE_MODE; - else - *cursor_mode = DM_CURSOR_OVERLAY_MODE; - - return 0; -} - -/** - * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM. - * - * @dev: The DRM device - * @state: The atomic state to commit - * - * Validate that the given atomic state is programmable by DC into hardware. - * This involves constructing a &struct dc_state reflecting the new hardware - * state we wish to commit, then querying DC to see if it is programmable. It's - * important not to modify the existing DC state. Otherwise, atomic_check - * may unexpectedly commit hardware changes. - * - * When validating the DC state, it's important that the right locks are - * acquired. For full updates case which removes/adds/updates streams on one - * CRTC while flipping on another CRTC, acquiring global lock will guarantee - * that any such full update commit will wait for completion of any outstanding - * flip using DRMs synchronization events. - * - * Note that DM adds the affected connectors for all CRTCs in state, when that - * might not seem necessary. This is because DC stream creation requires the - * DC sink, which is tied to the DRM connector state. Cleaning this up should - * be possible but non-trivial - a possible TODO item. - * - * Return: -Error code if validation failed. - */ -static int amdgpu_dm_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct dm_atomic_state *dm_state = NULL; - struct dc *dc = adev->dm.dc; - struct drm_connector *connector; - struct drm_connector_state *old_con_state, *new_con_state; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct drm_plane *plane; - struct drm_plane_state *old_plane_state, *new_plane_state, *new_cursor_state; - enum dc_status status; - int ret, i; - bool lock_and_validation_needed = false; - bool is_top_most_overlay = true; - struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; - struct drm_dp_mst_topology_mgr *mgr; - struct drm_dp_mst_topology_state *mst_state; - struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0}; - - trace_amdgpu_dm_atomic_check_begin(state); - - ret = drm_atomic_helper_check_modeset(dev, state); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_helper_check_modeset() failed\n"); - goto fail; - } - - /* Check connector changes */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - - /* Skip connectors that are disabled or part of modeset already. */ - if (!new_con_state->crtc) - continue; - - new_crtc_state = drm_atomic_get_crtc_state(state, new_con_state->crtc); - if (IS_ERR(new_crtc_state)) { - drm_dbg_atomic(dev, "drm_atomic_get_crtc_state() failed\n"); - ret = PTR_ERR(new_crtc_state); - goto fail; - } - - if (dm_old_con_state->abm_level != dm_new_con_state->abm_level || - dm_old_con_state->scaling != dm_new_con_state->scaling) - new_crtc_state->connectors_changed = true; - } - - if (dc_resource_is_dsc_encoding_supported(dc)) { - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { - ret = add_affected_mst_dsc_crtcs(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "add_affected_mst_dsc_crtcs() failed\n"); - goto fail; - } - } - } - } - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - if (!drm_atomic_crtc_needs_modeset(new_crtc_state) && - !new_crtc_state->color_mgmt_changed && - old_crtc_state->vrr_enabled == new_crtc_state->vrr_enabled && - dm_old_crtc_state->dsc_force_changed == false) - continue; - - ret = amdgpu_dm_verify_lut_sizes(new_crtc_state); - if (ret) { - drm_dbg_atomic(dev, "amdgpu_dm_verify_lut_sizes() failed\n"); - goto fail; - } - - if (!new_crtc_state->enable) - continue; - - ret = drm_atomic_add_affected_connectors(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_add_affected_connectors() failed\n"); - goto fail; - } - - ret = drm_atomic_add_affected_planes(state, crtc); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_add_affected_planes() failed\n"); - goto fail; - } - - if (dm_old_crtc_state->dsc_force_changed) - new_crtc_state->mode_changed = true; - } - - /* - * Add all primary and overlay planes on the CRTC to the state - * whenever a plane is enabled to maintain correct z-ordering - * and to enable fast surface updates. - */ - drm_for_each_crtc(crtc, dev) { - bool modified = false; - - for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - if (new_plane_state->crtc == crtc || - old_plane_state->crtc == crtc) { - modified = true; - break; - } - } - - if (!modified) - continue; - - drm_for_each_plane_mask(plane, state->dev, crtc->state->plane_mask) { - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - new_plane_state = - drm_atomic_get_plane_state(state, plane); - - if (IS_ERR(new_plane_state)) { - ret = PTR_ERR(new_plane_state); - drm_dbg_atomic(dev, "new_plane_state is BAD\n"); - goto fail; - } - } - } - - /* - * DC consults the zpos (layer_index in DC terminology) to determine the - * hw plane on which to enable the hw cursor (see - * `dcn10_can_pipe_disable_cursor`). By now, all modified planes are in - * atomic state, so call drm helper to normalize zpos. - */ - ret = drm_atomic_normalize_zpos(dev, state); - if (ret) { - drm_dbg(dev, "drm_atomic_normalize_zpos() failed\n"); - goto fail; - } - - /* - * Determine whether cursors on each CRTC should be enabled in native or - * overlay mode. - */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - - ret = dm_crtc_get_cursor_mode(adev, state, dm_new_crtc_state, - &dm_new_crtc_state->cursor_mode); - if (ret) { - drm_dbg(dev, "Failed to determine cursor mode\n"); - goto fail; - } - } - - /* Remove exiting planes if they are modified */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, new_plane_state) { - if (old_plane_state->fb && new_plane_state->fb && - get_mem_type(old_plane_state->fb) != - get_mem_type(new_plane_state->fb)) - lock_and_validation_needed = true; - - ret = dm_update_plane_state(dc, state, plane, - old_plane_state, - new_plane_state, - false, - &lock_and_validation_needed, - &is_top_most_overlay); - if (ret) { - drm_dbg_atomic(dev, "dm_update_plane_state() failed\n"); - goto fail; - } - } - - /* Disable all crtcs which require disable */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = dm_update_crtc_state(&adev->dm, state, crtc, - old_crtc_state, - new_crtc_state, - false, - &lock_and_validation_needed); - if (ret) { - drm_dbg_atomic(dev, "DISABLE: dm_update_crtc_state() failed\n"); - goto fail; - } - } - - /* Enable all crtcs which require enable */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = dm_update_crtc_state(&adev->dm, state, crtc, - old_crtc_state, - new_crtc_state, - true, - &lock_and_validation_needed); - if (ret) { - drm_dbg_atomic(dev, "ENABLE: dm_update_crtc_state() failed\n"); - goto fail; - } - } - - /* Add new/modified planes */ - for_each_oldnew_plane_in_descending_zpos(state, plane, old_plane_state, new_plane_state) { - ret = dm_update_plane_state(dc, state, plane, - old_plane_state, - new_plane_state, - true, - &lock_and_validation_needed, - &is_top_most_overlay); - if (ret) { - drm_dbg_atomic(dev, "dm_update_plane_state() failed\n"); - goto fail; - } - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - if (dc_resource_is_dsc_encoding_supported(dc)) { - ret = pre_validate_dsc(state, &dm_state, vars); - if (ret != 0) - goto fail; - } -#endif - - /* Run this here since we want to validate the streams we created */ - ret = drm_atomic_helper_check_planes(dev, state); - if (ret) { - drm_dbg_atomic(dev, "drm_atomic_helper_check_planes() failed\n"); - goto fail; - } - - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->mpo_requested) - drm_dbg_atomic(dev, "MPO enablement requested on crtc:[%p]\n", crtc); - } - - /* Check cursor restrictions */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - enum amdgpu_dm_cursor_mode required_cursor_mode; - int is_rotated, is_scaled; - - /* Overlay cusor not subject to native cursor restrictions */ - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) - continue; - - /* Check if rotation or scaling is enabled on DCN401 */ - if ((drm_plane_mask(crtc->cursor) & new_crtc_state->plane_mask) && - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { - new_cursor_state = drm_atomic_get_new_plane_state(state, crtc->cursor); - - is_rotated = new_cursor_state && - ((new_cursor_state->rotation & DRM_MODE_ROTATE_MASK) != DRM_MODE_ROTATE_0); - is_scaled = new_cursor_state && ((new_cursor_state->src_w >> 16 != new_cursor_state->crtc_w) || - (new_cursor_state->src_h >> 16 != new_cursor_state->crtc_h)); - - if (is_rotated || is_scaled) { - drm_dbg_driver( - crtc->dev, - "[CRTC:%d:%s] cannot enable hardware cursor due to rotation/scaling\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - } - - /* If HW can only do native cursor, check restrictions again */ - ret = dm_crtc_get_cursor_mode(adev, state, dm_new_crtc_state, - &required_cursor_mode); - if (ret) { - drm_dbg_driver(crtc->dev, - "[CRTC:%d:%s] Checking cursor mode failed\n", - crtc->base.id, crtc->name); - goto fail; - } else if (required_cursor_mode == DM_CURSOR_OVERLAY_MODE) { - drm_dbg_driver(crtc->dev, - "[CRTC:%d:%s] Cannot enable native cursor due to scaling or YUV restrictions\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - } - - if (state->legacy_cursor_update) { - /* - * This is a fast cursor update coming from the plane update - * helper, check if it can be done asynchronously for better - * performance. - */ - state->async_update = - !drm_atomic_helper_async_check(dev, state); - - /* - * Skip the remaining global validation if this is an async - * update. Cursor updates can be done without affecting - * state or bandwidth calcs and this avoids the performance - * penalty of locking the private state object and - * allocating a new dc_state. - */ - if (state->async_update) - return 0; - } - - /* Check scaling and underscan changes*/ - /* TODO Removed scaling changes validation due to inability to commit - * new stream into context w\o causing full reset. Need to - * decide how to handle. - */ - for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) { - struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); - struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); - struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - - /* Skip any modesets/resets */ - if (!acrtc || drm_atomic_crtc_needs_modeset( - drm_atomic_get_new_crtc_state(state, &acrtc->base))) - continue; - - /* Skip any thing not scale or underscan changes */ - if (!is_scaling_state_different(dm_new_con_state, dm_old_con_state)) - continue; - - lock_and_validation_needed = true; - } - - /* set the slot info for each mst_state based on the link encoding format */ - for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) { - struct amdgpu_dm_connector *aconnector; - struct drm_connector *connector; - struct drm_connector_list_iter iter; - u8 link_coding_cap; - - drm_connector_list_iter_begin(dev, &iter); - drm_for_each_connector_iter(connector, &iter) { - if (connector->index == mst_state->mgr->conn_base_id) { - aconnector = to_amdgpu_dm_connector(connector); - link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(aconnector->dc_link); - drm_dp_mst_update_slots(mst_state, link_coding_cap); - - break; - } - } - drm_connector_list_iter_end(&iter); - } - - /** - * Streams and planes are reset when there are changes that affect - * bandwidth. Anything that affects bandwidth needs to go through - * DC global validation to ensure that the configuration can be applied - * to hardware. - * - * We have to currently stall out here in atomic_check for outstanding - * commits to finish in this case because our IRQ handlers reference - * DRM state directly - we can end up disabling interrupts too early - * if we don't. - * - * TODO: Remove this stall and drop DM state private objects. - */ - if (lock_and_validation_needed) { - ret = dm_atomic_get_state(state, &dm_state); - if (ret) { - drm_dbg_atomic(dev, "dm_atomic_get_state() failed\n"); - goto fail; - } - - ret = do_aquire_global_lock(dev, state); - if (ret) { - drm_dbg_atomic(dev, "do_aquire_global_lock() failed\n"); - goto fail; - } - -#if defined(CONFIG_DRM_AMD_DC_FP) - if (dc_resource_is_dsc_encoding_supported(dc)) { - ret = compute_mst_dsc_configs_for_state(state, dm_state->context, vars); - if (ret) { - drm_dbg_atomic(dev, "compute_mst_dsc_configs_for_state() failed\n"); - ret = -EINVAL; - goto fail; - } - } -#endif - - ret = dm_update_mst_vcpi_slots_for_dsc(state, dm_state->context, vars); - if (ret) { - drm_dbg_atomic(dev, "dm_update_mst_vcpi_slots_for_dsc() failed\n"); - goto fail; - } - - /* - * Perform validation of MST topology in the state: - * We need to perform MST atomic check before calling - * dc_validate_global_state(), or there is a chance - * to get stuck in an infinite loop and hang eventually. - */ - ret = drm_dp_mst_atomic_check(state); - if (ret) { - drm_dbg_atomic(dev, "drm_dp_mst_atomic_check() failed\n"); - goto fail; - } - status = dc_validate_global_state(dc, dm_state->context, true); - if (status != DC_OK) { - drm_dbg_atomic(dev, "DC global validation failure: %s (%d)", - dc_status_to_str(status), status); - ret = -EINVAL; - goto fail; - } - } else { - /* - * The commit is a fast update. Fast updates shouldn't change - * the DC context, affect global validation, and can have their - * commit work done in parallel with other commits not touching - * the same resource. If we have a new DC context as part of - * the DM atomic state from validation we need to free it and - * retain the existing one instead. - * - * Furthermore, since the DM atomic state only contains the DC - * context and can safely be annulled, we can free the state - * and clear the associated private object now to free - * some memory and avoid a possible use-after-free later. - */ - - for (i = 0; i < state->num_private_objs; i++) { - struct drm_private_obj *obj = state->private_objs[i].ptr; - - if (obj->funcs == adev->dm.atomic_obj.funcs) { - int j = state->num_private_objs-1; - - dm_atomic_destroy_state(obj, - state->private_objs[i].state); - - /* If i is not at the end of the array then the - * last element needs to be moved to where i was - * before the array can safely be truncated. - */ - if (i != j) - state->private_objs[i] = - state->private_objs[j]; - - state->private_objs[j].ptr = NULL; - state->private_objs[j].state = NULL; - state->private_objs[j].old_state = NULL; - state->private_objs[j].new_state = NULL; - - state->num_private_objs = j; - break; - } - } - } - - /* Store the overall update type for use later in atomic check. */ - for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - struct dm_crtc_state *dm_new_crtc_state = - to_dm_crtc_state(new_crtc_state); - - /* - * Only allow async flips for fast updates that don't change - * the FB pitch, the DCC state, rotation, etc. - */ - if (new_crtc_state->async_flip && lock_and_validation_needed) { - drm_dbg_atomic(crtc->dev, - "[CRTC:%d:%s] async flips are only supported for fast updates\n", - crtc->base.id, crtc->name); - ret = -EINVAL; - goto fail; - } - - dm_new_crtc_state->update_type = lock_and_validation_needed ? - UPDATE_TYPE_FULL : UPDATE_TYPE_FAST; - } - - /* Must be success */ - WARN_ON(ret); - - trace_amdgpu_dm_atomic_check_finish(state, ret); - - return ret; - -fail: - if (ret == -EDEADLK) - drm_dbg_atomic(dev, "Atomic check stopped to avoid deadlock.\n"); - else if (ret == -EINTR || ret == -EAGAIN || ret == -ERESTARTSYS) - drm_dbg_atomic(dev, "Atomic check stopped due to signal.\n"); - else - drm_dbg_atomic(dev, "Atomic check failed with err: %d\n", ret); - - trace_amdgpu_dm_atomic_check_finish(state, ret); - - return ret; -} - -static bool dm_edid_parser_send_cea(struct amdgpu_display_manager *dm, - unsigned int offset, - unsigned int total_length, - u8 *data, - unsigned int length, - struct amdgpu_hdmi_vsdb_info *vsdb) -{ - bool res; - union dmub_rb_cmd cmd; - struct dmub_cmd_send_edid_cea *input; - struct dmub_cmd_edid_cea_output *output; - - if (length > DMUB_EDID_CEA_DATA_CHUNK_BYTES) - return false; - - memset(&cmd, 0, sizeof(cmd)); - - input = &cmd.edid_cea.data.input; - - cmd.edid_cea.header.type = DMUB_CMD__EDID_CEA; - cmd.edid_cea.header.sub_type = 0; - cmd.edid_cea.header.payload_bytes = - sizeof(cmd.edid_cea) - sizeof(cmd.edid_cea.header); - input->offset = offset; - input->length = length; - input->cea_total_length = total_length; - memcpy(input->payload, data, length); - - res = dc_wake_and_execute_dmub_cmd(dm->dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY); - if (!res) { - DRM_ERROR("EDID CEA parser failed\n"); - return false; - } - - output = &cmd.edid_cea.data.output; - - if (output->type == DMUB_CMD__EDID_CEA_ACK) { - if (!output->ack.success) { - DRM_ERROR("EDID CEA ack failed at offset %d\n", - output->ack.offset); - } - } else if (output->type == DMUB_CMD__EDID_CEA_AMD_VSDB) { - if (!output->amd_vsdb.vsdb_found) - return false; - - vsdb->freesync_supported = output->amd_vsdb.freesync_supported; - vsdb->amd_vsdb_version = output->amd_vsdb.amd_vsdb_version; - vsdb->min_refresh_rate_hz = output->amd_vsdb.min_frame_rate; - vsdb->max_refresh_rate_hz = output->amd_vsdb.max_frame_rate; - } else { - DRM_WARN("Unknown EDID CEA parser results\n"); - return false; - } - - return true; -} - -static bool parse_edid_cea_dmcu(struct amdgpu_display_manager *dm, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - int i; - - /* send extension block to DMCU for parsing */ - for (i = 0; i < len; i += 8) { - bool res; - int offset; - - /* send 8 bytes a time */ - if (!dc_edid_parser_send_cea(dm->dc, i, len, &edid_ext[i], 8)) - return false; - - if (i+8 == len) { - /* EDID block sent completed, expect result */ - int version, min_rate, max_rate; - - res = dc_edid_parser_recv_amd_vsdb(dm->dc, &version, &min_rate, &max_rate); - if (res) { - /* amd vsdb found */ - vsdb_info->freesync_supported = 1; - vsdb_info->amd_vsdb_version = version; - vsdb_info->min_refresh_rate_hz = min_rate; - vsdb_info->max_refresh_rate_hz = max_rate; - return true; - } - /* not amd vsdb */ - return false; - } - - /* check for ack*/ - res = dc_edid_parser_recv_cea_ack(dm->dc, &offset); - if (!res) - return false; - } - - return false; -} - -static bool parse_edid_cea_dmub(struct amdgpu_display_manager *dm, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - int i; - - /* send extension block to DMCU for parsing */ - for (i = 0; i < len; i += 8) { - /* send 8 bytes a time */ - if (!dm_edid_parser_send_cea(dm, i, len, &edid_ext[i], 8, vsdb_info)) - return false; - } - - return vsdb_info->freesync_supported; -} - -static bool parse_edid_cea(struct amdgpu_dm_connector *aconnector, - u8 *edid_ext, int len, - struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev); - bool ret; - - mutex_lock(&adev->dm.dc_lock); - if (adev->dm.dmub_srv) - ret = parse_edid_cea_dmub(&adev->dm, edid_ext, len, vsdb_info); - else - ret = parse_edid_cea_dmcu(&adev->dm, edid_ext, len, vsdb_info); - mutex_unlock(&adev->dm.dc_lock); - return ret; -} - -static void parse_edid_displayid_vrr(struct drm_connector *connector, - struct edid *edid) -{ - u8 *edid_ext = NULL; - int i; - int j = 0; - u16 min_vfreq; - u16 max_vfreq; - - if (edid == NULL || edid->extensions == 0) - return; - - /* Find DisplayID extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (void *)(edid + (i + 1)); - if (edid_ext[0] == DISPLAYID_EXT) - break; - } - - if (edid_ext == NULL) - return; - - while (j < EDID_LENGTH) { - /* Get dynamic video timing range from DisplayID if available */ - if (EDID_LENGTH - j > 13 && edid_ext[j] == 0x25 && - (edid_ext[j+1] & 0xFE) == 0 && (edid_ext[j+2] == 9)) { - min_vfreq = edid_ext[j+9]; - if (edid_ext[j+1] & 7) - max_vfreq = edid_ext[j+10] + ((edid_ext[j+11] & 3) << 8); - else - max_vfreq = edid_ext[j+10]; - - if (max_vfreq && min_vfreq) { - connector->display_info.monitor_range.max_vfreq = max_vfreq; - connector->display_info.monitor_range.min_vfreq = min_vfreq; - - return; - } - } - j++; - } -} - -static int parse_amd_vsdb(struct amdgpu_dm_connector *aconnector, - struct edid *edid, struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - u8 *edid_ext = NULL; - int i; - int j = 0; - - if (edid == NULL || edid->extensions == 0) - return -ENODEV; - - /* Find DisplayID extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (void *)(edid + (i + 1)); - if (edid_ext[0] == DISPLAYID_EXT) - break; - } - - while (j < EDID_LENGTH) { - struct amd_vsdb_block *amd_vsdb = (struct amd_vsdb_block *)&edid_ext[j]; - unsigned int ieeeId = (amd_vsdb->ieee_id[2] << 16) | (amd_vsdb->ieee_id[1] << 8) | (amd_vsdb->ieee_id[0]); - - if (ieeeId == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_IEEE_REGISTRATION_ID && - amd_vsdb->version == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3) { - vsdb_info->replay_mode = (amd_vsdb->feature_caps & AMD_VSDB_VERSION_3_FEATURECAP_REPLAYMODE) ? true : false; - vsdb_info->amd_vsdb_version = HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3; - DRM_DEBUG_KMS("Panel supports Replay Mode: %d\n", vsdb_info->replay_mode); - - return true; - } - j++; - } - - return false; -} - -static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector, - struct edid *edid, struct amdgpu_hdmi_vsdb_info *vsdb_info) -{ - u8 *edid_ext = NULL; - int i; - bool valid_vsdb_found = false; - - /*----- drm_find_cea_extension() -----*/ - /* No EDID or EDID extensions */ - if (edid == NULL || edid->extensions == 0) - return -ENODEV; - - /* Find CEA extension */ - for (i = 0; i < edid->extensions; i++) { - edid_ext = (uint8_t *)edid + EDID_LENGTH * (i + 1); - if (edid_ext[0] == CEA_EXT) - break; - } - - if (i == edid->extensions) - return -ENODEV; - - /*----- cea_db_offsets() -----*/ - if (edid_ext[0] != CEA_EXT) - return -ENODEV; - - valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info); - - return valid_vsdb_found ? i : -ENODEV; -} - -/** - * amdgpu_dm_update_freesync_caps - Update Freesync capabilities - * - * @connector: Connector to query. - * @edid: EDID from monitor - * - * Amdgpu supports Freesync in DP and HDMI displays, and it is required to keep - * track of some of the display information in the internal data struct used by - * amdgpu_dm. This function checks which type of connector we need to set the - * FreeSync parameters. - */ -void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, - struct edid *edid) -{ - int i = 0; - struct detailed_timing *timing; - struct detailed_non_pixel *data; - struct detailed_data_monitor_range *range; - struct amdgpu_dm_connector *amdgpu_dm_connector = - to_amdgpu_dm_connector(connector); - struct dm_connector_state *dm_con_state = NULL; - struct dc_sink *sink; - - struct amdgpu_device *adev = drm_to_adev(connector->dev); - struct amdgpu_hdmi_vsdb_info vsdb_info = {0}; - bool freesync_capable = false; - enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE; - - if (!connector->state) { - DRM_ERROR("%s - Connector has no state", __func__); - goto update; - } - - sink = amdgpu_dm_connector->dc_sink ? - amdgpu_dm_connector->dc_sink : - amdgpu_dm_connector->dc_em_sink; - - if (!edid || !sink) { - dm_con_state = to_dm_connector_state(connector->state); - - amdgpu_dm_connector->min_vfreq = 0; - amdgpu_dm_connector->max_vfreq = 0; - connector->display_info.monitor_range.min_vfreq = 0; - connector->display_info.monitor_range.max_vfreq = 0; - freesync_capable = false; - - goto update; - } - - dm_con_state = to_dm_connector_state(connector->state); - - if (!adev->dm.freesync_module) - goto update; - - /* Some eDP panels only have the refresh rate range info in DisplayID */ - if ((connector->display_info.monitor_range.min_vfreq == 0 || - connector->display_info.monitor_range.max_vfreq == 0)) - parse_edid_displayid_vrr(connector, edid); - - if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { - bool edid_check_required = false; - - if (amdgpu_dm_connector->dc_link && - amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) { - if (edid->features & DRM_EDID_FEATURE_CONTINUOUS_FREQ) { - amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq; - amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq; - if (amdgpu_dm_connector->max_vfreq - - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - } else { - edid_check_required = edid->version > 1 || - (edid->version == 1 && - edid->revision > 1); - } - } - - if (edid_check_required) { - for (i = 0; i < 4; i++) { - - timing = &edid->detailed_timings[i]; - data = &timing->data.other_data; - range = &data->data.range; - /* - * Check if monitor has continuous frequency mode - */ - if (data->type != EDID_DETAIL_MONITOR_RANGE) - continue; - /* - * Check for flag range limits only. If flag == 1 then - * no additional timing information provided. - * Default GTF, GTF Secondary curve and CVT are not - * supported - */ - if (range->flags != 1) - continue; - - connector->display_info.monitor_range.min_vfreq = range->min_vfreq; - connector->display_info.monitor_range.max_vfreq = range->max_vfreq; - - if (edid->revision >= 4) { - if (data->pad2 & DRM_EDID_RANGE_OFFSET_MIN_VFREQ) - connector->display_info.monitor_range.min_vfreq += 255; - if (data->pad2 & DRM_EDID_RANGE_OFFSET_MAX_VFREQ) - connector->display_info.monitor_range.max_vfreq += 255; - } - - amdgpu_dm_connector->min_vfreq = - connector->display_info.monitor_range.min_vfreq; - amdgpu_dm_connector->max_vfreq = - connector->display_info.monitor_range.max_vfreq; - - break; - } - - if (amdgpu_dm_connector->max_vfreq - - amdgpu_dm_connector->min_vfreq > 10) { - - freesync_capable = true; - } - } - parse_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - - if (vsdb_info.replay_mode) { - amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode; - amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version; - amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP; - } - - } else if (edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0 && vsdb_info.freesync_supported) { - timing = &edid->detailed_timings[i]; - data = &timing->data.other_data; - - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } - } - - if (amdgpu_dm_connector->dc_link) - as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link); - - if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0 && vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) { - - amdgpu_dm_connector->pack_sdp_v1_3 = true; - amdgpu_dm_connector->as_type = as_type; - amdgpu_dm_connector->vsdb_info = vsdb_info; - - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } - } - -update: - if (dm_con_state) - dm_con_state->freesync_capable = freesync_capable; - - if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable && - amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) { - amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false; - amdgpu_dm_connector->dc_link->replay_settings.replay_feature_enabled = false; - } - - if (connector->vrr_capable_property) - drm_connector_set_vrr_capable_property(connector, - freesync_capable); -} - -void amdgpu_dm_trigger_timing_sync(struct drm_device *dev) -{ - struct amdgpu_device *adev = drm_to_adev(dev); - struct dc *dc = adev->dm.dc; - int i; - - mutex_lock(&adev->dm.dc_lock); - if (dc->current_state) { - for (i = 0; i < dc->current_state->stream_count; ++i) - dc->current_state->streams[i] - ->triggered_crtc_reset.enabled = - adev->dm.force_timing_sync; - - dm_enable_per_frame_crtc_master_sync(dc->current_state); - dc_trigger_sync(dc, dc->current_state); - } - mutex_unlock(&adev->dm.dc_lock); -} - -static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc) -{ - if (dc->ctx->dmub_srv && !dc->ctx->dmub_srv->idle_exit_counter) - dc_exit_ips_for_hw_access(dc); -} - -void dm_write_reg_func(const struct dc_context *ctx, uint32_t address, - u32 value, const char *func_name) -{ -#ifdef DM_CHECK_ADDR_0 - if (address == 0) { - drm_err(adev_to_drm(ctx->driver_context), - "invalid register write. address = 0"); - return; - } -#endif - - amdgpu_dm_exit_ips_for_hw_access(ctx->dc); - cgs_write_register(ctx->cgs_device, address, value); - trace_amdgpu_dc_wreg(&ctx->perf_trace->write_count, address, value); -} - -uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address, - const char *func_name) -{ - u32 value; -#ifdef DM_CHECK_ADDR_0 - if (address == 0) { - drm_err(adev_to_drm(ctx->driver_context), - "invalid register read; address = 0\n"); - return 0; - } -#endif - - if (ctx->dmub_srv && - ctx->dmub_srv->reg_helper_offload.gather_in_progress && - !ctx->dmub_srv->reg_helper_offload.should_burst_write) { - ASSERT(false); - return 0; - } - - amdgpu_dm_exit_ips_for_hw_access(ctx->dc); - - value = cgs_read_register(ctx->cgs_device, address); - - trace_amdgpu_dc_rreg(&ctx->perf_trace->read_count, address, value); - - return value; -} - -int amdgpu_dm_process_dmub_aux_transfer_sync( - struct dc_context *ctx, - unsigned int link_index, - struct aux_payload *payload, - enum aux_return_code_type *operation_result) -{ - struct amdgpu_device *adev = ctx->driver_context; - struct dmub_notification *p_notify = adev->dm.dmub_notify; - int ret = -1; - - mutex_lock(&adev->dm.dpia_aux_lock); - if (!dc_process_dmub_aux_transfer_async(ctx->dc, link_index, payload)) { - *operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE; - goto out; - } - - if (!wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) { - DRM_ERROR("wait_for_completion_timeout timeout!"); - *operation_result = AUX_RET_ERROR_TIMEOUT; - goto out; - } - - if (p_notify->result != AUX_RET_SUCCESS) { - /* - * Transient states before tunneling is enabled could - * lead to this error. We can ignore this for now. - */ - if (p_notify->result != AUX_RET_ERROR_PROTOCOL_ERROR) { - DRM_WARN("DPIA AUX failed on 0x%x(%d), error %d\n", - payload->address, payload->length, - p_notify->result); - } - *operation_result = AUX_RET_ERROR_INVALID_REPLY; - goto out; - } - - - payload->reply[0] = adev->dm.dmub_notify->aux_reply.command; - if (!payload->write && p_notify->aux_reply.length && - (payload->reply[0] == AUX_TRANSACTION_REPLY_AUX_ACK)) { - - if (payload->length != p_notify->aux_reply.length) { - DRM_WARN("invalid read length %d from DPIA AUX 0x%x(%d)!\n", - p_notify->aux_reply.length, - payload->address, payload->length); - *operation_result = AUX_RET_ERROR_INVALID_REPLY; - goto out; - } - - memcpy(payload->data, p_notify->aux_reply.data, - p_notify->aux_reply.length); - } - - /* success */ - ret = p_notify->aux_reply.length; - *operation_result = p_notify->result; -out: - reinit_completion(&adev->dm.dmub_aux_transfer_done); - mutex_unlock(&adev->dm.dpia_aux_lock); - return ret; -} - -int amdgpu_dm_process_dmub_set_config_sync( - struct dc_context *ctx, - unsigned int link_index, - struct set_config_cmd_payload *payload, - enum set_config_status *operation_result) -{ - struct amdgpu_device *adev = ctx->driver_context; - bool is_cmd_complete; - int ret; - - mutex_lock(&adev->dm.dpia_aux_lock); - is_cmd_complete = dc_process_dmub_set_config_async(ctx->dc, - link_index, payload, adev->dm.dmub_notify); - - if (is_cmd_complete || wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) { - ret = 0; - *operation_result = adev->dm.dmub_notify->sc_status; - } else { - DRM_ERROR("wait_for_completion_timeout timeout!"); - ret = -1; - *operation_result = SET_CONFIG_UNKNOWN_ERROR; - } - - if (!is_cmd_complete) - reinit_completion(&adev->dm.dmub_aux_transfer_done); - mutex_unlock(&adev->dm.dpia_aux_lock); - return ret; -} - -bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type) -{ - return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type); -} - -bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type) -{ - return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type); -} diff --git a/rr-cache/3b5fa8b49e38141113421fdc2503fb1e1928096c/preimage b/rr-cache/3b5fa8b49e38141113421fdc2503fb1e1928096c/preimage deleted file mode 100644 index 454f2a4c2cbe..000000000000 --- a/rr-cache/3b5fa8b49e38141113421fdc2503fb1e1928096c/preimage +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef __INTEL_DP_MST_H__ -#define __INTEL_DP_MST_H__ - -#include <linux/types.h> - -struct intel_atomic_state; -struct intel_crtc; -struct intel_crtc_state; -struct intel_digital_port; -struct intel_dp; -struct intel_link_bw_limits; - -int intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_id); -void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port); -int intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port); -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_source_support(struct intel_dp *intel_dp); -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits); -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc); -<<<<<<< -======= -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp); -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp); ->>>>>>> - -#endif /* __INTEL_DP_MST_H__ */ diff --git a/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage b/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage deleted file mode 100644 index 232d29e9a561..000000000000 --- a/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage +++ /dev/null @@ -1,2234 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2022 Intel Corporation - */ - -#include <linux/dma-fence-chain.h> - -#include "xe_pt.h" - -#include "regs/xe_gtt_defs.h" -#include "xe_bo.h" -#include "xe_device.h" -#include "xe_drm_client.h" -#include "xe_gt.h" -#include "xe_gt_tlb_invalidation.h" -#include "xe_migrate.h" -#include "xe_pt_types.h" -#include "xe_pt_walk.h" -#include "xe_res_cursor.h" -#include "xe_trace.h" -#include "xe_ttm_stolen_mgr.h" -#include "xe_vm.h" - -struct xe_pt_dir { - struct xe_pt pt; - /** @children: Array of page-table child nodes */ - struct xe_ptw *children[XE_PDES]; -}; - -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) -#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr)) -#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr) -#else -#define xe_pt_set_addr(__xe_pt, __addr) -#define xe_pt_addr(__xe_pt) 0ull -#endif - -static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48}; -static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48}; - -#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1) - -static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) -{ - return container_of(pt, struct xe_pt_dir, pt); -} - -static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index) -{ - return container_of(pt_dir->children[index], struct xe_pt, base); -} - -static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, - unsigned int level) -{ - struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; - u8 id = tile->id; - - if (!xe_vm_has_scratch(vm)) - return 0; - - if (level > MAX_HUGEPTE_LEVEL) - return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, - 0, pat_index); - - return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | - XE_PTE_NULL; -} - -static void xe_pt_free(struct xe_pt *pt) -{ - if (pt->level) - kfree(as_xe_pt_dir(pt)); - else - kfree(pt); -} - -/** - * xe_pt_create() - Create a page-table. - * @vm: The vm to create for. - * @tile: The tile to create for. - * @level: The page-table level. - * - * Allocate and initialize a single struct xe_pt metadata structure. Also - * create the corresponding page-table bo, but don't initialize it. If the - * level is grater than zero, then it's assumed to be a directory page- - * table and the directory structure is also allocated and initialized to - * NULL pointers. - * - * Return: A valid struct xe_pt pointer on success, Pointer error code on - * error. - */ -struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, - unsigned int level) -{ - struct xe_pt *pt; - struct xe_bo *bo; - int err; - - if (level) { - struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL); - - pt = (dir) ? &dir->pt : NULL; - } else { - pt = kzalloc(sizeof(*pt), GFP_KERNEL); - } - if (!pt) - return ERR_PTR(-ENOMEM); - - pt->level = level; - bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, - ttm_bo_type_kernel, - XE_BO_FLAG_VRAM_IF_DGFX(tile) | - XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE | - XE_BO_FLAG_PINNED | - XE_BO_FLAG_NO_RESV_EVICT | - XE_BO_FLAG_PAGETABLE); - if (IS_ERR(bo)) { - err = PTR_ERR(bo); - goto err_kfree; - } - pt->bo = bo; - pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; - - if (vm->xef) - xe_drm_client_add_bo(vm->xef->client, pt->bo); - xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL); - - return pt; - -err_kfree: - xe_pt_free(pt); - return ERR_PTR(err); -} - -/** - * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero - * entries. - * @tile: The tile the scratch pagetable of which to use. - * @vm: The vm we populate for. - * @pt: The pagetable the bo of which to initialize. - * - * Populate the page-table bo of @pt with entries pointing into the tile's - * scratch page-table tree if any. Otherwise populate with zeros. - */ -void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, - struct xe_pt *pt) -{ - struct iosys_map *map = &pt->bo->vmap; - u64 empty; - int i; - - if (!xe_vm_has_scratch(vm)) { - /* - * FIXME: Some memory is allocated already allocated to zero? - * Find out which memory that is and avoid this memset... - */ - xe_map_memset(vm->xe, map, 0, 0, SZ_4K); - } else { - empty = __xe_pt_empty_pte(tile, vm, pt->level); - for (i = 0; i < XE_PDES; i++) - xe_pt_write(vm->xe, map, i, empty); - } -} - -/** - * xe_pt_shift() - Return the ilog2 value of the size of the address range of - * a page-table at a certain level. - * @level: The level. - * - * Return: The ilog2 value of the size of the address range of a page-table - * at level @level. - */ -unsigned int xe_pt_shift(unsigned int level) -{ - return XE_PTE_SHIFT + XE_PDE_SHIFT * level; -} - -/** - * xe_pt_destroy() - Destroy a page-table tree. - * @pt: The root of the page-table tree to destroy. - * @flags: vm flags. Currently unused. - * @deferred: List head of lockless list for deferred putting. NULL for - * immediate putting. - * - * Puts the page-table bo, recursively calls xe_pt_destroy on all children - * and finally frees @pt. TODO: Can we remove the @flags argument? - */ -void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) -{ - int i; - - if (!pt) - return; - - XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); - xe_bo_unpin(pt->bo); - xe_bo_put_deferred(pt->bo, deferred); - - if (pt->level > 0 && pt->num_live) { - struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); - - for (i = 0; i < XE_PDES; i++) { - if (xe_pt_entry(pt_dir, i)) - xe_pt_destroy(xe_pt_entry(pt_dir, i), flags, - deferred); - } - } - xe_pt_free(pt); -} - -/** - * DOC: Pagetable building - * - * Below we use the term "page-table" for both page-directories, containing - * pointers to lower level page-directories or page-tables, and level 0 - * page-tables that contain only page-table-entries pointing to memory pages. - * - * When inserting an address range in an already existing page-table tree - * there will typically be a set of page-tables that are shared with other - * address ranges, and a set that are private to this address range. - * The set of shared page-tables can be at most two per level, - * and those can't be updated immediately because the entries of those - * page-tables may still be in use by the gpu for other mappings. Therefore - * when inserting entries into those, we instead stage those insertions by - * adding insertion data into struct xe_vm_pgtable_update structures. This - * data, (subtrees for the cpu and page-table-entries for the gpu) is then - * added in a separate commit step. CPU-data is committed while still under the - * vm lock, the object lock and for userptr, the notifier lock in read mode. - * The GPU async data is committed either by the GPU or CPU after fulfilling - * relevant dependencies. - * For non-shared page-tables (and, in fact, for shared ones that aren't - * existing at the time of staging), we add the data in-place without the - * special update structures. This private part of the page-table tree will - * remain disconnected from the vm page-table tree until data is committed to - * the shared page tables of the vm tree in the commit phase. - */ - -struct xe_pt_update { - /** @update: The update structure we're building for this parent. */ - struct xe_vm_pgtable_update *update; - /** @parent: The parent. Used to detect a parent change. */ - struct xe_pt *parent; - /** @preexisting: Whether the parent was pre-existing or allocated */ - bool preexisting; -}; - -struct xe_pt_stage_bind_walk { - /** base: The base class. */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @vm: The vm we're building for. */ - struct xe_vm *vm; - /** @tile: The tile we're building for. */ - struct xe_tile *tile; - /** @default_pte: PTE flag only template. No address is associated */ - u64 default_pte; - /** @dma_offset: DMA offset to add to the PTE. */ - u64 dma_offset; - /** - * @needs_64k: This address range enforces 64K alignment and - * granularity. - */ - bool needs_64K; - /** - * @vma: VMA being mapped - */ - struct xe_vma *vma; - - /* Also input, but is updated during the walk*/ - /** @curs: The DMA address cursor. */ - struct xe_res_cursor *curs; - /** @va_curs_start: The Virtual address coresponding to @curs->start */ - u64 va_curs_start; - - /* Output */ - struct xe_walk_update { - /** @wupd.entries: Caller provided storage. */ - struct xe_vm_pgtable_update *entries; - /** @wupd.num_used_entries: Number of update @entries used. */ - unsigned int num_used_entries; - /** @wupd.updates: Tracks the update entry at a given level */ - struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1]; - } wupd; - - /* Walk state */ - /** - * @l0_end_addr: The end address of the current l0 leaf. Used for - * 64K granularity detection. - */ - u64 l0_end_addr; - /** @addr_64K: The start address of the current 64K chunk. */ - u64 addr_64K; - /** @found_64: Whether @add_64K actually points to a 64K chunk. */ - bool found_64K; -}; - -static int -xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, - pgoff_t offset, bool alloc_entries) -{ - struct xe_pt_update *upd = &wupd->updates[parent->level]; - struct xe_vm_pgtable_update *entry; - - /* - * For *each level*, we could only have one active - * struct xt_pt_update at any one time. Once we move on to a - * new parent and page-directory, the old one is complete, and - * updates are either already stored in the build tree or in - * @wupd->entries - */ - if (likely(upd->parent == parent)) - return 0; - - upd->parent = parent; - upd->preexisting = true; - - if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) - return -EINVAL; - - entry = wupd->entries + wupd->num_used_entries++; - upd->update = entry; - entry->ofs = offset; - entry->pt_bo = parent->bo; - entry->pt = parent; - entry->flags = 0; - entry->qwords = 0; - - if (alloc_entries) { - entry->pt_entries = kmalloc_array(XE_PDES, - sizeof(*entry->pt_entries), - GFP_KERNEL); - if (!entry->pt_entries) - return -ENOMEM; - } - - return 0; -} - -/* - * NOTE: This is a very frequently called function so we allow ourselves - * to annotate (using branch prediction hints) the fastpath of updating a - * non-pre-existing pagetable with leaf ptes. - */ -static int -xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, - pgoff_t offset, struct xe_pt *xe_child, u64 pte) -{ - struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; - struct xe_pt_update *child_upd = xe_child ? - &xe_walk->wupd.updates[xe_child->level] : NULL; - int ret; - - ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); - if (unlikely(ret)) - return ret; - - /* - * Register this new pagetable so that it won't be recognized as - * a shared pagetable by a subsequent insertion. - */ - if (unlikely(child_upd)) { - child_upd->update = NULL; - child_upd->parent = xe_child; - child_upd->preexisting = false; - } - - if (likely(!upd->preexisting)) { - /* Continue building a non-connected subtree. */ - struct iosys_map *map = &parent->bo->vmap; - - if (unlikely(xe_child)) - parent->base.children[offset] = &xe_child->base; - - xe_pt_write(xe_walk->vm->xe, map, offset, pte); - parent->num_live++; - } else { - /* Shared pt. Stage update. */ - unsigned int idx; - struct xe_vm_pgtable_update *entry = upd->update; - - idx = offset - entry->ofs; - entry->pt_entries[idx].pt = xe_child; - entry->pt_entries[idx].pte = pte; - entry->qwords++; - } - - return 0; -} - -static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, - struct xe_pt_stage_bind_walk *xe_walk) -{ - u64 size, dma; - - if (level > MAX_HUGEPTE_LEVEL) - return false; - - /* Does the virtual range requested cover a huge pte? */ - if (!xe_pt_covers(addr, next, level, &xe_walk->base)) - return false; - - /* Does the DMA segment cover the whole pte? */ - if (next - xe_walk->va_curs_start > xe_walk->curs->size) - return false; - - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) - return true; - - /* Is the DMA address huge PTE size aligned? */ - size = next - addr; - dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); - - return IS_ALIGNED(dma, size); -} - -/* - * Scan the requested mapping to check whether it can be done entirely - * with 64K PTEs. - */ -static bool -xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) -{ - struct xe_res_cursor curs = *xe_walk->curs; - - if (!IS_ALIGNED(addr, SZ_64K)) - return false; - - if (next > xe_walk->l0_end_addr) - return false; - - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) - return true; - - xe_res_next(&curs, addr - xe_walk->va_curs_start); - for (; addr < next; addr += SZ_64K) { - if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) - return false; - - xe_res_next(&curs, SZ_64K); - } - - return addr == next; -} - -/* - * For non-compact "normal" 4K level-0 pagetables, we want to try to group - * addresses together in 64K-contigous regions to add a 64K TLB hint for the - * device to the PTE. - * This function determines whether the address is part of such a - * segment. For VRAM in normal pagetables, this is strictly necessary on - * some devices. - */ -static bool -xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) -{ - /* Address is within an already found 64k region */ - if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) - return true; - - xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); - xe_walk->addr_64K = addr; - - return xe_walk->found_64K; -} - -static int -xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_bind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - u16 pat_index = xe_walk->vma->pat_index; - struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); - struct xe_vm *vm = xe_walk->vm; - struct xe_pt *xe_child; - bool covers; - int ret = 0; - u64 pte; - - /* Is this a leaf entry ?*/ - if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) { - struct xe_res_cursor *curs = xe_walk->curs; - bool is_null = xe_vma_is_null(xe_walk->vma); - - XE_WARN_ON(xe_walk->va_curs_start != addr); - - pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : - xe_res_dma(curs) + xe_walk->dma_offset, - xe_walk->vma, pat_index, level); - pte |= xe_walk->default_pte; - - /* - * Set the XE_PTE_PS64 hint if possible, otherwise if - * this device *requires* 64K PTE size for VRAM, fail. - */ - if (level == 0 && !xe_parent->is_compact) { - if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) { - xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K; - pte |= XE_PTE_PS64; - } else if (XE_WARN_ON(xe_walk->needs_64K)) { - return -EINVAL; - } - } - - ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte); - if (unlikely(ret)) - return ret; - - if (!is_null) - xe_res_next(curs, next - addr); - xe_walk->va_curs_start = next; - xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); - *action = ACTION_CONTINUE; - - return ret; - } - - /* - * Descending to lower level. Determine if we need to allocate a - * new page table or -directory, which we do if there is no - * previous one or there is one we can completely replace. - */ - if (level == 1) { - walk->shifts = xe_normal_pt_shifts; - xe_walk->l0_end_addr = next; - } - - covers = xe_pt_covers(addr, next, level, &xe_walk->base); - if (covers || !*child) { - u64 flags = 0; - - xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); - if (IS_ERR(xe_child)) - return PTR_ERR(xe_child); - - xe_pt_set_addr(xe_child, - round_down(addr, 1ull << walk->shifts[level])); - - if (!covers) - xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); - - *child = &xe_child->base; - - /* - * Prefer the compact pagetable layout for L0 if possible. Only - * possible if VMA covers entire 2MB region as compact 64k and - * 4k pages cannot be mixed within a 2MB region. - * TODO: Suballocate the pt bo to avoid wasting a lot of - * memory. - */ - if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && - covers && xe_pt_scan_64K(addr, next, xe_walk)) { - walk->shifts = xe_compact_pt_shifts; - xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; - flags |= XE_PDE_64K; - xe_child->is_compact = true; - } - - pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; - ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, - pte); - } - - *action = ACTION_SUBTREE; - return ret; -} - -static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { - .pt_entry = xe_pt_stage_bind_entry, -}; - -/** - * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address - * range. - * @tile: The tile we're building for. - * @vma: The vma indicating the address range. - * @entries: Storage for the update entries used for connecting the tree to - * the main tree at commit time. - * @num_entries: On output contains the number of @entries used. - * - * This function builds a disconnected page-table tree for a given address - * range. The tree is connected to the main vm tree for the gpu using - * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind(). - * The function builds xe_vm_pgtable_update structures for already existing - * shared page-tables, and non-existing shared and non-shared page-tables - * are built and populated directly. - * - * Return 0 on success, negative error code on error. - */ -static int -xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 *num_entries) -{ - struct xe_device *xe = tile_to_xe(tile); - struct xe_bo *bo = xe_vma_bo(vma); - bool is_devmem = !xe_vma_is_userptr(vma) && bo && - (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)); - struct xe_res_cursor curs; - struct xe_pt_stage_bind_walk xe_walk = { - .base = { - .ops = &xe_pt_stage_bind_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .vm = xe_vma_vm(vma), - .tile = tile, - .curs = &curs, - .va_curs_start = xe_vma_start(vma), - .vma = vma, - .wupd.entries = entries, - .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - int ret; - - /** - * Default atomic expectations for different allocation scenarios are as follows: - * - * 1. Traditional API: When the VM is not in LR mode: - * - Device atomics are expected to function with all allocations. - * - * 2. Compute/SVM API: When the VM is in LR mode: - * - Device atomics are the default behavior when the bo is placed in a single region. - * - In all other cases device atomics will be disabled with AE=0 until an application - * request differently using a ioctl like madvise. - */ - if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { - if (xe_vm_in_lr_mode(xe_vma_vm(vma))) { - if (bo && xe_bo_has_single_placement(bo)) - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - /** - * If a SMEM+LMEM allocation is backed by SMEM, a device - * atomics will cause a gpu page fault and which then - * gets migrated to LMEM, bind such allocations with - * device atomics enabled. - */ - else if (is_devmem && !xe_bo_has_single_placement(bo)) - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - } else { - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - } - - /** - * Unset AE if the platform(PVC) doesn't support it on an - * allocation - */ - if (!xe->info.has_device_atomics_on_smem && !is_devmem) - xe_walk.default_pte &= ~XE_USM_PPGTT_PTE_AE; - } - - if (is_devmem) { - xe_walk.default_pte |= XE_PPGTT_PTE_DM; - xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); - } - - if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo)) - xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo)); - - xe_bo_assert_held(bo); - - if (!xe_vma_is_null(vma)) { - if (xe_vma_is_userptr(vma)) - xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, - xe_vma_size(vma), &curs); - else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) - xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), - xe_vma_size(vma), &curs); - else - xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma), - xe_vma_size(vma), &curs); - } else { - curs.size = xe_vma_size(vma); - } - - ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - *num_entries = xe_walk.wupd.num_used_entries; - return ret; -} - -/** - * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a - * shared pagetable. - * @addr: The start address within the non-shared pagetable. - * @end: The end address within the non-shared pagetable. - * @level: The level of the non-shared pagetable. - * @walk: Walk info. The function adjusts the walk action. - * @action: next action to perform (see enum page_walk_action) - * @offset: Ignored on input, First non-shared entry on output. - * @end_offset: Ignored on input, Last non-shared entry + 1 on output. - * - * A non-shared page-table has some entries that belong to the address range - * and others that don't. This function determines the entries that belong - * fully to the address range. Depending on level, some entries may - * partially belong to the address range (that can't happen at level 0). - * The function detects that and adjust those offsets to not include those - * partial entries. Iff it does detect partial entries, we know that there must - * be shared page tables also at lower levels, so it adjusts the walk action - * accordingly. - * - * Return: true if there were non-shared entries, false otherwise. - */ -static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level, - struct xe_pt_walk *walk, - enum page_walk_action *action, - pgoff_t *offset, pgoff_t *end_offset) -{ - u64 size = 1ull << walk->shifts[level]; - - *offset = xe_pt_offset(addr, level, walk); - *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset; - - if (!level) - return true; - - /* - * If addr or next are not size aligned, there are shared pts at lower - * level, so in that case traverse down the subtree - */ - *action = ACTION_CONTINUE; - if (!IS_ALIGNED(addr, size)) { - *action = ACTION_SUBTREE; - (*offset)++; - } - - if (!IS_ALIGNED(end, size)) { - *action = ACTION_SUBTREE; - (*end_offset)--; - } - - return *end_offset > *offset; -} - -struct xe_pt_zap_ptes_walk { - /** @base: The walk base-class */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @tile: The tile we're building for */ - struct xe_tile *tile; - - /* Output */ - /** @needs_invalidate: Whether we need to invalidate TLB*/ - bool needs_invalidate; -}; - -static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_zap_ptes_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - pgoff_t end_offset; - - XE_WARN_ON(!*child); - XE_WARN_ON(!level); - - /* - * Note that we're called from an entry callback, and we're dealing - * with the child of that entry rather than the parent, so need to - * adjust level down. - */ - if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, - &end_offset)) { - xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, - offset * sizeof(u64), 0, - (end_offset - offset) * sizeof(u64)); - xe_walk->needs_invalidate = true; - } - - return 0; -} - -static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { - .pt_entry = xe_pt_zap_ptes_entry, -}; - -/** - * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range - * @tile: The tile we're zapping for. - * @vma: GPU VMA detailing address range. - * - * Eviction and Userptr invalidation needs to be able to zap the - * gpu ptes of a given address range in pagefaulting mode. - * In order to be able to do that, that function needs access to the shared - * page-table entrieaso it can either clear the leaf PTEs or - * clear the pointers to lower-level page-tables. The caller is required - * to hold the necessary locks to ensure neither the page-table connectivity - * nor the page-table entries of the range is updated from under us. - * - * Return: Whether ptes were actually updated and a TLB invalidation is - * required. - */ -bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) -{ - struct xe_pt_zap_ptes_walk xe_walk = { - .base = { - .ops = &xe_pt_zap_ptes_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .tile = tile, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); - - if (!(pt_mask & BIT(tile->id))) - return false; - - (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - return xe_walk.needs_invalidate; -} - -static void -xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, - struct iosys_map *map, void *data, - u32 qword_ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update) -{ - struct xe_pt_entry *ptes = update->pt_entries; - u64 *ptr = data; - u32 i; - - for (i = 0; i < num_qwords; i++) { - if (map) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, ptes[i].pte); - else - ptr[i] = ptes[i].pte; - } -} - -static void xe_pt_abort_bind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, - u32 num_entries) -{ - u32 i, j; - - for (i = 0; i < num_entries; i++) { - if (!entries[i].pt_entries) - continue; - - for (j = 0; j < entries[i].qwords; j++) - xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL); - kfree(entries[i].pt_entries); - } -} - -static void xe_pt_commit_locks_assert(struct xe_vma *vma) -{ - struct xe_vm *vm = xe_vma_vm(vma); - - lockdep_assert_held(&vm->lock); - - if (xe_vma_is_userptr(vma)) - lockdep_assert_held_read(&vm->userptr.notifier_lock); - else if (!xe_vma_is_null(vma)) - dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); - - xe_vm_assert_held(vm); -} - -static void xe_pt_commit_bind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, - u32 num_entries, bool rebind, - struct llist_head *deferred) -{ - u32 i, j; - - xe_pt_commit_locks_assert(vma); - - for (i = 0; i < num_entries; i++) { - struct xe_pt *pt = entries[i].pt; - struct xe_pt_dir *pt_dir; - - if (!rebind) - pt->num_live += entries[i].qwords; - - if (!pt->level) { - kfree(entries[i].pt_entries); - continue; - } - - pt_dir = as_xe_pt_dir(pt); - for (j = 0; j < entries[i].qwords; j++) { - u32 j_ = j + entries[i].ofs; - struct xe_pt *newpte = entries[i].pt_entries[j].pt; - - if (xe_pt_entry(pt_dir, j_)) - xe_pt_destroy(xe_pt_entry(pt_dir, j_), - xe_vma_vm(vma)->flags, deferred); - - pt_dir->children[j_] = &newpte->base; - } - kfree(entries[i].pt_entries); - } -} - -static int -xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 *num_entries) -{ - int err; - - *num_entries = 0; - err = xe_pt_stage_bind(tile, vma, entries, num_entries); - if (!err) - xe_tile_assert(tile, *num_entries); - else /* abort! */ - xe_pt_abort_bind(vma, entries, *num_entries); - - return err; -} - -static void xe_vm_dbg_print_entries(struct xe_device *xe, - const struct xe_vm_pgtable_update *entries, - unsigned int num_entries) -#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) -{ - unsigned int i; - - vm_dbg(&xe->drm, "%u entries to update\n", num_entries); - for (i = 0; i < num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &entries[i]; - struct xe_pt *xe_pt = entry->pt; - u64 page_size = 1ull << xe_pt_shift(xe_pt->level); - u64 end; - u64 start; - - xe_assert(xe, !entry->pt->is_compact); - start = entry->ofs * page_size; - end = start + page_size * entry->qwords; - vm_dbg(&xe->drm, - "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n", - i, xe_pt->level, entry->ofs, entry->qwords, - xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0); - } -} -#else -{} -#endif - -#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT - -static int xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) -{ - u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; - static u32 count; - - if (count++ % divisor == divisor - 1) { - struct xe_vm *vm = xe_vma_vm(&uvma->vma); - - uvma->userptr.divisor = divisor << 1; - spin_lock(&vm->userptr.invalidated_lock); - list_move_tail(&uvma->userptr.invalidate_link, - &vm->userptr.invalidated); - spin_unlock(&vm->userptr.invalidated_lock); - return true; - } - - return false; -} - -#else - -static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) -{ - return false; -} - -#endif - -/** - * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks - * @base: Base we derive from. - * @bind: Whether this is a bind or an unbind operation. A bind operation - * makes the pre-commit callback error with -EAGAIN if it detects a - * pending invalidation. - * @locked: Whether the pre-commit callback locked the userptr notifier lock - * and it needs unlocking. - */ -struct xe_pt_migrate_pt_update { - struct xe_migrate_pt_update base; - bool bind; - bool locked; -}; - -/* - * This function adds the needed dependencies to a page-table update job - * to make sure racing jobs for separate bind engines don't race writing - * to the same page-table range, wreaking havoc. Initially use a single - * fence for the entire VM. An optimization would use smaller granularity. - */ -static int xe_pt_vm_dependencies(struct xe_sched_job *job, - struct xe_range_fence_tree *rftree, - u64 start, u64 last) -{ - struct xe_range_fence *rtfence; - struct dma_fence *fence; - int err; - - rtfence = xe_range_fence_tree_first(rftree, start, last); - while (rtfence) { - fence = rtfence->fence; - - if (!dma_fence_is_signaled(fence)) { - /* - * Is this a CPU update? GPU is busy updating, so return - * an error - */ - if (!job) - return -ETIME; - - dma_fence_get(fence); - err = drm_sched_job_add_dependency(&job->drm, fence); - if (err) - return err; - } - - rtfence = xe_range_fence_tree_next(rtfence, start, last); - } - - return 0; -} - -static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) -{ - struct xe_range_fence_tree *rftree = - &xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id]; - - return xe_pt_vm_dependencies(pt_update->job, rftree, - pt_update->start, pt_update->last); -} - -static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) -{ - struct xe_pt_migrate_pt_update *userptr_update = - container_of(pt_update, typeof(*userptr_update), base); - struct xe_userptr_vma *uvma = to_userptr_vma(pt_update->vma); - unsigned long notifier_seq = uvma->userptr.notifier_seq; - struct xe_vm *vm = xe_vma_vm(&uvma->vma); - int err = xe_pt_vm_dependencies(pt_update->job, - &vm->rftree[pt_update->tile_id], - pt_update->start, - pt_update->last); - - if (err) - return err; - - userptr_update->locked = false; - - /* - * Wait until nobody is running the invalidation notifier, and - * since we're exiting the loop holding the notifier lock, - * nobody can proceed invalidating either. - * - * Note that we don't update the vma->userptr.notifier_seq since - * we don't update the userptr pages. - */ - do { - down_read(&vm->userptr.notifier_lock); - if (!mmu_interval_read_retry(&uvma->userptr.notifier, - notifier_seq)) - break; - - up_read(&vm->userptr.notifier_lock); - - if (userptr_update->bind) - return -EAGAIN; - - notifier_seq = mmu_interval_read_begin(&uvma->userptr.notifier); - } while (true); - - /* Inject errors to test_whether they are handled correctly */ - if (userptr_update->bind && xe_pt_userptr_inject_eagain(uvma)) { - up_read(&vm->userptr.notifier_lock); - return -EAGAIN; - } - - userptr_update->locked = true; - - return 0; -} - -static const struct xe_migrate_pt_update_ops bind_ops = { - .populate = xe_vm_populate_pgtable, - .pre_commit = xe_pt_pre_commit, -}; - -static const struct xe_migrate_pt_update_ops userptr_bind_ops = { - .populate = xe_vm_populate_pgtable, - .pre_commit = xe_pt_userptr_pre_commit, -}; - -struct invalidation_fence { - struct xe_gt_tlb_invalidation_fence base; - struct xe_gt *gt; - struct dma_fence *fence; - struct dma_fence_cb cb; - struct work_struct work; - u64 start; - u64 end; - u32 asid; -}; - -static void invalidation_fence_cb(struct dma_fence *fence, - struct dma_fence_cb *cb) -{ - struct invalidation_fence *ifence = - container_of(cb, struct invalidation_fence, cb); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); - if (!ifence->fence->error) { - queue_work(system_wq, &ifence->work); - } else { - ifence->base.base.error = ifence->fence->error; - dma_fence_signal(&ifence->base.base); - dma_fence_put(&ifence->base.base); - } - dma_fence_put(ifence->fence); -} - -static void invalidation_fence_work_func(struct work_struct *w) -{ - struct invalidation_fence *ifence = - container_of(w, struct invalidation_fence, work); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); - xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, - ifence->end, ifence->asid); -} - -static int invalidation_fence_init(struct xe_gt *gt, - struct invalidation_fence *ifence, - struct dma_fence *fence, - u64 start, u64 end, u32 asid) -{ - int ret; - - trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); - - xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); - - ifence->fence = fence; - ifence->gt = gt; - ifence->start = start; - ifence->end = end; - ifence->asid = asid; - - INIT_WORK(&ifence->work, invalidation_fence_work_func); - ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); - if (ret == -ENOENT) { - dma_fence_put(ifence->fence); /* Usually dropped in CB */ - invalidation_fence_work_func(&ifence->work); - } else if (ret) { - dma_fence_put(&ifence->base.base); /* Caller ref */ - dma_fence_put(&ifence->base.base); /* Creation ref */ - } - - xe_gt_assert(gt, !ret || ret == -ENOENT); - - return ret && ret != -ENOENT ? ret : 0; -} - -static void xe_pt_calc_rfence_interval(struct xe_vma *vma, - struct xe_pt_migrate_pt_update *update, - struct xe_vm_pgtable_update *entries, - u32 num_entries) -{ - int i, level = 0; - - for (i = 0; i < num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &entries[i]; - - if (entry->pt->level > level) - level = entry->pt->level; - } - - /* Greedy (non-optimal) calculation but simple */ - update->base.start = ALIGN_DOWN(xe_vma_start(vma), - 0x1ull << xe_pt_shift(level)); - update->base.last = ALIGN(xe_vma_end(vma), - 0x1ull << xe_pt_shift(level)) - 1; -} - -/** - * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma - * address range. - * @tile: The tile to bind for. - * @vma: The vma to bind. - * @q: The exec_queue with which to do pipelined page-table updates. - * @syncs: Entries to sync on before binding the built tree to the live vm tree. - * @num_syncs: Number of @sync entries. - * @rebind: Whether we're rebinding this vma to the same address range without - * an unbind in-between. - * - * This function builds a page-table tree (see xe_pt_stage_bind() for more - * information on page-table building), and the xe_vm_pgtable_update entries - * abstracting the operations needed to attach it to the main vm tree. It - * then takes the relevant locks and updates the metadata side of the main - * vm tree and submits the operations for pipelined attachment of the - * gpu page-table to the vm main tree, (which can be done either by the - * cpu and the GPU). - * - * Return: A valid dma-fence representing the pipelined attachment operation - * on success, an error pointer on error. - */ -struct dma_fence * -__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, - struct xe_sync_entry *syncs, u32 num_syncs, - bool rebind) -{ - struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; - struct xe_pt_migrate_pt_update bind_pt_update = { - .base = { - .ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops, - .vma = vma, - .tile_id = tile->id, - }, - .bind = true, - }; - struct xe_vm *vm = xe_vma_vm(vma); - u32 num_entries; - struct dma_fence *fence; - struct invalidation_fence *ifence = NULL; - struct xe_range_fence *rfence; - int err; - - bind_pt_update.locked = false; - xe_bo_assert_held(xe_vma_bo(vma)); - xe_vm_assert_held(vm); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing bind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma), q); - - err = xe_pt_prepare_bind(tile, vma, entries, &num_entries); - if (err) - goto err; - - err = dma_resv_reserve_fences(xe_vm_resv(vm), 1); - if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1); - if (err) - goto err; - - xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries)); - - xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); - xe_pt_calc_rfence_interval(vma, &bind_pt_update, entries, - num_entries); - - /* - * If rebind, we have to invalidate TLB on !LR vms to invalidate - * cached PTEs point to freed memory. on LR vms this is done - * automatically when the context is re-enabled by the rebind worker, - * or in fault mode it was invalidated on PTE zapping. - * - * If !rebind, and scratch enabled VMs, there is a chance the scratch - * PTE is already cached in the TLB so it needs to be invalidated. - * on !LR VMs this is done in the ring ops preceding a batch, but on - * non-faulting LR, in particular on user-space batch buffer chaining, - * it needs to be done here. - */ - if ((!rebind && xe_vm_has_scratch(vm) && xe_vm_in_preempt_fence_mode(vm))) { - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) - return ERR_PTR(-ENOMEM); - } else if (rebind && !xe_vm_in_lr_mode(vm)) { - /* We bump also if batch_invalidate_tlb is true */ - vm->tlb_flush_seqno++; - } - - rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); - if (!rfence) { - kfree(ifence); - return ERR_PTR(-ENOMEM); - } - - fence = xe_migrate_update_pgtables(tile->migrate, - vm, xe_vma_bo(vma), q, - entries, num_entries, - syncs, num_syncs, - &bind_pt_update.base); - if (!IS_ERR(fence)) { - bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND; - LLIST_HEAD(deferred); - int err; - - err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - bind_pt_update.base.start, - bind_pt_update.base.last, fence); - if (err) - dma_fence_wait(fence, false); - - /* TLB invalidation must be done before signaling rebind */ - if (ifence) { - int err = invalidation_fence_init(tile->primary_gt, - ifence, fence, - xe_vma_start(vma), - xe_vma_end(vma), - xe_vma_vm(vma)->usm.asid); - if (err) { - dma_fence_put(fence); - kfree(ifence); - return ERR_PTR(err); - } - fence = &ifence->base.base; - } - - /* add shared fence now for pagetable delayed destroy */ - dma_resv_add_fence(xe_vm_resv(vm), fence, rebind || - last_munmap_rebind ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - DMA_RESV_USAGE_BOOKKEEP); - xe_pt_commit_bind(vma, entries, num_entries, rebind, - bind_pt_update.locked ? &deferred : NULL); - - /* This vma is live (again?) now */ - vma->tile_present |= BIT(tile->id); - - if (bind_pt_update.locked) { - to_userptr_vma(vma)->userptr.initial_bind = true; - up_read(&vm->userptr.notifier_lock); - xe_bo_put_commit(&deferred); - } - if (!rebind && last_munmap_rebind && - xe_vm_in_preempt_fence_mode(vm)) - xe_vm_queue_rebind_worker(vm); - } else { - kfree(rfence); - kfree(ifence); - if (bind_pt_update.locked) - up_read(&vm->userptr.notifier_lock); - xe_pt_abort_bind(vma, entries, num_entries); - } - - return fence; - -err: - return ERR_PTR(err); -} - -struct xe_pt_stage_unbind_walk { - /** @base: The pagewalk base-class. */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @tile: The tile we're unbinding from. */ - struct xe_tile *tile; - - /** - * @modified_start: Walk range start, modified to include any - * shared pagetables that we're the only user of and can thus - * treat as private. - */ - u64 modified_start; - /** @modified_end: Walk range start, modified like @modified_start. */ - u64 modified_end; - - /* Output */ - /* @wupd: Structure to track the page-table updates we're building */ - struct xe_walk_update wupd; -}; - -/* - * Check whether this range is the only one populating this pagetable, - * and in that case, update the walk range checks so that higher levels don't - * view us as a shared pagetable. - */ -static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, - const struct xe_pt *child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_unbind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - unsigned int shift = walk->shifts[level]; - u64 size = 1ull << shift; - - if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) && - ((next - addr) >> shift) == child->num_live) { - u64 size = 1ull << walk->shifts[level + 1]; - - *action = ACTION_CONTINUE; - - if (xe_walk->modified_start >= addr) - xe_walk->modified_start = round_down(addr, size); - if (xe_walk->modified_end <= next) - xe_walk->modified_end = round_up(next, size); - - return true; - } - - return false; -} - -static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - - XE_WARN_ON(!*child); - XE_WARN_ON(!level); - - xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); - - return 0; -} - -static int -xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_unbind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - pgoff_t end_offset; - u64 size = 1ull << walk->shifts[--level]; - - if (!IS_ALIGNED(addr, size)) - addr = xe_walk->modified_start; - if (!IS_ALIGNED(next, size)) - next = xe_walk->modified_end; - - /* Parent == *child is the root pt. Don't kill it. */ - if (parent != *child && - xe_pt_check_kill(addr, next, level, xe_child, action, walk)) - return 0; - - if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset, - &end_offset)) - return 0; - - (void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false); - xe_walk->wupd.updates[level].update->qwords = end_offset - offset; - - return 0; -} - -static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { - .pt_entry = xe_pt_stage_unbind_entry, - .pt_post_descend = xe_pt_stage_unbind_post_descend, -}; - -/** - * xe_pt_stage_unbind() - Build page-table update structures for an unbind - * operation - * @tile: The tile we're unbinding for. - * @vma: The vma we're unbinding. - * @entries: Caller-provided storage for the update structures. - * - * Builds page-table update structures for an unbind operation. The function - * will attempt to remove all page-tables that we're the only user - * of, and for that to work, the unbind operation must be committed in the - * same critical section that blocks racing binds to the same page-table tree. - * - * Return: The number of entries used. - */ -static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries) -{ - struct xe_pt_stage_unbind_walk xe_walk = { - .base = { - .ops = &xe_pt_stage_unbind_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .tile = tile, - .modified_start = xe_vma_start(vma), - .modified_end = xe_vma_end(vma), - .wupd.entries = entries, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - - (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - return xe_walk.wupd.num_used_entries; -} - -static void -xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, - struct xe_tile *tile, struct iosys_map *map, - void *ptr, u32 qword_ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update) -{ - struct xe_vma *vma = pt_update->vma; - u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level); - int i; - - if (map && map->is_iomem) - for (i = 0; i < num_qwords; ++i) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, empty); - else if (map) - memset64(map->vaddr + qword_ofs * sizeof(u64), empty, - num_qwords); - else - memset64(ptr, empty, num_qwords); -} - -static void -xe_pt_commit_unbind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 num_entries, - struct llist_head *deferred) -{ - u32 j; - - xe_pt_commit_locks_assert(vma); - - for (j = 0; j < num_entries; ++j) { - struct xe_vm_pgtable_update *entry = &entries[j]; - struct xe_pt *pt = entry->pt; - - pt->num_live -= entry->qwords; - if (pt->level) { - struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); - u32 i; - - for (i = entry->ofs; i < entry->ofs + entry->qwords; - i++) { - if (xe_pt_entry(pt_dir, i)) - xe_pt_destroy(xe_pt_entry(pt_dir, i), - xe_vma_vm(vma)->flags, deferred); - - pt_dir->children[i] = NULL; - } - } - } -} - -<<<<<<< -static const struct xe_migrate_pt_update_ops unbind_ops = { - .populate = xe_migrate_clear_pgtable_callback, -======= -static void -xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int i, level = 0; - u64 start, last; - - for (i = 0; i < pt_op->num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; - - if (entry->pt->level > level) - level = entry->pt->level; - } - - /* Greedy (non-optimal) calculation but simple */ - start = ALIGN_DOWN(xe_vma_start(vma), 0x1ull << xe_pt_shift(level)); - last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1; - - if (start < pt_update_ops->start) - pt_update_ops->start = start; - if (last > pt_update_ops->last) - pt_update_ops->last = last; -} - -static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, - xe->info.tile_count); - - return 0; -} - -static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int err; - - xe_bo_assert_held(xe_vma_bo(vma)); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing bind, with range [%llx...%llx)\n", - xe_vma_start(vma), xe_vma_end(vma) - 1); - - pt_op->vma = NULL; - pt_op->bind = true; - pt_op->rebind = BIT(tile->id) & vma->tile_present; - - err = vma_reserve_fences(tile_to_xe(tile), vma); - if (err) - return err; - - err = xe_pt_prepare_bind(tile, vma, pt_op->entries, - &pt_op->num_entries); - if (!err) { - xe_tile_assert(tile, pt_op->num_entries <= - ARRAY_SIZE(pt_op->entries)); - xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, - pt_op->num_entries, true); - - xe_pt_update_ops_rfence_interval(pt_update_ops, vma); - ++pt_update_ops->current_op; - pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); - - /* - * If rebind, we have to invalidate TLB on !LR vms to invalidate - * cached PTEs point to freed memory. On LR vms this is done - * automatically when the context is re-enabled by the rebind worker, - * or in fault mode it was invalidated on PTE zapping. - * - * If !rebind, and scratch enabled VMs, there is a chance the scratch - * PTE is already cached in the TLB so it needs to be invalidated. - * On !LR VMs this is done in the ring ops preceding a batch, but on - * non-faulting LR, in particular on user-space batch buffer chaining, - * it needs to be done here. - */ - if ((!pt_op->rebind && xe_vm_has_scratch(vm) && - xe_vm_in_preempt_fence_mode(vm))) - pt_update_ops->needs_invalidation = true; - else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) - /* We bump also if batch_invalidate_tlb is true */ - vm->tlb_flush_seqno++; - - vma->tile_staged |= BIT(tile->id); - pt_op->vma = vma; - xe_pt_commit_prepare_bind(vma, pt_op->entries, - pt_op->num_entries, pt_op->rebind); - } else { - xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); - } - - return err; -} - -static int unbind_op_prepare(struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int err; - - if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) - return 0; - - xe_bo_assert_held(xe_vma_bo(vma)); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing unbind, with range [%llx...%llx)\n", - xe_vma_start(vma), xe_vma_end(vma) - 1); - - /* - * Wait for invalidation to complete. Can corrupt internal page table - * state if an invalidation is running while preparing an unbind. - */ - if (xe_vma_is_userptr(vma) && xe_vm_in_fault_mode(xe_vma_vm(vma))) - mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier); - - pt_op->vma = vma; - pt_op->bind = false; - pt_op->rebind = false; - - err = vma_reserve_fences(tile_to_xe(tile), vma); - if (err) - return err; - - pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries); - - xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, - pt_op->num_entries, false); - xe_pt_update_ops_rfence_interval(pt_update_ops, vma); - ++pt_update_ops->current_op; - pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); - pt_update_ops->needs_invalidation = true; - - xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); - - return 0; -} - -static int op_prepare(struct xe_vm *vm, - struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op) -{ - int err = 0; - - xe_vm_assert_held(vm); - - switch (op->base.op) { - case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) - break; - - err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma); - pt_update_ops->wait_vm_kernel = true; - break; - case DRM_GPUVA_OP_REMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va)); - - if (!err && op->remap.prev) { - err = bind_op_prepare(vm, tile, pt_update_ops, - op->remap.prev); - pt_update_ops->wait_vm_bookkeep = true; - } - if (!err && op->remap.next) { - err = bind_op_prepare(vm, tile, pt_update_ops, - op->remap.next); - pt_update_ops->wait_vm_bookkeep = true; - } - break; - case DRM_GPUVA_OP_UNMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va)); - break; - case DRM_GPUVA_OP_PREFETCH: - err = bind_op_prepare(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va)); - pt_update_ops->wait_vm_kernel = true; - break; - default: - drm_warn(&vm->xe->drm, "NOT POSSIBLE"); - } - - return err; -} - -static void -xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) -{ - init_llist_head(&pt_update_ops->deferred); - pt_update_ops->start = ~0x0ull; - pt_update_ops->last = 0x0ull; -} - -/** - * xe_pt_update_ops_prepare() - Prepare PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operationa - * - * Prepare PT update operations which includes updating internal PT state, - * allocate memory for page tables, populate page table being pruned in, and - * create PT update operations for leaf insertion / removal. - * - * Return: 0 on success, negative error code on error. - */ -int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - struct xe_vma_op *op; - int err; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - xe_pt_update_ops_init(pt_update_ops); - - err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), - tile_to_xe(tile)->info.tile_count); - if (err) - return err; - - list_for_each_entry(op, &vops->list, link) { - err = op_prepare(vops->vm, tile, pt_update_ops, op); - - if (err) - return err; - } - - xe_tile_assert(tile, pt_update_ops->current_op <= - pt_update_ops->num_ops); - -#ifdef TEST_VM_OPS_ERROR - if (vops->inject_error && - vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) - return -ENOSPC; -#endif - - return 0; -} - -static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - } - vma->tile_present |= BIT(tile->id); - vma->tile_staged &= ~BIT(tile->id); - if (xe_vma_is_userptr(vma)) { - lockdep_assert_held_read(&vm->userptr.notifier_lock); - to_userptr_vma(vma)->userptr.initial_bind = true; - } - - /* - * Kick rebind worker if this bind triggers preempt fences and not in - * the rebind worker - */ - if (pt_update_ops->wait_vm_bookkeep && - xe_vm_in_preempt_fence_mode(vm) && - !current->mm) - xe_vm_queue_rebind_worker(vm); -} - -static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - } - vma->tile_present &= ~BIT(tile->id); - if (!vma->tile_present) { - list_del_init(&vma->combined_links.rebind); - if (xe_vma_is_userptr(vma)) { - lockdep_assert_held_read(&vm->userptr.notifier_lock); - - spin_lock(&vm->userptr.invalidated_lock); - list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); - spin_unlock(&vm->userptr.invalidated_lock); - } - } -} - -static void op_commit(struct xe_vm *vm, - struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op, struct dma_fence *fence, - struct dma_fence *fence2) -{ - xe_vm_assert_held(vm); - - switch (op->base.op) { - case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) - break; - - bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, - fence2); - break; - case DRM_GPUVA_OP_REMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va), fence, - fence2); - - if (op->remap.prev) - bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, - fence, fence2); - if (op->remap.next) - bind_op_commit(vm, tile, pt_update_ops, op->remap.next, - fence, fence2); - break; - case DRM_GPUVA_OP_UNMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va), fence, fence2); - break; - case DRM_GPUVA_OP_PREFETCH: - bind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va), fence, fence2); - break; - default: - drm_warn(&vm->xe->drm, "NOT POSSIBLE"); - } -} - -static const struct xe_migrate_pt_update_ops migrate_ops = { - .populate = xe_vm_populate_pgtable, - .clear = xe_migrate_clear_pgtable_callback, ->>>>>>> - .pre_commit = xe_pt_pre_commit, -}; - -static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { - .populate = xe_migrate_clear_pgtable_callback, - .pre_commit = xe_pt_userptr_pre_commit, -}; - -/** - * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma - * address range. - * @tile: The tile to unbind for. - * @vma: The vma to unbind. - * @q: The exec_queue with which to do pipelined page-table updates. - * @syncs: Entries to sync on before disconnecting the tree to be destroyed. - * @num_syncs: Number of @sync entries. - * - * This function builds a the xe_vm_pgtable_update entries abstracting the - * operations needed to detach the page-table tree to be destroyed from the - * man vm tree. - * It then takes the relevant locks and submits the operations for - * pipelined detachment of the gpu page-table from the vm main tree, - * (which can be done either by the cpu and the GPU), Finally it frees the - * detached page-table tree. - * - * Return: A valid dma-fence representing the pipelined detachment operation - * on success, an error pointer on error. - */ -struct dma_fence * -__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, - struct xe_sync_entry *syncs, u32 num_syncs) -{ -<<<<<<< - struct xe_vm *vm = vops->vm; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - struct dma_fence *fence; - struct invalidation_fence *ifence = NULL, *mfence = NULL; - struct dma_fence_chain *chain_fence = NULL; - struct xe_range_fence *rfence; - struct xe_vma_op *op; - int err = 0, i; - struct xe_migrate_pt_update update = { - .ops = pt_update_ops->needs_userptr_lock ? - &userptr_migrate_ops : - &migrate_ops, - .vops = vops, - .tile_id = tile->id, -======= - struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; - struct xe_pt_migrate_pt_update unbind_pt_update = { - .base = { - .ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops : - &unbind_ops, - .vma = vma, - .tile_id = tile->id, - }, ->>>>>>> - }; - struct xe_vm *vm = xe_vma_vm(vma); - u32 num_entries; - struct dma_fence *fence = NULL; - struct invalidation_fence *ifence; - struct xe_range_fence *rfence; - int err; - - LLIST_HEAD(deferred); - - xe_bo_assert_held(xe_vma_bo(vma)); - xe_vm_assert_held(vm); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing unbind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma), q); - - num_entries = xe_pt_stage_unbind(tile, vma, entries); - xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries)); - - xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); - xe_pt_calc_rfence_interval(vma, &unbind_pt_update, entries, - num_entries); - -<<<<<<< - err = dma_resv_reserve_fences(xe_vm_resv(vm), 1); - if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1); - if (err) - return ERR_PTR(err); - - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) - return ERR_PTR(-ENOMEM); -======= - if (pt_update_ops->needs_invalidation) { - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) { - err = -ENOMEM; - goto kill_vm_tile1; - } - if (tile->media_gt) { - mfence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!mfence) { - err = -ENOMEM; - goto free_ifence; - } - chain_fence = dma_fence_chain_alloc(); - if (!chain_fence) { - err = -ENOMEM; - goto free_ifence; - } - } - } ->>>>>>> - - rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); - if (!rfence) { - kfree(ifence); - return ERR_PTR(-ENOMEM); - } - - /* - * Even if we were already evicted and unbind to destroy, we need to - * clear again here. The eviction may have updated pagetables at a - * lower level, because it needs to be more conservative. - */ - fence = xe_migrate_update_pgtables(tile->migrate, - vm, NULL, q ? q : - vm->q[tile->id], - entries, num_entries, - syncs, num_syncs, - &unbind_pt_update.base); - if (!IS_ERR(fence)) { - int err; - - err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - unbind_pt_update.base.start, - unbind_pt_update.base.last, fence); - if (err) - dma_fence_wait(fence, false); - -<<<<<<< - /* TLB invalidation must be done before signaling unbind */ - err = invalidation_fence_init(tile->primary_gt, ifence, fence, - xe_vma_start(vma), - xe_vma_end(vma), - xe_vma_vm(vma)->usm.asid); - if (err) { - dma_fence_put(fence); - kfree(ifence); - return ERR_PTR(err); - } - fence = &ifence->base.base; - - /* add shared fence now for pagetable delayed destroy */ - dma_resv_add_fence(xe_vm_resv(vm), fence, - DMA_RESV_USAGE_BOOKKEEP); - - /* This fence will be installed by caller when doing eviction */ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - DMA_RESV_USAGE_BOOKKEEP); - xe_pt_commit_unbind(vma, entries, num_entries, - unbind_pt_update.locked ? &deferred : NULL); - vma->tile_present &= ~BIT(tile->id); - } else { - kfree(rfence); - kfree(ifence); - } - - if (!vma->tile_present) - list_del_init(&vma->combined_links.rebind); - - if (unbind_pt_update.locked) { - xe_tile_assert(tile, xe_vma_is_userptr(vma)); -======= - xe_pt_commit(pt_op->vma, pt_op->entries, - pt_op->num_entries, &pt_update_ops->deferred); - pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ - } - - if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - pt_update_ops->start, - pt_update_ops->last, fence)) - dma_fence_wait(fence, false); - - /* tlb invalidation must be done before signaling rebind */ - if (ifence) { - if (mfence) - dma_fence_get(fence); - invalidation_fence_init(tile->primary_gt, ifence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - if (mfence) { - invalidation_fence_init(tile->media_gt, mfence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - dma_fence_chain_init(chain_fence, &ifence->base.base, - &mfence->base.base, 0); - fence = &chain_fence->base; - } else { - fence = &ifence->base.base; - } - } - - if (!mfence) { - dma_resv_add_fence(xe_vm_resv(vm), fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); - } else { - dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, - &ifence->base.base, &mfence->base.base); - } ->>>>>>> - - if (!vma->tile_present) { - spin_lock(&vm->userptr.invalidated_lock); - list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); - spin_unlock(&vm->userptr.invalidated_lock); - } - up_read(&vm->userptr.notifier_lock); - xe_bo_put_commit(&deferred); - } - - return fence; -<<<<<<< -======= - -free_rfence: - kfree(rfence); -free_ifence: - dma_fence_chain_free(chain_fence); - kfree(mfence); - kfree(ifence); -kill_vm_tile1: - if (err != -EAGAIN && tile->id) - xe_vm_kill(vops->vm, false); - - return ERR_PTR(err); -} - -/** - * xe_pt_update_ops_fini() - Finish PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operations - * - * Finish PT update operations by committing to destroy page table memory - */ -void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - for (i = 0; i < pt_update_ops->current_op; ++i) { - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; - - xe_pt_free_bind(pt_op->entries, pt_op->num_entries); - } - xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); -} - -/** - * xe_pt_update_ops_abort() - Abort PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operationa - * - * Abort PT update operations by unwinding internal PT state - */ -void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - - if (!pt_op->vma || i >= pt_update_ops->current_op) - continue; - - if (pt_op->bind) - xe_pt_abort_bind(pt_op->vma, pt_op->entries, - pt_op->num_entries, - pt_op->rebind); - else - xe_pt_abort_unbind(pt_op->vma, pt_op->entries, - pt_op->num_entries); - } - - xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); ->>>>>>> -} diff --git a/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage.1 b/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage.1 deleted file mode 100644 index 232d29e9a561..000000000000 --- a/rr-cache/41549dd6cc337627199acbe6749c5685c7e927d3/preimage.1 +++ /dev/null @@ -1,2234 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2022 Intel Corporation - */ - -#include <linux/dma-fence-chain.h> - -#include "xe_pt.h" - -#include "regs/xe_gtt_defs.h" -#include "xe_bo.h" -#include "xe_device.h" -#include "xe_drm_client.h" -#include "xe_gt.h" -#include "xe_gt_tlb_invalidation.h" -#include "xe_migrate.h" -#include "xe_pt_types.h" -#include "xe_pt_walk.h" -#include "xe_res_cursor.h" -#include "xe_trace.h" -#include "xe_ttm_stolen_mgr.h" -#include "xe_vm.h" - -struct xe_pt_dir { - struct xe_pt pt; - /** @children: Array of page-table child nodes */ - struct xe_ptw *children[XE_PDES]; -}; - -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) -#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr)) -#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr) -#else -#define xe_pt_set_addr(__xe_pt, __addr) -#define xe_pt_addr(__xe_pt) 0ull -#endif - -static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48}; -static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48}; - -#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1) - -static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) -{ - return container_of(pt, struct xe_pt_dir, pt); -} - -static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index) -{ - return container_of(pt_dir->children[index], struct xe_pt, base); -} - -static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, - unsigned int level) -{ - struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; - u8 id = tile->id; - - if (!xe_vm_has_scratch(vm)) - return 0; - - if (level > MAX_HUGEPTE_LEVEL) - return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, - 0, pat_index); - - return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | - XE_PTE_NULL; -} - -static void xe_pt_free(struct xe_pt *pt) -{ - if (pt->level) - kfree(as_xe_pt_dir(pt)); - else - kfree(pt); -} - -/** - * xe_pt_create() - Create a page-table. - * @vm: The vm to create for. - * @tile: The tile to create for. - * @level: The page-table level. - * - * Allocate and initialize a single struct xe_pt metadata structure. Also - * create the corresponding page-table bo, but don't initialize it. If the - * level is grater than zero, then it's assumed to be a directory page- - * table and the directory structure is also allocated and initialized to - * NULL pointers. - * - * Return: A valid struct xe_pt pointer on success, Pointer error code on - * error. - */ -struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, - unsigned int level) -{ - struct xe_pt *pt; - struct xe_bo *bo; - int err; - - if (level) { - struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL); - - pt = (dir) ? &dir->pt : NULL; - } else { - pt = kzalloc(sizeof(*pt), GFP_KERNEL); - } - if (!pt) - return ERR_PTR(-ENOMEM); - - pt->level = level; - bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, - ttm_bo_type_kernel, - XE_BO_FLAG_VRAM_IF_DGFX(tile) | - XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE | - XE_BO_FLAG_PINNED | - XE_BO_FLAG_NO_RESV_EVICT | - XE_BO_FLAG_PAGETABLE); - if (IS_ERR(bo)) { - err = PTR_ERR(bo); - goto err_kfree; - } - pt->bo = bo; - pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; - - if (vm->xef) - xe_drm_client_add_bo(vm->xef->client, pt->bo); - xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL); - - return pt; - -err_kfree: - xe_pt_free(pt); - return ERR_PTR(err); -} - -/** - * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero - * entries. - * @tile: The tile the scratch pagetable of which to use. - * @vm: The vm we populate for. - * @pt: The pagetable the bo of which to initialize. - * - * Populate the page-table bo of @pt with entries pointing into the tile's - * scratch page-table tree if any. Otherwise populate with zeros. - */ -void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, - struct xe_pt *pt) -{ - struct iosys_map *map = &pt->bo->vmap; - u64 empty; - int i; - - if (!xe_vm_has_scratch(vm)) { - /* - * FIXME: Some memory is allocated already allocated to zero? - * Find out which memory that is and avoid this memset... - */ - xe_map_memset(vm->xe, map, 0, 0, SZ_4K); - } else { - empty = __xe_pt_empty_pte(tile, vm, pt->level); - for (i = 0; i < XE_PDES; i++) - xe_pt_write(vm->xe, map, i, empty); - } -} - -/** - * xe_pt_shift() - Return the ilog2 value of the size of the address range of - * a page-table at a certain level. - * @level: The level. - * - * Return: The ilog2 value of the size of the address range of a page-table - * at level @level. - */ -unsigned int xe_pt_shift(unsigned int level) -{ - return XE_PTE_SHIFT + XE_PDE_SHIFT * level; -} - -/** - * xe_pt_destroy() - Destroy a page-table tree. - * @pt: The root of the page-table tree to destroy. - * @flags: vm flags. Currently unused. - * @deferred: List head of lockless list for deferred putting. NULL for - * immediate putting. - * - * Puts the page-table bo, recursively calls xe_pt_destroy on all children - * and finally frees @pt. TODO: Can we remove the @flags argument? - */ -void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) -{ - int i; - - if (!pt) - return; - - XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); - xe_bo_unpin(pt->bo); - xe_bo_put_deferred(pt->bo, deferred); - - if (pt->level > 0 && pt->num_live) { - struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); - - for (i = 0; i < XE_PDES; i++) { - if (xe_pt_entry(pt_dir, i)) - xe_pt_destroy(xe_pt_entry(pt_dir, i), flags, - deferred); - } - } - xe_pt_free(pt); -} - -/** - * DOC: Pagetable building - * - * Below we use the term "page-table" for both page-directories, containing - * pointers to lower level page-directories or page-tables, and level 0 - * page-tables that contain only page-table-entries pointing to memory pages. - * - * When inserting an address range in an already existing page-table tree - * there will typically be a set of page-tables that are shared with other - * address ranges, and a set that are private to this address range. - * The set of shared page-tables can be at most two per level, - * and those can't be updated immediately because the entries of those - * page-tables may still be in use by the gpu for other mappings. Therefore - * when inserting entries into those, we instead stage those insertions by - * adding insertion data into struct xe_vm_pgtable_update structures. This - * data, (subtrees for the cpu and page-table-entries for the gpu) is then - * added in a separate commit step. CPU-data is committed while still under the - * vm lock, the object lock and for userptr, the notifier lock in read mode. - * The GPU async data is committed either by the GPU or CPU after fulfilling - * relevant dependencies. - * For non-shared page-tables (and, in fact, for shared ones that aren't - * existing at the time of staging), we add the data in-place without the - * special update structures. This private part of the page-table tree will - * remain disconnected from the vm page-table tree until data is committed to - * the shared page tables of the vm tree in the commit phase. - */ - -struct xe_pt_update { - /** @update: The update structure we're building for this parent. */ - struct xe_vm_pgtable_update *update; - /** @parent: The parent. Used to detect a parent change. */ - struct xe_pt *parent; - /** @preexisting: Whether the parent was pre-existing or allocated */ - bool preexisting; -}; - -struct xe_pt_stage_bind_walk { - /** base: The base class. */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @vm: The vm we're building for. */ - struct xe_vm *vm; - /** @tile: The tile we're building for. */ - struct xe_tile *tile; - /** @default_pte: PTE flag only template. No address is associated */ - u64 default_pte; - /** @dma_offset: DMA offset to add to the PTE. */ - u64 dma_offset; - /** - * @needs_64k: This address range enforces 64K alignment and - * granularity. - */ - bool needs_64K; - /** - * @vma: VMA being mapped - */ - struct xe_vma *vma; - - /* Also input, but is updated during the walk*/ - /** @curs: The DMA address cursor. */ - struct xe_res_cursor *curs; - /** @va_curs_start: The Virtual address coresponding to @curs->start */ - u64 va_curs_start; - - /* Output */ - struct xe_walk_update { - /** @wupd.entries: Caller provided storage. */ - struct xe_vm_pgtable_update *entries; - /** @wupd.num_used_entries: Number of update @entries used. */ - unsigned int num_used_entries; - /** @wupd.updates: Tracks the update entry at a given level */ - struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1]; - } wupd; - - /* Walk state */ - /** - * @l0_end_addr: The end address of the current l0 leaf. Used for - * 64K granularity detection. - */ - u64 l0_end_addr; - /** @addr_64K: The start address of the current 64K chunk. */ - u64 addr_64K; - /** @found_64: Whether @add_64K actually points to a 64K chunk. */ - bool found_64K; -}; - -static int -xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, - pgoff_t offset, bool alloc_entries) -{ - struct xe_pt_update *upd = &wupd->updates[parent->level]; - struct xe_vm_pgtable_update *entry; - - /* - * For *each level*, we could only have one active - * struct xt_pt_update at any one time. Once we move on to a - * new parent and page-directory, the old one is complete, and - * updates are either already stored in the build tree or in - * @wupd->entries - */ - if (likely(upd->parent == parent)) - return 0; - - upd->parent = parent; - upd->preexisting = true; - - if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) - return -EINVAL; - - entry = wupd->entries + wupd->num_used_entries++; - upd->update = entry; - entry->ofs = offset; - entry->pt_bo = parent->bo; - entry->pt = parent; - entry->flags = 0; - entry->qwords = 0; - - if (alloc_entries) { - entry->pt_entries = kmalloc_array(XE_PDES, - sizeof(*entry->pt_entries), - GFP_KERNEL); - if (!entry->pt_entries) - return -ENOMEM; - } - - return 0; -} - -/* - * NOTE: This is a very frequently called function so we allow ourselves - * to annotate (using branch prediction hints) the fastpath of updating a - * non-pre-existing pagetable with leaf ptes. - */ -static int -xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, - pgoff_t offset, struct xe_pt *xe_child, u64 pte) -{ - struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; - struct xe_pt_update *child_upd = xe_child ? - &xe_walk->wupd.updates[xe_child->level] : NULL; - int ret; - - ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); - if (unlikely(ret)) - return ret; - - /* - * Register this new pagetable so that it won't be recognized as - * a shared pagetable by a subsequent insertion. - */ - if (unlikely(child_upd)) { - child_upd->update = NULL; - child_upd->parent = xe_child; - child_upd->preexisting = false; - } - - if (likely(!upd->preexisting)) { - /* Continue building a non-connected subtree. */ - struct iosys_map *map = &parent->bo->vmap; - - if (unlikely(xe_child)) - parent->base.children[offset] = &xe_child->base; - - xe_pt_write(xe_walk->vm->xe, map, offset, pte); - parent->num_live++; - } else { - /* Shared pt. Stage update. */ - unsigned int idx; - struct xe_vm_pgtable_update *entry = upd->update; - - idx = offset - entry->ofs; - entry->pt_entries[idx].pt = xe_child; - entry->pt_entries[idx].pte = pte; - entry->qwords++; - } - - return 0; -} - -static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, - struct xe_pt_stage_bind_walk *xe_walk) -{ - u64 size, dma; - - if (level > MAX_HUGEPTE_LEVEL) - return false; - - /* Does the virtual range requested cover a huge pte? */ - if (!xe_pt_covers(addr, next, level, &xe_walk->base)) - return false; - - /* Does the DMA segment cover the whole pte? */ - if (next - xe_walk->va_curs_start > xe_walk->curs->size) - return false; - - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) - return true; - - /* Is the DMA address huge PTE size aligned? */ - size = next - addr; - dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); - - return IS_ALIGNED(dma, size); -} - -/* - * Scan the requested mapping to check whether it can be done entirely - * with 64K PTEs. - */ -static bool -xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) -{ - struct xe_res_cursor curs = *xe_walk->curs; - - if (!IS_ALIGNED(addr, SZ_64K)) - return false; - - if (next > xe_walk->l0_end_addr) - return false; - - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) - return true; - - xe_res_next(&curs, addr - xe_walk->va_curs_start); - for (; addr < next; addr += SZ_64K) { - if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) - return false; - - xe_res_next(&curs, SZ_64K); - } - - return addr == next; -} - -/* - * For non-compact "normal" 4K level-0 pagetables, we want to try to group - * addresses together in 64K-contigous regions to add a 64K TLB hint for the - * device to the PTE. - * This function determines whether the address is part of such a - * segment. For VRAM in normal pagetables, this is strictly necessary on - * some devices. - */ -static bool -xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) -{ - /* Address is within an already found 64k region */ - if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) - return true; - - xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); - xe_walk->addr_64K = addr; - - return xe_walk->found_64K; -} - -static int -xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_bind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - u16 pat_index = xe_walk->vma->pat_index; - struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); - struct xe_vm *vm = xe_walk->vm; - struct xe_pt *xe_child; - bool covers; - int ret = 0; - u64 pte; - - /* Is this a leaf entry ?*/ - if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) { - struct xe_res_cursor *curs = xe_walk->curs; - bool is_null = xe_vma_is_null(xe_walk->vma); - - XE_WARN_ON(xe_walk->va_curs_start != addr); - - pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : - xe_res_dma(curs) + xe_walk->dma_offset, - xe_walk->vma, pat_index, level); - pte |= xe_walk->default_pte; - - /* - * Set the XE_PTE_PS64 hint if possible, otherwise if - * this device *requires* 64K PTE size for VRAM, fail. - */ - if (level == 0 && !xe_parent->is_compact) { - if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) { - xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K; - pte |= XE_PTE_PS64; - } else if (XE_WARN_ON(xe_walk->needs_64K)) { - return -EINVAL; - } - } - - ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte); - if (unlikely(ret)) - return ret; - - if (!is_null) - xe_res_next(curs, next - addr); - xe_walk->va_curs_start = next; - xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); - *action = ACTION_CONTINUE; - - return ret; - } - - /* - * Descending to lower level. Determine if we need to allocate a - * new page table or -directory, which we do if there is no - * previous one or there is one we can completely replace. - */ - if (level == 1) { - walk->shifts = xe_normal_pt_shifts; - xe_walk->l0_end_addr = next; - } - - covers = xe_pt_covers(addr, next, level, &xe_walk->base); - if (covers || !*child) { - u64 flags = 0; - - xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); - if (IS_ERR(xe_child)) - return PTR_ERR(xe_child); - - xe_pt_set_addr(xe_child, - round_down(addr, 1ull << walk->shifts[level])); - - if (!covers) - xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); - - *child = &xe_child->base; - - /* - * Prefer the compact pagetable layout for L0 if possible. Only - * possible if VMA covers entire 2MB region as compact 64k and - * 4k pages cannot be mixed within a 2MB region. - * TODO: Suballocate the pt bo to avoid wasting a lot of - * memory. - */ - if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && - covers && xe_pt_scan_64K(addr, next, xe_walk)) { - walk->shifts = xe_compact_pt_shifts; - xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; - flags |= XE_PDE_64K; - xe_child->is_compact = true; - } - - pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; - ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, - pte); - } - - *action = ACTION_SUBTREE; - return ret; -} - -static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { - .pt_entry = xe_pt_stage_bind_entry, -}; - -/** - * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address - * range. - * @tile: The tile we're building for. - * @vma: The vma indicating the address range. - * @entries: Storage for the update entries used for connecting the tree to - * the main tree at commit time. - * @num_entries: On output contains the number of @entries used. - * - * This function builds a disconnected page-table tree for a given address - * range. The tree is connected to the main vm tree for the gpu using - * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind(). - * The function builds xe_vm_pgtable_update structures for already existing - * shared page-tables, and non-existing shared and non-shared page-tables - * are built and populated directly. - * - * Return 0 on success, negative error code on error. - */ -static int -xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 *num_entries) -{ - struct xe_device *xe = tile_to_xe(tile); - struct xe_bo *bo = xe_vma_bo(vma); - bool is_devmem = !xe_vma_is_userptr(vma) && bo && - (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)); - struct xe_res_cursor curs; - struct xe_pt_stage_bind_walk xe_walk = { - .base = { - .ops = &xe_pt_stage_bind_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .vm = xe_vma_vm(vma), - .tile = tile, - .curs = &curs, - .va_curs_start = xe_vma_start(vma), - .vma = vma, - .wupd.entries = entries, - .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - int ret; - - /** - * Default atomic expectations for different allocation scenarios are as follows: - * - * 1. Traditional API: When the VM is not in LR mode: - * - Device atomics are expected to function with all allocations. - * - * 2. Compute/SVM API: When the VM is in LR mode: - * - Device atomics are the default behavior when the bo is placed in a single region. - * - In all other cases device atomics will be disabled with AE=0 until an application - * request differently using a ioctl like madvise. - */ - if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { - if (xe_vm_in_lr_mode(xe_vma_vm(vma))) { - if (bo && xe_bo_has_single_placement(bo)) - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - /** - * If a SMEM+LMEM allocation is backed by SMEM, a device - * atomics will cause a gpu page fault and which then - * gets migrated to LMEM, bind such allocations with - * device atomics enabled. - */ - else if (is_devmem && !xe_bo_has_single_placement(bo)) - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - } else { - xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE; - } - - /** - * Unset AE if the platform(PVC) doesn't support it on an - * allocation - */ - if (!xe->info.has_device_atomics_on_smem && !is_devmem) - xe_walk.default_pte &= ~XE_USM_PPGTT_PTE_AE; - } - - if (is_devmem) { - xe_walk.default_pte |= XE_PPGTT_PTE_DM; - xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); - } - - if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo)) - xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo)); - - xe_bo_assert_held(bo); - - if (!xe_vma_is_null(vma)) { - if (xe_vma_is_userptr(vma)) - xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, - xe_vma_size(vma), &curs); - else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) - xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), - xe_vma_size(vma), &curs); - else - xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma), - xe_vma_size(vma), &curs); - } else { - curs.size = xe_vma_size(vma); - } - - ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - *num_entries = xe_walk.wupd.num_used_entries; - return ret; -} - -/** - * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a - * shared pagetable. - * @addr: The start address within the non-shared pagetable. - * @end: The end address within the non-shared pagetable. - * @level: The level of the non-shared pagetable. - * @walk: Walk info. The function adjusts the walk action. - * @action: next action to perform (see enum page_walk_action) - * @offset: Ignored on input, First non-shared entry on output. - * @end_offset: Ignored on input, Last non-shared entry + 1 on output. - * - * A non-shared page-table has some entries that belong to the address range - * and others that don't. This function determines the entries that belong - * fully to the address range. Depending on level, some entries may - * partially belong to the address range (that can't happen at level 0). - * The function detects that and adjust those offsets to not include those - * partial entries. Iff it does detect partial entries, we know that there must - * be shared page tables also at lower levels, so it adjusts the walk action - * accordingly. - * - * Return: true if there were non-shared entries, false otherwise. - */ -static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level, - struct xe_pt_walk *walk, - enum page_walk_action *action, - pgoff_t *offset, pgoff_t *end_offset) -{ - u64 size = 1ull << walk->shifts[level]; - - *offset = xe_pt_offset(addr, level, walk); - *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset; - - if (!level) - return true; - - /* - * If addr or next are not size aligned, there are shared pts at lower - * level, so in that case traverse down the subtree - */ - *action = ACTION_CONTINUE; - if (!IS_ALIGNED(addr, size)) { - *action = ACTION_SUBTREE; - (*offset)++; - } - - if (!IS_ALIGNED(end, size)) { - *action = ACTION_SUBTREE; - (*end_offset)--; - } - - return *end_offset > *offset; -} - -struct xe_pt_zap_ptes_walk { - /** @base: The walk base-class */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @tile: The tile we're building for */ - struct xe_tile *tile; - - /* Output */ - /** @needs_invalidate: Whether we need to invalidate TLB*/ - bool needs_invalidate; -}; - -static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_zap_ptes_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - pgoff_t end_offset; - - XE_WARN_ON(!*child); - XE_WARN_ON(!level); - - /* - * Note that we're called from an entry callback, and we're dealing - * with the child of that entry rather than the parent, so need to - * adjust level down. - */ - if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, - &end_offset)) { - xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, - offset * sizeof(u64), 0, - (end_offset - offset) * sizeof(u64)); - xe_walk->needs_invalidate = true; - } - - return 0; -} - -static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { - .pt_entry = xe_pt_zap_ptes_entry, -}; - -/** - * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range - * @tile: The tile we're zapping for. - * @vma: GPU VMA detailing address range. - * - * Eviction and Userptr invalidation needs to be able to zap the - * gpu ptes of a given address range in pagefaulting mode. - * In order to be able to do that, that function needs access to the shared - * page-table entrieaso it can either clear the leaf PTEs or - * clear the pointers to lower-level page-tables. The caller is required - * to hold the necessary locks to ensure neither the page-table connectivity - * nor the page-table entries of the range is updated from under us. - * - * Return: Whether ptes were actually updated and a TLB invalidation is - * required. - */ -bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) -{ - struct xe_pt_zap_ptes_walk xe_walk = { - .base = { - .ops = &xe_pt_zap_ptes_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .tile = tile, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); - - if (!(pt_mask & BIT(tile->id))) - return false; - - (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - return xe_walk.needs_invalidate; -} - -static void -xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, - struct iosys_map *map, void *data, - u32 qword_ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update) -{ - struct xe_pt_entry *ptes = update->pt_entries; - u64 *ptr = data; - u32 i; - - for (i = 0; i < num_qwords; i++) { - if (map) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, ptes[i].pte); - else - ptr[i] = ptes[i].pte; - } -} - -static void xe_pt_abort_bind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, - u32 num_entries) -{ - u32 i, j; - - for (i = 0; i < num_entries; i++) { - if (!entries[i].pt_entries) - continue; - - for (j = 0; j < entries[i].qwords; j++) - xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL); - kfree(entries[i].pt_entries); - } -} - -static void xe_pt_commit_locks_assert(struct xe_vma *vma) -{ - struct xe_vm *vm = xe_vma_vm(vma); - - lockdep_assert_held(&vm->lock); - - if (xe_vma_is_userptr(vma)) - lockdep_assert_held_read(&vm->userptr.notifier_lock); - else if (!xe_vma_is_null(vma)) - dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); - - xe_vm_assert_held(vm); -} - -static void xe_pt_commit_bind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, - u32 num_entries, bool rebind, - struct llist_head *deferred) -{ - u32 i, j; - - xe_pt_commit_locks_assert(vma); - - for (i = 0; i < num_entries; i++) { - struct xe_pt *pt = entries[i].pt; - struct xe_pt_dir *pt_dir; - - if (!rebind) - pt->num_live += entries[i].qwords; - - if (!pt->level) { - kfree(entries[i].pt_entries); - continue; - } - - pt_dir = as_xe_pt_dir(pt); - for (j = 0; j < entries[i].qwords; j++) { - u32 j_ = j + entries[i].ofs; - struct xe_pt *newpte = entries[i].pt_entries[j].pt; - - if (xe_pt_entry(pt_dir, j_)) - xe_pt_destroy(xe_pt_entry(pt_dir, j_), - xe_vma_vm(vma)->flags, deferred); - - pt_dir->children[j_] = &newpte->base; - } - kfree(entries[i].pt_entries); - } -} - -static int -xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 *num_entries) -{ - int err; - - *num_entries = 0; - err = xe_pt_stage_bind(tile, vma, entries, num_entries); - if (!err) - xe_tile_assert(tile, *num_entries); - else /* abort! */ - xe_pt_abort_bind(vma, entries, *num_entries); - - return err; -} - -static void xe_vm_dbg_print_entries(struct xe_device *xe, - const struct xe_vm_pgtable_update *entries, - unsigned int num_entries) -#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) -{ - unsigned int i; - - vm_dbg(&xe->drm, "%u entries to update\n", num_entries); - for (i = 0; i < num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &entries[i]; - struct xe_pt *xe_pt = entry->pt; - u64 page_size = 1ull << xe_pt_shift(xe_pt->level); - u64 end; - u64 start; - - xe_assert(xe, !entry->pt->is_compact); - start = entry->ofs * page_size; - end = start + page_size * entry->qwords; - vm_dbg(&xe->drm, - "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n", - i, xe_pt->level, entry->ofs, entry->qwords, - xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0); - } -} -#else -{} -#endif - -#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT - -static int xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) -{ - u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; - static u32 count; - - if (count++ % divisor == divisor - 1) { - struct xe_vm *vm = xe_vma_vm(&uvma->vma); - - uvma->userptr.divisor = divisor << 1; - spin_lock(&vm->userptr.invalidated_lock); - list_move_tail(&uvma->userptr.invalidate_link, - &vm->userptr.invalidated); - spin_unlock(&vm->userptr.invalidated_lock); - return true; - } - - return false; -} - -#else - -static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) -{ - return false; -} - -#endif - -/** - * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks - * @base: Base we derive from. - * @bind: Whether this is a bind or an unbind operation. A bind operation - * makes the pre-commit callback error with -EAGAIN if it detects a - * pending invalidation. - * @locked: Whether the pre-commit callback locked the userptr notifier lock - * and it needs unlocking. - */ -struct xe_pt_migrate_pt_update { - struct xe_migrate_pt_update base; - bool bind; - bool locked; -}; - -/* - * This function adds the needed dependencies to a page-table update job - * to make sure racing jobs for separate bind engines don't race writing - * to the same page-table range, wreaking havoc. Initially use a single - * fence for the entire VM. An optimization would use smaller granularity. - */ -static int xe_pt_vm_dependencies(struct xe_sched_job *job, - struct xe_range_fence_tree *rftree, - u64 start, u64 last) -{ - struct xe_range_fence *rtfence; - struct dma_fence *fence; - int err; - - rtfence = xe_range_fence_tree_first(rftree, start, last); - while (rtfence) { - fence = rtfence->fence; - - if (!dma_fence_is_signaled(fence)) { - /* - * Is this a CPU update? GPU is busy updating, so return - * an error - */ - if (!job) - return -ETIME; - - dma_fence_get(fence); - err = drm_sched_job_add_dependency(&job->drm, fence); - if (err) - return err; - } - - rtfence = xe_range_fence_tree_next(rtfence, start, last); - } - - return 0; -} - -static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) -{ - struct xe_range_fence_tree *rftree = - &xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id]; - - return xe_pt_vm_dependencies(pt_update->job, rftree, - pt_update->start, pt_update->last); -} - -static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) -{ - struct xe_pt_migrate_pt_update *userptr_update = - container_of(pt_update, typeof(*userptr_update), base); - struct xe_userptr_vma *uvma = to_userptr_vma(pt_update->vma); - unsigned long notifier_seq = uvma->userptr.notifier_seq; - struct xe_vm *vm = xe_vma_vm(&uvma->vma); - int err = xe_pt_vm_dependencies(pt_update->job, - &vm->rftree[pt_update->tile_id], - pt_update->start, - pt_update->last); - - if (err) - return err; - - userptr_update->locked = false; - - /* - * Wait until nobody is running the invalidation notifier, and - * since we're exiting the loop holding the notifier lock, - * nobody can proceed invalidating either. - * - * Note that we don't update the vma->userptr.notifier_seq since - * we don't update the userptr pages. - */ - do { - down_read(&vm->userptr.notifier_lock); - if (!mmu_interval_read_retry(&uvma->userptr.notifier, - notifier_seq)) - break; - - up_read(&vm->userptr.notifier_lock); - - if (userptr_update->bind) - return -EAGAIN; - - notifier_seq = mmu_interval_read_begin(&uvma->userptr.notifier); - } while (true); - - /* Inject errors to test_whether they are handled correctly */ - if (userptr_update->bind && xe_pt_userptr_inject_eagain(uvma)) { - up_read(&vm->userptr.notifier_lock); - return -EAGAIN; - } - - userptr_update->locked = true; - - return 0; -} - -static const struct xe_migrate_pt_update_ops bind_ops = { - .populate = xe_vm_populate_pgtable, - .pre_commit = xe_pt_pre_commit, -}; - -static const struct xe_migrate_pt_update_ops userptr_bind_ops = { - .populate = xe_vm_populate_pgtable, - .pre_commit = xe_pt_userptr_pre_commit, -}; - -struct invalidation_fence { - struct xe_gt_tlb_invalidation_fence base; - struct xe_gt *gt; - struct dma_fence *fence; - struct dma_fence_cb cb; - struct work_struct work; - u64 start; - u64 end; - u32 asid; -}; - -static void invalidation_fence_cb(struct dma_fence *fence, - struct dma_fence_cb *cb) -{ - struct invalidation_fence *ifence = - container_of(cb, struct invalidation_fence, cb); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); - if (!ifence->fence->error) { - queue_work(system_wq, &ifence->work); - } else { - ifence->base.base.error = ifence->fence->error; - dma_fence_signal(&ifence->base.base); - dma_fence_put(&ifence->base.base); - } - dma_fence_put(ifence->fence); -} - -static void invalidation_fence_work_func(struct work_struct *w) -{ - struct invalidation_fence *ifence = - container_of(w, struct invalidation_fence, work); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); - xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, - ifence->end, ifence->asid); -} - -static int invalidation_fence_init(struct xe_gt *gt, - struct invalidation_fence *ifence, - struct dma_fence *fence, - u64 start, u64 end, u32 asid) -{ - int ret; - - trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); - - xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); - - ifence->fence = fence; - ifence->gt = gt; - ifence->start = start; - ifence->end = end; - ifence->asid = asid; - - INIT_WORK(&ifence->work, invalidation_fence_work_func); - ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); - if (ret == -ENOENT) { - dma_fence_put(ifence->fence); /* Usually dropped in CB */ - invalidation_fence_work_func(&ifence->work); - } else if (ret) { - dma_fence_put(&ifence->base.base); /* Caller ref */ - dma_fence_put(&ifence->base.base); /* Creation ref */ - } - - xe_gt_assert(gt, !ret || ret == -ENOENT); - - return ret && ret != -ENOENT ? ret : 0; -} - -static void xe_pt_calc_rfence_interval(struct xe_vma *vma, - struct xe_pt_migrate_pt_update *update, - struct xe_vm_pgtable_update *entries, - u32 num_entries) -{ - int i, level = 0; - - for (i = 0; i < num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &entries[i]; - - if (entry->pt->level > level) - level = entry->pt->level; - } - - /* Greedy (non-optimal) calculation but simple */ - update->base.start = ALIGN_DOWN(xe_vma_start(vma), - 0x1ull << xe_pt_shift(level)); - update->base.last = ALIGN(xe_vma_end(vma), - 0x1ull << xe_pt_shift(level)) - 1; -} - -/** - * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma - * address range. - * @tile: The tile to bind for. - * @vma: The vma to bind. - * @q: The exec_queue with which to do pipelined page-table updates. - * @syncs: Entries to sync on before binding the built tree to the live vm tree. - * @num_syncs: Number of @sync entries. - * @rebind: Whether we're rebinding this vma to the same address range without - * an unbind in-between. - * - * This function builds a page-table tree (see xe_pt_stage_bind() for more - * information on page-table building), and the xe_vm_pgtable_update entries - * abstracting the operations needed to attach it to the main vm tree. It - * then takes the relevant locks and updates the metadata side of the main - * vm tree and submits the operations for pipelined attachment of the - * gpu page-table to the vm main tree, (which can be done either by the - * cpu and the GPU). - * - * Return: A valid dma-fence representing the pipelined attachment operation - * on success, an error pointer on error. - */ -struct dma_fence * -__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, - struct xe_sync_entry *syncs, u32 num_syncs, - bool rebind) -{ - struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; - struct xe_pt_migrate_pt_update bind_pt_update = { - .base = { - .ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops, - .vma = vma, - .tile_id = tile->id, - }, - .bind = true, - }; - struct xe_vm *vm = xe_vma_vm(vma); - u32 num_entries; - struct dma_fence *fence; - struct invalidation_fence *ifence = NULL; - struct xe_range_fence *rfence; - int err; - - bind_pt_update.locked = false; - xe_bo_assert_held(xe_vma_bo(vma)); - xe_vm_assert_held(vm); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing bind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma), q); - - err = xe_pt_prepare_bind(tile, vma, entries, &num_entries); - if (err) - goto err; - - err = dma_resv_reserve_fences(xe_vm_resv(vm), 1); - if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1); - if (err) - goto err; - - xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries)); - - xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); - xe_pt_calc_rfence_interval(vma, &bind_pt_update, entries, - num_entries); - - /* - * If rebind, we have to invalidate TLB on !LR vms to invalidate - * cached PTEs point to freed memory. on LR vms this is done - * automatically when the context is re-enabled by the rebind worker, - * or in fault mode it was invalidated on PTE zapping. - * - * If !rebind, and scratch enabled VMs, there is a chance the scratch - * PTE is already cached in the TLB so it needs to be invalidated. - * on !LR VMs this is done in the ring ops preceding a batch, but on - * non-faulting LR, in particular on user-space batch buffer chaining, - * it needs to be done here. - */ - if ((!rebind && xe_vm_has_scratch(vm) && xe_vm_in_preempt_fence_mode(vm))) { - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) - return ERR_PTR(-ENOMEM); - } else if (rebind && !xe_vm_in_lr_mode(vm)) { - /* We bump also if batch_invalidate_tlb is true */ - vm->tlb_flush_seqno++; - } - - rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); - if (!rfence) { - kfree(ifence); - return ERR_PTR(-ENOMEM); - } - - fence = xe_migrate_update_pgtables(tile->migrate, - vm, xe_vma_bo(vma), q, - entries, num_entries, - syncs, num_syncs, - &bind_pt_update.base); - if (!IS_ERR(fence)) { - bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND; - LLIST_HEAD(deferred); - int err; - - err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - bind_pt_update.base.start, - bind_pt_update.base.last, fence); - if (err) - dma_fence_wait(fence, false); - - /* TLB invalidation must be done before signaling rebind */ - if (ifence) { - int err = invalidation_fence_init(tile->primary_gt, - ifence, fence, - xe_vma_start(vma), - xe_vma_end(vma), - xe_vma_vm(vma)->usm.asid); - if (err) { - dma_fence_put(fence); - kfree(ifence); - return ERR_PTR(err); - } - fence = &ifence->base.base; - } - - /* add shared fence now for pagetable delayed destroy */ - dma_resv_add_fence(xe_vm_resv(vm), fence, rebind || - last_munmap_rebind ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - DMA_RESV_USAGE_BOOKKEEP); - xe_pt_commit_bind(vma, entries, num_entries, rebind, - bind_pt_update.locked ? &deferred : NULL); - - /* This vma is live (again?) now */ - vma->tile_present |= BIT(tile->id); - - if (bind_pt_update.locked) { - to_userptr_vma(vma)->userptr.initial_bind = true; - up_read(&vm->userptr.notifier_lock); - xe_bo_put_commit(&deferred); - } - if (!rebind && last_munmap_rebind && - xe_vm_in_preempt_fence_mode(vm)) - xe_vm_queue_rebind_worker(vm); - } else { - kfree(rfence); - kfree(ifence); - if (bind_pt_update.locked) - up_read(&vm->userptr.notifier_lock); - xe_pt_abort_bind(vma, entries, num_entries); - } - - return fence; - -err: - return ERR_PTR(err); -} - -struct xe_pt_stage_unbind_walk { - /** @base: The pagewalk base-class. */ - struct xe_pt_walk base; - - /* Input parameters for the walk */ - /** @tile: The tile we're unbinding from. */ - struct xe_tile *tile; - - /** - * @modified_start: Walk range start, modified to include any - * shared pagetables that we're the only user of and can thus - * treat as private. - */ - u64 modified_start; - /** @modified_end: Walk range start, modified like @modified_start. */ - u64 modified_end; - - /* Output */ - /* @wupd: Structure to track the page-table updates we're building */ - struct xe_walk_update wupd; -}; - -/* - * Check whether this range is the only one populating this pagetable, - * and in that case, update the walk range checks so that higher levels don't - * view us as a shared pagetable. - */ -static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, - const struct xe_pt *child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_unbind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - unsigned int shift = walk->shifts[level]; - u64 size = 1ull << shift; - - if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) && - ((next - addr) >> shift) == child->num_live) { - u64 size = 1ull << walk->shifts[level + 1]; - - *action = ACTION_CONTINUE; - - if (xe_walk->modified_start >= addr) - xe_walk->modified_start = round_down(addr, size); - if (xe_walk->modified_end <= next) - xe_walk->modified_end = round_up(next, size); - - return true; - } - - return false; -} - -static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - - XE_WARN_ON(!*child); - XE_WARN_ON(!level); - - xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); - - return 0; -} - -static int -xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, - unsigned int level, u64 addr, u64 next, - struct xe_ptw **child, - enum page_walk_action *action, - struct xe_pt_walk *walk) -{ - struct xe_pt_stage_unbind_walk *xe_walk = - container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); - pgoff_t end_offset; - u64 size = 1ull << walk->shifts[--level]; - - if (!IS_ALIGNED(addr, size)) - addr = xe_walk->modified_start; - if (!IS_ALIGNED(next, size)) - next = xe_walk->modified_end; - - /* Parent == *child is the root pt. Don't kill it. */ - if (parent != *child && - xe_pt_check_kill(addr, next, level, xe_child, action, walk)) - return 0; - - if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset, - &end_offset)) - return 0; - - (void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false); - xe_walk->wupd.updates[level].update->qwords = end_offset - offset; - - return 0; -} - -static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { - .pt_entry = xe_pt_stage_unbind_entry, - .pt_post_descend = xe_pt_stage_unbind_post_descend, -}; - -/** - * xe_pt_stage_unbind() - Build page-table update structures for an unbind - * operation - * @tile: The tile we're unbinding for. - * @vma: The vma we're unbinding. - * @entries: Caller-provided storage for the update structures. - * - * Builds page-table update structures for an unbind operation. The function - * will attempt to remove all page-tables that we're the only user - * of, and for that to work, the unbind operation must be committed in the - * same critical section that blocks racing binds to the same page-table tree. - * - * Return: The number of entries used. - */ -static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma, - struct xe_vm_pgtable_update *entries) -{ - struct xe_pt_stage_unbind_walk xe_walk = { - .base = { - .ops = &xe_pt_stage_unbind_ops, - .shifts = xe_normal_pt_shifts, - .max_level = XE_PT_HIGHEST_LEVEL, - }, - .tile = tile, - .modified_start = xe_vma_start(vma), - .modified_end = xe_vma_end(vma), - .wupd.entries = entries, - }; - struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; - - (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), - xe_vma_end(vma), &xe_walk.base); - - return xe_walk.wupd.num_used_entries; -} - -static void -xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, - struct xe_tile *tile, struct iosys_map *map, - void *ptr, u32 qword_ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update) -{ - struct xe_vma *vma = pt_update->vma; - u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level); - int i; - - if (map && map->is_iomem) - for (i = 0; i < num_qwords; ++i) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, empty); - else if (map) - memset64(map->vaddr + qword_ofs * sizeof(u64), empty, - num_qwords); - else - memset64(ptr, empty, num_qwords); -} - -static void -xe_pt_commit_unbind(struct xe_vma *vma, - struct xe_vm_pgtable_update *entries, u32 num_entries, - struct llist_head *deferred) -{ - u32 j; - - xe_pt_commit_locks_assert(vma); - - for (j = 0; j < num_entries; ++j) { - struct xe_vm_pgtable_update *entry = &entries[j]; - struct xe_pt *pt = entry->pt; - - pt->num_live -= entry->qwords; - if (pt->level) { - struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); - u32 i; - - for (i = entry->ofs; i < entry->ofs + entry->qwords; - i++) { - if (xe_pt_entry(pt_dir, i)) - xe_pt_destroy(xe_pt_entry(pt_dir, i), - xe_vma_vm(vma)->flags, deferred); - - pt_dir->children[i] = NULL; - } - } - } -} - -<<<<<<< -static const struct xe_migrate_pt_update_ops unbind_ops = { - .populate = xe_migrate_clear_pgtable_callback, -======= -static void -xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int i, level = 0; - u64 start, last; - - for (i = 0; i < pt_op->num_entries; i++) { - const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; - - if (entry->pt->level > level) - level = entry->pt->level; - } - - /* Greedy (non-optimal) calculation but simple */ - start = ALIGN_DOWN(xe_vma_start(vma), 0x1ull << xe_pt_shift(level)); - last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1; - - if (start < pt_update_ops->start) - pt_update_ops->start = start; - if (last > pt_update_ops->last) - pt_update_ops->last = last; -} - -static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, - xe->info.tile_count); - - return 0; -} - -static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int err; - - xe_bo_assert_held(xe_vma_bo(vma)); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing bind, with range [%llx...%llx)\n", - xe_vma_start(vma), xe_vma_end(vma) - 1); - - pt_op->vma = NULL; - pt_op->bind = true; - pt_op->rebind = BIT(tile->id) & vma->tile_present; - - err = vma_reserve_fences(tile_to_xe(tile), vma); - if (err) - return err; - - err = xe_pt_prepare_bind(tile, vma, pt_op->entries, - &pt_op->num_entries); - if (!err) { - xe_tile_assert(tile, pt_op->num_entries <= - ARRAY_SIZE(pt_op->entries)); - xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, - pt_op->num_entries, true); - - xe_pt_update_ops_rfence_interval(pt_update_ops, vma); - ++pt_update_ops->current_op; - pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); - - /* - * If rebind, we have to invalidate TLB on !LR vms to invalidate - * cached PTEs point to freed memory. On LR vms this is done - * automatically when the context is re-enabled by the rebind worker, - * or in fault mode it was invalidated on PTE zapping. - * - * If !rebind, and scratch enabled VMs, there is a chance the scratch - * PTE is already cached in the TLB so it needs to be invalidated. - * On !LR VMs this is done in the ring ops preceding a batch, but on - * non-faulting LR, in particular on user-space batch buffer chaining, - * it needs to be done here. - */ - if ((!pt_op->rebind && xe_vm_has_scratch(vm) && - xe_vm_in_preempt_fence_mode(vm))) - pt_update_ops->needs_invalidation = true; - else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) - /* We bump also if batch_invalidate_tlb is true */ - vm->tlb_flush_seqno++; - - vma->tile_staged |= BIT(tile->id); - pt_op->vma = vma; - xe_pt_commit_prepare_bind(vma, pt_op->entries, - pt_op->num_entries, pt_op->rebind); - } else { - xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); - } - - return err; -} - -static int unbind_op_prepare(struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma) -{ - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; - int err; - - if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) - return 0; - - xe_bo_assert_held(xe_vma_bo(vma)); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing unbind, with range [%llx...%llx)\n", - xe_vma_start(vma), xe_vma_end(vma) - 1); - - /* - * Wait for invalidation to complete. Can corrupt internal page table - * state if an invalidation is running while preparing an unbind. - */ - if (xe_vma_is_userptr(vma) && xe_vm_in_fault_mode(xe_vma_vm(vma))) - mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier); - - pt_op->vma = vma; - pt_op->bind = false; - pt_op->rebind = false; - - err = vma_reserve_fences(tile_to_xe(tile), vma); - if (err) - return err; - - pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries); - - xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, - pt_op->num_entries, false); - xe_pt_update_ops_rfence_interval(pt_update_ops, vma); - ++pt_update_ops->current_op; - pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); - pt_update_ops->needs_invalidation = true; - - xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); - - return 0; -} - -static int op_prepare(struct xe_vm *vm, - struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op) -{ - int err = 0; - - xe_vm_assert_held(vm); - - switch (op->base.op) { - case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) - break; - - err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma); - pt_update_ops->wait_vm_kernel = true; - break; - case DRM_GPUVA_OP_REMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va)); - - if (!err && op->remap.prev) { - err = bind_op_prepare(vm, tile, pt_update_ops, - op->remap.prev); - pt_update_ops->wait_vm_bookkeep = true; - } - if (!err && op->remap.next) { - err = bind_op_prepare(vm, tile, pt_update_ops, - op->remap.next); - pt_update_ops->wait_vm_bookkeep = true; - } - break; - case DRM_GPUVA_OP_UNMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va)); - break; - case DRM_GPUVA_OP_PREFETCH: - err = bind_op_prepare(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va)); - pt_update_ops->wait_vm_kernel = true; - break; - default: - drm_warn(&vm->xe->drm, "NOT POSSIBLE"); - } - - return err; -} - -static void -xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) -{ - init_llist_head(&pt_update_ops->deferred); - pt_update_ops->start = ~0x0ull; - pt_update_ops->last = 0x0ull; -} - -/** - * xe_pt_update_ops_prepare() - Prepare PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operationa - * - * Prepare PT update operations which includes updating internal PT state, - * allocate memory for page tables, populate page table being pruned in, and - * create PT update operations for leaf insertion / removal. - * - * Return: 0 on success, negative error code on error. - */ -int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - struct xe_vma_op *op; - int err; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - xe_pt_update_ops_init(pt_update_ops); - - err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), - tile_to_xe(tile)->info.tile_count); - if (err) - return err; - - list_for_each_entry(op, &vops->list, link) { - err = op_prepare(vops->vm, tile, pt_update_ops, op); - - if (err) - return err; - } - - xe_tile_assert(tile, pt_update_ops->current_op <= - pt_update_ops->num_ops); - -#ifdef TEST_VM_OPS_ERROR - if (vops->inject_error && - vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) - return -ENOSPC; -#endif - - return 0; -} - -static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - } - vma->tile_present |= BIT(tile->id); - vma->tile_staged &= ~BIT(tile->id); - if (xe_vma_is_userptr(vma)) { - lockdep_assert_held_read(&vm->userptr.notifier_lock); - to_userptr_vma(vma)->userptr.initial_bind = true; - } - - /* - * Kick rebind worker if this bind triggers preempt fences and not in - * the rebind worker - */ - if (pt_update_ops->wait_vm_bookkeep && - xe_vm_in_preempt_fence_mode(vm) && - !current->mm) - xe_vm_queue_rebind_worker(vm); -} - -static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2) -{ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - } - vma->tile_present &= ~BIT(tile->id); - if (!vma->tile_present) { - list_del_init(&vma->combined_links.rebind); - if (xe_vma_is_userptr(vma)) { - lockdep_assert_held_read(&vm->userptr.notifier_lock); - - spin_lock(&vm->userptr.invalidated_lock); - list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); - spin_unlock(&vm->userptr.invalidated_lock); - } - } -} - -static void op_commit(struct xe_vm *vm, - struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op, struct dma_fence *fence, - struct dma_fence *fence2) -{ - xe_vm_assert_held(vm); - - switch (op->base.op) { - case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) - break; - - bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, - fence2); - break; - case DRM_GPUVA_OP_REMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va), fence, - fence2); - - if (op->remap.prev) - bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, - fence, fence2); - if (op->remap.next) - bind_op_commit(vm, tile, pt_update_ops, op->remap.next, - fence, fence2); - break; - case DRM_GPUVA_OP_UNMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va), fence, fence2); - break; - case DRM_GPUVA_OP_PREFETCH: - bind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va), fence, fence2); - break; - default: - drm_warn(&vm->xe->drm, "NOT POSSIBLE"); - } -} - -static const struct xe_migrate_pt_update_ops migrate_ops = { - .populate = xe_vm_populate_pgtable, - .clear = xe_migrate_clear_pgtable_callback, ->>>>>>> - .pre_commit = xe_pt_pre_commit, -}; - -static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { - .populate = xe_migrate_clear_pgtable_callback, - .pre_commit = xe_pt_userptr_pre_commit, -}; - -/** - * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma - * address range. - * @tile: The tile to unbind for. - * @vma: The vma to unbind. - * @q: The exec_queue with which to do pipelined page-table updates. - * @syncs: Entries to sync on before disconnecting the tree to be destroyed. - * @num_syncs: Number of @sync entries. - * - * This function builds a the xe_vm_pgtable_update entries abstracting the - * operations needed to detach the page-table tree to be destroyed from the - * man vm tree. - * It then takes the relevant locks and submits the operations for - * pipelined detachment of the gpu page-table from the vm main tree, - * (which can be done either by the cpu and the GPU), Finally it frees the - * detached page-table tree. - * - * Return: A valid dma-fence representing the pipelined detachment operation - * on success, an error pointer on error. - */ -struct dma_fence * -__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, - struct xe_sync_entry *syncs, u32 num_syncs) -{ -<<<<<<< - struct xe_vm *vm = vops->vm; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - struct dma_fence *fence; - struct invalidation_fence *ifence = NULL, *mfence = NULL; - struct dma_fence_chain *chain_fence = NULL; - struct xe_range_fence *rfence; - struct xe_vma_op *op; - int err = 0, i; - struct xe_migrate_pt_update update = { - .ops = pt_update_ops->needs_userptr_lock ? - &userptr_migrate_ops : - &migrate_ops, - .vops = vops, - .tile_id = tile->id, -======= - struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; - struct xe_pt_migrate_pt_update unbind_pt_update = { - .base = { - .ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops : - &unbind_ops, - .vma = vma, - .tile_id = tile->id, - }, ->>>>>>> - }; - struct xe_vm *vm = xe_vma_vm(vma); - u32 num_entries; - struct dma_fence *fence = NULL; - struct invalidation_fence *ifence; - struct xe_range_fence *rfence; - int err; - - LLIST_HEAD(deferred); - - xe_bo_assert_held(xe_vma_bo(vma)); - xe_vm_assert_held(vm); - - vm_dbg(&xe_vma_vm(vma)->xe->drm, - "Preparing unbind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma), q); - - num_entries = xe_pt_stage_unbind(tile, vma, entries); - xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries)); - - xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); - xe_pt_calc_rfence_interval(vma, &unbind_pt_update, entries, - num_entries); - -<<<<<<< - err = dma_resv_reserve_fences(xe_vm_resv(vm), 1); - if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1); - if (err) - return ERR_PTR(err); - - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) - return ERR_PTR(-ENOMEM); -======= - if (pt_update_ops->needs_invalidation) { - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) { - err = -ENOMEM; - goto kill_vm_tile1; - } - if (tile->media_gt) { - mfence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!mfence) { - err = -ENOMEM; - goto free_ifence; - } - chain_fence = dma_fence_chain_alloc(); - if (!chain_fence) { - err = -ENOMEM; - goto free_ifence; - } - } - } ->>>>>>> - - rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); - if (!rfence) { - kfree(ifence); - return ERR_PTR(-ENOMEM); - } - - /* - * Even if we were already evicted and unbind to destroy, we need to - * clear again here. The eviction may have updated pagetables at a - * lower level, because it needs to be more conservative. - */ - fence = xe_migrate_update_pgtables(tile->migrate, - vm, NULL, q ? q : - vm->q[tile->id], - entries, num_entries, - syncs, num_syncs, - &unbind_pt_update.base); - if (!IS_ERR(fence)) { - int err; - - err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - unbind_pt_update.base.start, - unbind_pt_update.base.last, fence); - if (err) - dma_fence_wait(fence, false); - -<<<<<<< - /* TLB invalidation must be done before signaling unbind */ - err = invalidation_fence_init(tile->primary_gt, ifence, fence, - xe_vma_start(vma), - xe_vma_end(vma), - xe_vma_vm(vma)->usm.asid); - if (err) { - dma_fence_put(fence); - kfree(ifence); - return ERR_PTR(err); - } - fence = &ifence->base.base; - - /* add shared fence now for pagetable delayed destroy */ - dma_resv_add_fence(xe_vm_resv(vm), fence, - DMA_RESV_USAGE_BOOKKEEP); - - /* This fence will be installed by caller when doing eviction */ - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - DMA_RESV_USAGE_BOOKKEEP); - xe_pt_commit_unbind(vma, entries, num_entries, - unbind_pt_update.locked ? &deferred : NULL); - vma->tile_present &= ~BIT(tile->id); - } else { - kfree(rfence); - kfree(ifence); - } - - if (!vma->tile_present) - list_del_init(&vma->combined_links.rebind); - - if (unbind_pt_update.locked) { - xe_tile_assert(tile, xe_vma_is_userptr(vma)); -======= - xe_pt_commit(pt_op->vma, pt_op->entries, - pt_op->num_entries, &pt_update_ops->deferred); - pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ - } - - if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, - &xe_range_fence_kfree_ops, - pt_update_ops->start, - pt_update_ops->last, fence)) - dma_fence_wait(fence, false); - - /* tlb invalidation must be done before signaling rebind */ - if (ifence) { - if (mfence) - dma_fence_get(fence); - invalidation_fence_init(tile->primary_gt, ifence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - if (mfence) { - invalidation_fence_init(tile->media_gt, mfence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - dma_fence_chain_init(chain_fence, &ifence->base.base, - &mfence->base.base, 0); - fence = &chain_fence->base; - } else { - fence = &ifence->base.base; - } - } - - if (!mfence) { - dma_resv_add_fence(xe_vm_resv(vm), fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); - } else { - dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, - &ifence->base.base, &mfence->base.base); - } ->>>>>>> - - if (!vma->tile_present) { - spin_lock(&vm->userptr.invalidated_lock); - list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); - spin_unlock(&vm->userptr.invalidated_lock); - } - up_read(&vm->userptr.notifier_lock); - xe_bo_put_commit(&deferred); - } - - return fence; -<<<<<<< -======= - -free_rfence: - kfree(rfence); -free_ifence: - dma_fence_chain_free(chain_fence); - kfree(mfence); - kfree(ifence); -kill_vm_tile1: - if (err != -EAGAIN && tile->id) - xe_vm_kill(vops->vm, false); - - return ERR_PTR(err); -} - -/** - * xe_pt_update_ops_fini() - Finish PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operations - * - * Finish PT update operations by committing to destroy page table memory - */ -void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - for (i = 0; i < pt_update_ops->current_op; ++i) { - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; - - xe_pt_free_bind(pt_op->entries, pt_op->num_entries); - } - xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); -} - -/** - * xe_pt_update_ops_abort() - Abort PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operationa - * - * Abort PT update operations by unwinding internal PT state - */ -void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops) -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; - - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); - - for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - - if (!pt_op->vma || i >= pt_update_ops->current_op) - continue; - - if (pt_op->bind) - xe_pt_abort_bind(pt_op->vma, pt_op->entries, - pt_op->num_entries, - pt_op->rebind); - else - xe_pt_abort_unbind(pt_op->vma, pt_op->entries, - pt_op->num_entries); - } - - xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); ->>>>>>> -} diff --git a/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage b/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage new file mode 100644 index 000000000000..ef95bec8d180 --- /dev/null +++ b/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage @@ -0,0 +1,1231 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/** + * DOC: atomic plane helpers + * + * The functions here are used by the atomic plane helper functions to + * implement legacy plane updates (i.e., drm_plane->update_plane() and + * drm_plane->disable_plane()). This allows plane updates to use the + * atomic state infrastructure and perform plane updates as separate + * prepare/check/commit/cleanup steps. + */ + +#include <linux/dma-fence-chain.h> +#include <linux/dma-resv.h> + +#include <drm/drm_atomic_helper.h> +#include <drm/drm_blend.h> +#include <drm/drm_fourcc.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_atomic_helper.h> + +#include "i915_config.h" +#include "i9xx_plane_regs.h" +#include "intel_atomic_plane.h" +#include "intel_cdclk.h" +#include "intel_cursor.h" +#include "intel_display_rps.h" +#include "intel_display_trace.h" +#include "intel_display_types.h" +#include "intel_fb.h" +#include "intel_fb_pin.h" +#include "skl_scaler.h" +#include "skl_watermark.h" + +static void intel_plane_state_reset(struct intel_plane_state *plane_state, + struct intel_plane *plane) +{ + memset(plane_state, 0, sizeof(*plane_state)); + + __drm_atomic_helper_plane_state_reset(&plane_state->uapi, &plane->base); + + plane_state->scaler_id = -1; +} + +struct intel_plane *intel_plane_alloc(void) +{ + struct intel_plane_state *plane_state; + struct intel_plane *plane; + + plane = kzalloc(sizeof(*plane), GFP_KERNEL); + if (!plane) + return ERR_PTR(-ENOMEM); + + plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); + if (!plane_state) { + kfree(plane); + return ERR_PTR(-ENOMEM); + } + + intel_plane_state_reset(plane_state, plane); + + plane->base.state = &plane_state->uapi; + + return plane; +} + +void intel_plane_free(struct intel_plane *plane) +{ + intel_plane_destroy_state(&plane->base, plane->base.state); + kfree(plane); +} + +/** + * intel_plane_duplicate_state - duplicate plane state + * @plane: drm plane + * + * Allocates and returns a copy of the plane state (both common and + * Intel-specific) for the specified plane. + * + * Returns: The newly allocated plane state, or NULL on failure. + */ +struct drm_plane_state * +intel_plane_duplicate_state(struct drm_plane *plane) +{ + struct intel_plane_state *intel_state; + + intel_state = to_intel_plane_state(plane->state); + intel_state = kmemdup(intel_state, sizeof(*intel_state), GFP_KERNEL); + + if (!intel_state) + return NULL; + + __drm_atomic_helper_plane_duplicate_state(plane, &intel_state->uapi); + + intel_state->ggtt_vma = NULL; + intel_state->dpt_vma = NULL; + intel_state->flags = 0; + + /* add reference to fb */ + if (intel_state->hw.fb) + drm_framebuffer_get(intel_state->hw.fb); + + return &intel_state->uapi; +} + +/** + * intel_plane_destroy_state - destroy plane state + * @plane: drm plane + * @state: state object to destroy + * + * Destroys the plane state (both common and Intel-specific) for the + * specified plane. + */ +void +intel_plane_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct intel_plane_state *plane_state = to_intel_plane_state(state); + + drm_WARN_ON(plane->dev, plane_state->ggtt_vma); + drm_WARN_ON(plane->dev, plane_state->dpt_vma); + + __drm_atomic_helper_plane_destroy_state(&plane_state->uapi); + if (plane_state->hw.fb) + drm_framebuffer_put(plane_state->hw.fb); + kfree(plane_state); +} + +bool intel_plane_needs_physical(struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return plane->id == PLANE_CURSOR && + DISPLAY_INFO(i915)->cursor_needs_physical; +} + +unsigned int intel_adjusted_rate(const struct drm_rect *src, + const struct drm_rect *dst, + unsigned int rate) +{ + unsigned int src_w, src_h, dst_w, dst_h; + + src_w = drm_rect_width(src) >> 16; + src_h = drm_rect_height(src) >> 16; + dst_w = drm_rect_width(dst); + dst_h = drm_rect_height(dst); + + /* Downscaling limits the maximum pixel rate */ + dst_w = min(src_w, dst_w); + dst_h = min(src_h, dst_h); + + return DIV_ROUND_UP_ULL(mul_u32_u32(rate, src_w * src_h), + dst_w * dst_h); +} + +unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + /* + * Note we don't check for plane visibility here as + * we want to use this when calculating the cursor + * watermarks even if the cursor is fully offscreen. + * That depends on the src/dst rectangles being + * correctly populated whenever the watermark code + * considers the cursor to be visible, whether or not + * it is actually visible. + * + * See: intel_wm_plane_visible() and intel_check_cursor() + */ + + return intel_adjusted_rate(&plane_state->uapi.src, + &plane_state->uapi.dst, + crtc_state->pixel_rate); +} + +unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int color_plane) +{ + const struct drm_framebuffer *fb = plane_state->hw.fb; + + if (!plane_state->uapi.visible) + return 0; + + return intel_plane_pixel_rate(crtc_state, plane_state) * + fb->format->cpp[color_plane]; +} + +static bool +use_min_ddb(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return DISPLAY_VER(i915) >= 13 && + crtc_state->uapi.async_flip && + plane->async_flip; +} + +static unsigned int +intel_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int color_plane) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + const struct drm_framebuffer *fb = plane_state->hw.fb; + int width, height; + unsigned int rel_data_rate; + + if (plane->id == PLANE_CURSOR) + return 0; + + if (!plane_state->uapi.visible) + return 0; + + /* + * We calculate extra ddb based on ratio plane rate/total data rate + * in case, in some cases we should not allocate extra ddb for the plane, + * so do not count its data rate, if this is the case. + */ + if (use_min_ddb(crtc_state, plane)) + return 0; + + /* + * Src coordinates are already rotated by 270 degrees for + * the 90/270 degree plane rotation cases (to match the + * GTT mapping), hence no need to account for rotation here. + */ + width = drm_rect_width(&plane_state->uapi.src) >> 16; + height = drm_rect_height(&plane_state->uapi.src) >> 16; + + /* UV plane does 1/2 pixel sub-sampling */ + if (color_plane == 1) { + width /= 2; + height /= 2; + } + + rel_data_rate = width * height * fb->format->cpp[color_plane]; + + return intel_adjusted_rate(&plane_state->uapi.src, + &plane_state->uapi.dst, + rel_data_rate); +} + +int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, + struct intel_plane *plane, + bool *need_cdclk_calc) +{ + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + const struct intel_plane_state *plane_state = + intel_atomic_get_new_plane_state(state, plane); + struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc); + const struct intel_cdclk_state *cdclk_state; + const struct intel_crtc_state *old_crtc_state; + struct intel_crtc_state *new_crtc_state; + + if (!plane_state->uapi.visible || !plane->min_cdclk) + return 0; + + old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); + new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + + new_crtc_state->min_cdclk[plane->id] = + plane->min_cdclk(new_crtc_state, plane_state); + + /* + * No need to check against the cdclk state if + * the min cdclk for the plane doesn't increase. + * + * Ie. we only ever increase the cdclk due to plane + * requirements. This can reduce back and forth + * display blinking due to constant cdclk changes. + */ + if (new_crtc_state->min_cdclk[plane->id] <= + old_crtc_state->min_cdclk[plane->id]) + return 0; + + cdclk_state = intel_atomic_get_cdclk_state(state); + if (IS_ERR(cdclk_state)) + return PTR_ERR(cdclk_state); + + /* + * No need to recalculate the cdclk state if + * the min cdclk for the pipe doesn't increase. + * + * Ie. we only ever increase the cdclk due to plane + * requirements. This can reduce back and forth + * display blinking due to constant cdclk changes. + */ + if (new_crtc_state->min_cdclk[plane->id] <= + cdclk_state->min_cdclk[crtc->pipe]) + return 0; + + drm_dbg_kms(&dev_priv->drm, + "[PLANE:%d:%s] min cdclk (%d kHz) > [CRTC:%d:%s] min cdclk (%d kHz)\n", + plane->base.base.id, plane->base.name, + new_crtc_state->min_cdclk[plane->id], + crtc->base.base.id, crtc->base.name, + cdclk_state->min_cdclk[crtc->pipe]); + *need_cdclk_calc = true; + + return 0; +} + +static void intel_plane_clear_hw_state(struct intel_plane_state *plane_state) +{ + if (plane_state->hw.fb) + drm_framebuffer_put(plane_state->hw.fb); + + memset(&plane_state->hw, 0, sizeof(plane_state->hw)); +} + +void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state, + struct intel_crtc *crtc) +{ + intel_plane_clear_hw_state(plane_state); + + /* + * For the joiner secondary uapi.crtc will point at + * the primary crtc. So we explicitly assign the right + * secondary crtc to hw.crtc. uapi.crtc!=NULL simply + * indicates the plane is logically enabled on the uapi level. + */ + plane_state->hw.crtc = from_plane_state->uapi.crtc ? &crtc->base : NULL; + + plane_state->hw.fb = from_plane_state->uapi.fb; + if (plane_state->hw.fb) + drm_framebuffer_get(plane_state->hw.fb); + + plane_state->hw.alpha = from_plane_state->uapi.alpha; + plane_state->hw.pixel_blend_mode = + from_plane_state->uapi.pixel_blend_mode; + plane_state->hw.rotation = from_plane_state->uapi.rotation; + plane_state->hw.color_encoding = from_plane_state->uapi.color_encoding; + plane_state->hw.color_range = from_plane_state->uapi.color_range; + plane_state->hw.scaling_filter = from_plane_state->uapi.scaling_filter; + + plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi); + plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi); +} + +void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state) +{ + intel_plane_clear_hw_state(plane_state); + + memcpy(&plane_state->hw, &from_plane_state->hw, + sizeof(plane_state->hw)); + + if (plane_state->hw.fb) + drm_framebuffer_get(plane_state->hw.fb); +} + +void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + + crtc_state->active_planes &= ~BIT(plane->id); + crtc_state->scaled_planes &= ~BIT(plane->id); + crtc_state->nv12_planes &= ~BIT(plane->id); + crtc_state->c8_planes &= ~BIT(plane->id); + crtc_state->async_flip_planes &= ~BIT(plane->id); + crtc_state->data_rate[plane->id] = 0; + crtc_state->data_rate_y[plane->id] = 0; + crtc_state->rel_data_rate[plane->id] = 0; + crtc_state->rel_data_rate_y[plane->id] = 0; + crtc_state->min_cdclk[plane->id] = 0; + + plane_state->uapi.visible = false; +} + +static bool intel_plane_is_scaled(const struct intel_plane_state *plane_state) +{ + int src_w = drm_rect_width(&plane_state->uapi.src) >> 16; + int src_h = drm_rect_height(&plane_state->uapi.src) >> 16; + int dst_w = drm_rect_width(&plane_state->uapi.dst); + int dst_h = drm_rect_height(&plane_state->uapi.dst); + + return src_w != dst_w || src_h != dst_h; +} + +static bool intel_plane_do_async_flip(struct intel_plane *plane, + const struct intel_crtc_state *old_crtc_state, + const struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + if (!plane->async_flip) + return false; + + if (!new_crtc_state->uapi.async_flip) + return false; + + /* + * In platforms after DISPLAY13, we might need to override + * first async flip in order to change watermark levels + * as part of optimization. + * + * And let's do this for all skl+ so that we can eg. change the + * modifier as well. + * + * TODO: For older platforms there is less reason to do this as + * only X-tile is supported with async flips, though we could + * extend this so other scanout parameters (stride/etc) could + * be changed as well... + */ + return DISPLAY_VER(i915) < 9 || old_crtc_state->uapi.async_flip; +} + +static bool i9xx_must_disable_cxsr(const struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + const struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + bool old_visible = old_plane_state->uapi.visible; + bool new_visible = new_plane_state->uapi.visible; + u32 old_ctl = old_plane_state->ctl; + u32 new_ctl = new_plane_state->ctl; + bool modeset, turn_on, turn_off; + + if (plane->id == PLANE_CURSOR) + return false; + + modeset = intel_crtc_needs_modeset(new_crtc_state); + turn_off = old_visible && (!new_visible || modeset); + turn_on = new_visible && (!old_visible || modeset); + + /* Must disable CxSR around plane enable/disable */ + if (turn_on || turn_off) + return true; + + if (!old_visible || !new_visible) + return false; + + /* + * Most plane control register updates are blocked while in CxSR. + * + * Tiling mode is one exception where the primary plane can + * apparently handle it, whereas the sprites can not (the + * sprite issue being only relevant on VLV/CHV where CxSR + * is actually possible with a sprite enabled). + */ + if (plane->id == PLANE_PRIMARY) { + old_ctl &= ~DISP_TILED; + new_ctl &= ~DISP_TILED; + } + + return old_ctl != new_ctl; +} + +static bool ilk_must_disable_cxsr(const struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + const struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + bool old_visible = old_plane_state->uapi.visible; + bool new_visible = new_plane_state->uapi.visible; + bool modeset, turn_on; + + if (plane->id == PLANE_CURSOR) + return false; + + modeset = intel_crtc_needs_modeset(new_crtc_state); + turn_on = new_visible && (!old_visible || modeset); + + /* + * ILK/SNB DVSACNTR/Sprite Enable + * IVB SPR_CTL/Sprite Enable + * "When in Self Refresh Big FIFO mode, a write to enable the + * plane will be internally buffered and delayed while Big FIFO + * mode is exiting." + * + * Which means that enabling the sprite can take an extra frame + * when we start in big FIFO mode (LP1+). Thus we need to drop + * down to LP0 and wait for vblank in order to make sure the + * sprite gets enabled on the next vblank after the register write. + * Doing otherwise would risk enabling the sprite one frame after + * we've already signalled flip completion. We can resume LP1+ + * once the sprite has been enabled. + * + * With experimental results seems this is needed also for primary + * plane, not only sprite plane. + */ + if (turn_on) + return true; + + /* + * WaCxSRDisabledForSpriteScaling:ivb + * IVB SPR_SCALE/Scaling Enable + * "Low Power watermarks must be disabled for at least one + * frame before enabling sprite scaling, and kept disabled + * until sprite scaling is disabled." + * + * ILK/SNB DVSASCALE/Scaling Enable + * "When in Self Refresh Big FIFO mode, scaling enable will be + * masked off while Big FIFO mode is exiting." + * + * Despite the w/a only being listed for IVB we assume that + * the ILK/SNB note has similar ramifications, hence we apply + * the w/a on all three platforms. + */ + return !intel_plane_is_scaled(old_plane_state) && + intel_plane_is_scaled(new_plane_state); +} + +static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + bool mode_changed = intel_crtc_needs_modeset(new_crtc_state); + bool was_crtc_enabled = old_crtc_state->hw.active; + bool is_crtc_enabled = new_crtc_state->hw.active; + bool turn_off, turn_on, visible, was_visible; + int ret; + + if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { + ret = skl_update_scaler_plane(new_crtc_state, new_plane_state); + if (ret) + return ret; + } + + was_visible = old_plane_state->uapi.visible; + visible = new_plane_state->uapi.visible; + + if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) + was_visible = false; + + /* + * Visibility is calculated as if the crtc was on, but + * after scaler setup everything depends on it being off + * when the crtc isn't active. + * + * FIXME this is wrong for watermarks. Watermarks should also + * be computed as if the pipe would be active. Perhaps move + * per-plane wm computation to the .check_plane() hook, and + * only combine the results from all planes in the current place? + */ + if (!is_crtc_enabled) { + intel_plane_set_invisible(new_crtc_state, new_plane_state); + visible = false; + } + + if (!was_visible && !visible) + return 0; + + turn_off = was_visible && (!visible || mode_changed); + turn_on = visible && (!was_visible || mode_changed); + + drm_dbg_atomic(&dev_priv->drm, + "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", + crtc->base.base.id, crtc->base.name, + plane->base.base.id, plane->base.name, + was_visible, visible, + turn_off, turn_on, mode_changed); + + if (visible || was_visible) + new_crtc_state->fb_bits |= plane->frontbuffer_bit; + + if (HAS_GMCH(dev_priv) && + i9xx_must_disable_cxsr(new_crtc_state, old_plane_state, new_plane_state)) + new_crtc_state->disable_cxsr = true; + + if ((IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) && + ilk_must_disable_cxsr(new_crtc_state, old_plane_state, new_plane_state)) + new_crtc_state->disable_cxsr = true; + + if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state)) { + new_crtc_state->do_async_flip = true; + new_crtc_state->async_flip_planes |= BIT(plane->id); + } else if (plane->need_async_flip_toggle_wa && + new_crtc_state->uapi.async_flip) { + /* + * On platforms with double buffered async flip bit we + * set the bit already one frame early during the sync + * flip (see {i9xx,skl}_plane_update_arm()). The + * hardware will therefore be ready to perform a real + * async flip during the next commit, without having + * to wait yet another frame for the bit to latch. + */ + new_crtc_state->async_flip_planes |= BIT(plane->id); + } + + return 0; +} + +int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + const struct drm_framebuffer *fb = new_plane_state->hw.fb; + int ret; + + intel_plane_set_invisible(new_crtc_state, new_plane_state); + new_crtc_state->enabled_planes &= ~BIT(plane->id); + + if (!new_plane_state->hw.crtc && !old_plane_state->hw.crtc) + return 0; + + ret = plane->check_plane(new_crtc_state, new_plane_state); + if (ret) + return ret; + + if (fb) + new_crtc_state->enabled_planes |= BIT(plane->id); + + /* FIXME pre-g4x don't work like this */ + if (new_plane_state->uapi.visible) + new_crtc_state->active_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_plane_is_scaled(new_plane_state)) + new_crtc_state->scaled_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) + new_crtc_state->nv12_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + fb->format->format == DRM_FORMAT_C8) + new_crtc_state->c8_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible || old_plane_state->uapi.visible) + new_crtc_state->update_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { + new_crtc_state->data_rate_y[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 0); + new_crtc_state->data_rate[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 1); + + new_crtc_state->rel_data_rate_y[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 0); + new_crtc_state->rel_data_rate[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 1); + } else if (new_plane_state->uapi.visible) { + new_crtc_state->data_rate[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 0); + + new_crtc_state->rel_data_rate[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 0); + } + + return intel_plane_atomic_calc_changes(old_crtc_state, new_crtc_state, + old_plane_state, new_plane_state); +} + +static struct intel_plane * +intel_crtc_get_plane(struct intel_crtc *crtc, enum plane_id plane_id) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_plane *plane; + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + if (plane->id == plane_id) + return plane; + } + + return NULL; +} + +int intel_plane_atomic_check(struct intel_atomic_state *state, + struct intel_plane *plane) +{ + struct intel_display *display = to_intel_display(state); + struct intel_plane_state *new_plane_state = + intel_atomic_get_new_plane_state(state, plane); + const struct intel_plane_state *old_plane_state = + intel_atomic_get_old_plane_state(state, plane); + const struct intel_plane_state *new_primary_crtc_plane_state; + struct intel_crtc *crtc = intel_crtc_for_pipe(display, plane->pipe); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (new_crtc_state && intel_crtc_is_joiner_secondary(new_crtc_state)) { + struct intel_crtc *primary_crtc = + intel_primary_crtc(new_crtc_state); + struct intel_plane *primary_crtc_plane = + intel_crtc_get_plane(primary_crtc, plane->id); + + new_primary_crtc_plane_state = + intel_atomic_get_new_plane_state(state, primary_crtc_plane); + } else { + new_primary_crtc_plane_state = new_plane_state; + } + + intel_plane_copy_uapi_to_hw_state(new_plane_state, + new_primary_crtc_plane_state, + crtc); + + new_plane_state->uapi.visible = false; + if (!new_crtc_state) + return 0; + + return intel_plane_atomic_check_with_state(old_crtc_state, + new_crtc_state, + old_plane_state, + new_plane_state); +} + +static struct intel_plane * +skl_next_plane_to_commit(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct skl_ddb_entry ddb[I915_MAX_PLANES], + struct skl_ddb_entry ddb_y[I915_MAX_PLANES], + unsigned int *update_mask) +{ + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct intel_plane_state __maybe_unused *plane_state; + struct intel_plane *plane; + int i; + + if (*update_mask == 0) + return NULL; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + enum plane_id plane_id = plane->id; + + if (crtc->pipe != plane->pipe || + !(*update_mask & BIT(plane_id))) + continue; + + if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb[plane_id], + ddb, I915_MAX_PLANES, plane_id) || + skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_y[plane_id], + ddb_y, I915_MAX_PLANES, plane_id)) + continue; + + *update_mask &= ~BIT(plane_id); + ddb[plane_id] = crtc_state->wm.skl.plane_ddb[plane_id]; + ddb_y[plane_id] = crtc_state->wm.skl.plane_ddb_y[plane_id]; + + return plane; + } + + /* should never happen */ + drm_WARN_ON(state->base.dev, 1); + + return NULL; +} + +void intel_plane_update_noarm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_update_noarm(plane, crtc); + + if (plane->update_noarm) + plane->update_noarm(dsb, plane, crtc_state, plane_state); +} + +void intel_plane_async_flip(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + bool async_flip) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_async_flip(plane, crtc, async_flip); + plane->async_flip(dsb, plane, crtc_state, plane_state, async_flip); +} + +void intel_plane_update_arm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + if (crtc_state->do_async_flip && plane->async_flip) { + intel_plane_async_flip(dsb, plane, crtc_state, plane_state, true); + return; + } + + trace_intel_plane_update_arm(plane, crtc); + plane->update_arm(dsb, plane, crtc_state, plane_state); +} + +void intel_plane_disable_arm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_disable_arm(plane, crtc); + plane->disable_arm(dsb, plane, crtc_state); +} + +void intel_crtc_planes_update_noarm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + if (new_crtc_state->do_async_flip) + return; + + /* + * Since we only write non-arming registers here, + * the order does not matter even for skl+. + */ + for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; + + /* TODO: for mailbox updates this should be skipped */ + if (new_plane_state->uapi.visible || + new_plane_state->planar_slave) + intel_plane_update_noarm(dsb, plane, + new_crtc_state, new_plane_state); + } +} + +static void skl_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct skl_ddb_entry ddb[I915_MAX_PLANES]; + struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane *plane; + + memcpy(ddb, old_crtc_state->wm.skl.plane_ddb, + sizeof(old_crtc_state->wm.skl.plane_ddb)); + memcpy(ddb_y, old_crtc_state->wm.skl.plane_ddb_y, + sizeof(old_crtc_state->wm.skl.plane_ddb_y)); + + while ((plane = skl_next_plane_to_commit(state, crtc, ddb, ddb_y, &update_mask))) { + struct intel_plane_state *new_plane_state = + intel_atomic_get_new_plane_state(state, plane); + + /* + * TODO: for mailbox updates intel_plane_update_noarm() + * would have to be called here as well. + */ + if (new_plane_state->uapi.visible || + new_plane_state->planar_slave) + intel_plane_update_arm(dsb, plane, new_crtc_state, new_plane_state); + else + intel_plane_disable_arm(dsb, plane, new_crtc_state); + } +} + +static void i9xx_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; + + /* + * TODO: for mailbox updates intel_plane_update_noarm() + * would have to be called here as well. + */ + if (new_plane_state->uapi.visible) + intel_plane_update_arm(dsb, plane, new_crtc_state, new_plane_state); + else + intel_plane_disable_arm(dsb, plane, new_crtc_state); + } +} + +void intel_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + + if (DISPLAY_VER(i915) >= 9) + skl_crtc_planes_update_arm(dsb, state, crtc); + else + i9xx_crtc_planes_update_arm(dsb, state, crtc); +} + +int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, + struct intel_crtc_state *crtc_state, + int min_scale, int max_scale, + bool can_position) +{ + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + struct drm_framebuffer *fb = plane_state->hw.fb; + struct drm_rect *src = &plane_state->uapi.src; + struct drm_rect *dst = &plane_state->uapi.dst; + const struct drm_rect *clip = &crtc_state->pipe_src; + unsigned int rotation = plane_state->hw.rotation; + int hscale, vscale; + + if (!fb) { + plane_state->uapi.visible = false; + return 0; + } + + drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation); + + /* Check scaling */ + hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); + vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); + if (hscale < 0 || vscale < 0) { + drm_dbg_kms(&i915->drm, "Invalid scaling of plane\n"); + drm_rect_debug_print("src: ", src, true); + drm_rect_debug_print("dst: ", dst, false); + return -ERANGE; + } + + /* + * FIXME: This might need further adjustment for seamless scaling + * with phase information, for the 2p2 and 2p1 scenarios. + */ + plane_state->uapi.visible = drm_rect_clip_scaled(src, dst, clip); + + drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation); + + if (!can_position && plane_state->uapi.visible && + !drm_rect_equals(dst, clip)) { + drm_dbg_kms(&i915->drm, "Plane must cover entire CRTC\n"); + drm_rect_debug_print("dst: ", dst, false); + drm_rect_debug_print("clip: ", clip, false); + return -EINVAL; + } + + /* final plane coordinates will be relative to the plane's pipe */ + drm_rect_translate(dst, -clip->x1, -clip->y1); + + return 0; +} + +int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) +{ + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + const struct drm_framebuffer *fb = plane_state->hw.fb; + struct drm_rect *src = &plane_state->uapi.src; + u32 src_x, src_y, src_w, src_h, hsub, vsub; + bool rotated = drm_rotation_90_or_270(plane_state->hw.rotation); + + /* + * FIXME hsub/vsub vs. block size is a mess. Pre-tgl CCS + * abuses hsub/vsub so we can't use them here. But as they + * are limited to 32bpp RGB formats we don't actually need + * to check anything. + */ + if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS || + fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) + return 0; + + /* + * Hardware doesn't handle subpixel coordinates. + * Adjust to (macro)pixel boundary, but be careful not to + * increase the source viewport size, because that could + * push the downscaling factor out of bounds. + */ + src_x = src->x1 >> 16; + src_w = drm_rect_width(src) >> 16; + src_y = src->y1 >> 16; + src_h = drm_rect_height(src) >> 16; + + drm_rect_init(src, src_x << 16, src_y << 16, + src_w << 16, src_h << 16); + + if (fb->format->format == DRM_FORMAT_RGB565 && rotated) { + hsub = 2; + vsub = 2; + } else if (DISPLAY_VER(i915) >= 20 && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { + /* + * This allows NV12 and P0xx formats to have odd size and/or odd + * source coordinates on DISPLAY_VER(i915) >= 20 + */ + hsub = 1; + vsub = 1; + + /* Wa_16023981245 */ + if ((DISPLAY_VER_FULL(i915) == IP_VER(20, 0) || +<<<<<<< + DISPLAY_VER_FULL(i915) == IP_VER(30, 0)) && + src_x % 2 != 0) +======= + DISPLAY_VER_FULL(i915) == IP_VER(30, 0)) && + src_x % 2 != 0) +>>>>>>> + hsub = 2; + } else { + hsub = fb->format->hsub; + vsub = fb->format->vsub; + } + + if (rotated) + hsub = vsub = max(hsub, vsub); + + if (src_x % hsub || src_w % hsub) { + drm_dbg_kms(&i915->drm, "src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_x, src_w, hsub, str_yes_no(rotated)); + return -EINVAL; + } + + if (src_y % vsub || src_h % vsub) { + drm_dbg_kms(&i915->drm, "src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_y, src_h, vsub, str_yes_no(rotated)); + return -EINVAL; + } + + return 0; +} + +static int add_dma_resv_fences(struct dma_resv *resv, + struct drm_plane_state *new_plane_state) +{ + struct dma_fence *fence = dma_fence_get(new_plane_state->fence); + struct dma_fence *new; + int ret; + + ret = dma_resv_get_singleton(resv, dma_resv_usage_rw(false), &new); + if (ret) + goto error; + + if (new && fence) { + struct dma_fence_chain *chain = dma_fence_chain_alloc(); + + if (!chain) { + ret = -ENOMEM; + goto error; + } + + dma_fence_chain_init(chain, fence, new, 1); + fence = &chain->base; + + } else if (new) { + fence = new; + } + + dma_fence_put(new_plane_state->fence); + new_plane_state->fence = fence; + return 0; + +error: + dma_fence_put(fence); + return ret; +} + +/** + * intel_prepare_plane_fb - Prepare fb for usage on plane + * @_plane: drm plane to prepare for + * @_new_plane_state: the plane state being prepared + * + * Prepares a framebuffer for usage on a display plane. Generally this + * involves pinning the underlying object and updating the frontbuffer tracking + * bits. Some older platforms need special physical address handling for + * cursor planes. + * + * Returns 0 on success, negative error code on failure. + */ +static int +intel_prepare_plane_fb(struct drm_plane *_plane, + struct drm_plane_state *_new_plane_state) +{ + struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY }; + struct intel_plane *plane = to_intel_plane(_plane); + struct intel_plane_state *new_plane_state = + to_intel_plane_state(_new_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(new_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + struct intel_plane_state *old_plane_state = + intel_atomic_get_old_plane_state(state, plane); + struct drm_gem_object *obj = intel_fb_bo(new_plane_state->hw.fb); + struct drm_gem_object *old_obj = intel_fb_bo(old_plane_state->hw.fb); + int ret; + + if (old_obj) { + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, + to_intel_crtc(old_plane_state->hw.crtc)); + + /* Big Hammer, we also need to ensure that any pending + * MI_WAIT_FOR_EVENT inside a user batch buffer on the + * current scanout is retired before unpinning the old + * framebuffer. Note that we rely on userspace rendering + * into the buffer attached to the pipe they are waiting + * on. If not, userspace generates a GPU hang with IPEHR + * point to the MI_WAIT_FOR_EVENT. + * + * This should only fail upon a hung GPU, in which case we + * can safely continue. + */ + if (new_crtc_state && intel_crtc_needs_modeset(new_crtc_state)) { + ret = add_dma_resv_fences(old_obj->resv, + &new_plane_state->uapi); + if (ret < 0) + return ret; + } + } + + if (!obj) + return 0; + + ret = intel_plane_pin_fb(new_plane_state); + if (ret) + return ret; + + ret = drm_gem_plane_helper_prepare_fb(&plane->base, &new_plane_state->uapi); + if (ret < 0) + goto unpin_fb; + + if (new_plane_state->uapi.fence) { + i915_gem_fence_wait_priority(new_plane_state->uapi.fence, + &attr); + + intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc, + new_plane_state->uapi.fence); + } + + /* + * We declare pageflips to be interactive and so merit a small bias + * towards upclocking to deliver the frame on time. By only changing + * the RPS thresholds to sample more regularly and aim for higher + * clocks we can hopefully deliver low power workloads (like kodi) + * that are not quite steady state without resorting to forcing + * maximum clocks following a vblank miss (see do_rps_boost()). + */ + intel_display_rps_mark_interactive(dev_priv, state, true); + + return 0; + +unpin_fb: + intel_plane_unpin_fb(new_plane_state); + + return ret; +} + +/** + * intel_cleanup_plane_fb - Cleans up an fb after plane use + * @plane: drm plane to clean up for + * @_old_plane_state: the state from the previous modeset + * + * Cleans up a framebuffer that has just been removed from a plane. + */ +static void +intel_cleanup_plane_fb(struct drm_plane *plane, + struct drm_plane_state *_old_plane_state) +{ + struct intel_plane_state *old_plane_state = + to_intel_plane_state(_old_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(old_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->dev); + struct drm_gem_object *obj = intel_fb_bo(old_plane_state->hw.fb); + + if (!obj) + return; + + intel_display_rps_mark_interactive(dev_priv, state, false); + + intel_plane_unpin_fb(old_plane_state); +} + +static const struct drm_plane_helper_funcs intel_plane_helper_funcs = { + .prepare_fb = intel_prepare_plane_fb, + .cleanup_fb = intel_cleanup_plane_fb, +}; + +void intel_plane_helper_add(struct intel_plane *plane) +{ + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); +} + +void intel_plane_init_cursor_vblank_work(struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + if (!old_plane_state->ggtt_vma || + old_plane_state->ggtt_vma == new_plane_state->ggtt_vma) + return; + + drm_vblank_work_init(&old_plane_state->unpin_work, old_plane_state->uapi.crtc, + intel_cursor_unpin_work); +} diff --git a/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage.1 b/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage.1 new file mode 100644 index 000000000000..ef95bec8d180 --- /dev/null +++ b/rr-cache/44bc583546c40f3144ba90b705854f1ee09d13ea/preimage.1 @@ -0,0 +1,1231 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/** + * DOC: atomic plane helpers + * + * The functions here are used by the atomic plane helper functions to + * implement legacy plane updates (i.e., drm_plane->update_plane() and + * drm_plane->disable_plane()). This allows plane updates to use the + * atomic state infrastructure and perform plane updates as separate + * prepare/check/commit/cleanup steps. + */ + +#include <linux/dma-fence-chain.h> +#include <linux/dma-resv.h> + +#include <drm/drm_atomic_helper.h> +#include <drm/drm_blend.h> +#include <drm/drm_fourcc.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_atomic_helper.h> + +#include "i915_config.h" +#include "i9xx_plane_regs.h" +#include "intel_atomic_plane.h" +#include "intel_cdclk.h" +#include "intel_cursor.h" +#include "intel_display_rps.h" +#include "intel_display_trace.h" +#include "intel_display_types.h" +#include "intel_fb.h" +#include "intel_fb_pin.h" +#include "skl_scaler.h" +#include "skl_watermark.h" + +static void intel_plane_state_reset(struct intel_plane_state *plane_state, + struct intel_plane *plane) +{ + memset(plane_state, 0, sizeof(*plane_state)); + + __drm_atomic_helper_plane_state_reset(&plane_state->uapi, &plane->base); + + plane_state->scaler_id = -1; +} + +struct intel_plane *intel_plane_alloc(void) +{ + struct intel_plane_state *plane_state; + struct intel_plane *plane; + + plane = kzalloc(sizeof(*plane), GFP_KERNEL); + if (!plane) + return ERR_PTR(-ENOMEM); + + plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); + if (!plane_state) { + kfree(plane); + return ERR_PTR(-ENOMEM); + } + + intel_plane_state_reset(plane_state, plane); + + plane->base.state = &plane_state->uapi; + + return plane; +} + +void intel_plane_free(struct intel_plane *plane) +{ + intel_plane_destroy_state(&plane->base, plane->base.state); + kfree(plane); +} + +/** + * intel_plane_duplicate_state - duplicate plane state + * @plane: drm plane + * + * Allocates and returns a copy of the plane state (both common and + * Intel-specific) for the specified plane. + * + * Returns: The newly allocated plane state, or NULL on failure. + */ +struct drm_plane_state * +intel_plane_duplicate_state(struct drm_plane *plane) +{ + struct intel_plane_state *intel_state; + + intel_state = to_intel_plane_state(plane->state); + intel_state = kmemdup(intel_state, sizeof(*intel_state), GFP_KERNEL); + + if (!intel_state) + return NULL; + + __drm_atomic_helper_plane_duplicate_state(plane, &intel_state->uapi); + + intel_state->ggtt_vma = NULL; + intel_state->dpt_vma = NULL; + intel_state->flags = 0; + + /* add reference to fb */ + if (intel_state->hw.fb) + drm_framebuffer_get(intel_state->hw.fb); + + return &intel_state->uapi; +} + +/** + * intel_plane_destroy_state - destroy plane state + * @plane: drm plane + * @state: state object to destroy + * + * Destroys the plane state (both common and Intel-specific) for the + * specified plane. + */ +void +intel_plane_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct intel_plane_state *plane_state = to_intel_plane_state(state); + + drm_WARN_ON(plane->dev, plane_state->ggtt_vma); + drm_WARN_ON(plane->dev, plane_state->dpt_vma); + + __drm_atomic_helper_plane_destroy_state(&plane_state->uapi); + if (plane_state->hw.fb) + drm_framebuffer_put(plane_state->hw.fb); + kfree(plane_state); +} + +bool intel_plane_needs_physical(struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return plane->id == PLANE_CURSOR && + DISPLAY_INFO(i915)->cursor_needs_physical; +} + +unsigned int intel_adjusted_rate(const struct drm_rect *src, + const struct drm_rect *dst, + unsigned int rate) +{ + unsigned int src_w, src_h, dst_w, dst_h; + + src_w = drm_rect_width(src) >> 16; + src_h = drm_rect_height(src) >> 16; + dst_w = drm_rect_width(dst); + dst_h = drm_rect_height(dst); + + /* Downscaling limits the maximum pixel rate */ + dst_w = min(src_w, dst_w); + dst_h = min(src_h, dst_h); + + return DIV_ROUND_UP_ULL(mul_u32_u32(rate, src_w * src_h), + dst_w * dst_h); +} + +unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + /* + * Note we don't check for plane visibility here as + * we want to use this when calculating the cursor + * watermarks even if the cursor is fully offscreen. + * That depends on the src/dst rectangles being + * correctly populated whenever the watermark code + * considers the cursor to be visible, whether or not + * it is actually visible. + * + * See: intel_wm_plane_visible() and intel_check_cursor() + */ + + return intel_adjusted_rate(&plane_state->uapi.src, + &plane_state->uapi.dst, + crtc_state->pixel_rate); +} + +unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int color_plane) +{ + const struct drm_framebuffer *fb = plane_state->hw.fb; + + if (!plane_state->uapi.visible) + return 0; + + return intel_plane_pixel_rate(crtc_state, plane_state) * + fb->format->cpp[color_plane]; +} + +static bool +use_min_ddb(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return DISPLAY_VER(i915) >= 13 && + crtc_state->uapi.async_flip && + plane->async_flip; +} + +static unsigned int +intel_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int color_plane) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + const struct drm_framebuffer *fb = plane_state->hw.fb; + int width, height; + unsigned int rel_data_rate; + + if (plane->id == PLANE_CURSOR) + return 0; + + if (!plane_state->uapi.visible) + return 0; + + /* + * We calculate extra ddb based on ratio plane rate/total data rate + * in case, in some cases we should not allocate extra ddb for the plane, + * so do not count its data rate, if this is the case. + */ + if (use_min_ddb(crtc_state, plane)) + return 0; + + /* + * Src coordinates are already rotated by 270 degrees for + * the 90/270 degree plane rotation cases (to match the + * GTT mapping), hence no need to account for rotation here. + */ + width = drm_rect_width(&plane_state->uapi.src) >> 16; + height = drm_rect_height(&plane_state->uapi.src) >> 16; + + /* UV plane does 1/2 pixel sub-sampling */ + if (color_plane == 1) { + width /= 2; + height /= 2; + } + + rel_data_rate = width * height * fb->format->cpp[color_plane]; + + return intel_adjusted_rate(&plane_state->uapi.src, + &plane_state->uapi.dst, + rel_data_rate); +} + +int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, + struct intel_plane *plane, + bool *need_cdclk_calc) +{ + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + const struct intel_plane_state *plane_state = + intel_atomic_get_new_plane_state(state, plane); + struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc); + const struct intel_cdclk_state *cdclk_state; + const struct intel_crtc_state *old_crtc_state; + struct intel_crtc_state *new_crtc_state; + + if (!plane_state->uapi.visible || !plane->min_cdclk) + return 0; + + old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); + new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + + new_crtc_state->min_cdclk[plane->id] = + plane->min_cdclk(new_crtc_state, plane_state); + + /* + * No need to check against the cdclk state if + * the min cdclk for the plane doesn't increase. + * + * Ie. we only ever increase the cdclk due to plane + * requirements. This can reduce back and forth + * display blinking due to constant cdclk changes. + */ + if (new_crtc_state->min_cdclk[plane->id] <= + old_crtc_state->min_cdclk[plane->id]) + return 0; + + cdclk_state = intel_atomic_get_cdclk_state(state); + if (IS_ERR(cdclk_state)) + return PTR_ERR(cdclk_state); + + /* + * No need to recalculate the cdclk state if + * the min cdclk for the pipe doesn't increase. + * + * Ie. we only ever increase the cdclk due to plane + * requirements. This can reduce back and forth + * display blinking due to constant cdclk changes. + */ + if (new_crtc_state->min_cdclk[plane->id] <= + cdclk_state->min_cdclk[crtc->pipe]) + return 0; + + drm_dbg_kms(&dev_priv->drm, + "[PLANE:%d:%s] min cdclk (%d kHz) > [CRTC:%d:%s] min cdclk (%d kHz)\n", + plane->base.base.id, plane->base.name, + new_crtc_state->min_cdclk[plane->id], + crtc->base.base.id, crtc->base.name, + cdclk_state->min_cdclk[crtc->pipe]); + *need_cdclk_calc = true; + + return 0; +} + +static void intel_plane_clear_hw_state(struct intel_plane_state *plane_state) +{ + if (plane_state->hw.fb) + drm_framebuffer_put(plane_state->hw.fb); + + memset(&plane_state->hw, 0, sizeof(plane_state->hw)); +} + +void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state, + struct intel_crtc *crtc) +{ + intel_plane_clear_hw_state(plane_state); + + /* + * For the joiner secondary uapi.crtc will point at + * the primary crtc. So we explicitly assign the right + * secondary crtc to hw.crtc. uapi.crtc!=NULL simply + * indicates the plane is logically enabled on the uapi level. + */ + plane_state->hw.crtc = from_plane_state->uapi.crtc ? &crtc->base : NULL; + + plane_state->hw.fb = from_plane_state->uapi.fb; + if (plane_state->hw.fb) + drm_framebuffer_get(plane_state->hw.fb); + + plane_state->hw.alpha = from_plane_state->uapi.alpha; + plane_state->hw.pixel_blend_mode = + from_plane_state->uapi.pixel_blend_mode; + plane_state->hw.rotation = from_plane_state->uapi.rotation; + plane_state->hw.color_encoding = from_plane_state->uapi.color_encoding; + plane_state->hw.color_range = from_plane_state->uapi.color_range; + plane_state->hw.scaling_filter = from_plane_state->uapi.scaling_filter; + + plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi); + plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi); +} + +void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state) +{ + intel_plane_clear_hw_state(plane_state); + + memcpy(&plane_state->hw, &from_plane_state->hw, + sizeof(plane_state->hw)); + + if (plane_state->hw.fb) + drm_framebuffer_get(plane_state->hw.fb); +} + +void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + + crtc_state->active_planes &= ~BIT(plane->id); + crtc_state->scaled_planes &= ~BIT(plane->id); + crtc_state->nv12_planes &= ~BIT(plane->id); + crtc_state->c8_planes &= ~BIT(plane->id); + crtc_state->async_flip_planes &= ~BIT(plane->id); + crtc_state->data_rate[plane->id] = 0; + crtc_state->data_rate_y[plane->id] = 0; + crtc_state->rel_data_rate[plane->id] = 0; + crtc_state->rel_data_rate_y[plane->id] = 0; + crtc_state->min_cdclk[plane->id] = 0; + + plane_state->uapi.visible = false; +} + +static bool intel_plane_is_scaled(const struct intel_plane_state *plane_state) +{ + int src_w = drm_rect_width(&plane_state->uapi.src) >> 16; + int src_h = drm_rect_height(&plane_state->uapi.src) >> 16; + int dst_w = drm_rect_width(&plane_state->uapi.dst); + int dst_h = drm_rect_height(&plane_state->uapi.dst); + + return src_w != dst_w || src_h != dst_h; +} + +static bool intel_plane_do_async_flip(struct intel_plane *plane, + const struct intel_crtc_state *old_crtc_state, + const struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + if (!plane->async_flip) + return false; + + if (!new_crtc_state->uapi.async_flip) + return false; + + /* + * In platforms after DISPLAY13, we might need to override + * first async flip in order to change watermark levels + * as part of optimization. + * + * And let's do this for all skl+ so that we can eg. change the + * modifier as well. + * + * TODO: For older platforms there is less reason to do this as + * only X-tile is supported with async flips, though we could + * extend this so other scanout parameters (stride/etc) could + * be changed as well... + */ + return DISPLAY_VER(i915) < 9 || old_crtc_state->uapi.async_flip; +} + +static bool i9xx_must_disable_cxsr(const struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + const struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + bool old_visible = old_plane_state->uapi.visible; + bool new_visible = new_plane_state->uapi.visible; + u32 old_ctl = old_plane_state->ctl; + u32 new_ctl = new_plane_state->ctl; + bool modeset, turn_on, turn_off; + + if (plane->id == PLANE_CURSOR) + return false; + + modeset = intel_crtc_needs_modeset(new_crtc_state); + turn_off = old_visible && (!new_visible || modeset); + turn_on = new_visible && (!old_visible || modeset); + + /* Must disable CxSR around plane enable/disable */ + if (turn_on || turn_off) + return true; + + if (!old_visible || !new_visible) + return false; + + /* + * Most plane control register updates are blocked while in CxSR. + * + * Tiling mode is one exception where the primary plane can + * apparently handle it, whereas the sprites can not (the + * sprite issue being only relevant on VLV/CHV where CxSR + * is actually possible with a sprite enabled). + */ + if (plane->id == PLANE_PRIMARY) { + old_ctl &= ~DISP_TILED; + new_ctl &= ~DISP_TILED; + } + + return old_ctl != new_ctl; +} + +static bool ilk_must_disable_cxsr(const struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + const struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + bool old_visible = old_plane_state->uapi.visible; + bool new_visible = new_plane_state->uapi.visible; + bool modeset, turn_on; + + if (plane->id == PLANE_CURSOR) + return false; + + modeset = intel_crtc_needs_modeset(new_crtc_state); + turn_on = new_visible && (!old_visible || modeset); + + /* + * ILK/SNB DVSACNTR/Sprite Enable + * IVB SPR_CTL/Sprite Enable + * "When in Self Refresh Big FIFO mode, a write to enable the + * plane will be internally buffered and delayed while Big FIFO + * mode is exiting." + * + * Which means that enabling the sprite can take an extra frame + * when we start in big FIFO mode (LP1+). Thus we need to drop + * down to LP0 and wait for vblank in order to make sure the + * sprite gets enabled on the next vblank after the register write. + * Doing otherwise would risk enabling the sprite one frame after + * we've already signalled flip completion. We can resume LP1+ + * once the sprite has been enabled. + * + * With experimental results seems this is needed also for primary + * plane, not only sprite plane. + */ + if (turn_on) + return true; + + /* + * WaCxSRDisabledForSpriteScaling:ivb + * IVB SPR_SCALE/Scaling Enable + * "Low Power watermarks must be disabled for at least one + * frame before enabling sprite scaling, and kept disabled + * until sprite scaling is disabled." + * + * ILK/SNB DVSASCALE/Scaling Enable + * "When in Self Refresh Big FIFO mode, scaling enable will be + * masked off while Big FIFO mode is exiting." + * + * Despite the w/a only being listed for IVB we assume that + * the ILK/SNB note has similar ramifications, hence we apply + * the w/a on all three platforms. + */ + return !intel_plane_is_scaled(old_plane_state) && + intel_plane_is_scaled(new_plane_state); +} + +static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + bool mode_changed = intel_crtc_needs_modeset(new_crtc_state); + bool was_crtc_enabled = old_crtc_state->hw.active; + bool is_crtc_enabled = new_crtc_state->hw.active; + bool turn_off, turn_on, visible, was_visible; + int ret; + + if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { + ret = skl_update_scaler_plane(new_crtc_state, new_plane_state); + if (ret) + return ret; + } + + was_visible = old_plane_state->uapi.visible; + visible = new_plane_state->uapi.visible; + + if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) + was_visible = false; + + /* + * Visibility is calculated as if the crtc was on, but + * after scaler setup everything depends on it being off + * when the crtc isn't active. + * + * FIXME this is wrong for watermarks. Watermarks should also + * be computed as if the pipe would be active. Perhaps move + * per-plane wm computation to the .check_plane() hook, and + * only combine the results from all planes in the current place? + */ + if (!is_crtc_enabled) { + intel_plane_set_invisible(new_crtc_state, new_plane_state); + visible = false; + } + + if (!was_visible && !visible) + return 0; + + turn_off = was_visible && (!visible || mode_changed); + turn_on = visible && (!was_visible || mode_changed); + + drm_dbg_atomic(&dev_priv->drm, + "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", + crtc->base.base.id, crtc->base.name, + plane->base.base.id, plane->base.name, + was_visible, visible, + turn_off, turn_on, mode_changed); + + if (visible || was_visible) + new_crtc_state->fb_bits |= plane->frontbuffer_bit; + + if (HAS_GMCH(dev_priv) && + i9xx_must_disable_cxsr(new_crtc_state, old_plane_state, new_plane_state)) + new_crtc_state->disable_cxsr = true; + + if ((IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) && + ilk_must_disable_cxsr(new_crtc_state, old_plane_state, new_plane_state)) + new_crtc_state->disable_cxsr = true; + + if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state)) { + new_crtc_state->do_async_flip = true; + new_crtc_state->async_flip_planes |= BIT(plane->id); + } else if (plane->need_async_flip_toggle_wa && + new_crtc_state->uapi.async_flip) { + /* + * On platforms with double buffered async flip bit we + * set the bit already one frame early during the sync + * flip (see {i9xx,skl}_plane_update_arm()). The + * hardware will therefore be ready to perform a real + * async flip during the next commit, without having + * to wait yet another frame for the bit to latch. + */ + new_crtc_state->async_flip_planes |= BIT(plane->id); + } + + return 0; +} + +int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + const struct drm_framebuffer *fb = new_plane_state->hw.fb; + int ret; + + intel_plane_set_invisible(new_crtc_state, new_plane_state); + new_crtc_state->enabled_planes &= ~BIT(plane->id); + + if (!new_plane_state->hw.crtc && !old_plane_state->hw.crtc) + return 0; + + ret = plane->check_plane(new_crtc_state, new_plane_state); + if (ret) + return ret; + + if (fb) + new_crtc_state->enabled_planes |= BIT(plane->id); + + /* FIXME pre-g4x don't work like this */ + if (new_plane_state->uapi.visible) + new_crtc_state->active_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_plane_is_scaled(new_plane_state)) + new_crtc_state->scaled_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) + new_crtc_state->nv12_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + fb->format->format == DRM_FORMAT_C8) + new_crtc_state->c8_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible || old_plane_state->uapi.visible) + new_crtc_state->update_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { + new_crtc_state->data_rate_y[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 0); + new_crtc_state->data_rate[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 1); + + new_crtc_state->rel_data_rate_y[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 0); + new_crtc_state->rel_data_rate[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 1); + } else if (new_plane_state->uapi.visible) { + new_crtc_state->data_rate[plane->id] = + intel_plane_data_rate(new_crtc_state, new_plane_state, 0); + + new_crtc_state->rel_data_rate[plane->id] = + intel_plane_relative_data_rate(new_crtc_state, + new_plane_state, 0); + } + + return intel_plane_atomic_calc_changes(old_crtc_state, new_crtc_state, + old_plane_state, new_plane_state); +} + +static struct intel_plane * +intel_crtc_get_plane(struct intel_crtc *crtc, enum plane_id plane_id) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_plane *plane; + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + if (plane->id == plane_id) + return plane; + } + + return NULL; +} + +int intel_plane_atomic_check(struct intel_atomic_state *state, + struct intel_plane *plane) +{ + struct intel_display *display = to_intel_display(state); + struct intel_plane_state *new_plane_state = + intel_atomic_get_new_plane_state(state, plane); + const struct intel_plane_state *old_plane_state = + intel_atomic_get_old_plane_state(state, plane); + const struct intel_plane_state *new_primary_crtc_plane_state; + struct intel_crtc *crtc = intel_crtc_for_pipe(display, plane->pipe); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (new_crtc_state && intel_crtc_is_joiner_secondary(new_crtc_state)) { + struct intel_crtc *primary_crtc = + intel_primary_crtc(new_crtc_state); + struct intel_plane *primary_crtc_plane = + intel_crtc_get_plane(primary_crtc, plane->id); + + new_primary_crtc_plane_state = + intel_atomic_get_new_plane_state(state, primary_crtc_plane); + } else { + new_primary_crtc_plane_state = new_plane_state; + } + + intel_plane_copy_uapi_to_hw_state(new_plane_state, + new_primary_crtc_plane_state, + crtc); + + new_plane_state->uapi.visible = false; + if (!new_crtc_state) + return 0; + + return intel_plane_atomic_check_with_state(old_crtc_state, + new_crtc_state, + old_plane_state, + new_plane_state); +} + +static struct intel_plane * +skl_next_plane_to_commit(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct skl_ddb_entry ddb[I915_MAX_PLANES], + struct skl_ddb_entry ddb_y[I915_MAX_PLANES], + unsigned int *update_mask) +{ + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct intel_plane_state __maybe_unused *plane_state; + struct intel_plane *plane; + int i; + + if (*update_mask == 0) + return NULL; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + enum plane_id plane_id = plane->id; + + if (crtc->pipe != plane->pipe || + !(*update_mask & BIT(plane_id))) + continue; + + if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb[plane_id], + ddb, I915_MAX_PLANES, plane_id) || + skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_y[plane_id], + ddb_y, I915_MAX_PLANES, plane_id)) + continue; + + *update_mask &= ~BIT(plane_id); + ddb[plane_id] = crtc_state->wm.skl.plane_ddb[plane_id]; + ddb_y[plane_id] = crtc_state->wm.skl.plane_ddb_y[plane_id]; + + return plane; + } + + /* should never happen */ + drm_WARN_ON(state->base.dev, 1); + + return NULL; +} + +void intel_plane_update_noarm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_update_noarm(plane, crtc); + + if (plane->update_noarm) + plane->update_noarm(dsb, plane, crtc_state, plane_state); +} + +void intel_plane_async_flip(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + bool async_flip) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_async_flip(plane, crtc, async_flip); + plane->async_flip(dsb, plane, crtc_state, plane_state, async_flip); +} + +void intel_plane_update_arm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + if (crtc_state->do_async_flip && plane->async_flip) { + intel_plane_async_flip(dsb, plane, crtc_state, plane_state, true); + return; + } + + trace_intel_plane_update_arm(plane, crtc); + plane->update_arm(dsb, plane, crtc_state, plane_state); +} + +void intel_plane_disable_arm(struct intel_dsb *dsb, + struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + trace_intel_plane_disable_arm(plane, crtc); + plane->disable_arm(dsb, plane, crtc_state); +} + +void intel_crtc_planes_update_noarm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + if (new_crtc_state->do_async_flip) + return; + + /* + * Since we only write non-arming registers here, + * the order does not matter even for skl+. + */ + for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; + + /* TODO: for mailbox updates this should be skipped */ + if (new_plane_state->uapi.visible || + new_plane_state->planar_slave) + intel_plane_update_noarm(dsb, plane, + new_crtc_state, new_plane_state); + } +} + +static void skl_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct skl_ddb_entry ddb[I915_MAX_PLANES]; + struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane *plane; + + memcpy(ddb, old_crtc_state->wm.skl.plane_ddb, + sizeof(old_crtc_state->wm.skl.plane_ddb)); + memcpy(ddb_y, old_crtc_state->wm.skl.plane_ddb_y, + sizeof(old_crtc_state->wm.skl.plane_ddb_y)); + + while ((plane = skl_next_plane_to_commit(state, crtc, ddb, ddb_y, &update_mask))) { + struct intel_plane_state *new_plane_state = + intel_atomic_get_new_plane_state(state, plane); + + /* + * TODO: for mailbox updates intel_plane_update_noarm() + * would have to be called here as well. + */ + if (new_plane_state->uapi.visible || + new_plane_state->planar_slave) + intel_plane_update_arm(dsb, plane, new_crtc_state, new_plane_state); + else + intel_plane_disable_arm(dsb, plane, new_crtc_state); + } +} + +static void i9xx_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; + + /* + * TODO: for mailbox updates intel_plane_update_noarm() + * would have to be called here as well. + */ + if (new_plane_state->uapi.visible) + intel_plane_update_arm(dsb, plane, new_crtc_state, new_plane_state); + else + intel_plane_disable_arm(dsb, plane, new_crtc_state); + } +} + +void intel_crtc_planes_update_arm(struct intel_dsb *dsb, + struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + + if (DISPLAY_VER(i915) >= 9) + skl_crtc_planes_update_arm(dsb, state, crtc); + else + i9xx_crtc_planes_update_arm(dsb, state, crtc); +} + +int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, + struct intel_crtc_state *crtc_state, + int min_scale, int max_scale, + bool can_position) +{ + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + struct drm_framebuffer *fb = plane_state->hw.fb; + struct drm_rect *src = &plane_state->uapi.src; + struct drm_rect *dst = &plane_state->uapi.dst; + const struct drm_rect *clip = &crtc_state->pipe_src; + unsigned int rotation = plane_state->hw.rotation; + int hscale, vscale; + + if (!fb) { + plane_state->uapi.visible = false; + return 0; + } + + drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation); + + /* Check scaling */ + hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); + vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); + if (hscale < 0 || vscale < 0) { + drm_dbg_kms(&i915->drm, "Invalid scaling of plane\n"); + drm_rect_debug_print("src: ", src, true); + drm_rect_debug_print("dst: ", dst, false); + return -ERANGE; + } + + /* + * FIXME: This might need further adjustment for seamless scaling + * with phase information, for the 2p2 and 2p1 scenarios. + */ + plane_state->uapi.visible = drm_rect_clip_scaled(src, dst, clip); + + drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation); + + if (!can_position && plane_state->uapi.visible && + !drm_rect_equals(dst, clip)) { + drm_dbg_kms(&i915->drm, "Plane must cover entire CRTC\n"); + drm_rect_debug_print("dst: ", dst, false); + drm_rect_debug_print("clip: ", clip, false); + return -EINVAL; + } + + /* final plane coordinates will be relative to the plane's pipe */ + drm_rect_translate(dst, -clip->x1, -clip->y1); + + return 0; +} + +int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) +{ + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + const struct drm_framebuffer *fb = plane_state->hw.fb; + struct drm_rect *src = &plane_state->uapi.src; + u32 src_x, src_y, src_w, src_h, hsub, vsub; + bool rotated = drm_rotation_90_or_270(plane_state->hw.rotation); + + /* + * FIXME hsub/vsub vs. block size is a mess. Pre-tgl CCS + * abuses hsub/vsub so we can't use them here. But as they + * are limited to 32bpp RGB formats we don't actually need + * to check anything. + */ + if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS || + fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) + return 0; + + /* + * Hardware doesn't handle subpixel coordinates. + * Adjust to (macro)pixel boundary, but be careful not to + * increase the source viewport size, because that could + * push the downscaling factor out of bounds. + */ + src_x = src->x1 >> 16; + src_w = drm_rect_width(src) >> 16; + src_y = src->y1 >> 16; + src_h = drm_rect_height(src) >> 16; + + drm_rect_init(src, src_x << 16, src_y << 16, + src_w << 16, src_h << 16); + + if (fb->format->format == DRM_FORMAT_RGB565 && rotated) { + hsub = 2; + vsub = 2; + } else if (DISPLAY_VER(i915) >= 20 && + intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { + /* + * This allows NV12 and P0xx formats to have odd size and/or odd + * source coordinates on DISPLAY_VER(i915) >= 20 + */ + hsub = 1; + vsub = 1; + + /* Wa_16023981245 */ + if ((DISPLAY_VER_FULL(i915) == IP_VER(20, 0) || +<<<<<<< + DISPLAY_VER_FULL(i915) == IP_VER(30, 0)) && + src_x % 2 != 0) +======= + DISPLAY_VER_FULL(i915) == IP_VER(30, 0)) && + src_x % 2 != 0) +>>>>>>> + hsub = 2; + } else { + hsub = fb->format->hsub; + vsub = fb->format->vsub; + } + + if (rotated) + hsub = vsub = max(hsub, vsub); + + if (src_x % hsub || src_w % hsub) { + drm_dbg_kms(&i915->drm, "src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_x, src_w, hsub, str_yes_no(rotated)); + return -EINVAL; + } + + if (src_y % vsub || src_h % vsub) { + drm_dbg_kms(&i915->drm, "src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_y, src_h, vsub, str_yes_no(rotated)); + return -EINVAL; + } + + return 0; +} + +static int add_dma_resv_fences(struct dma_resv *resv, + struct drm_plane_state *new_plane_state) +{ + struct dma_fence *fence = dma_fence_get(new_plane_state->fence); + struct dma_fence *new; + int ret; + + ret = dma_resv_get_singleton(resv, dma_resv_usage_rw(false), &new); + if (ret) + goto error; + + if (new && fence) { + struct dma_fence_chain *chain = dma_fence_chain_alloc(); + + if (!chain) { + ret = -ENOMEM; + goto error; + } + + dma_fence_chain_init(chain, fence, new, 1); + fence = &chain->base; + + } else if (new) { + fence = new; + } + + dma_fence_put(new_plane_state->fence); + new_plane_state->fence = fence; + return 0; + +error: + dma_fence_put(fence); + return ret; +} + +/** + * intel_prepare_plane_fb - Prepare fb for usage on plane + * @_plane: drm plane to prepare for + * @_new_plane_state: the plane state being prepared + * + * Prepares a framebuffer for usage on a display plane. Generally this + * involves pinning the underlying object and updating the frontbuffer tracking + * bits. Some older platforms need special physical address handling for + * cursor planes. + * + * Returns 0 on success, negative error code on failure. + */ +static int +intel_prepare_plane_fb(struct drm_plane *_plane, + struct drm_plane_state *_new_plane_state) +{ + struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY }; + struct intel_plane *plane = to_intel_plane(_plane); + struct intel_plane_state *new_plane_state = + to_intel_plane_state(_new_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(new_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + struct intel_plane_state *old_plane_state = + intel_atomic_get_old_plane_state(state, plane); + struct drm_gem_object *obj = intel_fb_bo(new_plane_state->hw.fb); + struct drm_gem_object *old_obj = intel_fb_bo(old_plane_state->hw.fb); + int ret; + + if (old_obj) { + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, + to_intel_crtc(old_plane_state->hw.crtc)); + + /* Big Hammer, we also need to ensure that any pending + * MI_WAIT_FOR_EVENT inside a user batch buffer on the + * current scanout is retired before unpinning the old + * framebuffer. Note that we rely on userspace rendering + * into the buffer attached to the pipe they are waiting + * on. If not, userspace generates a GPU hang with IPEHR + * point to the MI_WAIT_FOR_EVENT. + * + * This should only fail upon a hung GPU, in which case we + * can safely continue. + */ + if (new_crtc_state && intel_crtc_needs_modeset(new_crtc_state)) { + ret = add_dma_resv_fences(old_obj->resv, + &new_plane_state->uapi); + if (ret < 0) + return ret; + } + } + + if (!obj) + return 0; + + ret = intel_plane_pin_fb(new_plane_state); + if (ret) + return ret; + + ret = drm_gem_plane_helper_prepare_fb(&plane->base, &new_plane_state->uapi); + if (ret < 0) + goto unpin_fb; + + if (new_plane_state->uapi.fence) { + i915_gem_fence_wait_priority(new_plane_state->uapi.fence, + &attr); + + intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc, + new_plane_state->uapi.fence); + } + + /* + * We declare pageflips to be interactive and so merit a small bias + * towards upclocking to deliver the frame on time. By only changing + * the RPS thresholds to sample more regularly and aim for higher + * clocks we can hopefully deliver low power workloads (like kodi) + * that are not quite steady state without resorting to forcing + * maximum clocks following a vblank miss (see do_rps_boost()). + */ + intel_display_rps_mark_interactive(dev_priv, state, true); + + return 0; + +unpin_fb: + intel_plane_unpin_fb(new_plane_state); + + return ret; +} + +/** + * intel_cleanup_plane_fb - Cleans up an fb after plane use + * @plane: drm plane to clean up for + * @_old_plane_state: the state from the previous modeset + * + * Cleans up a framebuffer that has just been removed from a plane. + */ +static void +intel_cleanup_plane_fb(struct drm_plane *plane, + struct drm_plane_state *_old_plane_state) +{ + struct intel_plane_state *old_plane_state = + to_intel_plane_state(_old_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(old_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->dev); + struct drm_gem_object *obj = intel_fb_bo(old_plane_state->hw.fb); + + if (!obj) + return; + + intel_display_rps_mark_interactive(dev_priv, state, false); + + intel_plane_unpin_fb(old_plane_state); +} + +static const struct drm_plane_helper_funcs intel_plane_helper_funcs = { + .prepare_fb = intel_prepare_plane_fb, + .cleanup_fb = intel_cleanup_plane_fb, +}; + +void intel_plane_helper_add(struct intel_plane *plane) +{ + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); +} + +void intel_plane_init_cursor_vblank_work(struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + if (!old_plane_state->ggtt_vma || + old_plane_state->ggtt_vma == new_plane_state->ggtt_vma) + return; + + drm_vblank_work_init(&old_plane_state->unpin_work, old_plane_state->uapi.crtc, + intel_cursor_unpin_work); +} diff --git a/rr-cache/607de968990f3d5df977e832f76f7e44097ae59c/preimage b/rr-cache/607de968990f3d5df977e832f76f7e44097ae59c/preimage deleted file mode 100644 index 5ddd36e5bc54..000000000000 --- a/rr-cache/607de968990f3d5df977e832f76f7e44097ae59c/preimage +++ /dev/null @@ -1,2073 +0,0 @@ -/* - * Copyright © 2008 Intel Corporation - * 2014 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_edid.h> -#include <drm/drm_fixed.h> -#include <drm/drm_probe_helper.h> - -#include "i915_drv.h" -#include "i915_reg.h" -#include "intel_atomic.h" -#include "intel_audio.h" -#include "intel_connector.h" -#include "intel_crtc.h" -#include "intel_ddi.h" -#include "intel_de.h" -#include "intel_display_driver.h" -#include "intel_display_types.h" -#include "intel_dp.h" -#include "intel_dp_hdcp.h" -#include "intel_dp_mst.h" -#include "intel_dp_tunnel.h" -#include "intel_dpio_phy.h" -#include "intel_hdcp.h" -#include "intel_hotplug.h" -#include "intel_link_bw.h" -#include "intel_psr.h" -#include "intel_vdsc.h" -#include "skl_scaler.h" - -static int intel_dp_mst_max_dpt_bpp(const struct intel_crtc_state *crtc_state, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - if (!intel_dp_is_uhbr(crtc_state) || DISPLAY_VER(i915) >= 20 || !dsc) - return INT_MAX; - - /* - * DSC->DPT interface width: - * ICL-MTL: 72 bits (each branch has 72 bits, only left branch is used) - * LNL+: 144 bits (not a bottleneck in any config) - * - * Bspec/49259 suggests that the FEC overhead needs to be - * applied here, though HW people claim that neither this FEC - * or any other overhead is applicable here (that is the actual - * available_bw is just symbol_clock * 72). However based on - * testing on MTL-P the - * - DELL U3224KBA display - * - Unigraf UCD-500 CTS test sink - * devices the - * - 5120x2880/995.59Mhz - * - 6016x3384/1357.23Mhz - * - 6144x3456/1413.39Mhz - * modes (all the ones having a DPT limit on the above devices), - * both the channel coding efficiency and an additional 3% - * overhead needs to be accounted for. - */ - return div64_u64(mul_u32_u32(intel_dp_link_symbol_clock(crtc_state->port_clock) * 72, - drm_dp_bw_channel_coding_efficiency(true)), - mul_u32_u32(adjusted_mode->crtc_clock, 1030000)); -} - -static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - bool ssc, bool dsc, int bpp_x16) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - unsigned long flags = DRM_DP_BW_OVERHEAD_MST; - int dsc_slice_count = 0; - int overhead; - - flags |= intel_dp_is_uhbr(crtc_state) ? DRM_DP_BW_OVERHEAD_UHBR : 0; - flags |= ssc ? DRM_DP_BW_OVERHEAD_SSC_REF_CLK : 0; - flags |= crtc_state->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0; - - if (dsc) { - flags |= DRM_DP_BW_OVERHEAD_DSC; - dsc_slice_count = intel_dp_dsc_get_slice_count(connector, - adjusted_mode->clock, - adjusted_mode->hdisplay, - crtc_state->joiner_pipes); - } - - overhead = drm_dp_bw_overhead(crtc_state->lane_count, - adjusted_mode->hdisplay, - dsc_slice_count, - bpp_x16, - flags); - - /* - * TODO: clarify whether a minimum required by the fixed FEC overhead - * in the bspec audio programming sequence is required here. - */ - return max(overhead, intel_dp_bw_fec_overhead(crtc_state->fec_enable)); -} - -static void intel_dp_mst_compute_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_connector *connector, - int overhead, - int bpp_x16, - struct intel_link_m_n *m_n) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - - /* TODO: Check WA 14013163432 to set data M/N for full BW utilization. */ - intel_link_compute_m_n(bpp_x16, crtc_state->lane_count, - adjusted_mode->crtc_clock, - crtc_state->port_clock, - overhead, - m_n); - - m_n->tu = DIV_ROUND_UP_ULL(mul_u32_u32(m_n->data_m, 64), m_n->data_n); -} - -static int intel_dp_mst_calc_pbn(int pixel_clock, int bpp_x16, int bw_overhead) -{ - int effective_data_rate = - intel_dp_effective_data_rate(pixel_clock, bpp_x16, bw_overhead); - - /* - * TODO: Use drm_dp_calc_pbn_mode() instead, once it's converted - * to calculate PBN with the BW overhead passed to it. - */ - return DIV_ROUND_UP(effective_data_rate * 64, 54 * 1000); -} - -static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - int max_bpp, - int min_bpp, - struct link_config_limits *limits, - struct drm_connector_state *conn_state, - int step, - bool dsc) -{ - struct drm_atomic_state *state = crtc_state->uapi.state; - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_state *mst_state; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - int bpp, slots = -EINVAL; - int max_dpt_bpp; - int ret = 0; - - mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - crtc_state->lane_count = limits->max_lane_count; - crtc_state->port_clock = limits->max_rate; - - if (dsc) { - if (!intel_dp_supports_fec(intel_dp, connector, crtc_state)) - return -EINVAL; - - crtc_state->fec_enable = !intel_dp_is_uhbr(crtc_state); - } - - mst_state->pbn_div = drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr, - crtc_state->port_clock, - crtc_state->lane_count); - - max_dpt_bpp = intel_dp_mst_max_dpt_bpp(crtc_state, dsc); - if (max_bpp > max_dpt_bpp) { - drm_dbg_kms(&i915->drm, "Limiting bpp to max DPT bpp (%d -> %d)\n", - max_bpp, max_dpt_bpp); - max_bpp = max_dpt_bpp; - } - - drm_dbg_kms(&i915->drm, "Looking for slots in range min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - for (bpp = max_bpp; bpp >= min_bpp; bpp -= step) { - int local_bw_overhead; - int remote_bw_overhead; - int link_bpp_x16; - int remote_tu; - fixed20_12 pbn; - - drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp); - - link_bpp_x16 = to_bpp_x16(dsc ? bpp : - intel_dp_output_bpp(crtc_state->output_format, bpp)); - - local_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - false, dsc, link_bpp_x16); - remote_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector, - true, dsc, link_bpp_x16); - - intel_dp_mst_compute_m_n(crtc_state, connector, - local_bw_overhead, - link_bpp_x16, - &crtc_state->dp_m_n); - - /* - * The TU size programmed to the HW determines which slots in - * an MTP frame are used for this stream, which needs to match - * the payload size programmed to the first downstream branch - * device's payload table. - * - * Note that atm the payload's PBN value DRM core sends via - * the ALLOCATE_PAYLOAD side-band message matches the payload - * size (which it calculates from the PBN value) it programs - * to the first branch device's payload table. The allocation - * in the payload table could be reduced though (to - * crtc_state->dp_m_n.tu), provided that the driver doesn't - * enable SSC on the corresponding link. - */ - pbn.full = dfixed_const(intel_dp_mst_calc_pbn(adjusted_mode->crtc_clock, - link_bpp_x16, - remote_bw_overhead)); - remote_tu = DIV_ROUND_UP(pbn.full, mst_state->pbn_div.full); - - /* - * Aligning the TUs ensures that symbols consisting of multiple - * (4) symbol cycles don't get split between two consecutive - * MTPs, as required by Bspec. - * TODO: remove the alignment restriction for 128b/132b links - * on some platforms, where Bspec allows this. - */ - remote_tu = ALIGN(remote_tu, 4 / crtc_state->lane_count); - - /* - * Also align PBNs accordingly, since MST core will derive its - * own copy of TU from the PBN in drm_dp_atomic_find_time_slots(). - * The above comment about the difference between the PBN - * allocated for the whole path and the TUs allocated for the - * first branch device's link also applies here. - */ - pbn.full = remote_tu * mst_state->pbn_div.full; - crtc_state->pbn = dfixed_trunc(pbn); - - drm_WARN_ON(&i915->drm, remote_tu < crtc_state->dp_m_n.tu); - crtc_state->dp_m_n.tu = remote_tu; - - slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr, - connector->port, - crtc_state->pbn); - if (slots == -EDEADLK) - return slots; - - if (slots >= 0) { - drm_WARN_ON(&i915->drm, slots != crtc_state->dp_m_n.tu); - - break; - } - } - - /* We failed to find a proper bpp/timeslots, return error */ - if (ret) - slots = ret; - - if (slots < 0) { - drm_dbg_kms(&i915->drm, "failed finding vcpi slots:%d\n", - slots); - } else { - if (!dsc) - crtc_state->pipe_bpp = bpp; - else - crtc_state->dsc.compressed_bpp_x16 = to_bpp_x16(bpp); - drm_dbg_kms(&i915->drm, "Got %d slots for pipe bpp %d dsc %d\n", slots, bpp, dsc); - } - - return slots; -} - -static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - int slots = -EINVAL; - - /* - * FIXME: allocate the BW according to link_bpp, which in the case of - * YUV420 is only half of the pipe bpp value. - */ - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, - to_bpp_int(limits->link.max_bpp_x16), - to_bpp_int(limits->link.min_bpp_x16), - limits, - conn_state, 2 * 3, false); - - if (slots < 0) - return slots; - - return 0; -} - -static int intel_dp_dsc_mst_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state, - struct link_config_limits *limits) -{ - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - int slots = -EINVAL; - int i, num_bpc; - u8 dsc_bpc[3] = {}; - int min_bpp, max_bpp, sink_min_bpp, sink_max_bpp; - u8 dsc_max_bpc; - int min_compressed_bpp, max_compressed_bpp; - - /* Max DSC Input BPC for ICL is 10 and for TGL+ is 12 */ - if (DISPLAY_VER(i915) >= 12) - dsc_max_bpc = min_t(u8, 12, conn_state->max_requested_bpc); - else - dsc_max_bpc = min_t(u8, 10, conn_state->max_requested_bpc); - - max_bpp = min_t(u8, dsc_max_bpc * 3, limits->pipe.max_bpp); - min_bpp = limits->pipe.min_bpp; - - num_bpc = drm_dp_dsc_sink_supported_input_bpcs(connector->dp.dsc_dpcd, - dsc_bpc); - - drm_dbg_kms(&i915->drm, "DSC Source supported min bpp %d max bpp %d\n", - min_bpp, max_bpp); - - sink_max_bpp = dsc_bpc[0] * 3; - sink_min_bpp = sink_max_bpp; - - for (i = 1; i < num_bpc; i++) { - if (sink_min_bpp > dsc_bpc[i] * 3) - sink_min_bpp = dsc_bpc[i] * 3; - if (sink_max_bpp < dsc_bpc[i] * 3) - sink_max_bpp = dsc_bpc[i] * 3; - } - - drm_dbg_kms(&i915->drm, "DSC Sink supported min bpp %d max bpp %d\n", - sink_min_bpp, sink_max_bpp); - - if (min_bpp < sink_min_bpp) - min_bpp = sink_min_bpp; - - if (max_bpp > sink_max_bpp) - max_bpp = sink_max_bpp; - - crtc_state->pipe_bpp = max_bpp; - - max_compressed_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector, - crtc_state, - max_bpp / 3); - max_compressed_bpp = min(max_compressed_bpp, - to_bpp_int(limits->link.max_bpp_x16)); - - min_compressed_bpp = intel_dp_dsc_sink_min_compressed_bpp(crtc_state); - min_compressed_bpp = max(min_compressed_bpp, - to_bpp_int_roundup(limits->link.min_bpp_x16)); - - drm_dbg_kms(&i915->drm, "DSC Sink supported compressed min bpp %d compressed max bpp %d\n", - min_compressed_bpp, max_compressed_bpp); - - /* Align compressed bpps according to our own constraints */ - max_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, max_compressed_bpp, - crtc_state->pipe_bpp); - min_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, min_compressed_bpp, - crtc_state->pipe_bpp); - - slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, max_compressed_bpp, - min_compressed_bpp, limits, - conn_state, 1, true); - - if (slots < 0) - return slots; - - return 0; -} -static int intel_dp_mst_update_slots(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_topology_state *topology_state; - u8 link_coding_cap = intel_dp_is_uhbr(crtc_state) ? - DP_CAP_ANSI_128B132B : DP_CAP_ANSI_8B10B; - - topology_state = drm_atomic_get_mst_topology_state(conn_state->state, mgr); - if (IS_ERR(topology_state)) { - drm_dbg_kms(&i915->drm, "slot update failed\n"); - return PTR_ERR(topology_state); - } - - drm_dp_mst_update_slots(topology_state, link_coding_cap); - - return 0; -} - -static int mode_hblank_period_ns(const struct drm_display_mode *mode) -{ - return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(mode->htotal - mode->hdisplay, - NSEC_PER_SEC / 1000), - mode->crtc_clock); -} - -static bool -hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - const struct link_config_limits *limits) -{ - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; - bool is_uhbr_sink = connector->mst_port && - drm_dp_128b132b_supported(connector->mst_port->dpcd); - int hblank_limit = is_uhbr_sink ? 500 : 300; - - if (!connector->dp.dsc_hblank_expansion_quirk) - return false; - - if (is_uhbr_sink && !drm_dp_is_uhbr_rate(limits->max_rate)) - return false; - - if (mode_hblank_period_ns(adjusted_mode) > hblank_limit) - return false; - - return true; -} - -static bool -adjust_limits_for_dsc_hblank_expansion_quirk(const struct intel_connector *connector, - const struct intel_crtc_state *crtc_state, - struct link_config_limits *limits, - bool dsc) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - int min_bpp_x16 = limits->link.min_bpp_x16; - - if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state, limits)) - return true; - - if (!dsc) { - if (intel_dp_supports_dsc(connector, crtc_state)) { - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] DSC needed by hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - return false; - } - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to 24 due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name); - - if (limits->link.max_bpp_x16 < to_bpp_x16(24)) - return false; - - limits->link.min_bpp_x16 = to_bpp_x16(24); - - return true; - } - - drm_WARN_ON(&i915->drm, limits->min_rate != limits->max_rate); - - if (limits->max_rate < 540000) - min_bpp_x16 = to_bpp_x16(13); - else if (limits->max_rate < 810000) - min_bpp_x16 = to_bpp_x16(10); - - if (limits->link.min_bpp_x16 >= min_bpp_x16) - return true; - - drm_dbg_kms(&i915->drm, - "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to " BPP_X16_FMT " in DSC mode due to hblank expansion quirk\n", - crtc->base.base.id, crtc->base.name, - connector->base.base.id, connector->base.name, - BPP_X16_ARGS(min_bpp_x16)); - - if (limits->link.max_bpp_x16 < min_bpp_x16) - return false; - - limits->link.min_bpp_x16 = min_bpp_x16; - - return true; -} - -static bool -intel_dp_mst_compute_config_limits(struct intel_dp *intel_dp, - const struct intel_connector *connector, - struct intel_crtc_state *crtc_state, - bool dsc, - struct link_config_limits *limits) -{ - /* - * for MST we always configure max link bw - the spec doesn't - * seem to suggest we should do otherwise. - */ - limits->min_rate = limits->max_rate = - intel_dp_max_link_rate(intel_dp); - - limits->min_lane_count = limits->max_lane_count = - intel_dp_max_lane_count(intel_dp); - - limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); - /* - * FIXME: If all the streams can't fit into the link with - * their current pipe_bpp we should reduce pipe_bpp across - * the board until things start to fit. Until then we - * limit to <= 8bpc since that's what was hardcoded for all - * MST streams previously. This hack should be removed once - * we have the proper retry logic in place. - */ - limits->pipe.max_bpp = min(crtc_state->pipe_bpp, 24); - - intel_dp_adjust_compliance_config(intel_dp, crtc_state, limits); - - if (!intel_dp_compute_config_link_bpp_limits(intel_dp, - crtc_state, - dsc, - limits)) - return false; - - return adjust_limits_for_dsc_hblank_expansion_quirk(connector, - crtc_state, - limits, - dsc); -} - -static int intel_dp_mst_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - struct link_config_limits limits; - bool dsc_needed, joiner_needs_dsc; - int ret = 0; - - if (pipe_config->fec_enable && - !intel_dp_supports_fec(intel_dp, connector, pipe_config)) - return -EINVAL; - - if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return -EINVAL; - - if (intel_dp_need_joiner(intel_dp, connector, - adjusted_mode->crtc_hdisplay, - adjusted_mode->crtc_clock)) - pipe_config->joiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe); - - pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - pipe_config->has_pch_encoder = false; - - joiner_needs_dsc = intel_dp_joiner_needs_dsc(dev_priv, pipe_config->joiner_pipes); - - dsc_needed = joiner_needs_dsc || intel_dp->force_dsc_en || - !intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - false, - &limits); - - if (!dsc_needed) { - ret = intel_dp_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - - if (ret == -EDEADLK) - return ret; - - if (ret) - dsc_needed = true; - } - - /* enable compression if the mode doesn't fit available BW */ - if (dsc_needed) { - drm_dbg_kms(&dev_priv->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n", - str_yes_no(ret), str_yes_no(joiner_needs_dsc), - str_yes_no(intel_dp->force_dsc_en)); - - if (!intel_dp_supports_dsc(connector, pipe_config)) - return -EINVAL; - - if (!intel_dp_mst_compute_config_limits(intel_dp, - connector, - pipe_config, - true, - &limits)) - return -EINVAL; - - /* - * FIXME: As bpc is hardcoded to 8, as mentioned above, - * WARN and ignore the debug flag force_dsc_bpc for now. - */ - drm_WARN(&dev_priv->drm, intel_dp->force_dsc_bpc, "Cannot Force BPC for MST\n"); - /* - * Try to get at least some timeslots and then see, if - * we can fit there with DSC. - */ - drm_dbg_kms(&dev_priv->drm, "Trying to find VCPI slots in DSC mode\n"); - - ret = intel_dp_dsc_mst_compute_link_config(encoder, pipe_config, - conn_state, &limits); - if (ret < 0) - return ret; - - ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, - conn_state, &limits, - pipe_config->dp_m_n.tu, false); - } - - if (ret) - return ret; - - ret = intel_dp_mst_update_slots(encoder, pipe_config, conn_state); - if (ret) - return ret; - - pipe_config->limited_color_range = - intel_dp_limited_color_range(pipe_config, conn_state); - - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - pipe_config->lane_lat_optim_mask = - bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); - - intel_dp_audio_compute_config(encoder, pipe_config, conn_state); - - intel_ddi_compute_min_voltage_level(pipe_config); - - intel_psr_compute_config(intel_dp, pipe_config, conn_state); - - return intel_dp_tunnel_atomic_compute_stream_bw(state, intel_dp, connector, - pipe_config); -} - -/* - * Iterate over all connectors and return a mask of - * all CPU transcoders streaming over the same DP link. - */ -static unsigned int -intel_dp_mst_transcoder_mask(struct intel_atomic_state *state, - struct intel_dp *mst_port) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 transcoders = 0; - int i; - - if (DISPLAY_VER(dev_priv) < 12) - return 0; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - const struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector->mst_port != mst_port || !conn_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_state->base.crtc); - crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - - if (!crtc_state->hw.active) - continue; - - transcoders |= BIT(crtc_state->cpu_transcoder); - } - - return transcoders; -} - -static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_port *parent_port) -{ - const struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - u8 mask = 0; - int i; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - if (!conn_state->base.crtc) - continue; - - if (&connector->mst_port->mst_mgr != mst_mgr) - continue; - - if (connector->port != parent_port && - !drm_dp_mst_port_downstream_of_parent(mst_mgr, - connector->port, - parent_port)) - continue; - - mask |= BIT(to_intel_crtc(conn_state->base.crtc)->pipe); - } - - return mask; -} - -static int intel_dp_mst_check_fec_change(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct intel_link_bw_limits *limits) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - struct intel_crtc *crtc; - u8 mst_pipe_mask; - u8 fec_pipe_mask = 0; - int ret; - - mst_pipe_mask = get_pipes_downstream_of_mst_port(state, mst_mgr, NULL); - - for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, mst_pipe_mask) { - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - /* Atomic connector check should've added all the MST CRTCs. */ - if (drm_WARN_ON(&i915->drm, !crtc_state)) - return -EINVAL; - - if (crtc_state->fec_enable) - fec_pipe_mask |= BIT(crtc->pipe); - } - - if (!fec_pipe_mask || mst_pipe_mask == fec_pipe_mask) - return 0; - - limits->force_fec_pipes |= mst_pipe_mask; - - ret = intel_modeset_pipes_in_mask_early(state, "MST FEC", - mst_pipe_mask); - - return ret ? : -EAGAIN; -} - -static int intel_dp_mst_check_bw(struct intel_atomic_state *state, - struct drm_dp_mst_topology_mgr *mst_mgr, - struct drm_dp_mst_topology_state *mst_state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_port *mst_port; - u8 mst_port_pipes; - int ret; - - ret = drm_dp_mst_atomic_check_mgr(&state->base, mst_mgr, mst_state, &mst_port); - if (ret != -ENOSPC) - return ret; - - mst_port_pipes = get_pipes_downstream_of_mst_port(state, mst_mgr, mst_port); - - ret = intel_link_bw_reduce_bpp(state, limits, - mst_port_pipes, "MST link BW"); - - return ret ? : -EAGAIN; -} - -/** - * intel_dp_mst_atomic_check_link - check all modeset MST link configuration - * @state: intel atomic state - * @limits: link BW limits - * - * Check the link configuration for all modeset MST outputs. If the - * configuration is invalid @limits will be updated if possible to - * reduce the total BW, after which the configuration for all CRTCs in - * @state must be recomputed with the updated @limits. - * - * Returns: - * - 0 if the confugration is valid - * - %-EAGAIN, if the configuration is invalid and @limits got updated - * with fallback values with which the configuration of all CRTCs in - * @state must be recomputed - * - Other negative error, if the configuration is invalid without a - * fallback possibility, or the check failed for another reason - */ -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits) -{ - struct drm_dp_mst_topology_mgr *mgr; - struct drm_dp_mst_topology_state *mst_state; - int ret; - int i; - - for_each_new_mst_mgr_in_state(&state->base, mgr, mst_state, i) { - ret = intel_dp_mst_check_fec_change(state, mgr, limits); - if (ret) - return ret; - - ret = intel_dp_mst_check_bw(state, mgr, mst_state, - limits); - if (ret) - return ret; - } - - return 0; -} - -static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state, - struct drm_connector_state *conn_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - /* lowest numbered transcoder will be designated master */ - crtc_state->mst_master_transcoder = - ffs(intel_dp_mst_transcoder_mask(state, intel_dp)) - 1; - - return 0; -} - -/* - * If one of the connectors in a MST stream needs a modeset, mark all CRTCs - * that shares the same MST stream as mode changed, - * intel_modeset_pipe_config()+intel_crtc_check_fastset() will take care to do - * a fastset when possible. - * - * On TGL+ this is required since each stream go through a master transcoder, - * so if the master transcoder needs modeset, all other streams in the - * topology need a modeset. All platforms need to add the atomic state - * for all streams in the topology, since a modeset on one may require - * changing the MST link BW usage of the others, which in turn needs a - * recomputation of the corresponding CRTC states. - */ -static int -intel_dp_mst_atomic_topology_check(struct intel_connector *connector, - struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct drm_connector_list_iter connector_list_iter; - struct intel_connector *connector_iter; - int ret = 0; - - if (!intel_connector_needs_modeset(state, &connector->base)) - return 0; - - drm_connector_list_iter_begin(&dev_priv->drm, &connector_list_iter); - for_each_intel_connector_iter(connector_iter, &connector_list_iter) { - struct intel_digital_connector_state *conn_iter_state; - struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector_iter->mst_port != connector->mst_port || - connector_iter == connector) - continue; - - conn_iter_state = intel_atomic_get_digital_connector_state(state, - connector_iter); - if (IS_ERR(conn_iter_state)) { - ret = PTR_ERR(conn_iter_state); - break; - } - - if (!conn_iter_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_iter_state->base.crtc); - crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - break; - } - - ret = drm_atomic_add_affected_planes(&state->base, &crtc->base); - if (ret) - break; - crtc_state->uapi.mode_changed = true; - } - drm_connector_list_iter_end(&connector_list_iter); - - return ret; -} - -static int -intel_dp_mst_atomic_check(struct drm_connector *connector, - struct drm_atomic_state *_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(_state); - struct intel_connector *intel_connector = - to_intel_connector(connector); - int ret; - - ret = intel_digital_connector_atomic_check(connector, &state->base); - if (ret) - return ret; - - ret = intel_dp_mst_atomic_topology_check(intel_connector, state); - if (ret) - return ret; - - if (intel_connector_needs_modeset(state, connector)) { - ret = intel_dp_tunnel_atomic_check_state(state, - intel_connector->mst_port, - intel_connector); - if (ret) - return ret; - } - - return drm_dp_atomic_release_time_slots(&state->base, - &intel_connector->mst_port->mst_mgr, - intel_connector->port); -} - -static void clear_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - - intel_de_write(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT); -} - -static void wait_for_act_sent(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_dp *intel_dp = &intel_mst->primary->dp; - - if (intel_de_wait_for_set(i915, dp_tp_status_reg(encoder, crtc_state), - DP_TP_STATUS_ACT_SENT, 1)) - drm_err(&i915->drm, "Timed out waiting for ACT sent\n"); - - drm_dp_check_act_status(&intel_dp->mst_mgr); -} - -static void intel_mst_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - - drm_dbg_kms(&i915->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (intel_dp->active_mst_links == 1) - intel_dp->link_trained = false; - - intel_hdcp_disable(intel_mst->connector); - - intel_dp_sink_disable_decompression(state, connector, old_crtc_state); -} - -static void intel_mst_post_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = - to_intel_connector(old_conn_state->connector); - struct drm_dp_mst_topology_state *old_mst_state = - drm_atomic_get_old_mst_topology_state(&state->base, &intel_dp->mst_mgr); - struct drm_dp_mst_topology_state *new_mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - const struct drm_dp_mst_atomic_payload *old_payload = - drm_atomic_get_mst_payload_state(old_mst_state, connector->port); - struct drm_dp_mst_atomic_payload *new_payload = - drm_atomic_get_mst_payload_state(new_mst_state, connector->port); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_crtc *pipe_crtc; - bool last_mst_stream; - - intel_dp->active_mst_links--; - last_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && last_mst_stream && - !intel_dp_mst_is_master_trans(old_crtc_state)); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_off(old_pipe_crtc_state); - } - - intel_disable_transcoder(old_crtc_state); - - drm_dp_remove_payload_part1(&intel_dp->mst_mgr, new_mst_state, new_payload); - - clear_act_sent(encoder, old_crtc_state); - - intel_de_rmw(dev_priv, - TRANS_DDI_FUNC_CTL(dev_priv, old_crtc_state->cpu_transcoder), - TRANS_DDI_DP_VC_PAYLOAD_ALLOC, 0); - - wait_for_act_sent(encoder, old_crtc_state); - - drm_dp_remove_payload_part2(&intel_dp->mst_mgr, new_mst_state, - old_payload, new_payload); - - intel_ddi_disable_transcoder_func(old_crtc_state); - - for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(old_crtc_state)) { - const struct intel_crtc_state *old_pipe_crtc_state = - intel_atomic_get_old_crtc_state(state, pipe_crtc); - - intel_dsc_disable(old_pipe_crtc_state); - - if (DISPLAY_VER(dev_priv) >= 9) - skl_scaler_disable(old_pipe_crtc_state); - else - ilk_pfit_disable(old_pipe_crtc_state); - } - - /* - * Power down mst path before disabling the port, otherwise we end - * up getting interrupts from the sink upon detecting link loss. - */ - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, - false); - - /* - * BSpec 4287: disable DIP after the transcoder is disabled and before - * the transcoder clock select is set to none. - */ - intel_dp_set_infoframes(&dig_port->base, false, - old_crtc_state, NULL); - /* - * From TGL spec: "If multi-stream slave transcoder: Configure - * Transcoder Clock Select to direct no clock to the transcoder" - * - * From older GENs spec: "Configure Transcoder Clock Select to direct - * no clock to the transcoder" - */ - if (DISPLAY_VER(dev_priv) < 12 || !last_mst_stream) - intel_ddi_disable_transcoder_clock(old_crtc_state); - - - intel_mst->connector = NULL; - if (last_mst_stream) - dig_port->base.post_disable(state, &dig_port->base, - old_crtc_state, NULL); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); -} - -static void intel_mst_post_pll_disable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0 && - dig_port->base.post_pll_disable) - dig_port->base.post_pll_disable(state, encoder, old_crtc_state, old_conn_state); -} - -static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - - if (intel_dp->active_mst_links == 0) - dig_port->base.pre_pll_enable(state, &dig_port->base, - pipe_config, NULL); - else - /* - * The port PLL state needs to get updated for secondary - * streams as for the primary stream. - */ - intel_ddi_update_active_dpll(state, &dig_port->base, - to_intel_crtc(pipe_config->uapi.crtc)); -} - -static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_connector *connector = - to_intel_connector(conn_state->connector); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - int ret; - bool first_mst_stream; - - /* MST encoders are bound to a crtc, not to a connector, - * force the mapping here for get_hw_state. - */ - connector->encoder = encoder; - intel_mst->connector = connector; - first_mst_stream = intel_dp->active_mst_links == 0; - drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 12 && first_mst_stream && - !intel_dp_mst_is_master_trans(pipe_config)); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - if (first_mst_stream) - intel_dp_set_power(intel_dp, DP_SET_POWER_D0); - - drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true); - - intel_dp_sink_enable_decompression(state, connector, pipe_config); - - if (first_mst_stream) - dig_port->base.pre_enable(state, &dig_port->base, - pipe_config, NULL); - - intel_dp->active_mst_links++; - - ret = drm_dp_add_payload_part1(&intel_dp->mst_mgr, mst_state, - drm_atomic_get_mst_payload_state(mst_state, connector->port)); - if (ret < 0) - drm_dbg_kms(&dev_priv->drm, "Failed to create MST payload for %s: %d\n", - connector->base.name, ret); - - /* - * Before Gen 12 this is not done as part of - * dig_port->base.pre_enable() and should be done here. For - * Gen 12+ the step in which this should be done is different for the - * first MST stream, so it's done on the DDI for the first stream and - * here for the following ones. - */ - if (DISPLAY_VER(dev_priv) < 12 || !first_mst_stream) - intel_ddi_enable_transcoder_clock(encoder, pipe_config); - - intel_dsc_dp_pps_write(&dig_port->base, pipe_config); - intel_ddi_set_dp_msa(pipe_config, conn_state); -} - -static void enable_bs_jitter_was(const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - u32 clear = 0; - u32 set = 0; - - if (!IS_ALDERLAKE_P(i915)) - return; - - if (!IS_DISPLAY_STEP(i915, STEP_D0, STEP_FOREVER)) - return; - - /* Wa_14013163432:adlp */ - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_FEC_BS_JITTER_WA(crtc_state->cpu_transcoder); - - /* Wa_14014143976:adlp */ - if (IS_DISPLAY_STEP(i915, STEP_E0, STEP_FOREVER)) { - if (intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - else if (crtc_state->fec_enable) - clear |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder); - - if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state)) - set |= DP_MST_DPT_DPTP_ALIGN_WA(crtc_state->cpu_transcoder); - } - - if (!clear && !set) - return; - - intel_de_rmw(i915, CHICKEN_MISC_3, clear, set); -} - -static void intel_mst_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - struct intel_dp *intel_dp = &dig_port->dp; - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct drm_dp_mst_topology_state *mst_state = - drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr); - enum transcoder trans = pipe_config->cpu_transcoder; - bool first_mst_stream = intel_dp->active_mst_links == 1; - struct intel_crtc *pipe_crtc; - - drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder); - - if (intel_dp_is_uhbr(pipe_config)) { - const struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; - u64 crtc_clock_hz = KHz(adjusted_mode->crtc_clock); - - intel_de_write(dev_priv, TRANS_DP2_VFREQHIGH(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz >> 24)); - intel_de_write(dev_priv, TRANS_DP2_VFREQLOW(pipe_config->cpu_transcoder), - TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz & 0xffffff)); - } - - enable_bs_jitter_was(pipe_config); - - intel_ddi_enable_transcoder_func(encoder, pipe_config); - - clear_act_sent(encoder, pipe_config); - - intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(dev_priv, trans), 0, - TRANS_DDI_DP_VC_PAYLOAD_ALLOC); - - drm_dbg_kms(&dev_priv->drm, "active links %d\n", - intel_dp->active_mst_links); - - wait_for_act_sent(encoder, pipe_config); - - if (first_mst_stream) - intel_ddi_wait_for_fec_status(encoder, pipe_config, true); - - drm_dp_add_payload_part2(&intel_dp->mst_mgr, - drm_atomic_get_mst_payload_state(mst_state, connector->port)); - - if (DISPLAY_VER(dev_priv) >= 12) - intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, trans), - FECSTALL_DIS_DPTSTREAM_DPTTG, - pipe_config->fec_enable ? FECSTALL_DIS_DPTSTREAM_DPTTG : 0); - - intel_audio_sdp_split_update(pipe_config); - - intel_enable_transcoder(pipe_config); - - for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc, - intel_crtc_joined_pipe_mask(pipe_config)) { - const struct intel_crtc_state *pipe_crtc_state = - intel_atomic_get_new_crtc_state(state, pipe_crtc); - - intel_crtc_vblank_on(pipe_crtc_state); - } - - intel_hdcp_enable(state, encoder, pipe_config, conn_state); -} - -static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder, - enum pipe *pipe) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - *pipe = intel_mst->pipe; - if (intel_mst->connector) - return true; - return false; -} - -static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - dig_port->base.get_config(&dig_port->base, pipe_config); -} - -static bool intel_dp_mst_initial_fastset_check(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); - struct intel_digital_port *dig_port = intel_mst->primary; - - return intel_dp_initial_fastset_check(&dig_port->base, crtc_state); -} - -static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - struct drm_i915_private *i915 = to_i915(intel_connector->base.dev); - struct intel_dp *intel_dp = intel_connector->mst_port; - const struct drm_edid *drm_edid; - int ret; - - if (drm_connector_is_unregistered(connector)) - return intel_connector_update_modes(connector, NULL); - - if (!intel_display_driver_check_access(i915)) - return drm_edid_connector_add_modes(connector); - - drm_edid = drm_dp_mst_edid_read(connector, &intel_dp->mst_mgr, intel_connector->port); - - ret = intel_connector_update_modes(connector, drm_edid); - - drm_edid_free(drm_edid); - - return ret; -} - -static int -intel_dp_mst_connector_late_register(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - int ret; - - ret = drm_dp_mst_connector_late_register(connector, - intel_connector->port); - if (ret < 0) - return ret; - - ret = intel_connector_register(connector); - if (ret < 0) - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); - - return ret; -} - -static void -intel_dp_mst_connector_early_unregister(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - intel_connector_unregister(connector); - drm_dp_mst_connector_early_unregister(connector, - intel_connector->port); -} - -static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { - .fill_modes = drm_helper_probe_single_connector_modes, - .atomic_get_property = intel_digital_connector_atomic_get_property, - .atomic_set_property = intel_digital_connector_atomic_set_property, - .late_register = intel_dp_mst_connector_late_register, - .early_unregister = intel_dp_mst_connector_early_unregister, - .destroy = intel_connector_destroy, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_duplicate_state = intel_digital_connector_duplicate_state, -}; - -static int intel_dp_mst_get_modes(struct drm_connector *connector) -{ - return intel_dp_mst_get_ddc_modes(connector); -} - -static int -intel_dp_mst_mode_valid_ctx(struct drm_connector *connector, - struct drm_display_mode *mode, - struct drm_modeset_acquire_ctx *ctx, - enum drm_mode_status *status) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; - struct drm_dp_mst_port *port = intel_connector->port; - const int min_bpp = 18; - int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq; - int max_rate, mode_rate, max_lanes, max_link_clock; - int ret; - bool dsc = false, joiner = false; - u16 dsc_max_compressed_bpp = 0; - u8 dsc_slice_count = 0; - int target_clock = mode->clock; - - if (drm_connector_is_unregistered(connector)) { - *status = MODE_ERROR; - return 0; - } - - *status = intel_cpu_transcoder_mode_valid(dev_priv, mode); - if (*status != MODE_OK) - return 0; - - if (mode->flags & DRM_MODE_FLAG_DBLCLK) { - *status = MODE_H_ILLEGAL; - return 0; - } - - if (mode->clock < 10000) { - *status = MODE_CLOCK_LOW; - return 0; - } - - max_link_clock = intel_dp_max_link_rate(intel_dp); - max_lanes = intel_dp_max_lane_count(intel_dp); - - max_rate = intel_dp_max_link_data_rate(intel_dp, - max_link_clock, max_lanes); - mode_rate = intel_dp_link_required(mode->clock, min_bpp); - - /* - * TODO: - * - Also check if compression would allow for the mode - * - Calculate the overhead using drm_dp_bw_overhead() / - * drm_dp_bw_channel_coding_efficiency(), similarly to the - * compute config code, as drm_dp_calc_pbn_mode() doesn't - * account with all the overheads. - * - Check here and during compute config the BW reported by - * DFP_Link_Available_Payload_Bandwidth_Number (or the - * corresponding link capabilities of the sink) in case the - * stream is uncompressed for it by the last branch device. - */ - if (intel_dp_need_joiner(intel_dp, intel_connector, - mode->hdisplay, target_clock)) { - joiner = true; - max_dotclk *= 2; - } - - ret = drm_modeset_lock(&mgr->base.lock, ctx); - if (ret) - return ret; - - if (mode_rate > max_rate || mode->clock > max_dotclk || - drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (intel_dp_has_dsc(intel_connector)) { - /* - * TBD pass the connector BPC, - * for now U8_MAX so that max BPC on that platform would be picked - */ - int pipe_bpp = intel_dp_dsc_compute_max_bpp(intel_connector, U8_MAX); - - if (drm_dp_sink_supports_fec(intel_connector->dp.fec_capability)) { - dsc_max_compressed_bpp = - intel_dp_dsc_get_max_compressed_bpp(dev_priv, - max_link_clock, - max_lanes, - target_clock, - mode->hdisplay, - joiner, - INTEL_OUTPUT_FORMAT_RGB, - pipe_bpp, 64); - dsc_slice_count = - intel_dp_dsc_get_slice_count(intel_connector, - target_clock, - mode->hdisplay, - joiner); - } - - dsc = dsc_max_compressed_bpp && dsc_slice_count; - } - - if (intel_dp_joiner_needs_dsc(dev_priv, joiner) && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - if (mode_rate > max_rate && !dsc) { - *status = MODE_CLOCK_HIGH; - return 0; - } - - *status = intel_mode_valid_max_plane_size(dev_priv, mode, joiner); - return 0; -} - -static struct drm_encoder *intel_mst_atomic_best_encoder(struct drm_connector *connector, - struct drm_atomic_state *state) -{ - struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state, - connector); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - struct intel_crtc *crtc = to_intel_crtc(connector_state->crtc); - - return &intel_dp->mst_encoders[crtc->pipe]->base.base; -} - -static int -intel_dp_mst_detect(struct drm_connector *connector, - struct drm_modeset_acquire_ctx *ctx, bool force) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_dp *intel_dp = intel_connector->mst_port; - - if (!intel_display_device_enabled(i915)) - return connector_status_disconnected; - - if (drm_connector_is_unregistered(connector)) - return connector_status_disconnected; - - if (!intel_display_driver_check_access(i915)) - return connector->status; - - return drm_dp_mst_detect_port(connector, ctx, &intel_dp->mst_mgr, - intel_connector->port); -} - -static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = { - .get_modes = intel_dp_mst_get_modes, - .mode_valid_ctx = intel_dp_mst_mode_valid_ctx, - .atomic_best_encoder = intel_mst_atomic_best_encoder, - .atomic_check = intel_dp_mst_atomic_check, - .detect_ctx = intel_dp_mst_detect, -}; - -static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder) -{ - struct intel_dp_mst_encoder *intel_mst = enc_to_mst(to_intel_encoder(encoder)); - - drm_encoder_cleanup(encoder); - kfree(intel_mst); -} - -static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = { - .destroy = intel_dp_mst_encoder_destroy, -}; - -static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) -{ - if (intel_attached_encoder(connector) && connector->base.state->crtc) { - enum pipe pipe; - if (!intel_attached_encoder(connector)->get_hw_state(intel_attached_encoder(connector), &pipe)) - return false; - return true; - } - return false; -} - -static int intel_dp_mst_add_properties(struct intel_dp *intel_dp, - struct drm_connector *connector, - const char *pathprop) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); - - drm_object_attach_property(&connector->base, - i915->drm.mode_config.path_property, 0); - drm_object_attach_property(&connector->base, - i915->drm.mode_config.tile_property, 0); - - intel_attach_force_audio_property(connector); - intel_attach_broadcast_rgb_property(connector); - - /* - * Reuse the prop from the SST connector because we're - * not allowed to create new props after device registration. - */ - connector->max_bpc_property = - intel_dp->attached_connector->base.max_bpc_property; - if (connector->max_bpc_property) - drm_connector_attach_max_bpc_property(connector, 6, 12); - - return drm_connector_set_path_property(connector, pathprop); -} - -static void -intel_dp_mst_read_decompression_port_dsc_caps(struct intel_dp *intel_dp, - struct intel_connector *connector) -{ - u8 dpcd_caps[DP_RECEIVER_CAP_SIZE]; - - if (!connector->dp.dsc_decompression_aux) - return; - - if (drm_dp_read_dpcd_caps(connector->dp.dsc_decompression_aux, dpcd_caps) < 0) - return; - - intel_dp_get_dsc_sink_cap(dpcd_caps[DP_DPCD_REV], connector); -} - -static bool detect_dsc_hblank_expansion_quirk(const struct intel_connector *connector) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct drm_dp_aux *aux = connector->dp.dsc_decompression_aux; - struct drm_dp_desc desc; - u8 dpcd[DP_RECEIVER_CAP_SIZE]; - - if (!aux) - return false; - - /* - * A logical port's OUI (at least for affected sinks) is all 0, so - * instead of that the parent port's OUI is used for identification. - */ - if (drm_dp_mst_port_is_logical(connector->port)) { - aux = drm_dp_mst_aux_for_parent(connector->port); - if (!aux) - aux = &connector->mst_port->aux; - } - - if (drm_dp_read_dpcd_caps(aux, dpcd) < 0) - return false; - - if (drm_dp_read_desc(aux, &desc, drm_dp_is_branch(dpcd)) < 0) - return false; - - if (!drm_dp_has_quirk(&desc, - DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC)) - return false; - - /* - * UHBR (MST sink) devices requiring this quirk don't advertise the - * HBLANK expansion support. Presuming that they perform HBLANK - * expansion internally, or are affected by this issue on modes with a - * short HBLANK for other reasons. - */ - if (!drm_dp_128b132b_supported(dpcd) && - !(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE)) - return false; - - drm_dbg_kms(&i915->drm, - "[CONNECTOR:%d:%s] DSC HBLANK expansion quirk detected\n", - connector->base.base.id, connector->base.name); - - return true; -} - -static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, - struct drm_dp_mst_port *port, - const char *pathprop) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_connector *intel_connector; - struct drm_connector *connector; - enum pipe pipe; - int ret; - - intel_connector = intel_connector_alloc(); - if (!intel_connector) - return NULL; - - intel_connector->get_hw_state = intel_dp_mst_get_hw_state; - intel_connector->sync_state = intel_dp_connector_sync_state; - intel_connector->mst_port = intel_dp; - intel_connector->port = port; - drm_dp_mst_get_port_malloc(port); - - intel_dp_init_modeset_retry_work(intel_connector); - - intel_connector->dp.dsc_decompression_aux = drm_dp_mst_dsc_aux_for_port(port); - intel_dp_mst_read_decompression_port_dsc_caps(intel_dp, intel_connector); - intel_connector->dp.dsc_hblank_expansion_quirk = - detect_dsc_hblank_expansion_quirk(intel_connector); - - connector = &intel_connector->base; - ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, - DRM_MODE_CONNECTOR_DisplayPort); - if (ret) { - drm_dp_mst_put_port_malloc(port); - intel_connector_free(intel_connector); - return NULL; - } - - drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs); - - for_each_pipe(dev_priv, pipe) { - struct drm_encoder *enc = - &intel_dp->mst_encoders[pipe]->base.base; - - ret = drm_connector_attach_encoder(&intel_connector->base, enc); - if (ret) - goto err; - } - - ret = intel_dp_mst_add_properties(intel_dp, connector, pathprop); - if (ret) - goto err; - - ret = intel_dp_hdcp_init(dig_port, intel_connector); - if (ret) - drm_dbg_kms(&dev_priv->drm, "[%s:%d] HDCP MST init failed, skipping.\n", - connector->name, connector->base.id); - - return connector; - -err: - drm_connector_cleanup(connector); - return NULL; -} - -static void -intel_dp_mst_poll_hpd_irq(struct drm_dp_mst_topology_mgr *mgr) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - - intel_hpd_trigger_irq(dp_to_dig_port(intel_dp)); -} - -static const struct drm_dp_mst_topology_cbs mst_cbs = { - .add_connector = intel_dp_add_mst_connector, - .poll_hpd_irq = intel_dp_mst_poll_hpd_irq, -}; - -static struct intel_dp_mst_encoder * -intel_dp_create_fake_mst_encoder(struct intel_digital_port *dig_port, enum pipe pipe) -{ - struct intel_dp_mst_encoder *intel_mst; - struct intel_encoder *intel_encoder; - struct drm_device *dev = dig_port->base.base.dev; - - intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL); - - if (!intel_mst) - return NULL; - - intel_mst->pipe = pipe; - intel_encoder = &intel_mst->base; - intel_mst->primary = dig_port; - - drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, - DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe)); - - intel_encoder->type = INTEL_OUTPUT_DP_MST; - intel_encoder->power_domain = dig_port->base.power_domain; - intel_encoder->port = dig_port->base.port; - intel_encoder->cloneable = 0; - /* - * This is wrong, but broken userspace uses the intersection - * of possible_crtcs of all the encoders of a given connector - * to figure out which crtcs can drive said connector. What - * should be used instead is the union of possible_crtcs. - * To keep such userspace functioning we must misconfigure - * this to make sure the intersection is not empty :( - */ - intel_encoder->pipe_mask = ~0; - - intel_encoder->compute_config = intel_dp_mst_compute_config; - intel_encoder->compute_config_late = intel_dp_mst_compute_config_late; - intel_encoder->disable = intel_mst_disable_dp; - intel_encoder->post_disable = intel_mst_post_disable_dp; - intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp; - intel_encoder->update_pipe = intel_ddi_update_pipe; - intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; - intel_encoder->pre_enable = intel_mst_pre_enable_dp; - intel_encoder->enable = intel_mst_enable_dp; - intel_encoder->audio_enable = intel_audio_codec_enable; - intel_encoder->audio_disable = intel_audio_codec_disable; - intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state; - intel_encoder->get_config = intel_dp_mst_enc_get_config; - intel_encoder->initial_fastset_check = intel_dp_mst_initial_fastset_check; - - return intel_mst; - -} - -static bool -intel_dp_create_fake_mst_encoders(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum pipe pipe; - - for_each_pipe(dev_priv, pipe) - intel_dp->mst_encoders[pipe] = intel_dp_create_fake_mst_encoder(dig_port, pipe); - return true; -} - -int -intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port) -{ - return dig_port->dp.active_mst_links; -} - -int -intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id) -{ - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - struct intel_dp *intel_dp = &dig_port->dp; - enum port port = dig_port->base.port; - int ret; - - if (!HAS_DP_MST(i915) || intel_dp_is_edp(intel_dp)) - return 0; - - if (DISPLAY_VER(i915) < 12 && port == PORT_A) - return 0; - - if (DISPLAY_VER(i915) < 11 && port == PORT_E) - return 0; - - intel_dp->mst_mgr.cbs = &mst_cbs; - - /* create encoders */ - intel_dp_create_fake_mst_encoders(dig_port); - ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, &i915->drm, - &intel_dp->aux, 16, 3, conn_base_id); - if (ret) { - intel_dp->mst_mgr.cbs = NULL; - return ret; - } - - return 0; -} - -bool intel_dp_mst_source_support(struct intel_dp *intel_dp) -{ - return intel_dp->mst_mgr.cbs; -} - -void -intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port) -{ - struct intel_dp *intel_dp = &dig_port->dp; - - if (!intel_dp_mst_source_support(intel_dp)) - return; - - drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr); - /* encoders will get killed by normal cleanup */ - - intel_dp->mst_mgr.cbs = NULL; -} - -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder == crtc_state->cpu_transcoder; -} - -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state) -{ - return crtc_state->mst_master_transcoder != INVALID_TRANSCODER && - crtc_state->mst_master_transcoder != crtc_state->cpu_transcoder; -} - -/** - * intel_dp_mst_add_topology_state_for_connector - add MST topology state for a connector - * @state: atomic state - * @connector: connector to add the state for - * @crtc: the CRTC @connector is attached to - * - * Add the MST topology state for @connector to @state. - * - * Returns 0 on success, negative error code on failure. - */ -static int -intel_dp_mst_add_topology_state_for_connector(struct intel_atomic_state *state, - struct intel_connector *connector, - struct intel_crtc *crtc) -{ - struct drm_dp_mst_topology_state *mst_state; - - if (!connector->mst_port) - return 0; - - mst_state = drm_atomic_get_mst_topology_state(&state->base, - &connector->mst_port->mst_mgr); - if (IS_ERR(mst_state)) - return PTR_ERR(mst_state); - - mst_state->pending_crtc_mask |= drm_crtc_mask(&crtc->base); - - return 0; -} - -/** - * intel_dp_mst_add_topology_state_for_crtc - add MST topology state for a CRTC - * @state: atomic state - * @crtc: CRTC to add the state for - * - * Add the MST topology state for @crtc to @state. - * - * Returns 0 on success, negative error code on failure. - */ -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_connector *_connector; - struct drm_connector_state *conn_state; - int i; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - struct intel_connector *connector = to_intel_connector(_connector); - int ret; - - if (conn_state->crtc != &crtc->base) - continue; - - ret = intel_dp_mst_add_topology_state_for_connector(state, connector, crtc); - if (ret) - return ret; - } - - return 0; -} - -static struct intel_connector * -get_connector_in_state_for_crtc(struct intel_atomic_state *state, - const struct intel_crtc *crtc) -{ - struct drm_connector_state *old_conn_state; - struct drm_connector_state *new_conn_state; - struct drm_connector *_connector; - int i; - - for_each_oldnew_connector_in_state(&state->base, _connector, - old_conn_state, new_conn_state, i) { - struct intel_connector *connector = - to_intel_connector(_connector); - - if (old_conn_state->crtc == &crtc->base || - new_conn_state->crtc == &crtc->base) - return connector; - } - - return NULL; -} - -/** - * intel_dp_mst_crtc_needs_modeset - check if changes in topology need to modeset the given CRTC - * @state: atomic state - * @crtc: CRTC for which to check the modeset requirement - * - * Check if any change in a MST topology requires a forced modeset on @crtc in - * this topology. One such change is enabling/disabling the DSC decompression - * state in the first branch device's UFP DPCD as required by one CRTC, while - * the other @crtc in the same topology is still active, requiring a full modeset - * on @crtc. - */ -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - const struct intel_connector *crtc_connector; - const struct drm_connector_state *conn_state; - const struct drm_connector *_connector; - int i; - - if (!intel_crtc_has_type(intel_atomic_get_new_crtc_state(state, crtc), - INTEL_OUTPUT_DP_MST)) - return false; - - crtc_connector = get_connector_in_state_for_crtc(state, crtc); - - if (!crtc_connector) - /* None of the connectors in the topology needs modeset */ - return false; - - for_each_new_connector_in_state(&state->base, _connector, conn_state, i) { - const struct intel_connector *connector = - to_intel_connector(_connector); - const struct intel_crtc_state *new_crtc_state; - const struct intel_crtc_state *old_crtc_state; - struct intel_crtc *crtc_iter; - - if (connector->mst_port != crtc_connector->mst_port || - !conn_state->crtc) - continue; - - crtc_iter = to_intel_crtc(conn_state->crtc); - - new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc_iter); - old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc_iter); - - if (!intel_crtc_needs_modeset(new_crtc_state)) - continue; - - if (old_crtc_state->dsc.compression_enable == - new_crtc_state->dsc.compression_enable) - continue; - /* - * Toggling the decompression flag because of this stream in - * the first downstream branch device's UFP DPCD may reset the - * whole branch device. To avoid the reset while other streams - * are also active modeset the whole MST topology in this - * case. - */ - if (connector->dp.dsc_decompression_aux == - &connector->mst_port->aux) - return true; - } - - return false; -} -<<<<<<< -======= - -/** - * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing - * @intel_dp: DP port object - * - * Prepare an MST link for topology probing, programming the target - * link parameters to DPCD. This step is a requirement of the enumaration - * of path resources during probing. - */ -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp) -{ - int link_rate = intel_dp_max_link_rate(intel_dp); - int lane_count = intel_dp_max_lane_count(intel_dp); - u8 rate_select; - u8 link_bw; - - if (intel_dp->link_trained) - return; - - if (intel_mst_probed_link_params_valid(intel_dp, link_rate, lane_count)) - return; - - intel_dp_compute_rate(intel_dp, link_rate, &link_bw, &rate_select); - - intel_dp_link_training_set_mode(intel_dp, link_rate, false); - intel_dp_link_training_set_bw(intel_dp, link_bw, rate_select, lane_count, - drm_dp_enhanced_frame_cap(intel_dp->dpcd)); - - intel_mst_set_probed_link_params(intel_dp, link_rate, lane_count); -} - -/* - * intel_dp_mst_verify_dpcd_state - verify the MST SW enabled state wrt. the DPCD - * @intel_dp: DP port object - * - * Verify if @intel_dp's MST enabled SW state matches the corresponding DPCD - * state. A long HPD pulse - not long enough to be detected as a disconnected - * state - could've reset the DPCD state, which requires tearing - * down/recreating the MST topology. - * - * Returns %true if the SW MST enabled and DPCD states match, %false - * otherwise. - */ -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp) -{ - struct intel_display *display = to_intel_display(intel_dp); - struct intel_connector *connector = intel_dp->attached_connector; - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *encoder = &dig_port->base; - int ret; - u8 val; - - if (!intel_dp->is_mst) - return true; - - ret = drm_dp_dpcd_readb(intel_dp->mst_mgr.aux, DP_MSTM_CTRL, &val); - - /* Adjust the expected register value for SST + SideBand. */ - if (ret < 0 || val != (DP_MST_EN | DP_UP_REQ_EN | DP_UPSTREAM_IS_SRC)) { - drm_dbg_kms(display->drm, - "[CONNECTOR:%d:%s][ENCODER:%d:%s] MST mode got reset, removing topology (ret=%d, ctrl=0x%02x)\n", - connector->base.base.id, connector->base.name, - encoder->base.base.id, encoder->base.name, - ret, val); - - return false; - } - - return true; -} ->>>>>>> diff --git a/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.2 b/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.2 deleted file mode 100644 index 42ffb0f8cccc..000000000000 --- a/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.2 +++ /dev/null @@ -1,982 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_exec_queue.h" - -#include <linux/nospec.h> - -#include <drm/drm_device.h> -#include <drm/drm_file.h> -#include <drm/xe_drm.h> - -#include "xe_device.h" -#include "xe_gt.h" -#include "xe_hw_engine_class_sysfs.h" -#include "xe_hw_fence.h" -#include "xe_lrc.h" -#include "xe_macros.h" -#include "xe_migrate.h" -#include "xe_pm.h" -#include "xe_ring_ops_types.h" -#include "xe_trace.h" -#include "xe_vm.h" - -enum xe_exec_queue_sched_prop { - XE_EXEC_QUEUE_JOB_TIMEOUT = 0, - XE_EXEC_QUEUE_TIMESLICE = 1, - XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, - XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, -}; - -static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number); - -static void __xe_exec_queue_free(struct xe_exec_queue *q) -{ - if (q->vm) - xe_vm_put(q->vm); - - if (q->xef) - xe_file_put(q->xef); - - kfree(q); -} - -static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, - struct xe_vm *vm, - u32 logical_mask, - u16 width, struct xe_hw_engine *hwe, - u32 flags, u64 extensions) -{ - struct xe_exec_queue *q; - struct xe_gt *gt = hwe->gt; - int err; - - /* only kernel queues can be permanent */ - XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); - - q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); - if (!q) - return ERR_PTR(-ENOMEM); - - kref_init(&q->refcount); - q->flags = flags; - q->hwe = hwe; - q->gt = gt; - q->class = hwe->class; - q->width = width; - q->logical_mask = logical_mask; - q->fence_irq = >->fence_irq[hwe->class]; - q->ring_ops = gt->ring_ops[hwe->class]; - q->ops = gt->exec_queue_ops; - INIT_LIST_HEAD(&q->lr.link); - INIT_LIST_HEAD(&q->multi_gt_link); - - q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; - q->sched_props.preempt_timeout_us = - hwe->eclass->sched_props.preempt_timeout_us; - q->sched_props.job_timeout_ms = - hwe->eclass->sched_props.job_timeout_ms; - if (q->flags & EXEC_QUEUE_FLAG_KERNEL && - q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) - q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; - else - q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; - - if (vm) - q->vm = xe_vm_get(vm); - - if (extensions) { - /* - * may set q->usm, must come before xe_lrc_create(), - * may overwrite q->sched_props, must come before q->ops->init() - */ - err = exec_queue_user_extensions(xe, q, extensions, 0); - if (err) { - __xe_exec_queue_free(q); - return ERR_PTR(err); - } - } - - return q; -} - -static int __xe_exec_queue_init(struct xe_exec_queue *q) -{ - struct xe_vm *vm = q->vm; - int i, err; - - if (vm) { - err = xe_vm_lock(vm, true); - if (err) - return err; - } - - for (i = 0; i < q->width; ++i) { - q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K); - if (IS_ERR(q->lrc[i])) { - err = PTR_ERR(q->lrc[i]); - goto err_unlock; - } - } - - if (vm) - xe_vm_unlock(vm); - - err = q->ops->init(q); - if (err) - goto err_lrc; - - return 0; - -err_unlock: - if (vm) - xe_vm_unlock(vm); -err_lrc: - for (i = i - 1; i >= 0; --i) - xe_lrc_put(q->lrc[i]); - return err; -} - -struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, - u32 logical_mask, u16 width, - struct xe_hw_engine *hwe, u32 flags, - u64 extensions) -{ - struct xe_exec_queue *q; - int err; - - q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, - extensions); - if (IS_ERR(q)) - return q; - - err = __xe_exec_queue_init(q); - if (err) - goto err_post_alloc; - - return q; - -err_post_alloc: - __xe_exec_queue_free(q); - return ERR_PTR(err); -} - -struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, - struct xe_vm *vm, - enum xe_engine_class class, - u32 flags, u64 extensions) -{ - struct xe_hw_engine *hwe, *hwe0 = NULL; - enum xe_hw_engine_id id; - u32 logical_mask = 0; - - for_each_hw_engine(hwe, gt, id) { - if (xe_hw_engine_is_reserved(hwe)) - continue; - - if (hwe->class == class) { - logical_mask |= BIT(hwe->logical_instance); - if (!hwe0) - hwe0 = hwe; - } - } - - if (!logical_mask) - return ERR_PTR(-ENODEV); - - return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); -} - -/** - * xe_exec_queue_create_bind() - Create bind exec queue. - * @xe: Xe device. - * @tile: tile which bind exec queue belongs to. - * @flags: exec queue creation flags - * @extensions: exec queue creation extensions - * - * Normalize bind exec queue creation. Bind exec queue is tied to migration VM - * for access to physical memory required for page table programming. On a - * faulting devices the reserved copy engine instance must be used to avoid - * deadlocking (user binds cannot get stuck behind faults as kernel binds which - * resolve faults depend on user binds). On non-faulting devices any copy engine - * can be used. - * - * Returns exec queue on success, ERR_PTR on failure - */ -struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, - struct xe_tile *tile, - u32 flags, u64 extensions) -{ - struct xe_gt *gt = tile->primary_gt; - struct xe_exec_queue *q; - struct xe_vm *migrate_vm; - - migrate_vm = xe_migrate_get_vm(tile->migrate); - if (xe->info.has_usm) { - struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, - XE_ENGINE_CLASS_COPY, - gt->usm.reserved_bcs_instance, - false); - - if (!hwe) - return ERR_PTR(-EINVAL); - - q = xe_exec_queue_create(xe, migrate_vm, - BIT(hwe->logical_instance), 1, hwe, - flags, extensions); - } else { - q = xe_exec_queue_create_class(xe, gt, migrate_vm, - XE_ENGINE_CLASS_COPY, flags, - extensions); - } - xe_vm_put(migrate_vm); - - return q; -} - -void xe_exec_queue_destroy(struct kref *ref) -{ - struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); - struct xe_exec_queue *eq, *next; - - xe_exec_queue_last_fence_put_unlocked(q); - if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { - list_for_each_entry_safe(eq, next, &q->multi_gt_list, - multi_gt_link) - xe_exec_queue_put(eq); - } - - q->ops->fini(q); -} - -void xe_exec_queue_fini(struct xe_exec_queue *q) -{ - int i; - - for (i = 0; i < q->width; ++i) - xe_lrc_put(q->lrc[i]); - __xe_exec_queue_free(q); -} - -void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) -{ - switch (q->class) { - case XE_ENGINE_CLASS_RENDER: - snprintf(q->name, sizeof(q->name), "rcs%d", instance); - break; - case XE_ENGINE_CLASS_VIDEO_DECODE: - snprintf(q->name, sizeof(q->name), "vcs%d", instance); - break; - case XE_ENGINE_CLASS_VIDEO_ENHANCE: - snprintf(q->name, sizeof(q->name), "vecs%d", instance); - break; - case XE_ENGINE_CLASS_COPY: - snprintf(q->name, sizeof(q->name), "bcs%d", instance); - break; - case XE_ENGINE_CLASS_COMPUTE: - snprintf(q->name, sizeof(q->name), "ccs%d", instance); - break; - case XE_ENGINE_CLASS_OTHER: - snprintf(q->name, sizeof(q->name), "gsccs%d", instance); - break; - default: - XE_WARN_ON(q->class); - } -} - -struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) -{ - struct xe_exec_queue *q; - - mutex_lock(&xef->exec_queue.lock); - q = xa_load(&xef->exec_queue.xa, id); - if (q) - xe_exec_queue_get(q); - mutex_unlock(&xef->exec_queue.lock); - - return q; -} - -enum xe_exec_queue_priority -xe_exec_queue_device_get_max_priority(struct xe_device *xe) -{ - return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : - XE_EXEC_QUEUE_PRIORITY_NORMAL; -} - -static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, - u64 value) -{ - if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) - return -EPERM; - - q->sched_props.priority = value; - return 0; -} - -static bool xe_exec_queue_enforce_schedule_limit(void) -{ -#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) - return true; -#else - return !capable(CAP_SYS_NICE); -#endif -} - -static void -xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, - enum xe_exec_queue_sched_prop prop, - u32 *min, u32 *max) -{ - switch (prop) { - case XE_EXEC_QUEUE_JOB_TIMEOUT: - *min = eclass->sched_props.job_timeout_min; - *max = eclass->sched_props.job_timeout_max; - break; - case XE_EXEC_QUEUE_TIMESLICE: - *min = eclass->sched_props.timeslice_min; - *max = eclass->sched_props.timeslice_max; - break; - case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: - *min = eclass->sched_props.preempt_timeout_min; - *max = eclass->sched_props.preempt_timeout_max; - break; - default: - break; - } -#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) - if (capable(CAP_SYS_NICE)) { - switch (prop) { - case XE_EXEC_QUEUE_JOB_TIMEOUT: - *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; - *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; - break; - case XE_EXEC_QUEUE_TIMESLICE: - *min = XE_HW_ENGINE_TIMESLICE_MIN; - *max = XE_HW_ENGINE_TIMESLICE_MAX; - break; - case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: - *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; - *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; - break; - default: - break; - } - } -#endif -} - -static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, - u64 value) -{ - u32 min = 0, max = 0; - - xe_exec_queue_get_prop_minmax(q->hwe->eclass, - XE_EXEC_QUEUE_TIMESLICE, &min, &max); - - if (xe_exec_queue_enforce_schedule_limit() && - !xe_hw_engine_timeout_in_range(value, min, max)) - return -EINVAL; - - q->sched_props.timeslice_us = value; - return 0; -} - -typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, - struct xe_exec_queue *q, - u64 value); - -static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { - [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, - [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, -}; - -static int exec_queue_user_ext_set_property(struct xe_device *xe, - struct xe_exec_queue *q, - u64 extension) -{ - u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_set_property ext; - int err; - u32 idx; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.property >= - ARRAY_SIZE(exec_queue_set_property_funcs)) || - XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && - ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) - return -EINVAL; - - idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); - if (!exec_queue_set_property_funcs[idx]) - return -EINVAL; - - return exec_queue_set_property_funcs[idx](xe, q, ext.value); -} - -typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, - struct xe_exec_queue *q, - u64 extension); - -static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { - [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, -}; - -#define MAX_USER_EXTENSIONS 16 -static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number) -{ - u64 __user *address = u64_to_user_ptr(extensions); - struct drm_xe_user_extension ext; - int err; - u32 idx; - - if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) - return -E2BIG; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.name >= - ARRAY_SIZE(exec_queue_user_extension_funcs))) - return -EINVAL; - - idx = array_index_nospec(ext.name, - ARRAY_SIZE(exec_queue_user_extension_funcs)); - err = exec_queue_user_extension_funcs[idx](xe, q, extensions); - if (XE_IOCTL_DBG(xe, err)) - return err; - - if (ext.next_extension) - return exec_queue_user_extensions(xe, q, ext.next_extension, - ++ext_number); - - return 0; -} - -<<<<<<< -======= -static const enum xe_engine_class user_to_xe_engine_class[] = { - [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, - [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, - [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, - [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, - [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, -}; - -static struct xe_hw_engine * -find_hw_engine(struct xe_device *xe, - struct drm_xe_engine_class_instance eci) -{ - u32 idx; - - if (eci.engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) - return NULL; - - if (eci.gt_id >= xe->info.gt_count) - return NULL; - - idx = array_index_nospec(eci.engine_class, - ARRAY_SIZE(user_to_xe_engine_class)); - - return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), - user_to_xe_engine_class[idx], - eci.engine_instance, true); -} - -static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, - struct drm_xe_engine_class_instance *eci, - u16 width, u16 num_placements) -{ - struct xe_hw_engine *hwe; - enum xe_hw_engine_id id; - u32 logical_mask = 0; - - if (XE_IOCTL_DBG(xe, width != 1)) - return 0; - if (XE_IOCTL_DBG(xe, num_placements != 1)) - return 0; - if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) - return 0; - - eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; - - for_each_hw_engine(hwe, gt, id) { - if (xe_hw_engine_is_reserved(hwe)) - continue; - - if (hwe->class == - user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) - logical_mask |= BIT(hwe->logical_instance); - } - - return logical_mask; -} - ->>>>>>> -static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, - struct drm_xe_engine_class_instance *eci, - u16 width, u16 num_placements) -{ - int len = width * num_placements; - int i, j, n; - u16 class; - u16 gt_id; - u32 return_mask = 0, prev_mask; - - if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && - len > 1)) - return 0; - - for (i = 0; i < width; ++i) { - u32 current_mask = 0; - - for (j = 0; j < num_placements; ++j) { - struct xe_hw_engine *hwe; - - n = j * width + i; - - hwe = find_hw_engine(xe, eci[n]); - if (XE_IOCTL_DBG(xe, !hwe)) - return 0; - - if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) - return 0; - - if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || - XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) - return 0; - - class = eci[n].engine_class; - gt_id = eci[n].gt_id; - - if (width == 1 || !i) - return_mask |= BIT(eci[n].engine_instance); - current_mask |= BIT(eci[n].engine_instance); - } - - /* Parallel submissions must be logically contiguous */ - if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) - return 0; - - prev_mask = current_mask; - } - - return return_mask; -} - -int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_create *args = data; - struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; - struct drm_xe_engine_class_instance __user *user_eci = - u64_to_user_ptr(args->instances); - struct xe_hw_engine *hwe; - struct xe_vm *vm; - struct xe_gt *gt; - struct xe_tile *tile; - struct xe_exec_queue *q = NULL; - u32 logical_mask; - u32 id; - u32 len; - int err; - - if (XE_IOCTL_DBG(xe, args->flags) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - len = args->width * args->num_placements; - if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) - return -EINVAL; - - err = __copy_from_user(eci, user_eci, - sizeof(struct drm_xe_engine_class_instance) * - len); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) - return -EINVAL; - - if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { - if (XE_IOCTL_DBG(xe, args->width != 1) || - XE_IOCTL_DBG(xe, args->num_placements != 1) || - XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) - return -EINVAL; - - for_each_tile(tile, xe, id) { - struct xe_exec_queue *new; - u32 flags = EXEC_QUEUE_FLAG_VM; - - if (id) - flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; - -<<<<<<< - eci[0].gt_id = gt->info.id; - logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, - args->width, - args->num_placements); - if (XE_IOCTL_DBG(xe, !logical_mask)) - return -EINVAL; - - hwe = find_hw_engine(xe, eci[0]); - if (XE_IOCTL_DBG(xe, !hwe)) - return -EINVAL; - - /* The migration vm doesn't hold rpm ref */ - xe_pm_runtime_get_noresume(xe); - - flags = EXEC_QUEUE_FLAG_VM | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0); - - migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); - new = xe_exec_queue_create(xe, migrate_vm, logical_mask, - args->width, hwe, flags, - args->extensions); - - xe_pm_runtime_put(xe); /* now held by engine */ - - xe_vm_put(migrate_vm); -======= - new = xe_exec_queue_create_bind(xe, tile, flags, - args->extensions); ->>>>>>> - if (IS_ERR(new)) { - err = PTR_ERR(new); - if (q) - goto put_exec_queue; - return err; - } - if (id == 0) - q = new; - else - list_add_tail(&new->multi_gt_list, - &q->multi_gt_link); - } - } else { - gt = xe_device_get_gt(xe, eci[0].gt_id); - logical_mask = calc_validate_logical_mask(xe, gt, eci, - args->width, - args->num_placements); - if (XE_IOCTL_DBG(xe, !logical_mask)) - return -EINVAL; - - hwe = find_hw_engine(xe, eci[0]); - if (XE_IOCTL_DBG(xe, !hwe)) - return -EINVAL; - - vm = xe_vm_lookup(xef, args->vm_id); - if (XE_IOCTL_DBG(xe, !vm)) - return -ENOENT; - - err = down_read_interruptible(&vm->lock); - if (err) { - xe_vm_put(vm); - return err; - } - - if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { - up_read(&vm->lock); - xe_vm_put(vm); - return -ENOENT; - } - - q = xe_exec_queue_create(xe, vm, logical_mask, - args->width, hwe, 0, - args->extensions); - up_read(&vm->lock); - xe_vm_put(vm); - if (IS_ERR(q)) - return PTR_ERR(q); - - if (xe_vm_in_preempt_fence_mode(vm)) { - q->lr.context = dma_fence_context_alloc(1); - - err = xe_vm_add_compute_exec_queue(vm, q); - if (XE_IOCTL_DBG(xe, err)) - goto put_exec_queue; - } - } - - mutex_lock(&xef->exec_queue.lock); - err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); - mutex_unlock(&xef->exec_queue.lock); - if (err) - goto kill_exec_queue; - - args->exec_queue_id = id; - q->xef = xe_file_get(xef); - - return 0; - -kill_exec_queue: - xe_exec_queue_kill(q); -put_exec_queue: - xe_exec_queue_put(q); - return err; -} - -int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_get_property *args = data; - struct xe_exec_queue *q; - int ret; - - if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - q = xe_exec_queue_lookup(xef, args->exec_queue_id); - if (XE_IOCTL_DBG(xe, !q)) - return -ENOENT; - - switch (args->property) { - case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: - args->value = q->ops->reset_status(q); - ret = 0; - break; - default: - ret = -EINVAL; - } - - xe_exec_queue_put(q); - - return ret; -} - -/** - * xe_exec_queue_is_lr() - Whether an exec_queue is long-running - * @q: The exec_queue - * - * Return: True if the exec_queue is long-running, false otherwise. - */ -bool xe_exec_queue_is_lr(struct xe_exec_queue *q) -{ - return q->vm && xe_vm_in_lr_mode(q->vm) && - !(q->flags & EXEC_QUEUE_FLAG_VM); -} - -static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) -{ - return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; -} - -/** - * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full - * @q: The exec_queue - * - * Return: True if the exec_queue's ring is full, false otherwise. - */ -bool xe_exec_queue_ring_full(struct xe_exec_queue *q) -{ - struct xe_lrc *lrc = q->lrc[0]; - s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; - - return xe_exec_queue_num_job_inflight(q) >= max_job; -} - -/** - * xe_exec_queue_is_idle() - Whether an exec_queue is idle. - * @q: The exec_queue - * - * FIXME: Need to determine what to use as the short-lived - * timeline lock for the exec_queues, so that the return value - * of this function becomes more than just an advisory - * snapshot in time. The timeline lock must protect the - * seqno from racing submissions on the same exec_queue. - * Typically vm->resv, but user-created timeline locks use the migrate vm - * and never grabs the migrate vm->resv so we have a race there. - * - * Return: True if the exec_queue is idle, false otherwise. - */ -bool xe_exec_queue_is_idle(struct xe_exec_queue *q) -{ - if (xe_exec_queue_is_parallel(q)) { - int i; - - for (i = 0; i < q->width; ++i) { - if (xe_lrc_seqno(q->lrc[i]) != - q->lrc[i]->fence_ctx.next_seqno - 1) - return false; - } - - return true; - } - - return xe_lrc_seqno(q->lrc[0]) == - q->lrc[0]->fence_ctx.next_seqno - 1; -} - -/** - * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue - * from hw - * @q: The exec queue - * - * Update the timestamp saved by HW for this exec queue and save run ticks - * calculated by using the delta from last update. - */ -void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) -{ - struct xe_file *xef; - struct xe_lrc *lrc; - u32 old_ts, new_ts; - - /* - * Jobs that are run during driver load may use an exec_queue, but are - * not associated with a user xe file, so avoid accumulating busyness - * for kernel specific work. - */ - if (!q->vm || !q->vm->xef) - return; - - xef = q->vm->xef; - - /* - * Only sample the first LRC. For parallel submission, all of them are - * scheduled together and we compensate that below by multiplying by - * width - this may introduce errors if that premise is not true and - * they don't exit 100% aligned. On the other hand, looping through - * the LRCs and reading them in different time could also introduce - * errors. - */ - lrc = q->lrc[0]; - new_ts = xe_lrc_update_timestamp(lrc, &old_ts); - xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; -} - -void xe_exec_queue_kill(struct xe_exec_queue *q) -{ - struct xe_exec_queue *eq = q, *next; - - list_for_each_entry_safe(eq, next, &eq->multi_gt_list, - multi_gt_link) { - q->ops->kill(eq); - xe_vm_remove_compute_exec_queue(q->vm, eq); - } - - q->ops->kill(q); - xe_vm_remove_compute_exec_queue(q->vm, q); -} - -int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_destroy *args = data; - struct xe_exec_queue *q; - - if (XE_IOCTL_DBG(xe, args->pad) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - mutex_lock(&xef->exec_queue.lock); - q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); - mutex_unlock(&xef->exec_queue.lock); - if (XE_IOCTL_DBG(xe, !q)) - return -ENOENT; - - xe_exec_queue_kill(q); - - trace_xe_exec_queue_close(q); - xe_exec_queue_put(q); - - return 0; -} - -static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, - struct xe_vm *vm) -{ - if (q->flags & EXEC_QUEUE_FLAG_VM) - lockdep_assert_held(&vm->lock); - else - xe_vm_assert_held(vm); -} - -/** - * xe_exec_queue_last_fence_put() - Drop ref to last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - */ -void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) -{ - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - if (q->last_fence) { - dma_fence_put(q->last_fence); - q->last_fence = NULL; - } -} - -/** - * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked - * @q: The exec queue - * - * Only safe to be called from xe_exec_queue_destroy(). - */ -void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) -{ - if (q->last_fence) { - dma_fence_put(q->last_fence); - q->last_fence = NULL; - } -} - -/** - * xe_exec_queue_last_fence_get() - Get last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - * - * Get last fence, takes a ref - * - * Returns: last fence if not signaled, dma fence stub if signaled - */ -struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, - struct xe_vm *vm) -{ - struct dma_fence *fence; - - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - if (q->last_fence && - test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) - xe_exec_queue_last_fence_put(q, vm); - - fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); - dma_fence_get(fence); - return fence; -} - -/** - * xe_exec_queue_last_fence_set() - Set last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - * @fence: The fence - * - * Set the last fence for the engine. Increases reference count for fence, when - * closing engine xe_exec_queue_last_fence_put should be called. - */ -void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, - struct dma_fence *fence) -{ - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - xe_exec_queue_last_fence_put(q, vm); - q->last_fence = dma_fence_get(fence); -} diff --git a/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.3 b/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.3 deleted file mode 100644 index 42ffb0f8cccc..000000000000 --- a/rr-cache/6e989852f5454d81ebf331bbd2c55116dc711575/preimage.3 +++ /dev/null @@ -1,982 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_exec_queue.h" - -#include <linux/nospec.h> - -#include <drm/drm_device.h> -#include <drm/drm_file.h> -#include <drm/xe_drm.h> - -#include "xe_device.h" -#include "xe_gt.h" -#include "xe_hw_engine_class_sysfs.h" -#include "xe_hw_fence.h" -#include "xe_lrc.h" -#include "xe_macros.h" -#include "xe_migrate.h" -#include "xe_pm.h" -#include "xe_ring_ops_types.h" -#include "xe_trace.h" -#include "xe_vm.h" - -enum xe_exec_queue_sched_prop { - XE_EXEC_QUEUE_JOB_TIMEOUT = 0, - XE_EXEC_QUEUE_TIMESLICE = 1, - XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, - XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, -}; - -static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number); - -static void __xe_exec_queue_free(struct xe_exec_queue *q) -{ - if (q->vm) - xe_vm_put(q->vm); - - if (q->xef) - xe_file_put(q->xef); - - kfree(q); -} - -static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, - struct xe_vm *vm, - u32 logical_mask, - u16 width, struct xe_hw_engine *hwe, - u32 flags, u64 extensions) -{ - struct xe_exec_queue *q; - struct xe_gt *gt = hwe->gt; - int err; - - /* only kernel queues can be permanent */ - XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); - - q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); - if (!q) - return ERR_PTR(-ENOMEM); - - kref_init(&q->refcount); - q->flags = flags; - q->hwe = hwe; - q->gt = gt; - q->class = hwe->class; - q->width = width; - q->logical_mask = logical_mask; - q->fence_irq = >->fence_irq[hwe->class]; - q->ring_ops = gt->ring_ops[hwe->class]; - q->ops = gt->exec_queue_ops; - INIT_LIST_HEAD(&q->lr.link); - INIT_LIST_HEAD(&q->multi_gt_link); - - q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; - q->sched_props.preempt_timeout_us = - hwe->eclass->sched_props.preempt_timeout_us; - q->sched_props.job_timeout_ms = - hwe->eclass->sched_props.job_timeout_ms; - if (q->flags & EXEC_QUEUE_FLAG_KERNEL && - q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) - q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; - else - q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; - - if (vm) - q->vm = xe_vm_get(vm); - - if (extensions) { - /* - * may set q->usm, must come before xe_lrc_create(), - * may overwrite q->sched_props, must come before q->ops->init() - */ - err = exec_queue_user_extensions(xe, q, extensions, 0); - if (err) { - __xe_exec_queue_free(q); - return ERR_PTR(err); - } - } - - return q; -} - -static int __xe_exec_queue_init(struct xe_exec_queue *q) -{ - struct xe_vm *vm = q->vm; - int i, err; - - if (vm) { - err = xe_vm_lock(vm, true); - if (err) - return err; - } - - for (i = 0; i < q->width; ++i) { - q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K); - if (IS_ERR(q->lrc[i])) { - err = PTR_ERR(q->lrc[i]); - goto err_unlock; - } - } - - if (vm) - xe_vm_unlock(vm); - - err = q->ops->init(q); - if (err) - goto err_lrc; - - return 0; - -err_unlock: - if (vm) - xe_vm_unlock(vm); -err_lrc: - for (i = i - 1; i >= 0; --i) - xe_lrc_put(q->lrc[i]); - return err; -} - -struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, - u32 logical_mask, u16 width, - struct xe_hw_engine *hwe, u32 flags, - u64 extensions) -{ - struct xe_exec_queue *q; - int err; - - q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, - extensions); - if (IS_ERR(q)) - return q; - - err = __xe_exec_queue_init(q); - if (err) - goto err_post_alloc; - - return q; - -err_post_alloc: - __xe_exec_queue_free(q); - return ERR_PTR(err); -} - -struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, - struct xe_vm *vm, - enum xe_engine_class class, - u32 flags, u64 extensions) -{ - struct xe_hw_engine *hwe, *hwe0 = NULL; - enum xe_hw_engine_id id; - u32 logical_mask = 0; - - for_each_hw_engine(hwe, gt, id) { - if (xe_hw_engine_is_reserved(hwe)) - continue; - - if (hwe->class == class) { - logical_mask |= BIT(hwe->logical_instance); - if (!hwe0) - hwe0 = hwe; - } - } - - if (!logical_mask) - return ERR_PTR(-ENODEV); - - return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); -} - -/** - * xe_exec_queue_create_bind() - Create bind exec queue. - * @xe: Xe device. - * @tile: tile which bind exec queue belongs to. - * @flags: exec queue creation flags - * @extensions: exec queue creation extensions - * - * Normalize bind exec queue creation. Bind exec queue is tied to migration VM - * for access to physical memory required for page table programming. On a - * faulting devices the reserved copy engine instance must be used to avoid - * deadlocking (user binds cannot get stuck behind faults as kernel binds which - * resolve faults depend on user binds). On non-faulting devices any copy engine - * can be used. - * - * Returns exec queue on success, ERR_PTR on failure - */ -struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, - struct xe_tile *tile, - u32 flags, u64 extensions) -{ - struct xe_gt *gt = tile->primary_gt; - struct xe_exec_queue *q; - struct xe_vm *migrate_vm; - - migrate_vm = xe_migrate_get_vm(tile->migrate); - if (xe->info.has_usm) { - struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, - XE_ENGINE_CLASS_COPY, - gt->usm.reserved_bcs_instance, - false); - - if (!hwe) - return ERR_PTR(-EINVAL); - - q = xe_exec_queue_create(xe, migrate_vm, - BIT(hwe->logical_instance), 1, hwe, - flags, extensions); - } else { - q = xe_exec_queue_create_class(xe, gt, migrate_vm, - XE_ENGINE_CLASS_COPY, flags, - extensions); - } - xe_vm_put(migrate_vm); - - return q; -} - -void xe_exec_queue_destroy(struct kref *ref) -{ - struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); - struct xe_exec_queue *eq, *next; - - xe_exec_queue_last_fence_put_unlocked(q); - if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { - list_for_each_entry_safe(eq, next, &q->multi_gt_list, - multi_gt_link) - xe_exec_queue_put(eq); - } - - q->ops->fini(q); -} - -void xe_exec_queue_fini(struct xe_exec_queue *q) -{ - int i; - - for (i = 0; i < q->width; ++i) - xe_lrc_put(q->lrc[i]); - __xe_exec_queue_free(q); -} - -void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) -{ - switch (q->class) { - case XE_ENGINE_CLASS_RENDER: - snprintf(q->name, sizeof(q->name), "rcs%d", instance); - break; - case XE_ENGINE_CLASS_VIDEO_DECODE: - snprintf(q->name, sizeof(q->name), "vcs%d", instance); - break; - case XE_ENGINE_CLASS_VIDEO_ENHANCE: - snprintf(q->name, sizeof(q->name), "vecs%d", instance); - break; - case XE_ENGINE_CLASS_COPY: - snprintf(q->name, sizeof(q->name), "bcs%d", instance); - break; - case XE_ENGINE_CLASS_COMPUTE: - snprintf(q->name, sizeof(q->name), "ccs%d", instance); - break; - case XE_ENGINE_CLASS_OTHER: - snprintf(q->name, sizeof(q->name), "gsccs%d", instance); - break; - default: - XE_WARN_ON(q->class); - } -} - -struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) -{ - struct xe_exec_queue *q; - - mutex_lock(&xef->exec_queue.lock); - q = xa_load(&xef->exec_queue.xa, id); - if (q) - xe_exec_queue_get(q); - mutex_unlock(&xef->exec_queue.lock); - - return q; -} - -enum xe_exec_queue_priority -xe_exec_queue_device_get_max_priority(struct xe_device *xe) -{ - return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : - XE_EXEC_QUEUE_PRIORITY_NORMAL; -} - -static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, - u64 value) -{ - if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) - return -EPERM; - - q->sched_props.priority = value; - return 0; -} - -static bool xe_exec_queue_enforce_schedule_limit(void) -{ -#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) - return true; -#else - return !capable(CAP_SYS_NICE); -#endif -} - -static void -xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, - enum xe_exec_queue_sched_prop prop, - u32 *min, u32 *max) -{ - switch (prop) { - case XE_EXEC_QUEUE_JOB_TIMEOUT: - *min = eclass->sched_props.job_timeout_min; - *max = eclass->sched_props.job_timeout_max; - break; - case XE_EXEC_QUEUE_TIMESLICE: - *min = eclass->sched_props.timeslice_min; - *max = eclass->sched_props.timeslice_max; - break; - case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: - *min = eclass->sched_props.preempt_timeout_min; - *max = eclass->sched_props.preempt_timeout_max; - break; - default: - break; - } -#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) - if (capable(CAP_SYS_NICE)) { - switch (prop) { - case XE_EXEC_QUEUE_JOB_TIMEOUT: - *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; - *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; - break; - case XE_EXEC_QUEUE_TIMESLICE: - *min = XE_HW_ENGINE_TIMESLICE_MIN; - *max = XE_HW_ENGINE_TIMESLICE_MAX; - break; - case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: - *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; - *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; - break; - default: - break; - } - } -#endif -} - -static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, - u64 value) -{ - u32 min = 0, max = 0; - - xe_exec_queue_get_prop_minmax(q->hwe->eclass, - XE_EXEC_QUEUE_TIMESLICE, &min, &max); - - if (xe_exec_queue_enforce_schedule_limit() && - !xe_hw_engine_timeout_in_range(value, min, max)) - return -EINVAL; - - q->sched_props.timeslice_us = value; - return 0; -} - -typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, - struct xe_exec_queue *q, - u64 value); - -static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { - [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, - [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, -}; - -static int exec_queue_user_ext_set_property(struct xe_device *xe, - struct xe_exec_queue *q, - u64 extension) -{ - u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_set_property ext; - int err; - u32 idx; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.property >= - ARRAY_SIZE(exec_queue_set_property_funcs)) || - XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && - ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) - return -EINVAL; - - idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); - if (!exec_queue_set_property_funcs[idx]) - return -EINVAL; - - return exec_queue_set_property_funcs[idx](xe, q, ext.value); -} - -typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, - struct xe_exec_queue *q, - u64 extension); - -static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { - [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, -}; - -#define MAX_USER_EXTENSIONS 16 -static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number) -{ - u64 __user *address = u64_to_user_ptr(extensions); - struct drm_xe_user_extension ext; - int err; - u32 idx; - - if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) - return -E2BIG; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.name >= - ARRAY_SIZE(exec_queue_user_extension_funcs))) - return -EINVAL; - - idx = array_index_nospec(ext.name, - ARRAY_SIZE(exec_queue_user_extension_funcs)); - err = exec_queue_user_extension_funcs[idx](xe, q, extensions); - if (XE_IOCTL_DBG(xe, err)) - return err; - - if (ext.next_extension) - return exec_queue_user_extensions(xe, q, ext.next_extension, - ++ext_number); - - return 0; -} - -<<<<<<< -======= -static const enum xe_engine_class user_to_xe_engine_class[] = { - [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, - [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, - [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, - [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, - [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, -}; - -static struct xe_hw_engine * -find_hw_engine(struct xe_device *xe, - struct drm_xe_engine_class_instance eci) -{ - u32 idx; - - if (eci.engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) - return NULL; - - if (eci.gt_id >= xe->info.gt_count) - return NULL; - - idx = array_index_nospec(eci.engine_class, - ARRAY_SIZE(user_to_xe_engine_class)); - - return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), - user_to_xe_engine_class[idx], - eci.engine_instance, true); -} - -static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, - struct drm_xe_engine_class_instance *eci, - u16 width, u16 num_placements) -{ - struct xe_hw_engine *hwe; - enum xe_hw_engine_id id; - u32 logical_mask = 0; - - if (XE_IOCTL_DBG(xe, width != 1)) - return 0; - if (XE_IOCTL_DBG(xe, num_placements != 1)) - return 0; - if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) - return 0; - - eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; - - for_each_hw_engine(hwe, gt, id) { - if (xe_hw_engine_is_reserved(hwe)) - continue; - - if (hwe->class == - user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) - logical_mask |= BIT(hwe->logical_instance); - } - - return logical_mask; -} - ->>>>>>> -static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, - struct drm_xe_engine_class_instance *eci, - u16 width, u16 num_placements) -{ - int len = width * num_placements; - int i, j, n; - u16 class; - u16 gt_id; - u32 return_mask = 0, prev_mask; - - if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && - len > 1)) - return 0; - - for (i = 0; i < width; ++i) { - u32 current_mask = 0; - - for (j = 0; j < num_placements; ++j) { - struct xe_hw_engine *hwe; - - n = j * width + i; - - hwe = find_hw_engine(xe, eci[n]); - if (XE_IOCTL_DBG(xe, !hwe)) - return 0; - - if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) - return 0; - - if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || - XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) - return 0; - - class = eci[n].engine_class; - gt_id = eci[n].gt_id; - - if (width == 1 || !i) - return_mask |= BIT(eci[n].engine_instance); - current_mask |= BIT(eci[n].engine_instance); - } - - /* Parallel submissions must be logically contiguous */ - if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) - return 0; - - prev_mask = current_mask; - } - - return return_mask; -} - -int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_create *args = data; - struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; - struct drm_xe_engine_class_instance __user *user_eci = - u64_to_user_ptr(args->instances); - struct xe_hw_engine *hwe; - struct xe_vm *vm; - struct xe_gt *gt; - struct xe_tile *tile; - struct xe_exec_queue *q = NULL; - u32 logical_mask; - u32 id; - u32 len; - int err; - - if (XE_IOCTL_DBG(xe, args->flags) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - len = args->width * args->num_placements; - if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) - return -EINVAL; - - err = __copy_from_user(eci, user_eci, - sizeof(struct drm_xe_engine_class_instance) * - len); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) - return -EINVAL; - - if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { - if (XE_IOCTL_DBG(xe, args->width != 1) || - XE_IOCTL_DBG(xe, args->num_placements != 1) || - XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) - return -EINVAL; - - for_each_tile(tile, xe, id) { - struct xe_exec_queue *new; - u32 flags = EXEC_QUEUE_FLAG_VM; - - if (id) - flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; - -<<<<<<< - eci[0].gt_id = gt->info.id; - logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, - args->width, - args->num_placements); - if (XE_IOCTL_DBG(xe, !logical_mask)) - return -EINVAL; - - hwe = find_hw_engine(xe, eci[0]); - if (XE_IOCTL_DBG(xe, !hwe)) - return -EINVAL; - - /* The migration vm doesn't hold rpm ref */ - xe_pm_runtime_get_noresume(xe); - - flags = EXEC_QUEUE_FLAG_VM | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0); - - migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); - new = xe_exec_queue_create(xe, migrate_vm, logical_mask, - args->width, hwe, flags, - args->extensions); - - xe_pm_runtime_put(xe); /* now held by engine */ - - xe_vm_put(migrate_vm); -======= - new = xe_exec_queue_create_bind(xe, tile, flags, - args->extensions); ->>>>>>> - if (IS_ERR(new)) { - err = PTR_ERR(new); - if (q) - goto put_exec_queue; - return err; - } - if (id == 0) - q = new; - else - list_add_tail(&new->multi_gt_list, - &q->multi_gt_link); - } - } else { - gt = xe_device_get_gt(xe, eci[0].gt_id); - logical_mask = calc_validate_logical_mask(xe, gt, eci, - args->width, - args->num_placements); - if (XE_IOCTL_DBG(xe, !logical_mask)) - return -EINVAL; - - hwe = find_hw_engine(xe, eci[0]); - if (XE_IOCTL_DBG(xe, !hwe)) - return -EINVAL; - - vm = xe_vm_lookup(xef, args->vm_id); - if (XE_IOCTL_DBG(xe, !vm)) - return -ENOENT; - - err = down_read_interruptible(&vm->lock); - if (err) { - xe_vm_put(vm); - return err; - } - - if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { - up_read(&vm->lock); - xe_vm_put(vm); - return -ENOENT; - } - - q = xe_exec_queue_create(xe, vm, logical_mask, - args->width, hwe, 0, - args->extensions); - up_read(&vm->lock); - xe_vm_put(vm); - if (IS_ERR(q)) - return PTR_ERR(q); - - if (xe_vm_in_preempt_fence_mode(vm)) { - q->lr.context = dma_fence_context_alloc(1); - - err = xe_vm_add_compute_exec_queue(vm, q); - if (XE_IOCTL_DBG(xe, err)) - goto put_exec_queue; - } - } - - mutex_lock(&xef->exec_queue.lock); - err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); - mutex_unlock(&xef->exec_queue.lock); - if (err) - goto kill_exec_queue; - - args->exec_queue_id = id; - q->xef = xe_file_get(xef); - - return 0; - -kill_exec_queue: - xe_exec_queue_kill(q); -put_exec_queue: - xe_exec_queue_put(q); - return err; -} - -int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_get_property *args = data; - struct xe_exec_queue *q; - int ret; - - if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - q = xe_exec_queue_lookup(xef, args->exec_queue_id); - if (XE_IOCTL_DBG(xe, !q)) - return -ENOENT; - - switch (args->property) { - case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: - args->value = q->ops->reset_status(q); - ret = 0; - break; - default: - ret = -EINVAL; - } - - xe_exec_queue_put(q); - - return ret; -} - -/** - * xe_exec_queue_is_lr() - Whether an exec_queue is long-running - * @q: The exec_queue - * - * Return: True if the exec_queue is long-running, false otherwise. - */ -bool xe_exec_queue_is_lr(struct xe_exec_queue *q) -{ - return q->vm && xe_vm_in_lr_mode(q->vm) && - !(q->flags & EXEC_QUEUE_FLAG_VM); -} - -static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) -{ - return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; -} - -/** - * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full - * @q: The exec_queue - * - * Return: True if the exec_queue's ring is full, false otherwise. - */ -bool xe_exec_queue_ring_full(struct xe_exec_queue *q) -{ - struct xe_lrc *lrc = q->lrc[0]; - s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; - - return xe_exec_queue_num_job_inflight(q) >= max_job; -} - -/** - * xe_exec_queue_is_idle() - Whether an exec_queue is idle. - * @q: The exec_queue - * - * FIXME: Need to determine what to use as the short-lived - * timeline lock for the exec_queues, so that the return value - * of this function becomes more than just an advisory - * snapshot in time. The timeline lock must protect the - * seqno from racing submissions on the same exec_queue. - * Typically vm->resv, but user-created timeline locks use the migrate vm - * and never grabs the migrate vm->resv so we have a race there. - * - * Return: True if the exec_queue is idle, false otherwise. - */ -bool xe_exec_queue_is_idle(struct xe_exec_queue *q) -{ - if (xe_exec_queue_is_parallel(q)) { - int i; - - for (i = 0; i < q->width; ++i) { - if (xe_lrc_seqno(q->lrc[i]) != - q->lrc[i]->fence_ctx.next_seqno - 1) - return false; - } - - return true; - } - - return xe_lrc_seqno(q->lrc[0]) == - q->lrc[0]->fence_ctx.next_seqno - 1; -} - -/** - * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue - * from hw - * @q: The exec queue - * - * Update the timestamp saved by HW for this exec queue and save run ticks - * calculated by using the delta from last update. - */ -void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) -{ - struct xe_file *xef; - struct xe_lrc *lrc; - u32 old_ts, new_ts; - - /* - * Jobs that are run during driver load may use an exec_queue, but are - * not associated with a user xe file, so avoid accumulating busyness - * for kernel specific work. - */ - if (!q->vm || !q->vm->xef) - return; - - xef = q->vm->xef; - - /* - * Only sample the first LRC. For parallel submission, all of them are - * scheduled together and we compensate that below by multiplying by - * width - this may introduce errors if that premise is not true and - * they don't exit 100% aligned. On the other hand, looping through - * the LRCs and reading them in different time could also introduce - * errors. - */ - lrc = q->lrc[0]; - new_ts = xe_lrc_update_timestamp(lrc, &old_ts); - xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; -} - -void xe_exec_queue_kill(struct xe_exec_queue *q) -{ - struct xe_exec_queue *eq = q, *next; - - list_for_each_entry_safe(eq, next, &eq->multi_gt_list, - multi_gt_link) { - q->ops->kill(eq); - xe_vm_remove_compute_exec_queue(q->vm, eq); - } - - q->ops->kill(q); - xe_vm_remove_compute_exec_queue(q->vm, q); -} - -int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_destroy *args = data; - struct xe_exec_queue *q; - - if (XE_IOCTL_DBG(xe, args->pad) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - mutex_lock(&xef->exec_queue.lock); - q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); - mutex_unlock(&xef->exec_queue.lock); - if (XE_IOCTL_DBG(xe, !q)) - return -ENOENT; - - xe_exec_queue_kill(q); - - trace_xe_exec_queue_close(q); - xe_exec_queue_put(q); - - return 0; -} - -static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, - struct xe_vm *vm) -{ - if (q->flags & EXEC_QUEUE_FLAG_VM) - lockdep_assert_held(&vm->lock); - else - xe_vm_assert_held(vm); -} - -/** - * xe_exec_queue_last_fence_put() - Drop ref to last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - */ -void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) -{ - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - if (q->last_fence) { - dma_fence_put(q->last_fence); - q->last_fence = NULL; - } -} - -/** - * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked - * @q: The exec queue - * - * Only safe to be called from xe_exec_queue_destroy(). - */ -void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) -{ - if (q->last_fence) { - dma_fence_put(q->last_fence); - q->last_fence = NULL; - } -} - -/** - * xe_exec_queue_last_fence_get() - Get last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - * - * Get last fence, takes a ref - * - * Returns: last fence if not signaled, dma fence stub if signaled - */ -struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, - struct xe_vm *vm) -{ - struct dma_fence *fence; - - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - if (q->last_fence && - test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) - xe_exec_queue_last_fence_put(q, vm); - - fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); - dma_fence_get(fence); - return fence; -} - -/** - * xe_exec_queue_last_fence_set() - Set last fence - * @q: The exec queue - * @vm: The VM the engine does a bind or exec for - * @fence: The fence - * - * Set the last fence for the engine. Increases reference count for fence, when - * closing engine xe_exec_queue_last_fence_put should be called. - */ -void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, - struct dma_fence *fence) -{ - xe_exec_queue_last_fence_lockdep_assert(q, vm); - - xe_exec_queue_last_fence_put(q, vm); - q->last_fence = dma_fence_get(fence); -} diff --git a/rr-cache/882012b4af13c77de852328870770019afd5505e/postimage b/rr-cache/882012b4af13c77de852328870770019afd5505e/postimage deleted file mode 100644 index 8343804ce3f8..000000000000 --- a/rr-cache/882012b4af13c77de852328870770019afd5505e/postimage +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef __INTEL_DP_MST_H__ -#define __INTEL_DP_MST_H__ - -#include <linux/types.h> - -struct intel_atomic_state; -struct intel_crtc; -struct intel_crtc_state; -struct intel_digital_port; -struct intel_dp; -struct intel_link_bw_limits; - -int intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_id); -void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port); -int intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port); -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_source_support(struct intel_dp *intel_dp); -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits); -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc); -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp); -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp); - -#endif /* __INTEL_DP_MST_H__ */ diff --git a/rr-cache/882012b4af13c77de852328870770019afd5505e/preimage b/rr-cache/882012b4af13c77de852328870770019afd5505e/preimage deleted file mode 100644 index 10b54afa1242..000000000000 --- a/rr-cache/882012b4af13c77de852328870770019afd5505e/preimage +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef __INTEL_DP_MST_H__ -#define __INTEL_DP_MST_H__ - -#include <linux/types.h> - -struct intel_atomic_state; -struct intel_crtc; -struct intel_crtc_state; -struct intel_digital_port; -struct intel_dp; -struct intel_link_bw_limits; - -int intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_id); -void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port); -int intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port); -bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state); -bool intel_dp_mst_source_support(struct intel_dp *intel_dp); -int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); -int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, - struct intel_link_bw_limits *limits); -bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, - struct intel_crtc *crtc); -<<<<<<< -bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp); -======= -void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp); ->>>>>>> - -#endif /* __INTEL_DP_MST_H__ */ diff --git a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.5 b/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.5 deleted file mode 100644 index 110e70f7ee7b..000000000000 --- a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.5 +++ /dev/null @@ -1,2244 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2022 Intel Corporation - */ - -#include "xe_guc_submit.h" - -#include <linux/bitfield.h> -#include <linux/bitmap.h> -#include <linux/circ_buf.h> -#include <linux/delay.h> -#include <linux/dma-fence-array.h> -#include <linux/math64.h> - -#include <drm/drm_managed.h> - -#include "abi/guc_actions_abi.h" -#include "abi/guc_klvs_abi.h" -#include "regs/xe_lrc_layout.h" -#include "xe_assert.h" -#include "xe_devcoredump.h" -#include "xe_device.h" -#include "xe_exec_queue.h" -#include "xe_force_wake.h" -#include "xe_gpu_scheduler.h" -#include "xe_gt.h" -#include "xe_gt_clock.h" -#include "xe_gt_printk.h" -#include "xe_guc.h" -#include "xe_guc_ct.h" -#include "xe_guc_exec_queue_types.h" -#include "xe_guc_id_mgr.h" -#include "xe_guc_submit_types.h" -#include "xe_hw_engine.h" -#include "xe_hw_fence.h" -#include "xe_lrc.h" -#include "xe_macros.h" -#include "xe_map.h" -#include "xe_mocs.h" -#include "xe_pm.h" -#include "xe_ring_ops_types.h" -#include "xe_sched_job.h" -#include "xe_trace.h" -#include "xe_vm.h" - -static struct xe_guc * -exec_queue_to_guc(struct xe_exec_queue *q) -{ - return &q->gt->uc.guc; -} - -/* - * Helpers for engine state, using an atomic as some of the bits can transition - * as the same time (e.g. a suspend can be happning at the same time as schedule - * engine done being processed). - */ -#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) -#define EXEC_QUEUE_STATE_ENABLED (1 << 1) -#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) -#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) -#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) -#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) -#define EXEC_QUEUE_STATE_RESET (1 << 6) -#define EXEC_QUEUE_STATE_KILLED (1 << 7) -#define EXEC_QUEUE_STATE_WEDGED (1 << 8) -#define EXEC_QUEUE_STATE_BANNED (1 << 9) -#define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10) -#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) - -static bool exec_queue_registered(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; -} - -static void set_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static void clear_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static bool exec_queue_enabled(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; -} - -static void set_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static void clear_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static bool exec_queue_pending_enable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; -} - -static void set_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static bool exec_queue_pending_disable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; -} - -static void set_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static bool exec_queue_destroyed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; -} - -static void set_exec_queue_destroyed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); -} - -static bool exec_queue_banned(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; -} - -static void set_exec_queue_banned(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); -} - -static bool exec_queue_suspended(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; -} - -static void set_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static void clear_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static bool exec_queue_reset(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; -} - -static void set_exec_queue_reset(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); -} - -static bool exec_queue_killed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; -} - -static void set_exec_queue_killed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); -} - -static bool exec_queue_wedged(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; -} - -static void set_exec_queue_wedged(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); -} - -static bool exec_queue_check_timeout(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT; -} - -static void set_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static void clear_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static bool exec_queue_extra_ref(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF; -} - -static void set_exec_queue_extra_ref(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); -} - -static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) -{ - return (atomic_read(&q->guc->state) & - (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | - EXEC_QUEUE_STATE_BANNED)); -} - -#ifdef CONFIG_PROVE_LOCKING -static int alloc_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) { - guc->submission_state.submit_wq_pool[i] = - alloc_ordered_workqueue("submit_wq", 0); - if (!guc->submission_state.submit_wq_pool[i]) - goto err_free; - } - - return 0; - -err_free: - while (i) - destroy_workqueue(guc->submission_state.submit_wq_pool[--i]); - - return -ENOMEM; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) - destroy_workqueue(guc->submission_state.submit_wq_pool[i]); -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - int idx = guc->submission_state.submit_wq_idx++ % NUM_SUBMIT_WQ; - - return guc->submission_state.submit_wq_pool[idx]; -} -#else -static int alloc_submit_wq(struct xe_guc *guc) -{ - return 0; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - return NULL; -} -#endif - -static void guc_submit_fini(struct drm_device *drm, void *arg) -{ - struct xe_guc *guc = arg; - - xa_destroy(&guc->submission_state.exec_queue_lookup); - free_submit_wq(guc); -} - -static void guc_submit_wedged_fini(void *arg) -{ - struct xe_guc *guc = arg; - struct xe_exec_queue *q; - unsigned long index; - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (exec_queue_wedged(q)) - xe_exec_queue_put(q); -} - -static const struct xe_exec_queue_ops guc_exec_queue_ops; - -static void primelockdep(struct xe_guc *guc) -{ - if (!IS_ENABLED(CONFIG_LOCKDEP)) - return; - - fs_reclaim_acquire(GFP_KERNEL); - - mutex_lock(&guc->submission_state.lock); - mutex_unlock(&guc->submission_state.lock); - - fs_reclaim_release(GFP_KERNEL); -} - -/** - * xe_guc_submit_init() - Initialize GuC submission. - * @guc: the &xe_guc to initialize - * @num_ids: number of GuC context IDs to use - * - * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all - * GuC context IDs supported by the GuC firmware should be used for submission. - * - * Only VF drivers will have to provide explicit number of GuC context IDs - * that they can use for submission. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_gt *gt = guc_to_gt(guc); - int err; - - err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); - if (err) - return err; - - err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); - if (err) - return err; - - err = alloc_submit_wq(guc); - if (err) - return err; - - gt->exec_queue_ops = &guc_exec_queue_ops; - - xa_init(&guc->submission_state.exec_queue_lookup); - - primelockdep(guc); - - return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); -} - -static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) -{ - int i; - - lockdep_assert_held(&guc->submission_state.lock); - - for (i = 0; i < xa_count; ++i) - xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); - - xe_guc_id_mgr_release_locked(&guc->submission_state.idm, - q->guc->id, q->width); -} - -static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - int ret; - void *ptr; - int i; - - /* - * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, - * worse case user gets -ENOMEM on engine create and has to try again. - * - * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent - * failure. - */ - lockdep_assert_held(&guc->submission_state.lock); - - ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, - q->width); - if (ret < 0) - return ret; - - q->guc->id = ret; - - for (i = 0; i < q->width; ++i) { - ptr = xa_store(&guc->submission_state.exec_queue_lookup, - q->guc->id + i, q, GFP_NOWAIT); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - goto err_release; - } - } - - return 0; - -err_release: - __release_guc_id(guc, q, i); - - return ret; -} - -static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - mutex_lock(&guc->submission_state.lock); - __release_guc_id(guc, q, q->width); - mutex_unlock(&guc->submission_state.lock); -} - -struct exec_queue_policy { - u32 count; - struct guc_update_exec_queue_policy h2g; -}; - -static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) -{ - size_t bytes = sizeof(policy->h2g.header) + - (sizeof(policy->h2g.klv[0]) * policy->count); - - return bytes / sizeof(u32); -} - -static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, - u16 guc_id) -{ - policy->h2g.header.action = - XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; - policy->h2g.header.guc_id = guc_id; - policy->count = 0; -} - -#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ -static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ - u32 data) \ -{ \ - XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ -\ - policy->h2g.klv[policy->count].kl = \ - FIELD_PREP(GUC_KLV_0_KEY, \ - GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ - FIELD_PREP(GUC_KLV_0_LEN, 1); \ - policy->h2g.klv[policy->count].value = data; \ - policy->count++; \ -} - -MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) -MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) -MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) -#undef MAKE_EXEC_QUEUE_POLICY_ADD - -static const int xe_exec_queue_prio_to_guc[] = { - [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, - [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, -}; - -static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - struct xe_device *xe = guc_to_xe(guc); - enum xe_exec_queue_priority prio = q->sched_props.priority; - u32 timeslice_us = q->sched_props.timeslice_us; - u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; - - xe_assert(xe, exec_queue_registered(q)); - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); - __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); - __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -#define parallel_read(xe_, map_, field_) \ - xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_) -#define parallel_write(xe_, map_, field_, val_) \ - xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_, val_) - -static void __register_mlrc_exec_queue(struct xe_guc *guc, - struct xe_exec_queue *q, - struct guc_ctxt_registration_info *info) -{ -#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) - struct xe_device *xe = guc_to_xe(guc); - u32 action[MAX_MLRC_REG_SIZE]; - int len = 0; - int i; - - xe_assert(xe, xe_exec_queue_is_parallel(q)); - - action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; - action[len++] = info->flags; - action[len++] = info->context_idx; - action[len++] = info->engine_class; - action[len++] = info->engine_submit_mask; - action[len++] = info->wq_desc_lo; - action[len++] = info->wq_desc_hi; - action[len++] = info->wq_base_lo; - action[len++] = info->wq_base_hi; - action[len++] = info->wq_size; - action[len++] = q->width; - action[len++] = info->hwlrca_lo; - action[len++] = info->hwlrca_hi; - - for (i = 1; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); - action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); - } - - xe_assert(xe, len <= MAX_MLRC_REG_SIZE); -#undef MAX_MLRC_REG_SIZE - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); -} - -static void __register_exec_queue(struct xe_guc *guc, - struct guc_ctxt_registration_info *info) -{ - u32 action[] = { - XE_GUC_ACTION_REGISTER_CONTEXT, - info->flags, - info->context_idx, - info->engine_class, - info->engine_submit_mask, - info->wq_desc_lo, - info->wq_desc_hi, - info->wq_base_lo, - info->wq_base_hi, - info->wq_size, - info->hwlrca_lo, - info->hwlrca_hi, - }; - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); -} - -static void register_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - struct guc_ctxt_registration_info info; - - xe_assert(xe, !exec_queue_registered(q)); - - memset(&info, 0, sizeof(info)); - info.context_idx = q->guc->id; - info.engine_class = xe_engine_class_to_guc_class(q->class); - info.engine_submit_mask = q->logical_mask; - info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); - info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); - info.flags = CONTEXT_REGISTRATION_FLAG_KMD; - - if (xe_exec_queue_is_parallel(q)) { - u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); - struct iosys_map map = xe_lrc_parallel_map(lrc); - - info.wq_desc_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_desc_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_base_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_base_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_size = WQ_SIZE; - - q->guc->wqi_head = 0; - q->guc->wqi_tail = 0; - xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); - parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); - } - - /* - * We must keep a reference for LR engines if engine is registered with - * the GuC as jobs signal immediately and can't destroy an engine if the - * GuC has a reference to it. - */ - if (xe_exec_queue_is_lr(q)) - xe_exec_queue_get(q); - - set_exec_queue_registered(q); - trace_xe_exec_queue_register(q); - if (xe_exec_queue_is_parallel(q)) - __register_mlrc_exec_queue(guc, q, &info); - else - __register_exec_queue(guc, &info); - init_policies(guc, q); -} - -static u32 wq_space_until_wrap(struct xe_exec_queue *q) -{ - return (WQ_SIZE - q->guc->wqi_tail); -} - -static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - unsigned int sleep_period_ms = 1; - -#define AVAILABLE_SPACE \ - CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) - if (wqi_size > AVAILABLE_SPACE) { -try_again: - q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); - if (wqi_size > AVAILABLE_SPACE) { - if (sleep_period_ms == 1024) { - xe_gt_reset_async(q->gt); - return -ENODEV; - } - - msleep(sleep_period_ms); - sleep_period_ms <<= 1; - goto try_again; - } - } -#undef AVAILABLE_SPACE - - return 0; -} - -static int wq_noop_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; - - if (wq_wait_for_space(q, wq_space_until_wrap(q))) - return -ENODEV; - - xe_assert(xe, FIELD_FIT(WQ_LEN_MASK, len_dw)); - - parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | - FIELD_PREP(WQ_LEN_MASK, len_dw)); - q->guc->wqi_tail = 0; - - return 0; -} - -static void wq_item_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); -#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ - u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; - u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); - u32 len_dw = (wqi_size / sizeof(u32)) - 1; - int i = 0, j; - - if (wqi_size > wq_space_until_wrap(q)) { - if (wq_noop_append(q)) - return; - } - if (wq_wait_for_space(q, wqi_size)) - return; - - wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | - FIELD_PREP(WQ_LEN_MASK, len_dw); - wqi[i++] = xe_lrc_descriptor(q->lrc[0]); - wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | - FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); - wqi[i++] = 0; - for (j = 1; j < q->width; ++j) { - struct xe_lrc *lrc = q->lrc[j]; - - wqi[i++] = lrc->ring.tail / sizeof(u64); - } - - xe_assert(xe, i == wqi_size / sizeof(u32)); - - iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, - wq[q->guc->wqi_tail / sizeof(u32)])); - xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); - q->guc->wqi_tail += wqi_size; - xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE); - - xe_device_wmb(xe); - - map = xe_lrc_parallel_map(q->lrc[0]); - parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); -} - -#define RESUME_PENDING ~0x0ull -static void submit_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - u32 action[3]; - u32 g2h_len = 0; - u32 num_g2h = 0; - int len = 0; - bool extra_submit = false; - - xe_assert(xe, exec_queue_registered(q)); - - if (xe_exec_queue_is_parallel(q)) - wq_item_append(q); - else - xe_lrc_set_ring_tail(lrc, lrc->ring.tail); - - if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) - return; - - if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; - action[len++] = q->guc->id; - action[len++] = GUC_CONTEXT_ENABLE; - g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; - num_g2h = 1; - if (xe_exec_queue_is_parallel(q)) - extra_submit = true; - - q->guc->resume_time = RESUME_PENDING; - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - } else { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - } - - xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); - - if (extra_submit) { - len = 0; - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); - } -} - -static struct dma_fence * -guc_exec_queue_run_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_exec_queue *q = job->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - bool lr = xe_exec_queue_is_lr(q); - - xe_assert(xe, !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || - exec_queue_banned(q) || exec_queue_suspended(q)); - - trace_xe_sched_job_run(job); - - if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) { - if (!exec_queue_registered(q)) - register_exec_queue(q); - if (!lr) /* LR jobs are emitted in the exec IOCTL */ - q->ring_ops->emit_job(job); - submit_exec_queue(q); - } - - if (lr) { - xe_sched_job_set_error(job, -EOPNOTSUPP); - return NULL; - } else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) { - return job->fence; - } else { - return dma_fence_get(job->fence); - } -} - -static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - - xe_exec_queue_update_run_ticks(job->q); - - trace_xe_sched_job_free(job); - xe_sched_job_put(job); -} - -static int guc_read_stopped(struct xe_guc *guc) -{ - return atomic_read(&guc->submission_state.stopped); -} - -#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ - u32 action[] = { \ - XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ - q->guc->id, \ - GUC_CONTEXT_##enable_disable, \ - } - -static void disable_scheduling_deregister(struct xe_guc *guc, - struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_device *xe = guc_to_xe(guc); - int ret; - - set_min_preemption_timeout(guc, q); - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - struct xe_gpu_scheduler *sched = &q->guc->sched; - - drm_warn(&xe->drm, "Pending enable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - return; - } - - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - set_exec_queue_destroyed(q); - trace_xe_exec_queue_scheduling_disable(q); - - /* - * Reserve space for both G2H here as the 2nd G2H is sent from a G2H - * handler and we are not allowed to reserved G2H space in handlers. - */ - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + - G2H_LEN_DW_DEREGISTER_CONTEXT, 2); -} - -static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ - wake_up_all(&xe->ufence_wq); - - if (xe_exec_queue_is_lr(q)) - queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); - else - xe_sched_tdr_queue_imm(&q->guc->sched); -} - -/** - * xe_guc_submit_wedge() - Wedge GuC submission - * @guc: the GuC object - * - * Save exec queue's registered with GuC state by taking a ref to each queue. - * Register a DRMM handler to drop refs upon driver unload. - */ -void xe_guc_submit_wedge(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - unsigned long index; - int err; - - xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); - - err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, - guc_submit_wedged_fini, guc); - if (err) { - drm_err(&xe->drm, "Failed to register xe_guc_submit clean-up on wedged.mode=2. Although device is wedged.\n"); - return; - } - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (xe_exec_queue_get_unless_zero(q)) - set_exec_queue_wedged(q); - mutex_unlock(&guc->submission_state.lock); -} - -static bool guc_submit_hint_wedged(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - - if (xe->wedged.mode != 2) - return false; - - if (xe_device_wedged(xe)) - return true; - - xe_device_declare_wedged(xe); - - return true; -} - -static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, lr_tdr); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_gpu_scheduler *sched = &ge->sched; - bool wedged; - - xe_assert(xe, xe_exec_queue_is_lr(q)); - trace_xe_exec_queue_lr_cleanup(q); - - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Kill the run_job / process_msg entry points */ - xe_sched_submission_stop(sched); - - /* - * Engine state now mostly stable, disable scheduling / deregister if - * needed. This cleanup routine might be called multiple times, where - * the actual async engine deregister drops the final engine ref. - * Calling disable_scheduling_deregister will mark the engine as - * destroyed and fire off the CT requests to disable scheduling / - * deregister, which we only want to do once. We also don't want to mark - * the engine as pending_disable again as this may race with the - * xe_guc_deregister_done_handler() which treats it as an unexpected - * state. - */ - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - set_exec_queue_banned(q); - disable_scheduling_deregister(guc, q); - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - drm_warn(&xe->drm, "Schedule disable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - return; - } - } - - xe_sched_submission_start(sched); -} - -#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) - -static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) -{ - struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); - u32 ctx_timestamp = xe_lrc_ctx_timestamp(q->lrc[0]); - u32 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); - u32 timeout_ms = q->sched_props.job_timeout_ms; - u32 diff; - u64 running_time_ms; - - /* - * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch - * possible overflows with a high timeout. - */ - xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); - - if (ctx_timestamp < ctx_job_timestamp) - diff = ctx_timestamp + U32_MAX - ctx_job_timestamp; - else - diff = ctx_timestamp - ctx_job_timestamp; - - /* - * Ensure timeout is within 5% to account for an GuC scheduling latency - */ - running_time_ms = - ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); - - xe_gt_dbg(gt, - "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, running_time_ms, timeout_ms, diff); - - return running_time_ms >= timeout_ms; -} - -static void enable_scheduling(struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); - - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { - xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(&q->guc->sched); - } -} - -static void disable_scheduling(struct xe_exec_queue *q, bool immediate) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - if (immediate) - set_min_preemption_timeout(guc, q); - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - trace_xe_exec_queue_scheduling_disable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); -} - -static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - set_exec_queue_destroyed(q); - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_DEREGISTER_CONTEXT, 1); -} - -static enum drm_gpu_sched_stat -guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_sched_job *tmp_job; - struct xe_exec_queue *q = job->q; - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc *guc = exec_queue_to_guc(q); -<<<<<<< -======= - const char *process_name = "no process"; ->>>>>>> - int err = -ETIME; - int i = 0; - bool wedged, skip_timeout_check; - - /* - * TDR has fired before free job worker. Common if exec queue - * immediately closed after last fence signaled. - */ - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { - guc_exec_queue_free_job(drm_job); - - return DRM_GPU_SCHED_STAT_NOMINAL; - } - - /* Kill the run_job entry point */ - xe_sched_submission_stop(sched); - - /* Must check all state after stopping scheduler */ - skip_timeout_check = exec_queue_reset(q) || - exec_queue_killed_or_banned_or_wedged(q) || - exec_queue_destroyed(q); - - /* Job hasn't started, can't be timed out */ - if (!skip_timeout_check && !xe_sched_job_started(job)) - goto rearm; - - /* - * XXX: Sampling timeout doesn't work in wedged mode as we have to - * modify scheduling state to read timestamp. We could read the - * timestamp from a register to accumulate current running time but this - * doesn't work for SRIOV. For now assuming timeouts in wedged mode are - * genuine timeouts. - */ - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Engine state now stable, disable scheduling to check timestamp */ - if (!wedged && exec_queue_registered(q)) { - int ret; - - if (exec_queue_reset(q)) - err = -EIO; - - if (!exec_queue_destroyed(q)) { - /* - * Wait for any pending G2H to flush out before - * modifying state - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) - goto trigger_reset; - - /* - * Flag communicates to G2H handler that schedule - * disable originated from a timeout check. The G2H then - * avoid triggering cleanup or deregistering the exec - * queue. - */ - set_exec_queue_check_timeout(q); - disable_scheduling(q, skip_timeout_check); - } - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - * - * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault - * error) messages which can cause the schedule disable to get - * lost. If this occurs, trigger a GT reset to recover. - */ - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { -trigger_reset: - if (!ret) - xe_gt_warn(guc_to_gt(guc), "Schedule disable failed to respond"); - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); /* GT reset owns this */ - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - goto rearm; - } - } - - /* - * Check if job is actually timed out, if so restart job execution and TDR - */ - if (!wedged && !skip_timeout_check && !check_timeout(q, job) && - !exec_queue_reset(q) && exec_queue_registered(q)) { - clear_exec_queue_check_timeout(q); - goto sched_enable; - } - -<<<<<<< - if (q->vm && q->vm->xef) { - process_name = q->vm->xef->process_name; - pid = q->vm->xef->pid; - } - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags, process_name, pid); - -======= - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags); ->>>>>>> - trace_xe_sched_job_timedout(job); - - if (!exec_queue_killed(q)) - xe_devcoredump(job); - - /* - * Kernel jobs should never fail, nor should VM jobs if they do - * somethings has gone wrong and the GT needs a reset - */ - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, - "Kernel-submitted job timed out\n"); - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), - "VM job timed out on non-killed execqueue\n"); - if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || - (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { - if (!xe_sched_invalidate_job(job, 2)) { - clear_exec_queue_check_timeout(q); - xe_gt_reset_async(q->gt); - goto rearm; - } - } - - /* Finish cleaning up exec queue via deregister */ - set_exec_queue_banned(q); - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); - __deregister_exec_queue(guc, q); - } - - /* Stop fence signaling */ - xe_hw_fence_irq_stop(q->fence_irq); - - /* - * Fence state now stable, stop / start scheduler which cleans up any - * fences that are complete - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - xe_guc_exec_queue_trigger_cleanup(q); - - /* Mark all outstanding jobs as bad, thus completing them */ - spin_lock(&sched->base.job_list_lock); - list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) - xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); - spin_unlock(&sched->base.job_list_lock); - - /* Start fence signaling */ - xe_hw_fence_irq_start(q->fence_irq); - - return DRM_GPU_SCHED_STAT_NOMINAL; - -sched_enable: - enable_scheduling(q); -rearm: - /* - * XXX: Ideally want to adjust timeout based on current exection time - * but there is not currently an easy way to do in DRM scheduler. With - * some thought, do this in a follow up. - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - return DRM_GPU_SCHED_STAT_NOMINAL; -} - -static void __guc_exec_queue_fini_async(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, fini_async); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_pm_runtime_get(guc_to_xe(guc)); - trace_xe_exec_queue_destroy(q); - - if (xe_exec_queue_is_lr(q)) - cancel_work_sync(&ge->lr_tdr); - release_guc_id(guc, q); - xe_sched_entity_fini(&ge->entity); - xe_sched_fini(&ge->sched); - - kfree(ge); - xe_exec_queue_fini(q); - xe_pm_runtime_put(guc_to_xe(guc)); -} - -static void guc_exec_queue_fini_async(struct xe_exec_queue *q) -{ - INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async); - - /* We must block on kernel engines so slabs are empty on driver unload */ - if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) - __guc_exec_queue_fini_async(&q->guc->fini_async); - else - queue_work(system_wq, &q->guc->fini_async); -} - -static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q) -{ - /* - * Might be done from within the GPU scheduler, need to do async as we - * fini the scheduler when the engine is fini'd, the scheduler can't - * complete fini within itself (circular dependency). Async resolves - * this we and don't really care when everything is fini'd, just that it - * is. - */ - guc_exec_queue_fini_async(q); -} - -static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); - trace_xe_exec_queue_cleanup_entity(q); - - if (exec_queue_registered(q)) - disable_scheduling_deregister(guc, q); - else - __guc_exec_queue_fini(guc, q); -} - -static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) -{ - return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); -} - -static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q)) - init_policies(guc, q); - kfree(msg); -} - -static void suspend_fence_signal(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, exec_queue_suspended(q) || exec_queue_killed(q) || - guc_read_stopped(guc)); - xe_assert(xe, q->guc->suspend_pending); - - q->guc->suspend_pending = false; - smp_wmb(); - wake_up(&q->guc->suspend_wait); -} - -static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && - exec_queue_enabled(q)) { - wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING || - guc_read_stopped(guc)); - - if (!guc_read_stopped(guc)) { - s64 since_resume_ms = - ktime_ms_delta(ktime_get(), - q->guc->resume_time); - s64 wait_ms = q->vm->preempt.min_run_period_ms - - since_resume_ms; - - if (wait_ms > 0 && q->guc->resume_time) - msleep(wait_ms); - - set_exec_queue_suspended(q); - disable_scheduling(q, false); - } - } else if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } -} - -static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - - if (guc_exec_queue_allowed_to_change_state(q)) { - q->guc->resume_time = RESUME_PENDING; - clear_exec_queue_suspended(q); - enable_scheduling(q); - } else { - clear_exec_queue_suspended(q); - } -} - -#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ -#define SET_SCHED_PROPS 2 -#define SUSPEND 3 -#define RESUME 4 - -static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) -{ - trace_xe_sched_msg_recv(msg); - - switch (msg->opcode) { - case CLEANUP: - __guc_exec_queue_process_msg_cleanup(msg); - break; - case SET_SCHED_PROPS: - __guc_exec_queue_process_msg_set_sched_props(msg); - break; - case SUSPEND: - __guc_exec_queue_process_msg_suspend(msg); - break; - case RESUME: - __guc_exec_queue_process_msg_resume(msg); - break; - default: - XE_WARN_ON("Unknown message type"); - } - - xe_pm_runtime_put(guc_to_xe(exec_queue_to_guc(msg->private_data))); -} - -static const struct drm_sched_backend_ops drm_sched_ops = { - .run_job = guc_exec_queue_run_job, - .free_job = guc_exec_queue_free_job, - .timedout_job = guc_exec_queue_timedout_job, -}; - -static const struct xe_sched_backend_ops xe_sched_ops = { - .process_msg = guc_exec_queue_process_msg, -}; - -static int guc_exec_queue_init(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_guc_exec_queue *ge; - long timeout; - int err; - - xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc))); - - ge = kzalloc(sizeof(*ge), GFP_KERNEL); - if (!ge) - return -ENOMEM; - - q->guc = ge; - ge->q = q; - init_waitqueue_head(&ge->suspend_wait); - - timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : - msecs_to_jiffies(q->sched_props.job_timeout_ms); - err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, - get_submit_wq(guc), - q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 64, - timeout, guc_to_gt(guc)->ordered_wq, NULL, - q->name, gt_to_xe(q->gt)->drm.dev); - if (err) - goto err_free; - - sched = &ge->sched; - err = xe_sched_entity_init(&ge->entity, sched); - if (err) - goto err_sched; - - if (xe_exec_queue_is_lr(q)) - INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); - - mutex_lock(&guc->submission_state.lock); - - err = alloc_guc_id(guc, q); - if (err) - goto err_entity; - - q->entity = &ge->entity; - - if (guc_read_stopped(guc)) - xe_sched_stop(sched); - - mutex_unlock(&guc->submission_state.lock); - - xe_exec_queue_assign_name(q, q->guc->id); - - trace_xe_exec_queue_create(q); - - return 0; - -err_entity: - mutex_unlock(&guc->submission_state.lock); - xe_sched_entity_fini(&ge->entity); -err_sched: - xe_sched_fini(&ge->sched); -err_free: - kfree(ge); - - return err; -} - -static void guc_exec_queue_kill(struct xe_exec_queue *q) -{ - trace_xe_exec_queue_kill(q); - set_exec_queue_killed(q); - xe_guc_exec_queue_trigger_cleanup(q); -} - -static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, - u32 opcode) -{ - xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); - - INIT_LIST_HEAD(&msg->link); - msg->opcode = opcode; - msg->private_data = q; - - trace_xe_sched_msg_add(msg); - xe_sched_add_msg(&q->guc->sched, msg); -} - -#define STATIC_MSG_CLEANUP 0 -#define STATIC_MSG_SUSPEND 1 -#define STATIC_MSG_RESUME 2 -static void guc_exec_queue_fini(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; - - if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) - guc_exec_queue_add_msg(q, msg, CLEANUP); - else - __guc_exec_queue_fini(exec_queue_to_guc(q), q); -} - -static int guc_exec_queue_set_priority(struct xe_exec_queue *q, - enum xe_exec_queue_priority priority) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.priority == priority || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.priority = priority; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.timeslice_us == timeslice_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.timeslice_us = timeslice_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, - u32 preempt_timeout_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.preempt_timeout_us == preempt_timeout_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.preempt_timeout_us = preempt_timeout_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_suspend(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; - - if (exec_queue_killed_or_banned_or_wedged(q) || q->guc->suspend_pending) - return -EINVAL; - - q->guc->suspend_pending = true; - guc_exec_queue_add_msg(q, msg, SUSPEND); - - return 0; -} - -static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - - wait_event(q->guc->suspend_wait, !q->guc->suspend_pending || - guc_read_stopped(guc)); -} - -static void guc_exec_queue_resume(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !q->guc->suspend_pending); - - guc_exec_queue_add_msg(q, msg, RESUME); -} - -static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) -{ - return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); -} - -/* - * All of these functions are an abstraction layer which other parts of XE can - * use to trap into the GuC backend. All of these functions, aside from init, - * really shouldn't do much other than trap into the DRM scheduler which - * synchronizes these operations. - */ -static const struct xe_exec_queue_ops guc_exec_queue_ops = { - .init = guc_exec_queue_init, - .kill = guc_exec_queue_kill, - .fini = guc_exec_queue_fini, - .set_priority = guc_exec_queue_set_priority, - .set_timeslice = guc_exec_queue_set_timeslice, - .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, - .suspend = guc_exec_queue_suspend, - .suspend_wait = guc_exec_queue_suspend_wait, - .resume = guc_exec_queue_resume, - .reset_status = guc_exec_queue_reset_status, -}; - -static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - /* Stop scheduling + flush any DRM scheduler operations */ - xe_sched_submission_stop(sched); - - /* Clean up lost G2H + reset engine state */ - if (exec_queue_registered(q)) { - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else if (exec_queue_destroyed(q)) - __guc_exec_queue_fini(guc, q); - } - if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } - atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | - EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | - EXEC_QUEUE_STATE_SUSPENDED, - &q->guc->state); - q->guc->resume_time = 0; - trace_xe_exec_queue_stop(q); - - /* - * Ban any engine (aside from kernel and engines used for VM ops) with a - * started but not complete job or if a job has gone through a GT reset - * more than twice. - */ - if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { - struct xe_sched_job *job = xe_sched_first_pending_job(sched); - bool ban = false; - - if (job) { - if ((xe_sched_job_started(job) && - !xe_sched_job_completed(job)) || - xe_sched_invalidate_job(job, 2)) { - trace_xe_sched_job_ban(job); - ban = true; - } - } else if (xe_exec_queue_is_lr(q) && - (xe_lrc_ring_head(q->lrc[0]) != xe_lrc_ring_tail(q->lrc[0]))) { - ban = true; - } - - if (ban) { - set_exec_queue_banned(q); - xe_guc_exec_queue_trigger_cleanup(q); - } - } -} - -int xe_guc_submit_reset_prepare(struct xe_guc *guc) -{ - int ret; - - /* - * Using an atomic here rather than submission_state.lock as this - * function can be called while holding the CT lock (engine reset - * failure). submission_state.lock needs the CT lock to resubmit jobs. - * Atomic is not ideal, but it works to prevent against concurrent reset - * and releasing any TDRs waiting on guc->submission_state.stopped. - */ - ret = atomic_fetch_or(1, &guc->submission_state.stopped); - smp_wmb(); - wake_up_all(&guc->ct.wq); - - return ret; -} - -void xe_guc_submit_reset_wait(struct xe_guc *guc) -{ - wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || - !guc_read_stopped(guc)); -} - -void xe_guc_submit_stop(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_stop(guc, q); - - mutex_unlock(&guc->submission_state.lock); - - /* - * No one can enter the backend at this point, aside from new engine - * creation which is protected by guc->submission_state.lock. - */ - -} - -static void guc_exec_queue_start(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - if (!exec_queue_killed_or_banned_or_wedged(q)) { - int i; - - trace_xe_exec_queue_resubmit(q); - for (i = 0; i < q->width; ++i) - xe_lrc_set_ring_head(q->lrc[i], q->lrc[i]->ring.tail); - xe_sched_resubmit_jobs(sched); - } - - xe_sched_submission_start(sched); -} - -int xe_guc_submit_start(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - atomic_dec(&guc->submission_state.stopped); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_start(q); - mutex_unlock(&guc->submission_state.lock); - - wake_up_all(&guc->ct.wq); - - return 0; -} - -static struct xe_exec_queue * -g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - - if (unlikely(guc_id >= GUC_ID_MAX)) { - drm_err(&xe->drm, "Invalid guc_id %u", guc_id); - return NULL; - } - - q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); - if (unlikely(!q)) { - drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id); - return NULL; - } - - xe_assert(xe, guc_id >= q->guc->id); - xe_assert(xe, guc_id < (q->guc->id + q->width)); - - return q; -} - -static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); -} - -static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, - u32 runnable_state) -{ - trace_xe_exec_queue_scheduling_done(q); - - if (runnable_state == 1) { - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); - - q->guc->resume_time = ktime_get(); - clear_exec_queue_pending_enable(q); - smp_wmb(); - wake_up_all(&guc->ct.wq); - } else { - bool check_timeout = exec_queue_check_timeout(q); - - xe_gt_assert(guc_to_gt(guc), runnable_state == 0); - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); - - clear_exec_queue_pending_disable(q); - if (q->guc->suspend_pending) { - suspend_fence_signal(q); - } else { - if (exec_queue_banned(q) || check_timeout) { - smp_wmb(); - wake_up_all(&guc->ct.wq); - } - if (!check_timeout) - deregister_exec_queue(guc, q); - } - } -} - -int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - u32 runnable_state = msg[1]; - - if (unlikely(len < 2)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (unlikely(!exec_queue_pending_enable(q) && - !exec_queue_pending_disable(q))) { - xe_gt_err(guc_to_gt(guc), - "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", - atomic_read(&q->guc->state), q->guc->id, - runnable_state); - return -EPROTO; - } - - handle_sched_done(guc, q, runnable_state); - - return 0; -} - -static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) -{ - trace_xe_exec_queue_deregister_done(q); - - clear_exec_queue_registered(q); - - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else - __guc_exec_queue_fini(guc, q); -} - -int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || - exec_queue_pending_enable(q) || exec_queue_enabled(q)) { - xe_gt_err(guc_to_gt(guc), - "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", - atomic_read(&q->guc->state), q->guc->id); - return -EPROTO; - } - - handle_deregister_done(guc, q); - - return 0; -} - -int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - /* FIXME: Do error capture, most likely async */ - - trace_xe_exec_queue_reset(q); - - /* - * A banned engine is a NOP at this point (came from - * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel - * jobs by setting timeout of the job to the minimum value kicking - * guc_exec_queue_timedout_job. - */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, - u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_dbg(gt, "Engine memory cat error: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - trace_xe_exec_queue_memory_cat_error(q); - - /* Treat the same as engine reset */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - u8 guc_class, instance; - u32 reason; - - if (unlikely(len != 3)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - guc_class = msg[0]; - instance = msg[1]; - reason = msg[2]; - - /* Unexpected failure of a hardware feature, log an actual error */ - drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X", - guc_class, instance, reason); - - xe_gt_reset_async(guc_to_gt(guc)); - - return 0; -} - -static void -guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, - struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - int i; - - snapshot->guc.wqi_head = q->guc->wqi_head; - snapshot->guc.wqi_tail = q->guc->wqi_tail; - snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); - snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); - snapshot->parallel.wq_desc.status = parallel_read(xe, map, - wq_desc.wq_status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - snapshot->parallel.wq[i / sizeof(u32)] = - parallel_read(xe, map, wq[i / sizeof(u32)]); - } -} - -static void -guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", - snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); - drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", - snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); - drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), - snapshot->parallel.wq[i / sizeof(u32)]); - } -} - -/** - * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. - * @q: faulty exec queue - * - * This can be printed out in a later stage like during dev_coredump - * analysis. - * - * Returns: a GuC Submit Engine snapshot object that must be freed by the - * caller, using `xe_guc_exec_queue_snapshot_free`. - */ -struct xe_guc_submit_exec_queue_snapshot * -xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc_submit_exec_queue_snapshot *snapshot; - int i; - - snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); - - if (!snapshot) - return NULL; - - snapshot->guc.id = q->guc->id; - memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); - snapshot->class = q->class; - snapshot->logical_mask = q->logical_mask; - snapshot->width = q->width; - snapshot->refcount = kref_read(&q->refcount); - snapshot->sched_timeout = sched->base.timeout; - snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; - snapshot->sched_props.preempt_timeout_us = - q->sched_props.preempt_timeout_us; - - snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *), - GFP_ATOMIC); - - if (snapshot->lrc) { - for (i = 0; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); - } - } - - snapshot->schedule_state = atomic_read(&q->guc->state); - snapshot->exec_queue_flags = q->flags; - - snapshot->parallel_execution = xe_exec_queue_is_parallel(q); - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_capture(q, snapshot); - - spin_lock(&sched->base.job_list_lock); - snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); - snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, - sizeof(struct pending_list_snapshot), - GFP_ATOMIC); - - if (snapshot->pending_list) { - struct xe_sched_job *job_iter; - - i = 0; - list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) { - snapshot->pending_list[i].seqno = - xe_sched_job_seqno(job_iter); - snapshot->pending_list[i].fence = - dma_fence_is_signaled(job_iter->fence) ? 1 : 0; - snapshot->pending_list[i].finished = - dma_fence_is_signaled(&job_iter->drm.s_fence->finished) - ? 1 : 0; - i++; - } - } - - spin_unlock(&sched->base.job_list_lock); - - return snapshot; -} - -/** - * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. - * @snapshot: Previously captured snapshot of job. - * - * This captures some data that requires taking some locks, so it cannot be done in signaling path. - */ -void -xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot || !snapshot->lrc) - return; - - for (i = 0; i < snapshot->width; ++i) - xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); -} - -/** - * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * @p: drm_printer where it will be printed out. - * - * This function prints out a given GuC Submit Engine snapshot object. - */ -void -xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - if (!snapshot) - return; - - drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id); - drm_printf(p, "\tName: %s\n", snapshot->name); - drm_printf(p, "\tClass: %d\n", snapshot->class); - drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); - drm_printf(p, "\tWidth: %d\n", snapshot->width); - drm_printf(p, "\tRef: %d\n", snapshot->refcount); - drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); - drm_printf(p, "\tTimeslice: %u (us)\n", - snapshot->sched_props.timeslice_us); - drm_printf(p, "\tPreempt timeout: %u (us)\n", - snapshot->sched_props.preempt_timeout_us); - - for (i = 0; snapshot->lrc && i < snapshot->width; ++i) - xe_lrc_snapshot_print(snapshot->lrc[i], p); - - drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); - drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); - - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_print(snapshot, p); - - for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; - i++) - drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", - snapshot->pending_list[i].seqno, - snapshot->pending_list[i].fence, - snapshot->pending_list[i].finished); -} - -/** - * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given - * snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * - * This function free all the memory that needed to be allocated at capture - * time. - */ -void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot) - return; - - if (snapshot->lrc) { - for (i = 0; i < snapshot->width; i++) - xe_lrc_snapshot_free(snapshot->lrc[i]); - kfree(snapshot->lrc); - } - kfree(snapshot->pending_list); - kfree(snapshot); -} - -static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) -{ - struct xe_guc_submit_exec_queue_snapshot *snapshot; - - snapshot = xe_guc_exec_queue_snapshot_capture(q); - xe_guc_exec_queue_snapshot_print(snapshot, p); - xe_guc_exec_queue_snapshot_free(snapshot); -} - -/** - * xe_guc_submit_print - GuC Submit Print. - * @guc: GuC. - * @p: drm_printer where it will be printed out. - * - * This function capture and prints snapshots of **all** GuC Engines. - */ -void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) -{ - struct xe_exec_queue *q; - unsigned long index; - - if (!xe_device_uc_enabled(guc_to_xe(guc))) - return; - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_print(q, p); - mutex_unlock(&guc->submission_state.lock); -} diff --git a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.6 b/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.6 deleted file mode 100644 index 110e70f7ee7b..000000000000 --- a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.6 +++ /dev/null @@ -1,2244 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2022 Intel Corporation - */ - -#include "xe_guc_submit.h" - -#include <linux/bitfield.h> -#include <linux/bitmap.h> -#include <linux/circ_buf.h> -#include <linux/delay.h> -#include <linux/dma-fence-array.h> -#include <linux/math64.h> - -#include <drm/drm_managed.h> - -#include "abi/guc_actions_abi.h" -#include "abi/guc_klvs_abi.h" -#include "regs/xe_lrc_layout.h" -#include "xe_assert.h" -#include "xe_devcoredump.h" -#include "xe_device.h" -#include "xe_exec_queue.h" -#include "xe_force_wake.h" -#include "xe_gpu_scheduler.h" -#include "xe_gt.h" -#include "xe_gt_clock.h" -#include "xe_gt_printk.h" -#include "xe_guc.h" -#include "xe_guc_ct.h" -#include "xe_guc_exec_queue_types.h" -#include "xe_guc_id_mgr.h" -#include "xe_guc_submit_types.h" -#include "xe_hw_engine.h" -#include "xe_hw_fence.h" -#include "xe_lrc.h" -#include "xe_macros.h" -#include "xe_map.h" -#include "xe_mocs.h" -#include "xe_pm.h" -#include "xe_ring_ops_types.h" -#include "xe_sched_job.h" -#include "xe_trace.h" -#include "xe_vm.h" - -static struct xe_guc * -exec_queue_to_guc(struct xe_exec_queue *q) -{ - return &q->gt->uc.guc; -} - -/* - * Helpers for engine state, using an atomic as some of the bits can transition - * as the same time (e.g. a suspend can be happning at the same time as schedule - * engine done being processed). - */ -#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) -#define EXEC_QUEUE_STATE_ENABLED (1 << 1) -#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) -#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) -#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) -#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) -#define EXEC_QUEUE_STATE_RESET (1 << 6) -#define EXEC_QUEUE_STATE_KILLED (1 << 7) -#define EXEC_QUEUE_STATE_WEDGED (1 << 8) -#define EXEC_QUEUE_STATE_BANNED (1 << 9) -#define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10) -#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) - -static bool exec_queue_registered(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; -} - -static void set_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static void clear_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static bool exec_queue_enabled(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; -} - -static void set_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static void clear_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static bool exec_queue_pending_enable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; -} - -static void set_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static bool exec_queue_pending_disable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; -} - -static void set_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static bool exec_queue_destroyed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; -} - -static void set_exec_queue_destroyed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); -} - -static bool exec_queue_banned(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; -} - -static void set_exec_queue_banned(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); -} - -static bool exec_queue_suspended(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; -} - -static void set_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static void clear_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static bool exec_queue_reset(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; -} - -static void set_exec_queue_reset(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); -} - -static bool exec_queue_killed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; -} - -static void set_exec_queue_killed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); -} - -static bool exec_queue_wedged(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; -} - -static void set_exec_queue_wedged(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); -} - -static bool exec_queue_check_timeout(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT; -} - -static void set_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static void clear_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static bool exec_queue_extra_ref(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF; -} - -static void set_exec_queue_extra_ref(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); -} - -static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) -{ - return (atomic_read(&q->guc->state) & - (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | - EXEC_QUEUE_STATE_BANNED)); -} - -#ifdef CONFIG_PROVE_LOCKING -static int alloc_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) { - guc->submission_state.submit_wq_pool[i] = - alloc_ordered_workqueue("submit_wq", 0); - if (!guc->submission_state.submit_wq_pool[i]) - goto err_free; - } - - return 0; - -err_free: - while (i) - destroy_workqueue(guc->submission_state.submit_wq_pool[--i]); - - return -ENOMEM; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) - destroy_workqueue(guc->submission_state.submit_wq_pool[i]); -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - int idx = guc->submission_state.submit_wq_idx++ % NUM_SUBMIT_WQ; - - return guc->submission_state.submit_wq_pool[idx]; -} -#else -static int alloc_submit_wq(struct xe_guc *guc) -{ - return 0; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - return NULL; -} -#endif - -static void guc_submit_fini(struct drm_device *drm, void *arg) -{ - struct xe_guc *guc = arg; - - xa_destroy(&guc->submission_state.exec_queue_lookup); - free_submit_wq(guc); -} - -static void guc_submit_wedged_fini(void *arg) -{ - struct xe_guc *guc = arg; - struct xe_exec_queue *q; - unsigned long index; - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (exec_queue_wedged(q)) - xe_exec_queue_put(q); -} - -static const struct xe_exec_queue_ops guc_exec_queue_ops; - -static void primelockdep(struct xe_guc *guc) -{ - if (!IS_ENABLED(CONFIG_LOCKDEP)) - return; - - fs_reclaim_acquire(GFP_KERNEL); - - mutex_lock(&guc->submission_state.lock); - mutex_unlock(&guc->submission_state.lock); - - fs_reclaim_release(GFP_KERNEL); -} - -/** - * xe_guc_submit_init() - Initialize GuC submission. - * @guc: the &xe_guc to initialize - * @num_ids: number of GuC context IDs to use - * - * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all - * GuC context IDs supported by the GuC firmware should be used for submission. - * - * Only VF drivers will have to provide explicit number of GuC context IDs - * that they can use for submission. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_gt *gt = guc_to_gt(guc); - int err; - - err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); - if (err) - return err; - - err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); - if (err) - return err; - - err = alloc_submit_wq(guc); - if (err) - return err; - - gt->exec_queue_ops = &guc_exec_queue_ops; - - xa_init(&guc->submission_state.exec_queue_lookup); - - primelockdep(guc); - - return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); -} - -static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) -{ - int i; - - lockdep_assert_held(&guc->submission_state.lock); - - for (i = 0; i < xa_count; ++i) - xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); - - xe_guc_id_mgr_release_locked(&guc->submission_state.idm, - q->guc->id, q->width); -} - -static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - int ret; - void *ptr; - int i; - - /* - * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, - * worse case user gets -ENOMEM on engine create and has to try again. - * - * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent - * failure. - */ - lockdep_assert_held(&guc->submission_state.lock); - - ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, - q->width); - if (ret < 0) - return ret; - - q->guc->id = ret; - - for (i = 0; i < q->width; ++i) { - ptr = xa_store(&guc->submission_state.exec_queue_lookup, - q->guc->id + i, q, GFP_NOWAIT); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - goto err_release; - } - } - - return 0; - -err_release: - __release_guc_id(guc, q, i); - - return ret; -} - -static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - mutex_lock(&guc->submission_state.lock); - __release_guc_id(guc, q, q->width); - mutex_unlock(&guc->submission_state.lock); -} - -struct exec_queue_policy { - u32 count; - struct guc_update_exec_queue_policy h2g; -}; - -static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) -{ - size_t bytes = sizeof(policy->h2g.header) + - (sizeof(policy->h2g.klv[0]) * policy->count); - - return bytes / sizeof(u32); -} - -static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, - u16 guc_id) -{ - policy->h2g.header.action = - XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; - policy->h2g.header.guc_id = guc_id; - policy->count = 0; -} - -#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ -static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ - u32 data) \ -{ \ - XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ -\ - policy->h2g.klv[policy->count].kl = \ - FIELD_PREP(GUC_KLV_0_KEY, \ - GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ - FIELD_PREP(GUC_KLV_0_LEN, 1); \ - policy->h2g.klv[policy->count].value = data; \ - policy->count++; \ -} - -MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) -MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) -MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) -#undef MAKE_EXEC_QUEUE_POLICY_ADD - -static const int xe_exec_queue_prio_to_guc[] = { - [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, - [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, -}; - -static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - struct xe_device *xe = guc_to_xe(guc); - enum xe_exec_queue_priority prio = q->sched_props.priority; - u32 timeslice_us = q->sched_props.timeslice_us; - u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; - - xe_assert(xe, exec_queue_registered(q)); - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); - __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); - __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -#define parallel_read(xe_, map_, field_) \ - xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_) -#define parallel_write(xe_, map_, field_, val_) \ - xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_, val_) - -static void __register_mlrc_exec_queue(struct xe_guc *guc, - struct xe_exec_queue *q, - struct guc_ctxt_registration_info *info) -{ -#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) - struct xe_device *xe = guc_to_xe(guc); - u32 action[MAX_MLRC_REG_SIZE]; - int len = 0; - int i; - - xe_assert(xe, xe_exec_queue_is_parallel(q)); - - action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; - action[len++] = info->flags; - action[len++] = info->context_idx; - action[len++] = info->engine_class; - action[len++] = info->engine_submit_mask; - action[len++] = info->wq_desc_lo; - action[len++] = info->wq_desc_hi; - action[len++] = info->wq_base_lo; - action[len++] = info->wq_base_hi; - action[len++] = info->wq_size; - action[len++] = q->width; - action[len++] = info->hwlrca_lo; - action[len++] = info->hwlrca_hi; - - for (i = 1; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); - action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); - } - - xe_assert(xe, len <= MAX_MLRC_REG_SIZE); -#undef MAX_MLRC_REG_SIZE - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); -} - -static void __register_exec_queue(struct xe_guc *guc, - struct guc_ctxt_registration_info *info) -{ - u32 action[] = { - XE_GUC_ACTION_REGISTER_CONTEXT, - info->flags, - info->context_idx, - info->engine_class, - info->engine_submit_mask, - info->wq_desc_lo, - info->wq_desc_hi, - info->wq_base_lo, - info->wq_base_hi, - info->wq_size, - info->hwlrca_lo, - info->hwlrca_hi, - }; - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); -} - -static void register_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - struct guc_ctxt_registration_info info; - - xe_assert(xe, !exec_queue_registered(q)); - - memset(&info, 0, sizeof(info)); - info.context_idx = q->guc->id; - info.engine_class = xe_engine_class_to_guc_class(q->class); - info.engine_submit_mask = q->logical_mask; - info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); - info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); - info.flags = CONTEXT_REGISTRATION_FLAG_KMD; - - if (xe_exec_queue_is_parallel(q)) { - u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); - struct iosys_map map = xe_lrc_parallel_map(lrc); - - info.wq_desc_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_desc_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_base_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_base_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_size = WQ_SIZE; - - q->guc->wqi_head = 0; - q->guc->wqi_tail = 0; - xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); - parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); - } - - /* - * We must keep a reference for LR engines if engine is registered with - * the GuC as jobs signal immediately and can't destroy an engine if the - * GuC has a reference to it. - */ - if (xe_exec_queue_is_lr(q)) - xe_exec_queue_get(q); - - set_exec_queue_registered(q); - trace_xe_exec_queue_register(q); - if (xe_exec_queue_is_parallel(q)) - __register_mlrc_exec_queue(guc, q, &info); - else - __register_exec_queue(guc, &info); - init_policies(guc, q); -} - -static u32 wq_space_until_wrap(struct xe_exec_queue *q) -{ - return (WQ_SIZE - q->guc->wqi_tail); -} - -static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - unsigned int sleep_period_ms = 1; - -#define AVAILABLE_SPACE \ - CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) - if (wqi_size > AVAILABLE_SPACE) { -try_again: - q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); - if (wqi_size > AVAILABLE_SPACE) { - if (sleep_period_ms == 1024) { - xe_gt_reset_async(q->gt); - return -ENODEV; - } - - msleep(sleep_period_ms); - sleep_period_ms <<= 1; - goto try_again; - } - } -#undef AVAILABLE_SPACE - - return 0; -} - -static int wq_noop_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; - - if (wq_wait_for_space(q, wq_space_until_wrap(q))) - return -ENODEV; - - xe_assert(xe, FIELD_FIT(WQ_LEN_MASK, len_dw)); - - parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | - FIELD_PREP(WQ_LEN_MASK, len_dw)); - q->guc->wqi_tail = 0; - - return 0; -} - -static void wq_item_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); -#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ - u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; - u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); - u32 len_dw = (wqi_size / sizeof(u32)) - 1; - int i = 0, j; - - if (wqi_size > wq_space_until_wrap(q)) { - if (wq_noop_append(q)) - return; - } - if (wq_wait_for_space(q, wqi_size)) - return; - - wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | - FIELD_PREP(WQ_LEN_MASK, len_dw); - wqi[i++] = xe_lrc_descriptor(q->lrc[0]); - wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | - FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); - wqi[i++] = 0; - for (j = 1; j < q->width; ++j) { - struct xe_lrc *lrc = q->lrc[j]; - - wqi[i++] = lrc->ring.tail / sizeof(u64); - } - - xe_assert(xe, i == wqi_size / sizeof(u32)); - - iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, - wq[q->guc->wqi_tail / sizeof(u32)])); - xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); - q->guc->wqi_tail += wqi_size; - xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE); - - xe_device_wmb(xe); - - map = xe_lrc_parallel_map(q->lrc[0]); - parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); -} - -#define RESUME_PENDING ~0x0ull -static void submit_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - u32 action[3]; - u32 g2h_len = 0; - u32 num_g2h = 0; - int len = 0; - bool extra_submit = false; - - xe_assert(xe, exec_queue_registered(q)); - - if (xe_exec_queue_is_parallel(q)) - wq_item_append(q); - else - xe_lrc_set_ring_tail(lrc, lrc->ring.tail); - - if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) - return; - - if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; - action[len++] = q->guc->id; - action[len++] = GUC_CONTEXT_ENABLE; - g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; - num_g2h = 1; - if (xe_exec_queue_is_parallel(q)) - extra_submit = true; - - q->guc->resume_time = RESUME_PENDING; - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - } else { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - } - - xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); - - if (extra_submit) { - len = 0; - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); - } -} - -static struct dma_fence * -guc_exec_queue_run_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_exec_queue *q = job->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - bool lr = xe_exec_queue_is_lr(q); - - xe_assert(xe, !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || - exec_queue_banned(q) || exec_queue_suspended(q)); - - trace_xe_sched_job_run(job); - - if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) { - if (!exec_queue_registered(q)) - register_exec_queue(q); - if (!lr) /* LR jobs are emitted in the exec IOCTL */ - q->ring_ops->emit_job(job); - submit_exec_queue(q); - } - - if (lr) { - xe_sched_job_set_error(job, -EOPNOTSUPP); - return NULL; - } else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) { - return job->fence; - } else { - return dma_fence_get(job->fence); - } -} - -static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - - xe_exec_queue_update_run_ticks(job->q); - - trace_xe_sched_job_free(job); - xe_sched_job_put(job); -} - -static int guc_read_stopped(struct xe_guc *guc) -{ - return atomic_read(&guc->submission_state.stopped); -} - -#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ - u32 action[] = { \ - XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ - q->guc->id, \ - GUC_CONTEXT_##enable_disable, \ - } - -static void disable_scheduling_deregister(struct xe_guc *guc, - struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_device *xe = guc_to_xe(guc); - int ret; - - set_min_preemption_timeout(guc, q); - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - struct xe_gpu_scheduler *sched = &q->guc->sched; - - drm_warn(&xe->drm, "Pending enable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - return; - } - - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - set_exec_queue_destroyed(q); - trace_xe_exec_queue_scheduling_disable(q); - - /* - * Reserve space for both G2H here as the 2nd G2H is sent from a G2H - * handler and we are not allowed to reserved G2H space in handlers. - */ - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + - G2H_LEN_DW_DEREGISTER_CONTEXT, 2); -} - -static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ - wake_up_all(&xe->ufence_wq); - - if (xe_exec_queue_is_lr(q)) - queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); - else - xe_sched_tdr_queue_imm(&q->guc->sched); -} - -/** - * xe_guc_submit_wedge() - Wedge GuC submission - * @guc: the GuC object - * - * Save exec queue's registered with GuC state by taking a ref to each queue. - * Register a DRMM handler to drop refs upon driver unload. - */ -void xe_guc_submit_wedge(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - unsigned long index; - int err; - - xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); - - err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, - guc_submit_wedged_fini, guc); - if (err) { - drm_err(&xe->drm, "Failed to register xe_guc_submit clean-up on wedged.mode=2. Although device is wedged.\n"); - return; - } - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (xe_exec_queue_get_unless_zero(q)) - set_exec_queue_wedged(q); - mutex_unlock(&guc->submission_state.lock); -} - -static bool guc_submit_hint_wedged(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - - if (xe->wedged.mode != 2) - return false; - - if (xe_device_wedged(xe)) - return true; - - xe_device_declare_wedged(xe); - - return true; -} - -static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, lr_tdr); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_gpu_scheduler *sched = &ge->sched; - bool wedged; - - xe_assert(xe, xe_exec_queue_is_lr(q)); - trace_xe_exec_queue_lr_cleanup(q); - - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Kill the run_job / process_msg entry points */ - xe_sched_submission_stop(sched); - - /* - * Engine state now mostly stable, disable scheduling / deregister if - * needed. This cleanup routine might be called multiple times, where - * the actual async engine deregister drops the final engine ref. - * Calling disable_scheduling_deregister will mark the engine as - * destroyed and fire off the CT requests to disable scheduling / - * deregister, which we only want to do once. We also don't want to mark - * the engine as pending_disable again as this may race with the - * xe_guc_deregister_done_handler() which treats it as an unexpected - * state. - */ - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - set_exec_queue_banned(q); - disable_scheduling_deregister(guc, q); - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - drm_warn(&xe->drm, "Schedule disable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - return; - } - } - - xe_sched_submission_start(sched); -} - -#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) - -static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) -{ - struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); - u32 ctx_timestamp = xe_lrc_ctx_timestamp(q->lrc[0]); - u32 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); - u32 timeout_ms = q->sched_props.job_timeout_ms; - u32 diff; - u64 running_time_ms; - - /* - * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch - * possible overflows with a high timeout. - */ - xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); - - if (ctx_timestamp < ctx_job_timestamp) - diff = ctx_timestamp + U32_MAX - ctx_job_timestamp; - else - diff = ctx_timestamp - ctx_job_timestamp; - - /* - * Ensure timeout is within 5% to account for an GuC scheduling latency - */ - running_time_ms = - ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); - - xe_gt_dbg(gt, - "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, running_time_ms, timeout_ms, diff); - - return running_time_ms >= timeout_ms; -} - -static void enable_scheduling(struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); - - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { - xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(&q->guc->sched); - } -} - -static void disable_scheduling(struct xe_exec_queue *q, bool immediate) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - if (immediate) - set_min_preemption_timeout(guc, q); - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - trace_xe_exec_queue_scheduling_disable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); -} - -static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - set_exec_queue_destroyed(q); - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_DEREGISTER_CONTEXT, 1); -} - -static enum drm_gpu_sched_stat -guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_sched_job *tmp_job; - struct xe_exec_queue *q = job->q; - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc *guc = exec_queue_to_guc(q); -<<<<<<< -======= - const char *process_name = "no process"; ->>>>>>> - int err = -ETIME; - int i = 0; - bool wedged, skip_timeout_check; - - /* - * TDR has fired before free job worker. Common if exec queue - * immediately closed after last fence signaled. - */ - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { - guc_exec_queue_free_job(drm_job); - - return DRM_GPU_SCHED_STAT_NOMINAL; - } - - /* Kill the run_job entry point */ - xe_sched_submission_stop(sched); - - /* Must check all state after stopping scheduler */ - skip_timeout_check = exec_queue_reset(q) || - exec_queue_killed_or_banned_or_wedged(q) || - exec_queue_destroyed(q); - - /* Job hasn't started, can't be timed out */ - if (!skip_timeout_check && !xe_sched_job_started(job)) - goto rearm; - - /* - * XXX: Sampling timeout doesn't work in wedged mode as we have to - * modify scheduling state to read timestamp. We could read the - * timestamp from a register to accumulate current running time but this - * doesn't work for SRIOV. For now assuming timeouts in wedged mode are - * genuine timeouts. - */ - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Engine state now stable, disable scheduling to check timestamp */ - if (!wedged && exec_queue_registered(q)) { - int ret; - - if (exec_queue_reset(q)) - err = -EIO; - - if (!exec_queue_destroyed(q)) { - /* - * Wait for any pending G2H to flush out before - * modifying state - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) - goto trigger_reset; - - /* - * Flag communicates to G2H handler that schedule - * disable originated from a timeout check. The G2H then - * avoid triggering cleanup or deregistering the exec - * queue. - */ - set_exec_queue_check_timeout(q); - disable_scheduling(q, skip_timeout_check); - } - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - * - * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault - * error) messages which can cause the schedule disable to get - * lost. If this occurs, trigger a GT reset to recover. - */ - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { -trigger_reset: - if (!ret) - xe_gt_warn(guc_to_gt(guc), "Schedule disable failed to respond"); - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); /* GT reset owns this */ - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - goto rearm; - } - } - - /* - * Check if job is actually timed out, if so restart job execution and TDR - */ - if (!wedged && !skip_timeout_check && !check_timeout(q, job) && - !exec_queue_reset(q) && exec_queue_registered(q)) { - clear_exec_queue_check_timeout(q); - goto sched_enable; - } - -<<<<<<< - if (q->vm && q->vm->xef) { - process_name = q->vm->xef->process_name; - pid = q->vm->xef->pid; - } - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags, process_name, pid); - -======= - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags); ->>>>>>> - trace_xe_sched_job_timedout(job); - - if (!exec_queue_killed(q)) - xe_devcoredump(job); - - /* - * Kernel jobs should never fail, nor should VM jobs if they do - * somethings has gone wrong and the GT needs a reset - */ - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, - "Kernel-submitted job timed out\n"); - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), - "VM job timed out on non-killed execqueue\n"); - if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || - (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { - if (!xe_sched_invalidate_job(job, 2)) { - clear_exec_queue_check_timeout(q); - xe_gt_reset_async(q->gt); - goto rearm; - } - } - - /* Finish cleaning up exec queue via deregister */ - set_exec_queue_banned(q); - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); - __deregister_exec_queue(guc, q); - } - - /* Stop fence signaling */ - xe_hw_fence_irq_stop(q->fence_irq); - - /* - * Fence state now stable, stop / start scheduler which cleans up any - * fences that are complete - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - xe_guc_exec_queue_trigger_cleanup(q); - - /* Mark all outstanding jobs as bad, thus completing them */ - spin_lock(&sched->base.job_list_lock); - list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) - xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); - spin_unlock(&sched->base.job_list_lock); - - /* Start fence signaling */ - xe_hw_fence_irq_start(q->fence_irq); - - return DRM_GPU_SCHED_STAT_NOMINAL; - -sched_enable: - enable_scheduling(q); -rearm: - /* - * XXX: Ideally want to adjust timeout based on current exection time - * but there is not currently an easy way to do in DRM scheduler. With - * some thought, do this in a follow up. - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - return DRM_GPU_SCHED_STAT_NOMINAL; -} - -static void __guc_exec_queue_fini_async(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, fini_async); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_pm_runtime_get(guc_to_xe(guc)); - trace_xe_exec_queue_destroy(q); - - if (xe_exec_queue_is_lr(q)) - cancel_work_sync(&ge->lr_tdr); - release_guc_id(guc, q); - xe_sched_entity_fini(&ge->entity); - xe_sched_fini(&ge->sched); - - kfree(ge); - xe_exec_queue_fini(q); - xe_pm_runtime_put(guc_to_xe(guc)); -} - -static void guc_exec_queue_fini_async(struct xe_exec_queue *q) -{ - INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async); - - /* We must block on kernel engines so slabs are empty on driver unload */ - if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) - __guc_exec_queue_fini_async(&q->guc->fini_async); - else - queue_work(system_wq, &q->guc->fini_async); -} - -static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q) -{ - /* - * Might be done from within the GPU scheduler, need to do async as we - * fini the scheduler when the engine is fini'd, the scheduler can't - * complete fini within itself (circular dependency). Async resolves - * this we and don't really care when everything is fini'd, just that it - * is. - */ - guc_exec_queue_fini_async(q); -} - -static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); - trace_xe_exec_queue_cleanup_entity(q); - - if (exec_queue_registered(q)) - disable_scheduling_deregister(guc, q); - else - __guc_exec_queue_fini(guc, q); -} - -static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) -{ - return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); -} - -static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q)) - init_policies(guc, q); - kfree(msg); -} - -static void suspend_fence_signal(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, exec_queue_suspended(q) || exec_queue_killed(q) || - guc_read_stopped(guc)); - xe_assert(xe, q->guc->suspend_pending); - - q->guc->suspend_pending = false; - smp_wmb(); - wake_up(&q->guc->suspend_wait); -} - -static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && - exec_queue_enabled(q)) { - wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING || - guc_read_stopped(guc)); - - if (!guc_read_stopped(guc)) { - s64 since_resume_ms = - ktime_ms_delta(ktime_get(), - q->guc->resume_time); - s64 wait_ms = q->vm->preempt.min_run_period_ms - - since_resume_ms; - - if (wait_ms > 0 && q->guc->resume_time) - msleep(wait_ms); - - set_exec_queue_suspended(q); - disable_scheduling(q, false); - } - } else if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } -} - -static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - - if (guc_exec_queue_allowed_to_change_state(q)) { - q->guc->resume_time = RESUME_PENDING; - clear_exec_queue_suspended(q); - enable_scheduling(q); - } else { - clear_exec_queue_suspended(q); - } -} - -#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ -#define SET_SCHED_PROPS 2 -#define SUSPEND 3 -#define RESUME 4 - -static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) -{ - trace_xe_sched_msg_recv(msg); - - switch (msg->opcode) { - case CLEANUP: - __guc_exec_queue_process_msg_cleanup(msg); - break; - case SET_SCHED_PROPS: - __guc_exec_queue_process_msg_set_sched_props(msg); - break; - case SUSPEND: - __guc_exec_queue_process_msg_suspend(msg); - break; - case RESUME: - __guc_exec_queue_process_msg_resume(msg); - break; - default: - XE_WARN_ON("Unknown message type"); - } - - xe_pm_runtime_put(guc_to_xe(exec_queue_to_guc(msg->private_data))); -} - -static const struct drm_sched_backend_ops drm_sched_ops = { - .run_job = guc_exec_queue_run_job, - .free_job = guc_exec_queue_free_job, - .timedout_job = guc_exec_queue_timedout_job, -}; - -static const struct xe_sched_backend_ops xe_sched_ops = { - .process_msg = guc_exec_queue_process_msg, -}; - -static int guc_exec_queue_init(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_guc_exec_queue *ge; - long timeout; - int err; - - xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc))); - - ge = kzalloc(sizeof(*ge), GFP_KERNEL); - if (!ge) - return -ENOMEM; - - q->guc = ge; - ge->q = q; - init_waitqueue_head(&ge->suspend_wait); - - timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : - msecs_to_jiffies(q->sched_props.job_timeout_ms); - err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, - get_submit_wq(guc), - q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 64, - timeout, guc_to_gt(guc)->ordered_wq, NULL, - q->name, gt_to_xe(q->gt)->drm.dev); - if (err) - goto err_free; - - sched = &ge->sched; - err = xe_sched_entity_init(&ge->entity, sched); - if (err) - goto err_sched; - - if (xe_exec_queue_is_lr(q)) - INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); - - mutex_lock(&guc->submission_state.lock); - - err = alloc_guc_id(guc, q); - if (err) - goto err_entity; - - q->entity = &ge->entity; - - if (guc_read_stopped(guc)) - xe_sched_stop(sched); - - mutex_unlock(&guc->submission_state.lock); - - xe_exec_queue_assign_name(q, q->guc->id); - - trace_xe_exec_queue_create(q); - - return 0; - -err_entity: - mutex_unlock(&guc->submission_state.lock); - xe_sched_entity_fini(&ge->entity); -err_sched: - xe_sched_fini(&ge->sched); -err_free: - kfree(ge); - - return err; -} - -static void guc_exec_queue_kill(struct xe_exec_queue *q) -{ - trace_xe_exec_queue_kill(q); - set_exec_queue_killed(q); - xe_guc_exec_queue_trigger_cleanup(q); -} - -static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, - u32 opcode) -{ - xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); - - INIT_LIST_HEAD(&msg->link); - msg->opcode = opcode; - msg->private_data = q; - - trace_xe_sched_msg_add(msg); - xe_sched_add_msg(&q->guc->sched, msg); -} - -#define STATIC_MSG_CLEANUP 0 -#define STATIC_MSG_SUSPEND 1 -#define STATIC_MSG_RESUME 2 -static void guc_exec_queue_fini(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; - - if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) - guc_exec_queue_add_msg(q, msg, CLEANUP); - else - __guc_exec_queue_fini(exec_queue_to_guc(q), q); -} - -static int guc_exec_queue_set_priority(struct xe_exec_queue *q, - enum xe_exec_queue_priority priority) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.priority == priority || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.priority = priority; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.timeslice_us == timeslice_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.timeslice_us = timeslice_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, - u32 preempt_timeout_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.preempt_timeout_us == preempt_timeout_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.preempt_timeout_us = preempt_timeout_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_suspend(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; - - if (exec_queue_killed_or_banned_or_wedged(q) || q->guc->suspend_pending) - return -EINVAL; - - q->guc->suspend_pending = true; - guc_exec_queue_add_msg(q, msg, SUSPEND); - - return 0; -} - -static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - - wait_event(q->guc->suspend_wait, !q->guc->suspend_pending || - guc_read_stopped(guc)); -} - -static void guc_exec_queue_resume(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !q->guc->suspend_pending); - - guc_exec_queue_add_msg(q, msg, RESUME); -} - -static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) -{ - return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); -} - -/* - * All of these functions are an abstraction layer which other parts of XE can - * use to trap into the GuC backend. All of these functions, aside from init, - * really shouldn't do much other than trap into the DRM scheduler which - * synchronizes these operations. - */ -static const struct xe_exec_queue_ops guc_exec_queue_ops = { - .init = guc_exec_queue_init, - .kill = guc_exec_queue_kill, - .fini = guc_exec_queue_fini, - .set_priority = guc_exec_queue_set_priority, - .set_timeslice = guc_exec_queue_set_timeslice, - .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, - .suspend = guc_exec_queue_suspend, - .suspend_wait = guc_exec_queue_suspend_wait, - .resume = guc_exec_queue_resume, - .reset_status = guc_exec_queue_reset_status, -}; - -static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - /* Stop scheduling + flush any DRM scheduler operations */ - xe_sched_submission_stop(sched); - - /* Clean up lost G2H + reset engine state */ - if (exec_queue_registered(q)) { - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else if (exec_queue_destroyed(q)) - __guc_exec_queue_fini(guc, q); - } - if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } - atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | - EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | - EXEC_QUEUE_STATE_SUSPENDED, - &q->guc->state); - q->guc->resume_time = 0; - trace_xe_exec_queue_stop(q); - - /* - * Ban any engine (aside from kernel and engines used for VM ops) with a - * started but not complete job or if a job has gone through a GT reset - * more than twice. - */ - if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { - struct xe_sched_job *job = xe_sched_first_pending_job(sched); - bool ban = false; - - if (job) { - if ((xe_sched_job_started(job) && - !xe_sched_job_completed(job)) || - xe_sched_invalidate_job(job, 2)) { - trace_xe_sched_job_ban(job); - ban = true; - } - } else if (xe_exec_queue_is_lr(q) && - (xe_lrc_ring_head(q->lrc[0]) != xe_lrc_ring_tail(q->lrc[0]))) { - ban = true; - } - - if (ban) { - set_exec_queue_banned(q); - xe_guc_exec_queue_trigger_cleanup(q); - } - } -} - -int xe_guc_submit_reset_prepare(struct xe_guc *guc) -{ - int ret; - - /* - * Using an atomic here rather than submission_state.lock as this - * function can be called while holding the CT lock (engine reset - * failure). submission_state.lock needs the CT lock to resubmit jobs. - * Atomic is not ideal, but it works to prevent against concurrent reset - * and releasing any TDRs waiting on guc->submission_state.stopped. - */ - ret = atomic_fetch_or(1, &guc->submission_state.stopped); - smp_wmb(); - wake_up_all(&guc->ct.wq); - - return ret; -} - -void xe_guc_submit_reset_wait(struct xe_guc *guc) -{ - wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || - !guc_read_stopped(guc)); -} - -void xe_guc_submit_stop(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_stop(guc, q); - - mutex_unlock(&guc->submission_state.lock); - - /* - * No one can enter the backend at this point, aside from new engine - * creation which is protected by guc->submission_state.lock. - */ - -} - -static void guc_exec_queue_start(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - if (!exec_queue_killed_or_banned_or_wedged(q)) { - int i; - - trace_xe_exec_queue_resubmit(q); - for (i = 0; i < q->width; ++i) - xe_lrc_set_ring_head(q->lrc[i], q->lrc[i]->ring.tail); - xe_sched_resubmit_jobs(sched); - } - - xe_sched_submission_start(sched); -} - -int xe_guc_submit_start(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - atomic_dec(&guc->submission_state.stopped); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_start(q); - mutex_unlock(&guc->submission_state.lock); - - wake_up_all(&guc->ct.wq); - - return 0; -} - -static struct xe_exec_queue * -g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - - if (unlikely(guc_id >= GUC_ID_MAX)) { - drm_err(&xe->drm, "Invalid guc_id %u", guc_id); - return NULL; - } - - q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); - if (unlikely(!q)) { - drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id); - return NULL; - } - - xe_assert(xe, guc_id >= q->guc->id); - xe_assert(xe, guc_id < (q->guc->id + q->width)); - - return q; -} - -static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); -} - -static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, - u32 runnable_state) -{ - trace_xe_exec_queue_scheduling_done(q); - - if (runnable_state == 1) { - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); - - q->guc->resume_time = ktime_get(); - clear_exec_queue_pending_enable(q); - smp_wmb(); - wake_up_all(&guc->ct.wq); - } else { - bool check_timeout = exec_queue_check_timeout(q); - - xe_gt_assert(guc_to_gt(guc), runnable_state == 0); - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); - - clear_exec_queue_pending_disable(q); - if (q->guc->suspend_pending) { - suspend_fence_signal(q); - } else { - if (exec_queue_banned(q) || check_timeout) { - smp_wmb(); - wake_up_all(&guc->ct.wq); - } - if (!check_timeout) - deregister_exec_queue(guc, q); - } - } -} - -int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - u32 runnable_state = msg[1]; - - if (unlikely(len < 2)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (unlikely(!exec_queue_pending_enable(q) && - !exec_queue_pending_disable(q))) { - xe_gt_err(guc_to_gt(guc), - "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", - atomic_read(&q->guc->state), q->guc->id, - runnable_state); - return -EPROTO; - } - - handle_sched_done(guc, q, runnable_state); - - return 0; -} - -static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) -{ - trace_xe_exec_queue_deregister_done(q); - - clear_exec_queue_registered(q); - - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else - __guc_exec_queue_fini(guc, q); -} - -int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || - exec_queue_pending_enable(q) || exec_queue_enabled(q)) { - xe_gt_err(guc_to_gt(guc), - "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", - atomic_read(&q->guc->state), q->guc->id); - return -EPROTO; - } - - handle_deregister_done(guc, q); - - return 0; -} - -int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - /* FIXME: Do error capture, most likely async */ - - trace_xe_exec_queue_reset(q); - - /* - * A banned engine is a NOP at this point (came from - * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel - * jobs by setting timeout of the job to the minimum value kicking - * guc_exec_queue_timedout_job. - */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, - u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_dbg(gt, "Engine memory cat error: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - trace_xe_exec_queue_memory_cat_error(q); - - /* Treat the same as engine reset */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - u8 guc_class, instance; - u32 reason; - - if (unlikely(len != 3)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - guc_class = msg[0]; - instance = msg[1]; - reason = msg[2]; - - /* Unexpected failure of a hardware feature, log an actual error */ - drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X", - guc_class, instance, reason); - - xe_gt_reset_async(guc_to_gt(guc)); - - return 0; -} - -static void -guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, - struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - int i; - - snapshot->guc.wqi_head = q->guc->wqi_head; - snapshot->guc.wqi_tail = q->guc->wqi_tail; - snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); - snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); - snapshot->parallel.wq_desc.status = parallel_read(xe, map, - wq_desc.wq_status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - snapshot->parallel.wq[i / sizeof(u32)] = - parallel_read(xe, map, wq[i / sizeof(u32)]); - } -} - -static void -guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", - snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); - drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", - snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); - drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), - snapshot->parallel.wq[i / sizeof(u32)]); - } -} - -/** - * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. - * @q: faulty exec queue - * - * This can be printed out in a later stage like during dev_coredump - * analysis. - * - * Returns: a GuC Submit Engine snapshot object that must be freed by the - * caller, using `xe_guc_exec_queue_snapshot_free`. - */ -struct xe_guc_submit_exec_queue_snapshot * -xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc_submit_exec_queue_snapshot *snapshot; - int i; - - snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); - - if (!snapshot) - return NULL; - - snapshot->guc.id = q->guc->id; - memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); - snapshot->class = q->class; - snapshot->logical_mask = q->logical_mask; - snapshot->width = q->width; - snapshot->refcount = kref_read(&q->refcount); - snapshot->sched_timeout = sched->base.timeout; - snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; - snapshot->sched_props.preempt_timeout_us = - q->sched_props.preempt_timeout_us; - - snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *), - GFP_ATOMIC); - - if (snapshot->lrc) { - for (i = 0; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); - } - } - - snapshot->schedule_state = atomic_read(&q->guc->state); - snapshot->exec_queue_flags = q->flags; - - snapshot->parallel_execution = xe_exec_queue_is_parallel(q); - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_capture(q, snapshot); - - spin_lock(&sched->base.job_list_lock); - snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); - snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, - sizeof(struct pending_list_snapshot), - GFP_ATOMIC); - - if (snapshot->pending_list) { - struct xe_sched_job *job_iter; - - i = 0; - list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) { - snapshot->pending_list[i].seqno = - xe_sched_job_seqno(job_iter); - snapshot->pending_list[i].fence = - dma_fence_is_signaled(job_iter->fence) ? 1 : 0; - snapshot->pending_list[i].finished = - dma_fence_is_signaled(&job_iter->drm.s_fence->finished) - ? 1 : 0; - i++; - } - } - - spin_unlock(&sched->base.job_list_lock); - - return snapshot; -} - -/** - * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. - * @snapshot: Previously captured snapshot of job. - * - * This captures some data that requires taking some locks, so it cannot be done in signaling path. - */ -void -xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot || !snapshot->lrc) - return; - - for (i = 0; i < snapshot->width; ++i) - xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); -} - -/** - * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * @p: drm_printer where it will be printed out. - * - * This function prints out a given GuC Submit Engine snapshot object. - */ -void -xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - if (!snapshot) - return; - - drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id); - drm_printf(p, "\tName: %s\n", snapshot->name); - drm_printf(p, "\tClass: %d\n", snapshot->class); - drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); - drm_printf(p, "\tWidth: %d\n", snapshot->width); - drm_printf(p, "\tRef: %d\n", snapshot->refcount); - drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); - drm_printf(p, "\tTimeslice: %u (us)\n", - snapshot->sched_props.timeslice_us); - drm_printf(p, "\tPreempt timeout: %u (us)\n", - snapshot->sched_props.preempt_timeout_us); - - for (i = 0; snapshot->lrc && i < snapshot->width; ++i) - xe_lrc_snapshot_print(snapshot->lrc[i], p); - - drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); - drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); - - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_print(snapshot, p); - - for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; - i++) - drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", - snapshot->pending_list[i].seqno, - snapshot->pending_list[i].fence, - snapshot->pending_list[i].finished); -} - -/** - * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given - * snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * - * This function free all the memory that needed to be allocated at capture - * time. - */ -void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot) - return; - - if (snapshot->lrc) { - for (i = 0; i < snapshot->width; i++) - xe_lrc_snapshot_free(snapshot->lrc[i]); - kfree(snapshot->lrc); - } - kfree(snapshot->pending_list); - kfree(snapshot); -} - -static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) -{ - struct xe_guc_submit_exec_queue_snapshot *snapshot; - - snapshot = xe_guc_exec_queue_snapshot_capture(q); - xe_guc_exec_queue_snapshot_print(snapshot, p); - xe_guc_exec_queue_snapshot_free(snapshot); -} - -/** - * xe_guc_submit_print - GuC Submit Print. - * @guc: GuC. - * @p: drm_printer where it will be printed out. - * - * This function capture and prints snapshots of **all** GuC Engines. - */ -void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) -{ - struct xe_exec_queue *q; - unsigned long index; - - if (!xe_device_uc_enabled(guc_to_xe(guc))) - return; - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_print(q, p); - mutex_unlock(&guc->submission_state.lock); -} diff --git a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/postimage.2 b/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/postimage.2 deleted file mode 100644 index 436faff09bac..000000000000 --- a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/postimage.2 +++ /dev/null @@ -1,377 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_sync.h" - -#include <linux/dma-fence-array.h> -#include <linux/kthread.h> -#include <linux/sched/mm.h> -#include <linux/uaccess.h> - -#include <drm/drm_print.h> -#include <drm/drm_syncobj.h> -#include <drm/xe_drm.h> - -#include "xe_device_types.h" -#include "xe_exec_queue.h" -#include "xe_macros.h" -#include "xe_sched_job_types.h" - -struct xe_user_fence { - struct xe_device *xe; - struct kref refcount; - struct dma_fence_cb cb; - struct work_struct worker; - struct mm_struct *mm; - u64 __user *addr; - u64 value; - int signalled; -}; - -static void user_fence_destroy(struct kref *kref) -{ - struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, - refcount); - - mmdrop(ufence->mm); - kfree(ufence); -} - -static void user_fence_get(struct xe_user_fence *ufence) -{ - kref_get(&ufence->refcount); -} - -static void user_fence_put(struct xe_user_fence *ufence) -{ - kref_put(&ufence->refcount, user_fence_destroy); -} - -static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, - u64 value) -{ - struct xe_user_fence *ufence; - u64 __user *ptr = u64_to_user_ptr(addr); - - if (!access_ok(ptr, sizeof(*ptr))) - return ERR_PTR(-EFAULT); - - ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); - if (!ufence) - return ERR_PTR(-ENOMEM); - - ufence->xe = xe; - kref_init(&ufence->refcount); - ufence->addr = ptr; - ufence->value = value; - ufence->mm = current->mm; - mmgrab(ufence->mm); - - return ufence; -} - -static void user_fence_worker(struct work_struct *w) -{ - struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); - - if (mmget_not_zero(ufence->mm)) { - kthread_use_mm(ufence->mm); - if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) - XE_WARN_ON("Copy to user failed"); - kthread_unuse_mm(ufence->mm); - mmput(ufence->mm); - } - - wake_up_all(&ufence->xe->ufence_wq); - WRITE_ONCE(ufence->signalled, 1); - user_fence_put(ufence); -} - -static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) -{ - INIT_WORK(&ufence->worker, user_fence_worker); - queue_work(ufence->xe->ordered_wq, &ufence->worker); - dma_fence_put(fence); -} - -static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); - - kick_ufence(ufence, fence); -} - -int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, - struct xe_sync_entry *sync, - struct drm_xe_sync __user *sync_user, - unsigned int flags) -{ - struct drm_xe_sync sync_in; - int err; - bool exec = flags & SYNC_PARSE_FLAG_EXEC; - bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; - bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; - bool signal; - - if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || - XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) - return -EINVAL; - - signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; - switch (sync_in.type) { - case DRM_XE_SYNC_TYPE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (!signal) { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - } - break; - - case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (signal) { - sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; - } else { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - - err = dma_fence_chain_find_seqno(&sync->fence, - sync_in.timeline_value); - if (err) - return err; - } - break; - - case DRM_XE_SYNC_TYPE_USER_FENCE: - if (XE_IOCTL_DBG(xe, disallow_user_fence)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, !signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) - return -EINVAL; - - if (exec) { - sync->addr = sync_in.addr; - } else { - sync->ufence = user_fence_create(xe, sync_in.addr, - sync_in.timeline_value); - if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) - return PTR_ERR(sync->ufence); - } - - break; - - default: - return -EINVAL; - } - - sync->type = sync_in.type; - sync->flags = sync_in.flags; - sync->timeline_value = sync_in.timeline_value; - - return 0; -} - -int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) -{ - if (sync->fence) - return drm_sched_job_add_dependency(&job->drm, - dma_fence_get(sync->fence)); - - return 0; -} - -void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) -{ - if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) - return; - - if (sync->chain_fence) { - drm_syncobj_add_point(sync->syncobj, sync->chain_fence, - fence, sync->timeline_value); - /* - * The chain's ownership is transferred to the - * timeline. - */ - sync->chain_fence = NULL; - } else if (sync->syncobj) { - drm_syncobj_replace_fence(sync->syncobj, fence); - } else if (sync->ufence) { - int err; - - dma_fence_get(fence); - user_fence_get(sync->ufence); - err = dma_fence_add_callback(fence, &sync->ufence->cb, - user_fence_cb); - if (err == -ENOENT) { - kick_ufence(sync->ufence, fence); - } else if (err) { - XE_WARN_ON("failed to add user fence"); - user_fence_put(sync->ufence); - dma_fence_put(fence); - } - } -} - -void xe_sync_entry_cleanup(struct xe_sync_entry *sync) -{ - if (sync->syncobj) - drm_syncobj_put(sync->syncobj); - dma_fence_put(sync->fence); - dma_fence_chain_free(sync->chain_fence); - if (sync->ufence) - user_fence_put(sync->ufence); -} - -/** - * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM - * @sync: input syncs - * @num_sync: number of syncs - * @q: exec queue - * @vm: VM - * - * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create - * and return a composite fence of all in-fences + last fence. If no in-fences - * return last fence on input exec queue. Caller must drop reference to - * returned fence. - * - * Return: fence on success, ERR_PTR(-ENOMEM) on failure - */ -struct dma_fence * -xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, - struct xe_exec_queue *q, struct xe_vm *vm) -{ - struct dma_fence **fences = NULL; - struct dma_fence_array *cf = NULL; - struct dma_fence *fence; - int i, num_in_fence = 0, current_fence = 0; - - lockdep_assert_held(&vm->lock); - - /* Count in-fences */ - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - ++num_in_fence; - fence = sync[i].fence; - } - } - - /* Easy case... */ - if (!num_in_fence) { - fence = xe_exec_queue_last_fence_get(q, vm); - return fence; - } - - /* Create composite fence */ - fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); - if (!fences) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - dma_fence_get(sync[i].fence); - fences[current_fence++] = sync[i].fence; - } - } - fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); - cf = dma_fence_array_create(num_in_fence, fences, - vm->composite_fence_ctx, - vm->composite_fence_seqno++, - false); - if (!cf) { - --vm->composite_fence_seqno; - goto err_out; - } - - return &cf->base; - -err_out: - while (current_fence) - dma_fence_put(fences[--current_fence]); - kfree(fences); - kfree(cf); - - return ERR_PTR(-ENOMEM); -} - -/** - * __xe_sync_ufence_get() - Get user fence from user fence - * @ufence: input user fence - * - * Get a user fence reference from user fence - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) -{ - user_fence_get(ufence); - - return ufence; -} - -/** - * xe_sync_ufence_get() - Get user fence from sync - * @sync: input sync - * - * Get a user fence reference from sync. - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) -{ - user_fence_get(sync->ufence); - - return sync->ufence; -} - -/** - * xe_sync_ufence_put() - Put user fence reference - * @ufence: user fence reference - * - */ -void xe_sync_ufence_put(struct xe_user_fence *ufence) -{ - user_fence_put(ufence); -} - -/** - * xe_sync_ufence_get_status() - Get user fence status - * @ufence: user fence - * - * Return: 1 if signalled, 0 not signalled, <0 on error - */ -int xe_sync_ufence_get_status(struct xe_user_fence *ufence) -{ - return READ_ONCE(ufence->signalled); -} diff --git a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage b/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage deleted file mode 100644 index c21acad49d14..000000000000 --- a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage +++ /dev/null @@ -1,384 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_sync.h" - -#include <linux/dma-fence-array.h> -#include <linux/kthread.h> -#include <linux/sched/mm.h> -#include <linux/uaccess.h> - -#include <drm/drm_print.h> -#include <drm/drm_syncobj.h> -#include <drm/xe_drm.h> - -#include "xe_device_types.h" -#include "xe_exec_queue.h" -#include "xe_macros.h" -#include "xe_sched_job_types.h" - -struct xe_user_fence { - struct xe_device *xe; - struct kref refcount; - struct dma_fence_cb cb; - struct work_struct worker; - struct mm_struct *mm; - u64 __user *addr; - u64 value; - int signalled; -}; - -static void user_fence_destroy(struct kref *kref) -{ - struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, - refcount); - - mmdrop(ufence->mm); - kfree(ufence); -} - -static void user_fence_get(struct xe_user_fence *ufence) -{ - kref_get(&ufence->refcount); -} - -static void user_fence_put(struct xe_user_fence *ufence) -{ - kref_put(&ufence->refcount, user_fence_destroy); -} - -static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, - u64 value) -{ - struct xe_user_fence *ufence; - u64 __user *ptr = u64_to_user_ptr(addr); - - if (!access_ok(ptr, sizeof(*ptr))) - return ERR_PTR(-EFAULT); - - ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); - if (!ufence) - return ERR_PTR(-ENOMEM); - - ufence->xe = xe; - kref_init(&ufence->refcount); - ufence->addr = ptr; - ufence->value = value; - ufence->mm = current->mm; - mmgrab(ufence->mm); - - return ufence; -} - -static void user_fence_worker(struct work_struct *w) -{ - struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); - - if (mmget_not_zero(ufence->mm)) { - kthread_use_mm(ufence->mm); - if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) - XE_WARN_ON("Copy to user failed"); - kthread_unuse_mm(ufence->mm); - mmput(ufence->mm); - } - - wake_up_all(&ufence->xe->ufence_wq); - WRITE_ONCE(ufence->signalled, 1); - user_fence_put(ufence); -} - -static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) -{ - INIT_WORK(&ufence->worker, user_fence_worker); - queue_work(ufence->xe->ordered_wq, &ufence->worker); - dma_fence_put(fence); -} - -static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); - - kick_ufence(ufence, fence); -} - -int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, - struct xe_sync_entry *sync, - struct drm_xe_sync __user *sync_user, - unsigned int flags) -{ - struct drm_xe_sync sync_in; - int err; - bool exec = flags & SYNC_PARSE_FLAG_EXEC; - bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; - bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; - bool signal; - - if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || - XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) - return -EINVAL; - - signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; - switch (sync_in.type) { - case DRM_XE_SYNC_TYPE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (!signal) { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - } - break; - - case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (signal) { - sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; - } else { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - - err = dma_fence_chain_find_seqno(&sync->fence, - sync_in.timeline_value); - if (err) - return err; - } - break; - - case DRM_XE_SYNC_TYPE_USER_FENCE: - if (XE_IOCTL_DBG(xe, disallow_user_fence)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, !signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) - return -EINVAL; - - if (exec) { - sync->addr = sync_in.addr; - } else { - sync->ufence = user_fence_create(xe, sync_in.addr, - sync_in.timeline_value); - if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) - return PTR_ERR(sync->ufence); - } - - break; - - default: - return -EINVAL; - } - - sync->type = sync_in.type; - sync->flags = sync_in.flags; - sync->timeline_value = sync_in.timeline_value; - - return 0; -} - -int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) -{ - if (sync->fence) - return drm_sched_job_add_dependency(&job->drm, - dma_fence_get(sync->fence)); - - return 0; -} - -void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) -{ - if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) - return; - - if (sync->chain_fence) { - drm_syncobj_add_point(sync->syncobj, sync->chain_fence, - fence, sync->timeline_value); - /* - * The chain's ownership is transferred to the - * timeline. - */ - sync->chain_fence = NULL; - } else if (sync->syncobj) { - drm_syncobj_replace_fence(sync->syncobj, fence); - } else if (sync->ufence) { - int err; - - dma_fence_get(fence); - user_fence_get(sync->ufence); - err = dma_fence_add_callback(fence, &sync->ufence->cb, - user_fence_cb); - if (err == -ENOENT) { - kick_ufence(sync->ufence, fence); - } else if (err) { - XE_WARN_ON("failed to add user fence"); - user_fence_put(sync->ufence); - dma_fence_put(fence); - } - } -} - -void xe_sync_entry_cleanup(struct xe_sync_entry *sync) -{ - if (sync->syncobj) - drm_syncobj_put(sync->syncobj); -<<<<<<< - dma_fence_put(sync->fence); - dma_fence_chain_free(sync->chain_fence); -======= - if (sync->fence) - dma_fence_put(sync->fence); - if (sync->chain_fence) - dma_fence_chain_free(sync->chain_fence); ->>>>>>> - if (sync->ufence) - user_fence_put(sync->ufence); -} - -/** - * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM - * @sync: input syncs - * @num_sync: number of syncs - * @q: exec queue - * @vm: VM - * - * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create - * and return a composite fence of all in-fences + last fence. If no in-fences - * return last fence on input exec queue. Caller must drop reference to - * returned fence. - * - * Return: fence on success, ERR_PTR(-ENOMEM) on failure - */ -struct dma_fence * -xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, - struct xe_exec_queue *q, struct xe_vm *vm) -{ - struct dma_fence **fences = NULL; - struct dma_fence_array *cf = NULL; - struct dma_fence *fence; - int i, num_in_fence = 0, current_fence = 0; - - lockdep_assert_held(&vm->lock); - - /* Count in-fences */ - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - ++num_in_fence; - fence = sync[i].fence; - } - } - - /* Easy case... */ - if (!num_in_fence) { - fence = xe_exec_queue_last_fence_get(q, vm); - return fence; - } - - /* Create composite fence */ - fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); - if (!fences) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - dma_fence_get(sync[i].fence); - fences[current_fence++] = sync[i].fence; - } - } - fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); - cf = dma_fence_array_create(num_in_fence, fences, - vm->composite_fence_ctx, - vm->composite_fence_seqno++, - false); - if (!cf) { - --vm->composite_fence_seqno; - goto err_out; - } - - return &cf->base; - -err_out: - while (current_fence) - dma_fence_put(fences[--current_fence]); - kfree(fences); - kfree(cf); - - return ERR_PTR(-ENOMEM); -} - -/** - * __xe_sync_ufence_get() - Get user fence from user fence - * @ufence: input user fence - * - * Get a user fence reference from user fence - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) -{ - user_fence_get(ufence); - - return ufence; -} - -/** - * xe_sync_ufence_get() - Get user fence from sync - * @sync: input sync - * - * Get a user fence reference from sync. - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) -{ - user_fence_get(sync->ufence); - - return sync->ufence; -} - -/** - * xe_sync_ufence_put() - Put user fence reference - * @ufence: user fence reference - * - */ -void xe_sync_ufence_put(struct xe_user_fence *ufence) -{ - user_fence_put(ufence); -} - -/** - * xe_sync_ufence_get_status() - Get user fence status - * @ufence: user fence - * - * Return: 1 if signalled, 0 not signalled, <0 on error - */ -int xe_sync_ufence_get_status(struct xe_user_fence *ufence) -{ - return READ_ONCE(ufence->signalled); -} diff --git a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.1 b/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.1 deleted file mode 100644 index c21acad49d14..000000000000 --- a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.1 +++ /dev/null @@ -1,384 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_sync.h" - -#include <linux/dma-fence-array.h> -#include <linux/kthread.h> -#include <linux/sched/mm.h> -#include <linux/uaccess.h> - -#include <drm/drm_print.h> -#include <drm/drm_syncobj.h> -#include <drm/xe_drm.h> - -#include "xe_device_types.h" -#include "xe_exec_queue.h" -#include "xe_macros.h" -#include "xe_sched_job_types.h" - -struct xe_user_fence { - struct xe_device *xe; - struct kref refcount; - struct dma_fence_cb cb; - struct work_struct worker; - struct mm_struct *mm; - u64 __user *addr; - u64 value; - int signalled; -}; - -static void user_fence_destroy(struct kref *kref) -{ - struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, - refcount); - - mmdrop(ufence->mm); - kfree(ufence); -} - -static void user_fence_get(struct xe_user_fence *ufence) -{ - kref_get(&ufence->refcount); -} - -static void user_fence_put(struct xe_user_fence *ufence) -{ - kref_put(&ufence->refcount, user_fence_destroy); -} - -static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, - u64 value) -{ - struct xe_user_fence *ufence; - u64 __user *ptr = u64_to_user_ptr(addr); - - if (!access_ok(ptr, sizeof(*ptr))) - return ERR_PTR(-EFAULT); - - ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); - if (!ufence) - return ERR_PTR(-ENOMEM); - - ufence->xe = xe; - kref_init(&ufence->refcount); - ufence->addr = ptr; - ufence->value = value; - ufence->mm = current->mm; - mmgrab(ufence->mm); - - return ufence; -} - -static void user_fence_worker(struct work_struct *w) -{ - struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); - - if (mmget_not_zero(ufence->mm)) { - kthread_use_mm(ufence->mm); - if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) - XE_WARN_ON("Copy to user failed"); - kthread_unuse_mm(ufence->mm); - mmput(ufence->mm); - } - - wake_up_all(&ufence->xe->ufence_wq); - WRITE_ONCE(ufence->signalled, 1); - user_fence_put(ufence); -} - -static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) -{ - INIT_WORK(&ufence->worker, user_fence_worker); - queue_work(ufence->xe->ordered_wq, &ufence->worker); - dma_fence_put(fence); -} - -static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); - - kick_ufence(ufence, fence); -} - -int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, - struct xe_sync_entry *sync, - struct drm_xe_sync __user *sync_user, - unsigned int flags) -{ - struct drm_xe_sync sync_in; - int err; - bool exec = flags & SYNC_PARSE_FLAG_EXEC; - bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; - bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; - bool signal; - - if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || - XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) - return -EINVAL; - - signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; - switch (sync_in.type) { - case DRM_XE_SYNC_TYPE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (!signal) { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - } - break; - - case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (signal) { - sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; - } else { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - - err = dma_fence_chain_find_seqno(&sync->fence, - sync_in.timeline_value); - if (err) - return err; - } - break; - - case DRM_XE_SYNC_TYPE_USER_FENCE: - if (XE_IOCTL_DBG(xe, disallow_user_fence)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, !signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) - return -EINVAL; - - if (exec) { - sync->addr = sync_in.addr; - } else { - sync->ufence = user_fence_create(xe, sync_in.addr, - sync_in.timeline_value); - if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) - return PTR_ERR(sync->ufence); - } - - break; - - default: - return -EINVAL; - } - - sync->type = sync_in.type; - sync->flags = sync_in.flags; - sync->timeline_value = sync_in.timeline_value; - - return 0; -} - -int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) -{ - if (sync->fence) - return drm_sched_job_add_dependency(&job->drm, - dma_fence_get(sync->fence)); - - return 0; -} - -void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) -{ - if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) - return; - - if (sync->chain_fence) { - drm_syncobj_add_point(sync->syncobj, sync->chain_fence, - fence, sync->timeline_value); - /* - * The chain's ownership is transferred to the - * timeline. - */ - sync->chain_fence = NULL; - } else if (sync->syncobj) { - drm_syncobj_replace_fence(sync->syncobj, fence); - } else if (sync->ufence) { - int err; - - dma_fence_get(fence); - user_fence_get(sync->ufence); - err = dma_fence_add_callback(fence, &sync->ufence->cb, - user_fence_cb); - if (err == -ENOENT) { - kick_ufence(sync->ufence, fence); - } else if (err) { - XE_WARN_ON("failed to add user fence"); - user_fence_put(sync->ufence); - dma_fence_put(fence); - } - } -} - -void xe_sync_entry_cleanup(struct xe_sync_entry *sync) -{ - if (sync->syncobj) - drm_syncobj_put(sync->syncobj); -<<<<<<< - dma_fence_put(sync->fence); - dma_fence_chain_free(sync->chain_fence); -======= - if (sync->fence) - dma_fence_put(sync->fence); - if (sync->chain_fence) - dma_fence_chain_free(sync->chain_fence); ->>>>>>> - if (sync->ufence) - user_fence_put(sync->ufence); -} - -/** - * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM - * @sync: input syncs - * @num_sync: number of syncs - * @q: exec queue - * @vm: VM - * - * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create - * and return a composite fence of all in-fences + last fence. If no in-fences - * return last fence on input exec queue. Caller must drop reference to - * returned fence. - * - * Return: fence on success, ERR_PTR(-ENOMEM) on failure - */ -struct dma_fence * -xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, - struct xe_exec_queue *q, struct xe_vm *vm) -{ - struct dma_fence **fences = NULL; - struct dma_fence_array *cf = NULL; - struct dma_fence *fence; - int i, num_in_fence = 0, current_fence = 0; - - lockdep_assert_held(&vm->lock); - - /* Count in-fences */ - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - ++num_in_fence; - fence = sync[i].fence; - } - } - - /* Easy case... */ - if (!num_in_fence) { - fence = xe_exec_queue_last_fence_get(q, vm); - return fence; - } - - /* Create composite fence */ - fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); - if (!fences) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - dma_fence_get(sync[i].fence); - fences[current_fence++] = sync[i].fence; - } - } - fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); - cf = dma_fence_array_create(num_in_fence, fences, - vm->composite_fence_ctx, - vm->composite_fence_seqno++, - false); - if (!cf) { - --vm->composite_fence_seqno; - goto err_out; - } - - return &cf->base; - -err_out: - while (current_fence) - dma_fence_put(fences[--current_fence]); - kfree(fences); - kfree(cf); - - return ERR_PTR(-ENOMEM); -} - -/** - * __xe_sync_ufence_get() - Get user fence from user fence - * @ufence: input user fence - * - * Get a user fence reference from user fence - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) -{ - user_fence_get(ufence); - - return ufence; -} - -/** - * xe_sync_ufence_get() - Get user fence from sync - * @sync: input sync - * - * Get a user fence reference from sync. - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) -{ - user_fence_get(sync->ufence); - - return sync->ufence; -} - -/** - * xe_sync_ufence_put() - Put user fence reference - * @ufence: user fence reference - * - */ -void xe_sync_ufence_put(struct xe_user_fence *ufence) -{ - user_fence_put(ufence); -} - -/** - * xe_sync_ufence_get_status() - Get user fence status - * @ufence: user fence - * - * Return: 1 if signalled, 0 not signalled, <0 on error - */ -int xe_sync_ufence_get_status(struct xe_user_fence *ufence) -{ - return READ_ONCE(ufence->signalled); -} diff --git a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.2 b/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.2 deleted file mode 100644 index c21acad49d14..000000000000 --- a/rr-cache/eff95ddb428389e7f45eee42234372282a8ebd37/preimage.2 +++ /dev/null @@ -1,384 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_sync.h" - -#include <linux/dma-fence-array.h> -#include <linux/kthread.h> -#include <linux/sched/mm.h> -#include <linux/uaccess.h> - -#include <drm/drm_print.h> -#include <drm/drm_syncobj.h> -#include <drm/xe_drm.h> - -#include "xe_device_types.h" -#include "xe_exec_queue.h" -#include "xe_macros.h" -#include "xe_sched_job_types.h" - -struct xe_user_fence { - struct xe_device *xe; - struct kref refcount; - struct dma_fence_cb cb; - struct work_struct worker; - struct mm_struct *mm; - u64 __user *addr; - u64 value; - int signalled; -}; - -static void user_fence_destroy(struct kref *kref) -{ - struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, - refcount); - - mmdrop(ufence->mm); - kfree(ufence); -} - -static void user_fence_get(struct xe_user_fence *ufence) -{ - kref_get(&ufence->refcount); -} - -static void user_fence_put(struct xe_user_fence *ufence) -{ - kref_put(&ufence->refcount, user_fence_destroy); -} - -static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, - u64 value) -{ - struct xe_user_fence *ufence; - u64 __user *ptr = u64_to_user_ptr(addr); - - if (!access_ok(ptr, sizeof(*ptr))) - return ERR_PTR(-EFAULT); - - ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); - if (!ufence) - return ERR_PTR(-ENOMEM); - - ufence->xe = xe; - kref_init(&ufence->refcount); - ufence->addr = ptr; - ufence->value = value; - ufence->mm = current->mm; - mmgrab(ufence->mm); - - return ufence; -} - -static void user_fence_worker(struct work_struct *w) -{ - struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); - - if (mmget_not_zero(ufence->mm)) { - kthread_use_mm(ufence->mm); - if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) - XE_WARN_ON("Copy to user failed"); - kthread_unuse_mm(ufence->mm); - mmput(ufence->mm); - } - - wake_up_all(&ufence->xe->ufence_wq); - WRITE_ONCE(ufence->signalled, 1); - user_fence_put(ufence); -} - -static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) -{ - INIT_WORK(&ufence->worker, user_fence_worker); - queue_work(ufence->xe->ordered_wq, &ufence->worker); - dma_fence_put(fence); -} - -static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); - - kick_ufence(ufence, fence); -} - -int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, - struct xe_sync_entry *sync, - struct drm_xe_sync __user *sync_user, - unsigned int flags) -{ - struct drm_xe_sync sync_in; - int err; - bool exec = flags & SYNC_PARSE_FLAG_EXEC; - bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; - bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; - bool signal; - - if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || - XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) - return -EINVAL; - - signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; - switch (sync_in.type) { - case DRM_XE_SYNC_TYPE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (!signal) { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - } - break; - - case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: - if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) - return -EINVAL; - - sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); - if (XE_IOCTL_DBG(xe, !sync->syncobj)) - return -ENOENT; - - if (signal) { - sync->chain_fence = dma_fence_chain_alloc(); - if (!sync->chain_fence) - return -ENOMEM; - } else { - sync->fence = drm_syncobj_fence_get(sync->syncobj); - if (XE_IOCTL_DBG(xe, !sync->fence)) - return -EINVAL; - - err = dma_fence_chain_find_seqno(&sync->fence, - sync_in.timeline_value); - if (err) - return err; - } - break; - - case DRM_XE_SYNC_TYPE_USER_FENCE: - if (XE_IOCTL_DBG(xe, disallow_user_fence)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, !signal)) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) - return -EINVAL; - - if (exec) { - sync->addr = sync_in.addr; - } else { - sync->ufence = user_fence_create(xe, sync_in.addr, - sync_in.timeline_value); - if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) - return PTR_ERR(sync->ufence); - } - - break; - - default: - return -EINVAL; - } - - sync->type = sync_in.type; - sync->flags = sync_in.flags; - sync->timeline_value = sync_in.timeline_value; - - return 0; -} - -int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) -{ - if (sync->fence) - return drm_sched_job_add_dependency(&job->drm, - dma_fence_get(sync->fence)); - - return 0; -} - -void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) -{ - if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) - return; - - if (sync->chain_fence) { - drm_syncobj_add_point(sync->syncobj, sync->chain_fence, - fence, sync->timeline_value); - /* - * The chain's ownership is transferred to the - * timeline. - */ - sync->chain_fence = NULL; - } else if (sync->syncobj) { - drm_syncobj_replace_fence(sync->syncobj, fence); - } else if (sync->ufence) { - int err; - - dma_fence_get(fence); - user_fence_get(sync->ufence); - err = dma_fence_add_callback(fence, &sync->ufence->cb, - user_fence_cb); - if (err == -ENOENT) { - kick_ufence(sync->ufence, fence); - } else if (err) { - XE_WARN_ON("failed to add user fence"); - user_fence_put(sync->ufence); - dma_fence_put(fence); - } - } -} - -void xe_sync_entry_cleanup(struct xe_sync_entry *sync) -{ - if (sync->syncobj) - drm_syncobj_put(sync->syncobj); -<<<<<<< - dma_fence_put(sync->fence); - dma_fence_chain_free(sync->chain_fence); -======= - if (sync->fence) - dma_fence_put(sync->fence); - if (sync->chain_fence) - dma_fence_chain_free(sync->chain_fence); ->>>>>>> - if (sync->ufence) - user_fence_put(sync->ufence); -} - -/** - * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM - * @sync: input syncs - * @num_sync: number of syncs - * @q: exec queue - * @vm: VM - * - * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create - * and return a composite fence of all in-fences + last fence. If no in-fences - * return last fence on input exec queue. Caller must drop reference to - * returned fence. - * - * Return: fence on success, ERR_PTR(-ENOMEM) on failure - */ -struct dma_fence * -xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, - struct xe_exec_queue *q, struct xe_vm *vm) -{ - struct dma_fence **fences = NULL; - struct dma_fence_array *cf = NULL; - struct dma_fence *fence; - int i, num_in_fence = 0, current_fence = 0; - - lockdep_assert_held(&vm->lock); - - /* Count in-fences */ - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - ++num_in_fence; - fence = sync[i].fence; - } - } - - /* Easy case... */ - if (!num_in_fence) { - fence = xe_exec_queue_last_fence_get(q, vm); - return fence; - } - - /* Create composite fence */ - fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); - if (!fences) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_sync; ++i) { - if (sync[i].fence) { - dma_fence_get(sync[i].fence); - fences[current_fence++] = sync[i].fence; - } - } - fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); - cf = dma_fence_array_create(num_in_fence, fences, - vm->composite_fence_ctx, - vm->composite_fence_seqno++, - false); - if (!cf) { - --vm->composite_fence_seqno; - goto err_out; - } - - return &cf->base; - -err_out: - while (current_fence) - dma_fence_put(fences[--current_fence]); - kfree(fences); - kfree(cf); - - return ERR_PTR(-ENOMEM); -} - -/** - * __xe_sync_ufence_get() - Get user fence from user fence - * @ufence: input user fence - * - * Get a user fence reference from user fence - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) -{ - user_fence_get(ufence); - - return ufence; -} - -/** - * xe_sync_ufence_get() - Get user fence from sync - * @sync: input sync - * - * Get a user fence reference from sync. - * - * Return: xe_user_fence pointer with reference - */ -struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) -{ - user_fence_get(sync->ufence); - - return sync->ufence; -} - -/** - * xe_sync_ufence_put() - Put user fence reference - * @ufence: user fence reference - * - */ -void xe_sync_ufence_put(struct xe_user_fence *ufence) -{ - user_fence_put(ufence); -} - -/** - * xe_sync_ufence_get_status() - Get user fence status - * @ufence: user fence - * - * Return: 1 if signalled, 0 not signalled, <0 on error - */ -int xe_sync_ufence_get_status(struct xe_user_fence *ufence) -{ - return READ_ONCE(ufence->signalled); -} diff --git a/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage b/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage deleted file mode 100644 index edcbe7fec8d5..000000000000 --- a/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage +++ /dev/null @@ -1,1974 +0,0 @@ -/* - * Copyright 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/delay.h> -#include <linux/firmware.h> -#include <linux/module.h> -#include <linux/pci.h> - -#include "amdgpu.h" -#include "amdgpu_ucode.h" -#include "amdgpu_trace.h" - -#include "gc/gc_10_3_0_offset.h" -#include "gc/gc_10_3_0_sh_mask.h" -#include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h" -#include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h" -#include "ivsrcid/sdma2/irqsrcs_sdma2_5_0.h" -#include "ivsrcid/sdma3/irqsrcs_sdma3_5_0.h" - -#include "soc15_common.h" -#include "soc15.h" -#include "navi10_sdma_pkt_open.h" -#include "nbio_v2_3.h" -#include "sdma_common.h" -#include "sdma_v5_2.h" - -MODULE_FIRMWARE("amdgpu/sienna_cichlid_sdma.bin"); -MODULE_FIRMWARE("amdgpu/navy_flounder_sdma.bin"); -MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_sdma.bin"); -MODULE_FIRMWARE("amdgpu/beige_goby_sdma.bin"); - -MODULE_FIRMWARE("amdgpu/vangogh_sdma.bin"); -MODULE_FIRMWARE("amdgpu/yellow_carp_sdma.bin"); -MODULE_FIRMWARE("amdgpu/sdma_5_2_6.bin"); -MODULE_FIRMWARE("amdgpu/sdma_5_2_7.bin"); - -#define SDMA1_REG_OFFSET 0x600 -#define SDMA3_REG_OFFSET 0x400 -#define SDMA0_HYP_DEC_REG_START 0x5880 -#define SDMA0_HYP_DEC_REG_END 0x5893 -#define SDMA1_HYP_DEC_REG_OFFSET 0x20 - -static const struct amdgpu_hwip_reg_entry sdma_reg_list_5_2[] = { - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_INT_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2) -}; - -static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev); - -static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset) -{ - u32 base; - - if (internal_offset >= SDMA0_HYP_DEC_REG_START && - internal_offset <= SDMA0_HYP_DEC_REG_END) { - base = adev->reg_offset[GC_HWIP][0][1]; - if (instance != 0) - internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance; - } else { - if (instance < 2) { - base = adev->reg_offset[GC_HWIP][0][0]; - if (instance == 1) - internal_offset += SDMA1_REG_OFFSET; - } else { - base = adev->reg_offset[GC_HWIP][0][2]; - if (instance == 3) - internal_offset += SDMA3_REG_OFFSET; - } - } - - return base + internal_offset; -} - -static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring, - uint64_t addr) -{ - unsigned ret; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE)); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, 1); - /* this is the offset we need patch later */ - ret = ring->wptr & ring->buf_mask; - /* insert dummy here and patch it later */ - amdgpu_ring_write(ring, 0); - - return ret; -} - -/** - * sdma_v5_2_ring_get_rptr - get the current read pointer - * - * @ring: amdgpu ring pointer - * - * Get the current rptr from the hardware (NAVI10+). - */ -static uint64_t sdma_v5_2_ring_get_rptr(struct amdgpu_ring *ring) -{ - u64 *rptr; - - /* XXX check if swapping is necessary on BE */ - rptr = (u64 *)ring->rptr_cpu_addr; - - DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr); - return ((*rptr) >> 2); -} - -/** - * sdma_v5_2_ring_get_wptr - get the current write pointer - * - * @ring: amdgpu ring pointer - * - * Get the current wptr from the hardware (NAVI10+). - */ -static uint64_t sdma_v5_2_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - u64 wptr; - - if (ring->use_doorbell) { - /* XXX check if swapping is necessary on BE */ - wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr)); - DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr); - } else { - wptr = RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI)); - wptr = wptr << 32; - wptr |= RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR)); - DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr); - } - - return wptr >> 2; -} - -/** - * sdma_v5_2_ring_set_wptr - commit the write pointer - * - * @ring: amdgpu ring pointer - * - * Write the wptr back to the hardware (NAVI10+). - */ -static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - DRM_DEBUG("Setting write pointer\n"); - if (ring->use_doorbell) { - DRM_DEBUG("Using doorbell -- " - "wptr_offs == 0x%08x " - "lower_32_bits(ring->wptr << 2) == 0x%08x " - "upper_32_bits(ring->wptr << 2) == 0x%08x\n", - ring->wptr_offs, - lower_32_bits(ring->wptr << 2), - upper_32_bits(ring->wptr << 2)); - /* XXX check if swapping is necessary on BE */ - atomic64_set((atomic64_t *)ring->wptr_cpu_addr, - ring->wptr << 2); - DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n", - ring->doorbell_index, ring->wptr << 2); - WDOORBELL64(ring->doorbell_index, ring->wptr << 2); -<<<<<<< - /* SDMA seems to miss doorbells sometimes when powergating kicks in. - * Updating the wptr directly will wake it. This is only safe because - * we disallow gfxoff in begin_use() and then allow it again in end_use(). - */ - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); -======= - if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(5, 2, 1)) { - /* SDMA seems to miss doorbells sometimes when powergating kicks in. - * Updating the wptr directly will wake it. This is only safe because - * we disallow gfxoff in begin_use() and then allow it again in end_use(). - */ - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); - } ->>>>>>> - } else { - DRM_DEBUG("Not using doorbell -- " - "mmSDMA%i_GFX_RB_WPTR == 0x%08x " - "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", - ring->me, - lower_32_bits(ring->wptr << 2), - ring->me, - upper_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); - } -} - -static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) -{ - struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - int i; - - for (i = 0; i < count; i++) - if (sdma && sdma->burst_nop && (i == 0)) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); -} - -/** - * sdma_v5_2_ring_emit_ib - Schedule an IB on the DMA engine - * - * @ring: amdgpu ring pointer - * @job: job to retrieve vmid from - * @ib: IB object to schedule - * @flags: unused - * - * Schedule an IB in the DMA ring. - */ -static void sdma_v5_2_ring_emit_ib(struct amdgpu_ring *ring, - struct amdgpu_job *job, - struct amdgpu_ib *ib, - uint32_t flags) -{ - unsigned vmid = AMDGPU_JOB_GET_VMID(job); - uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid); - - /* An IB packet must end on a 8 DW boundary--the next dword - * must be on a 8-dword boundary. Our IB packet below is 6 - * dwords long, thus add x number of NOPs, such that, in - * modular arithmetic, - * wptr + 6 + x = 8k, k >= 0, which in C is, - * (wptr + 6 + x) % 8 = 0. - * The expression below, is a solution of x. - */ - sdma_v5_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7); - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | - SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf)); - /* base must be 32 byte aligned */ - amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0); - amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); - amdgpu_ring_write(ring, ib->length_dw); - amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr)); - amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr)); -} - -/** - * sdma_v5_2_ring_emit_mem_sync - flush the IB by graphics cache rinse - * - * @ring: amdgpu ring pointer - * - * flush the IB by graphics cache rinse. - */ -static void sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring *ring) -{ - uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | - SDMA_GCR_GLM_INV | SDMA_GCR_GL1_INV | - SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV | - SDMA_GCR_GLI_INV(1); - - /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) | - SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) | - SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) | - SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0)); -} - -/** - * sdma_v5_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring - * - * @ring: amdgpu ring pointer - * - * Emit an hdp flush packet on the requested DMA ring. - */ -static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - u32 ref_and_mask = 0; - const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; - - if (ring->me > 1) { - amdgpu_asic_flush_hdp(adev, ring); - } else { - ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */ - amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2); - amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2); - amdgpu_ring_write(ring, ref_and_mask); /* reference */ - amdgpu_ring_write(ring, ref_and_mask); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ - } -} - -/** - * sdma_v5_2_ring_emit_fence - emit a fence on the DMA ring - * - * @ring: amdgpu ring pointer - * @addr: address - * @seq: sequence number - * @flags: fence related flags - * - * Add a DMA fence packet to the ring to write - * the fence seq number and DMA trap packet to generate - * an interrupt if needed. - */ -static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, - unsigned flags) -{ - bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; - /* write the fence */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | - SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ - /* zero in first two bits */ - BUG_ON(addr & 0x3); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, lower_32_bits(seq)); - - /* optionally write high bits as well */ - if (write64bit) { - addr += 4; - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | - SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); - /* zero in first two bits */ - BUG_ON(addr & 0x3); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(seq)); - } - - if ((flags & AMDGPU_FENCE_FLAG_INT)) { - uint32_t ctx = ring->is_mes_queue ? - (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0; - /* generate an interrupt */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP)); - amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx)); - } -} - - -/** - * sdma_v5_2_gfx_stop - stop the gfx async dma engines - * - * @adev: amdgpu_device pointer - * - * Stop the gfx async dma ring buffers. - */ -static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev) -{ - u32 rb_cntl, ib_cntl; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL)); - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl); - } -} - -/** - * sdma_v5_2_rlc_stop - stop the compute async dma engines - * - * @adev: amdgpu_device pointer - * - * Stop the compute async dma queues. - */ -static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev) -{ - /* XXX todo */ -} - -/** - * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch - * - * @adev: amdgpu_device pointer - * @enable: enable/disable the DMA MEs context switch. - * - * Halt or unhalt the async dma engines context switch. - */ -static void sdma_v5_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable) -{ - u32 f32_cntl, phase_quantum = 0; - int i; - - if (amdgpu_sdma_phase_quantum) { - unsigned value = amdgpu_sdma_phase_quantum; - unsigned unit = 0; - - while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> - SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) { - value = (value + 1) >> 1; - unit++; - } - if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> - SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) { - value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> - SDMA0_PHASE0_QUANTUM__VALUE__SHIFT); - unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> - SDMA0_PHASE0_QUANTUM__UNIT__SHIFT); - WARN_ONCE(1, - "clamping sdma_phase_quantum to %uK clock cycles\n", - value << unit); - } - phase_quantum = - value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT | - unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT; - } - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (enable && amdgpu_sdma_phase_quantum) { - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM), - phase_quantum); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM), - phase_quantum); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM), - phase_quantum); - } - - if (!amdgpu_sriov_vf(adev)) { - f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL)); - f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, - AUTO_CTXSW_ENABLE, enable ? 1 : 0); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl); - } - } - -} - -/** - * sdma_v5_2_enable - stop the async dma engines - * - * @adev: amdgpu_device pointer - * @enable: enable/disable the DMA MEs. - * - * Halt or unhalt the async dma engines. - */ -static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable) -{ - u32 f32_cntl; - int i; - - if (!enable) { - sdma_v5_2_gfx_stop(adev); - sdma_v5_2_rlc_stop(adev); - } - - if (!amdgpu_sriov_vf(adev)) { - for (i = 0; i < adev->sdma.num_instances; i++) { - f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL)); - f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl); - } - } -} - -/** - * sdma_v5_2_gfx_resume - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the gfx DMA ring buffers and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev) -{ - struct amdgpu_ring *ring; - u32 rb_cntl, ib_cntl; - u32 rb_bufsz; - u32 doorbell; - u32 doorbell_offset; - u32 temp; - u32 wptr_poll_cntl; - u64 wptr_gpu_addr; - int i, r; - - for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; - - if (!amdgpu_sriov_vf(adev)) - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0); - - /* Set ring buffer size in dwords */ - rb_bufsz = order_base_2(ring->ring_size / 4); - rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz); -#ifdef __BIG_ENDIAN - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, - RPTR_WRITEBACK_SWAP_ENABLE, 1); -#endif - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0); - - /* setup the wptr shadow polling */ - wptr_gpu_addr = ring->wptr_gpu_addr; - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO), - lower_32_bits(wptr_gpu_addr)); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI), - upper_32_bits(wptr_gpu_addr)); - wptr_poll_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, - mmSDMA0_GFX_RB_WPTR_POLL_CNTL)); - wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, - SDMA0_GFX_RB_WPTR_POLL_CNTL, - F32_POLL_ENABLE, 1); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), - wptr_poll_cntl); - - /* set the wb address whether it's enabled or not */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI), - upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO), - lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC); - - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1); - - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40); - - ring->wptr = 0; - - /* before programing wptr to a less value, need set minor_ptr_update first */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1); - - if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */ - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2)); - } - - doorbell = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL)); - doorbell_offset = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET)); - - if (ring->use_doorbell) { - doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1); - doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET, - OFFSET, ring->doorbell_index); - } else { - doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0); - } - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset); - - adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell, - ring->doorbell_index, - adev->doorbell_index.sdma_doorbell_range); - - if (amdgpu_sriov_vf(adev)) - sdma_v5_2_ring_set_wptr(ring); - - /* set minor_ptr_update to 0 after wptr programed */ - - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0); - - /* SRIOV VF has no control of any of registers below */ - if (!amdgpu_sriov_vf(adev)) { - /* set utc l1 enable flag always to 1 */ - temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1); - - /* enable MCBP */ - temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), temp); - - /* Set up RESP_MODE to non-copy addresses */ - temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3); - temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp); - - /* program default cache read and write policy */ - temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE)); - /* clean read policy and write policy bits */ - temp &= 0xFF0FFF; - temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) | - (CACHE_WRITE_POLICY_L2__DEFAULT << 14) | - SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp); - - /* unhalt engine */ - temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp); - } - - /* enable DMA RB */ - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - - ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL)); - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1); -#ifdef __BIG_ENDIAN - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1); -#endif - /* enable DMA IBs */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl); - - if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */ - sdma_v5_2_ctx_switch_enable(adev, true); - sdma_v5_2_enable(adev, true); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - - return 0; -} - -/** - * sdma_v5_2_rlc_resume - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the compute DMA queues and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev) -{ - return 0; -} - -/** - * sdma_v5_2_load_microcode - load the sDMA ME ucode - * - * @adev: amdgpu_device pointer - * - * Loads the sDMA0/1/2/3 ucode. - * Returns 0 for success, -EINVAL if the ucode is not available. - */ -static int sdma_v5_2_load_microcode(struct amdgpu_device *adev) -{ - const struct sdma_firmware_header_v1_0 *hdr; - const __le32 *fw_data; - u32 fw_size; - int i, j; - - /* halt the MEs */ - sdma_v5_2_enable(adev, false); - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (!adev->sdma.instance[i].fw) - return -EINVAL; - - hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; - amdgpu_ucode_print_sdma_hdr(&hdr->header); - fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; - - fw_data = (const __le32 *) - (adev->sdma.instance[i].fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes)); - - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0); - - for (j = 0; j < fw_size; j++) { - if (amdgpu_emu_mode == 1 && j % 500 == 0) - msleep(1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++)); - } - - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version); - } - - return 0; -} - -static int sdma_v5_2_soft_reset(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - u32 grbm_soft_reset; - u32 tmp; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - grbm_soft_reset = REG_SET_FIELD(0, - GRBM_SOFT_RESET, SOFT_RESET_SDMA0, - 1); - grbm_soft_reset <<= i; - - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - tmp |= grbm_soft_reset; - DRM_DEBUG("GRBM_SOFT_RESET=0x%08X\n", tmp); - WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - - udelay(50); - - tmp &= ~grbm_soft_reset; - WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - - udelay(50); - } - - return 0; -} - -/** - * sdma_v5_2_start - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the DMA engines and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_start(struct amdgpu_device *adev) -{ - int r = 0; - - if (amdgpu_sriov_vf(adev)) { - sdma_v5_2_ctx_switch_enable(adev, false); - sdma_v5_2_enable(adev, false); - - /* set RB registers */ - r = sdma_v5_2_gfx_resume(adev); - return r; - } - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { - r = sdma_v5_2_load_microcode(adev); - if (r) - return r; - - /* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */ - if (amdgpu_emu_mode == 1) - msleep(1000); - } - - sdma_v5_2_soft_reset(adev); - /* unhalt the MEs */ - sdma_v5_2_enable(adev, true); - /* enable sdma ring preemption */ - sdma_v5_2_ctx_switch_enable(adev, true); - - /* start the gfx rings and rlc compute queues */ - r = sdma_v5_2_gfx_resume(adev); - if (r) - return r; - r = sdma_v5_2_rlc_resume(adev); - - return r; -} - -static int sdma_v5_2_mqd_init(struct amdgpu_device *adev, void *mqd, - struct amdgpu_mqd_prop *prop) -{ - struct v10_sdma_mqd *m = mqd; - uint64_t wb_gpu_addr; - - m->sdmax_rlcx_rb_cntl = - order_base_2(prop->queue_size / 4) << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | - 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | - 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | - 1 << SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT; - - m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8); - m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8); - - m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0, - mmSDMA0_GFX_RB_WPTR_POLL_CNTL)); - - wb_gpu_addr = prop->wptr_gpu_addr; - m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr); - m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr); - - wb_gpu_addr = prop->rptr_gpu_addr; - m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr); - m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr); - - m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0, - mmSDMA0_GFX_IB_CNTL)); - - m->sdmax_rlcx_doorbell_offset = - prop->doorbell_index << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT; - - m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1); - - return 0; -} - -static void sdma_v5_2_set_mqd_funcs(struct amdgpu_device *adev) -{ - adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v10_sdma_mqd); - adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v5_2_mqd_init; -} - -/** - * sdma_v5_2_ring_test_ring - simple async dma engine test - * - * @ring: amdgpu_ring structure holding ring information - * - * Test the DMA engine by writing using it to write an - * value to memory. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - unsigned i; - unsigned index; - int r; - u32 tmp; - u64 gpu_addr; - volatile uint32_t *cpu_ptr = NULL; - - tmp = 0xCAFEDEAD; - - if (ring->is_mes_queue) { - uint32_t offset = 0; - offset = amdgpu_mes_ctx_get_offs(ring, - AMDGPU_MES_CTX_PADDING_OFFS); - gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - *cpu_ptr = tmp; - } else { - r = amdgpu_device_wb_get(adev, &index); - if (r) { - dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); - return r; - } - - gpu_addr = adev->wb.gpu_addr + (index * 4); - adev->wb.wb[index] = cpu_to_le32(tmp); - } - - r = amdgpu_ring_alloc(ring, 20); - if (r) { - DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r); - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - return r; - } - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)); - amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); - amdgpu_ring_write(ring, upper_32_bits(gpu_addr)); - amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0)); - amdgpu_ring_write(ring, 0xDEADBEEF); - amdgpu_ring_commit(ring); - - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->is_mes_queue) - tmp = le32_to_cpu(*cpu_ptr); - else - tmp = le32_to_cpu(adev->wb.wb[index]); - if (tmp == 0xDEADBEEF) - break; - if (amdgpu_emu_mode == 1) - msleep(1); - else - udelay(1); - } - - if (i >= adev->usec_timeout) - r = -ETIMEDOUT; - - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - - return r; -} - -/** - * sdma_v5_2_ring_test_ib - test an IB on the DMA engine - * - * @ring: amdgpu_ring structure holding ring information - * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT - * - * Test a simple IB in the DMA ring. - * Returns 0 on success, error on failure. - */ -static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout) -{ - struct amdgpu_device *adev = ring->adev; - struct amdgpu_ib ib; - struct dma_fence *f = NULL; - unsigned index; - long r; - u32 tmp = 0; - u64 gpu_addr; - volatile uint32_t *cpu_ptr = NULL; - - tmp = 0xCAFEDEAD; - memset(&ib, 0, sizeof(ib)); - - if (ring->is_mes_queue) { - uint32_t offset = 0; - offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS); - ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - - offset = amdgpu_mes_ctx_get_offs(ring, - AMDGPU_MES_CTX_PADDING_OFFS); - gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - *cpu_ptr = tmp; - } else { - r = amdgpu_device_wb_get(adev, &index); - if (r) { - dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); - return r; - } - - gpu_addr = adev->wb.gpu_addr + (index * 4); - adev->wb.wb[index] = cpu_to_le32(tmp); - - r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib); - if (r) { - DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); - goto err0; - } - } - - ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); - ib.ptr[1] = lower_32_bits(gpu_addr); - ib.ptr[2] = upper_32_bits(gpu_addr); - ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0); - ib.ptr[4] = 0xDEADBEEF; - ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.length_dw = 8; - - r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); - if (r) - goto err1; - - r = dma_fence_wait_timeout(f, false, timeout); - if (r == 0) { - DRM_ERROR("amdgpu: IB test timed out\n"); - r = -ETIMEDOUT; - goto err1; - } else if (r < 0) { - DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); - goto err1; - } - - if (ring->is_mes_queue) - tmp = le32_to_cpu(*cpu_ptr); - else - tmp = le32_to_cpu(adev->wb.wb[index]); - - if (tmp == 0xDEADBEEF) - r = 0; - else - r = -EINVAL; - -err1: - amdgpu_ib_free(adev, &ib, NULL); - dma_fence_put(f); -err0: - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - return r; -} - - -/** - * sdma_v5_2_vm_copy_pte - update PTEs by copying them from the GART - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @src: src addr to copy from - * @count: number of page entries to update - * - * Update PTEs by copying them from the GART using sDMA. - */ -static void sdma_v5_2_vm_copy_pte(struct amdgpu_ib *ib, - uint64_t pe, uint64_t src, - unsigned count) -{ - unsigned bytes = count * 8; - - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); - ib->ptr[ib->length_dw++] = bytes - 1; - ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ - ib->ptr[ib->length_dw++] = lower_32_bits(src); - ib->ptr[ib->length_dw++] = upper_32_bits(src); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - -} - -/** - * sdma_v5_2_vm_write_pte - update PTEs by writing them manually - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @value: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * - * Update PTEs by writing them manually using sDMA. - */ -static void sdma_v5_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, - uint64_t value, unsigned count, - uint32_t incr) -{ - unsigned ndw = count * 2; - - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - ib->ptr[ib->length_dw++] = ndw - 1; - for (; ndw > 0; ndw -= 2) { - ib->ptr[ib->length_dw++] = lower_32_bits(value); - ib->ptr[ib->length_dw++] = upper_32_bits(value); - value += incr; - } -} - -/** - * sdma_v5_2_vm_set_pte_pde - update the page tables using sDMA - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @addr: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * @flags: access flags - * - * Update the page tables using sDMA. - */ -static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib, - uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint64_t flags) -{ - /* for physically contiguous pages (vram) */ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */ - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ - ib->ptr[ib->length_dw++] = upper_32_bits(flags); - ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */ - ib->ptr[ib->length_dw++] = upper_32_bits(addr); - ib->ptr[ib->length_dw++] = incr; /* increment size */ - ib->ptr[ib->length_dw++] = 0; - ib->ptr[ib->length_dw++] = count - 1; /* number of entries */ -} - -/** - * sdma_v5_2_ring_pad_ib - pad the IB - * - * @ib: indirect buffer to fill with padding - * @ring: amdgpu_ring structure holding ring information - * - * Pad the IB with NOPs to a boundary multiple of 8. - */ -static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) -{ - struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - u32 pad_count; - int i; - - pad_count = (-ib->length_dw) & 0x7; - for (i = 0; i < pad_count; i++) - if (sdma && sdma->burst_nop && (i == 0)) - ib->ptr[ib->length_dw++] = - SDMA_PKT_HEADER_OP(SDMA_OP_NOP) | - SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1); - else - ib->ptr[ib->length_dw++] = - SDMA_PKT_HEADER_OP(SDMA_OP_NOP); -} - - -/** - * sdma_v5_2_ring_emit_pipeline_sync - sync the pipeline - * - * @ring: amdgpu_ring pointer - * - * Make sure all previous operations are completed (CIK). - */ -static void sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring) -{ - uint32_t seq = ring->fence_drv.sync_seq; - uint64_t addr = ring->fence_drv.gpu_addr; - - /* wait for idle */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */ - SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1)); - amdgpu_ring_write(ring, addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); - amdgpu_ring_write(ring, seq); /* reference */ - amdgpu_ring_write(ring, 0xffffffff); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ -} - - -/** - * sdma_v5_2_ring_emit_vm_flush - vm flush using sDMA - * - * @ring: amdgpu_ring pointer - * @vmid: vmid number to use - * @pd_addr: address - * - * Update the page table base and flush the VM TLB - * using sDMA. - */ -static void sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vmid, uint64_t pd_addr) -{ - amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); -} - -static void sdma_v5_2_ring_emit_wreg(struct amdgpu_ring *ring, - uint32_t reg, uint32_t val) -{ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | - SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); - amdgpu_ring_write(ring, reg); - amdgpu_ring_write(ring, val); -} - -static void sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */ - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, val); /* reference */ - amdgpu_ring_write(ring, mask); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); -} - -static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, - uint32_t reg0, uint32_t reg1, - uint32_t ref, uint32_t mask) -{ - amdgpu_ring_emit_wreg(ring, reg0, ref); - /* wait for a cycle to reset vm_inv_eng*_ack */ - amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0); - amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask); -} - -static int sdma_v5_2_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_sdma_init_microcode(adev, 0, true); - if (r) - return r; - - sdma_v5_2_set_ring_funcs(adev); - sdma_v5_2_set_buffer_funcs(adev); - sdma_v5_2_set_vm_pte_funcs(adev); - sdma_v5_2_set_irq_funcs(adev); - sdma_v5_2_set_mqd_funcs(adev); - - return 0; -} - -static unsigned sdma_v5_2_seq_to_irq_id(int seq_num) -{ - switch (seq_num) { - case 0: - return SOC15_IH_CLIENTID_SDMA0; - case 1: - return SOC15_IH_CLIENTID_SDMA1; - case 2: - return SOC15_IH_CLIENTID_SDMA2; - case 3: - return SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid; - default: - break; - } - return -EINVAL; -} - -static unsigned sdma_v5_2_seq_to_trap_id(int seq_num) -{ - switch (seq_num) { - case 0: - return SDMA0_5_0__SRCID__SDMA_TRAP; - case 1: - return SDMA1_5_0__SRCID__SDMA_TRAP; - case 2: - return SDMA2_5_0__SRCID__SDMA_TRAP; - case 3: - return SDMA3_5_0__SRCID__SDMA_TRAP; - default: - break; - } - return -EINVAL; -} - -static int sdma_v5_2_sw_init(void *handle) -{ - struct amdgpu_ring *ring; - int r, i; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - uint32_t *ptr; - - /* SDMA trap event */ - for (i = 0; i < adev->sdma.num_instances; i++) { - r = amdgpu_irq_add_id(adev, sdma_v5_2_seq_to_irq_id(i), - sdma_v5_2_seq_to_trap_id(i), - &adev->sdma.trap_irq); - if (r) - return r; - } - - for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; - ring->ring_obj = NULL; - ring->use_doorbell = true; - ring->me = i; - - DRM_INFO("use_doorbell being set to: [%s]\n", - ring->use_doorbell?"true":"false"); - - ring->doorbell_index = - (adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset - - ring->vm_hub = AMDGPU_GFXHUB(0); - sprintf(ring->name, "sdma%d", i); - r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq, - AMDGPU_SDMA_IRQ_INSTANCE0 + i, - AMDGPU_RING_PRIO_DEFAULT, NULL); - if (r) - return r; - } - - /* Allocate memory for SDMA IP Dump buffer */ - ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (ptr) - adev->sdma.ip_dump = ptr; - else - DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n"); - - return r; -} - -static int sdma_v5_2_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) - amdgpu_ring_fini(&adev->sdma.instance[i].ring); - - amdgpu_sdma_destroy_inst_ctx(adev, true); - - kfree(adev->sdma.ip_dump); - - return 0; -} - -static int sdma_v5_2_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_start(adev); -} - -static int sdma_v5_2_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (amdgpu_sriov_vf(adev)) - return 0; - - sdma_v5_2_ctx_switch_enable(adev, false); - sdma_v5_2_enable(adev, false); - - return 0; -} - -static int sdma_v5_2_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_hw_fini(adev); -} - -static int sdma_v5_2_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_hw_init(adev); -} - -static bool sdma_v5_2_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - u32 i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG)); - - if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK)) - return false; - } - - return true; -} - -static int sdma_v5_2_wait_for_idle(void *handle) -{ - unsigned i; - u32 sdma0, sdma1, sdma2, sdma3; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - for (i = 0; i < adev->usec_timeout; i++) { - sdma0 = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG)); - sdma1 = RREG32(sdma_v5_2_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG)); - sdma2 = RREG32(sdma_v5_2_get_reg_offset(adev, 2, mmSDMA0_STATUS_REG)); - sdma3 = RREG32(sdma_v5_2_get_reg_offset(adev, 3, mmSDMA0_STATUS_REG)); - - if (sdma0 & sdma1 & sdma2 & sdma3 & SDMA0_STATUS_REG__IDLE_MASK) - return 0; - udelay(1); - } - return -ETIMEDOUT; -} - -static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring) -{ - int i, r = 0; - struct amdgpu_device *adev = ring->adev; - u32 index = 0; - u64 sdma_gfx_preempt; - - amdgpu_sdma_get_index_from_ring(ring, &index); - sdma_gfx_preempt = - sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT); - - /* assert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, false); - - /* emit the trailing fence */ - ring->trail_seq += 1; - amdgpu_ring_alloc(ring, 10); - sdma_v5_2_ring_emit_fence(ring, ring->trail_fence_gpu_addr, - ring->trail_seq, 0); - amdgpu_ring_commit(ring); - - /* assert IB preemption */ - WREG32(sdma_gfx_preempt, 1); - - /* poll the trailing fence */ - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->trail_seq == - le32_to_cpu(*(ring->trail_fence_cpu_addr))) - break; - udelay(1); - } - - if (i >= adev->usec_timeout) { - r = -EINVAL; - DRM_ERROR("ring %d failed to be preempted\n", ring->idx); - } - - /* deassert IB preemption */ - WREG32(sdma_gfx_preempt, 0); - - /* deassert the preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, true); - return r; -} - -static int sdma_v5_2_set_trap_irq_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned type, - enum amdgpu_interrupt_state state) -{ - u32 sdma_cntl; - u32 reg_offset = sdma_v5_2_get_reg_offset(adev, type, mmSDMA0_CNTL); - - if (!amdgpu_sriov_vf(adev)) { - sdma_cntl = RREG32(reg_offset); - sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, - state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); - WREG32(reg_offset, sdma_cntl); - } - - return 0; -} - -static int sdma_v5_2_process_trap_irq(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t mes_queue_id = entry->src_data[0]; - - DRM_DEBUG("IH: SDMA trap\n"); - - if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) { - struct amdgpu_mes_queue *queue; - - mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK; - - spin_lock(&adev->mes.queue_id_lock); - queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id); - if (queue) { - DRM_DEBUG("process smda queue id = %d\n", mes_queue_id); - amdgpu_fence_process(queue->ring); - } - spin_unlock(&adev->mes.queue_id_lock); - return 0; - } - - switch (entry->client_id) { - case SOC15_IH_CLIENTID_SDMA0: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[0].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA1: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[1].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA2: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[2].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[3].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - } - return 0; -} - -static int sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - return 0; -} - -static bool sdma_v5_2_firmware_mgcg_support(struct amdgpu_device *adev, - int i) -{ - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 1): - if (adev->sdma.instance[i].fw_version < 70) - return false; - break; - case IP_VERSION(5, 2, 3): - if (adev->sdma.instance[i].fw_version < 47) - return false; - break; - case IP_VERSION(5, 2, 7): - if (adev->sdma.instance[i].fw_version < 9) - return false; - break; - default: - return true; - } - - return true; - -} - -static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t data, def; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - - if (!sdma_v5_2_firmware_mgcg_support(adev, i)) - adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG; - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { - /* Enable sdma clock gating */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL)); - data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK); - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data); - } else { - /* Disable sdma clock gating */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL)); - data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK); - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data); - } - } -} - -static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev, - bool enable) -{ - uint32_t data, def; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (adev->sdma.instance[i].fw_version < 70 && - amdgpu_ip_version(adev, SDMA0_HWIP, 0) == - IP_VERSION(5, 2, 1)) - adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS; - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) { - /* Enable sdma mem light sleep */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL)); - data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data); - - } else { - /* Disable sdma mem light sleep */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL)); - data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data); - - } - } -} - -static int sdma_v5_2_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (amdgpu_sriov_vf(adev)) - return 0; - - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 1): - case IP_VERSION(5, 2, 4): - case IP_VERSION(5, 2, 5): - case IP_VERSION(5, 2, 6): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 7): - sdma_v5_2_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE); - sdma_v5_2_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE); - break; - default: - break; - } - - return 0; -} - -static int sdma_v5_2_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -static void sdma_v5_2_get_clockgating_state(void *handle, u64 *flags) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int data; - - if (amdgpu_sriov_vf(adev)) - *flags = 0; - - /* AMD_CG_SUPPORT_SDMA_MGCG */ - data = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL)); - if (!(data & SDMA0_CLK_CTRL__CGCG_EN_OVERRIDE_MASK)) - *flags |= AMD_CG_SUPPORT_SDMA_MGCG; - - /* AMD_CG_SUPPORT_SDMA_LS */ - data = RREG32_KIQ(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL)); - if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK) - *flags |= AMD_CG_SUPPORT_SDMA_LS; -} - -static void sdma_v5_2_ring_begin_use(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - /* SDMA 5.2.3 (RMB) FW doesn't seem to properly - * disallow GFXOFF in some cases leading to - * hangs in SDMA. Disallow GFXOFF while SDMA is active. - * We can probably just limit this to 5.2.3, - * but it shouldn't hurt for other parts since - * this GFXOFF will be disallowed anyway when SDMA is - * active, this just makes it explicit. - * sdma_v5_2_ring_set_wptr() takes advantage of this - * to update the wptr because sometimes SDMA seems to miss - * doorbells when entering PG. If you remove this, update - * sdma_v5_2_ring_set_wptr() as well! - */ - amdgpu_gfx_off_ctrl(adev, false); -} - -static void sdma_v5_2_ring_end_use(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - /* SDMA 5.2.3 (RMB) FW doesn't seem to properly - * disallow GFXOFF in some cases leading to - * hangs in SDMA. Allow GFXOFF when SDMA is complete. - */ - amdgpu_gfx_off_ctrl(adev, true); -} - -static void sdma_v5_2_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - uint32_t instance_offset; - - if (!adev->sdma.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances); - for (i = 0; i < adev->sdma.num_instances; i++) { - instance_offset = i * reg_count; - drm_printf(p, "\nInstance:%d\n", i); - - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_5_2[j].reg_name, - adev->sdma.ip_dump[instance_offset + j]); - } -} - -static void sdma_v5_2_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t instance_offset; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - - if (!adev->sdma.ip_dump) - return; - - amdgpu_gfx_off_ctrl(adev, false); - for (i = 0; i < adev->sdma.num_instances; i++) { - instance_offset = i * reg_count; - for (j = 0; j < reg_count; j++) - adev->sdma.ip_dump[instance_offset + j] = - RREG32(sdma_v5_2_get_reg_offset(adev, i, - sdma_reg_list_5_2[j].reg_offset)); - } - amdgpu_gfx_off_ctrl(adev, true); -} - -static const struct amd_ip_funcs sdma_v5_2_ip_funcs = { - .name = "sdma_v5_2", - .early_init = sdma_v5_2_early_init, - .late_init = NULL, - .sw_init = sdma_v5_2_sw_init, - .sw_fini = sdma_v5_2_sw_fini, - .hw_init = sdma_v5_2_hw_init, - .hw_fini = sdma_v5_2_hw_fini, - .suspend = sdma_v5_2_suspend, - .resume = sdma_v5_2_resume, - .is_idle = sdma_v5_2_is_idle, - .wait_for_idle = sdma_v5_2_wait_for_idle, - .soft_reset = sdma_v5_2_soft_reset, - .set_clockgating_state = sdma_v5_2_set_clockgating_state, - .set_powergating_state = sdma_v5_2_set_powergating_state, - .get_clockgating_state = sdma_v5_2_get_clockgating_state, - .dump_ip_state = sdma_v5_2_dump_ip_state, - .print_ip_state = sdma_v5_2_print_ip_state, -}; - -static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = { - .type = AMDGPU_RING_TYPE_SDMA, - .align_mask = 0xf, - .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), - .support_64bit_ptrs = true, - .secure_submission_supported = true, - .get_rptr = sdma_v5_2_ring_get_rptr, - .get_wptr = sdma_v5_2_ring_get_wptr, - .set_wptr = sdma_v5_2_ring_set_wptr, - .emit_frame_size = - 5 + /* sdma_v5_2_ring_init_cond_exec */ - 6 + /* sdma_v5_2_ring_emit_hdp_flush */ - 3 + /* hdp_invalidate */ - 6 + /* sdma_v5_2_ring_emit_pipeline_sync */ - /* sdma_v5_2_ring_emit_vm_flush */ - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 + - 10 + 10 + 10, /* sdma_v5_2_ring_emit_fence x3 for user fence, vm fence */ - .emit_ib_size = 7 + 6, /* sdma_v5_2_ring_emit_ib */ - .emit_ib = sdma_v5_2_ring_emit_ib, - .emit_mem_sync = sdma_v5_2_ring_emit_mem_sync, - .emit_fence = sdma_v5_2_ring_emit_fence, - .emit_pipeline_sync = sdma_v5_2_ring_emit_pipeline_sync, - .emit_vm_flush = sdma_v5_2_ring_emit_vm_flush, - .emit_hdp_flush = sdma_v5_2_ring_emit_hdp_flush, - .test_ring = sdma_v5_2_ring_test_ring, - .test_ib = sdma_v5_2_ring_test_ib, - .insert_nop = sdma_v5_2_ring_insert_nop, - .pad_ib = sdma_v5_2_ring_pad_ib, - .begin_use = sdma_v5_2_ring_begin_use, - .end_use = sdma_v5_2_ring_end_use, - .emit_wreg = sdma_v5_2_ring_emit_wreg, - .emit_reg_wait = sdma_v5_2_ring_emit_reg_wait, - .emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait, - .init_cond_exec = sdma_v5_2_ring_init_cond_exec, - .preempt_ib = sdma_v5_2_ring_preempt_ib, -}; - -static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - adev->sdma.instance[i].ring.funcs = &sdma_v5_2_ring_funcs; - adev->sdma.instance[i].ring.me = i; - } -} - -static const struct amdgpu_irq_src_funcs sdma_v5_2_trap_irq_funcs = { - .set = sdma_v5_2_set_trap_irq_state, - .process = sdma_v5_2_process_trap_irq, -}; - -static const struct amdgpu_irq_src_funcs sdma_v5_2_illegal_inst_irq_funcs = { - .process = sdma_v5_2_process_illegal_inst_irq, -}; - -static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev) -{ - adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 + - adev->sdma.num_instances; - adev->sdma.trap_irq.funcs = &sdma_v5_2_trap_irq_funcs; - adev->sdma.illegal_inst_irq.funcs = &sdma_v5_2_illegal_inst_irq_funcs; -} - -/** - * sdma_v5_2_emit_copy_buffer - copy buffer using the sDMA engine - * - * @ib: indirect buffer to copy to - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @byte_count: number of bytes to xfer - * @copy_flags: copy flags for the buffers - * - * Copy GPU buffers using the DMA engine. - * Used by the amdgpu ttm implementation to move pages if - * registered as the asic copy callback. - */ -static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib, - uint64_t src_offset, - uint64_t dst_offset, - uint32_t byte_count, - uint32_t copy_flags) -{ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) | - SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0); - ib->ptr[ib->length_dw++] = byte_count - 1; - ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ - ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); - ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); -} - -/** - * sdma_v5_2_emit_fill_buffer - fill buffer using the sDMA engine - * - * @ib: indirect buffer to fill - * @src_data: value to write to buffer - * @dst_offset: dst GPU address - * @byte_count: number of bytes to xfer - * - * Fill GPU buffers using the DMA engine. - */ -static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib, - uint32_t src_data, - uint64_t dst_offset, - uint32_t byte_count) -{ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL); - ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = src_data; - ib->ptr[ib->length_dw++] = byte_count - 1; -} - -static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { - .copy_max_bytes = 0x400000, - .copy_num_dw = 7, - .emit_copy_buffer = sdma_v5_2_emit_copy_buffer, - - .fill_max_bytes = 0x400000, - .fill_num_dw = 5, - .emit_fill_buffer = sdma_v5_2_emit_fill_buffer, -}; - -static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev) -{ - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } -} - -static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = { - .copy_pte_num_dw = 7, - .copy_pte = sdma_v5_2_vm_copy_pte, - .write_pte = sdma_v5_2_vm_write_pte, - .set_pte_pde = sdma_v5_2_vm_set_pte_pde, -}; - -static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev) -{ - unsigned i; - - if (adev->vm_manager.vm_pte_funcs == NULL) { - adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs; - for (i = 0; i < adev->sdma.num_instances; i++) { - adev->vm_manager.vm_pte_scheds[i] = - &adev->sdma.instance[i].ring.sched; - } - adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances; - } -} - -const struct amdgpu_ip_block_version sdma_v5_2_ip_block = { - .type = AMD_IP_BLOCK_TYPE_SDMA, - .major = 5, - .minor = 2, - .rev = 0, - .funcs = &sdma_v5_2_ip_funcs, -}; diff --git a/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage.1 b/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage.1 deleted file mode 100644 index edcbe7fec8d5..000000000000 --- a/rr-cache/f013472cad3d9dd7cdbe39311eced4b9997712a7/preimage.1 +++ /dev/null @@ -1,1974 +0,0 @@ -/* - * Copyright 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/delay.h> -#include <linux/firmware.h> -#include <linux/module.h> -#include <linux/pci.h> - -#include "amdgpu.h" -#include "amdgpu_ucode.h" -#include "amdgpu_trace.h" - -#include "gc/gc_10_3_0_offset.h" -#include "gc/gc_10_3_0_sh_mask.h" -#include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h" -#include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h" -#include "ivsrcid/sdma2/irqsrcs_sdma2_5_0.h" -#include "ivsrcid/sdma3/irqsrcs_sdma3_5_0.h" - -#include "soc15_common.h" -#include "soc15.h" -#include "navi10_sdma_pkt_open.h" -#include "nbio_v2_3.h" -#include "sdma_common.h" -#include "sdma_v5_2.h" - -MODULE_FIRMWARE("amdgpu/sienna_cichlid_sdma.bin"); -MODULE_FIRMWARE("amdgpu/navy_flounder_sdma.bin"); -MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_sdma.bin"); -MODULE_FIRMWARE("amdgpu/beige_goby_sdma.bin"); - -MODULE_FIRMWARE("amdgpu/vangogh_sdma.bin"); -MODULE_FIRMWARE("amdgpu/yellow_carp_sdma.bin"); -MODULE_FIRMWARE("amdgpu/sdma_5_2_6.bin"); -MODULE_FIRMWARE("amdgpu/sdma_5_2_7.bin"); - -#define SDMA1_REG_OFFSET 0x600 -#define SDMA3_REG_OFFSET 0x400 -#define SDMA0_HYP_DEC_REG_START 0x5880 -#define SDMA0_HYP_DEC_REG_END 0x5893 -#define SDMA1_HYP_DEC_REG_OFFSET 0x20 - -static const struct amdgpu_hwip_reg_entry sdma_reg_list_5_2[] = { - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_INT_STATUS), - SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL), - SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2) -}; - -static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev); -static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev); - -static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset) -{ - u32 base; - - if (internal_offset >= SDMA0_HYP_DEC_REG_START && - internal_offset <= SDMA0_HYP_DEC_REG_END) { - base = adev->reg_offset[GC_HWIP][0][1]; - if (instance != 0) - internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance; - } else { - if (instance < 2) { - base = adev->reg_offset[GC_HWIP][0][0]; - if (instance == 1) - internal_offset += SDMA1_REG_OFFSET; - } else { - base = adev->reg_offset[GC_HWIP][0][2]; - if (instance == 3) - internal_offset += SDMA3_REG_OFFSET; - } - } - - return base + internal_offset; -} - -static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring, - uint64_t addr) -{ - unsigned ret; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE)); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, 1); - /* this is the offset we need patch later */ - ret = ring->wptr & ring->buf_mask; - /* insert dummy here and patch it later */ - amdgpu_ring_write(ring, 0); - - return ret; -} - -/** - * sdma_v5_2_ring_get_rptr - get the current read pointer - * - * @ring: amdgpu ring pointer - * - * Get the current rptr from the hardware (NAVI10+). - */ -static uint64_t sdma_v5_2_ring_get_rptr(struct amdgpu_ring *ring) -{ - u64 *rptr; - - /* XXX check if swapping is necessary on BE */ - rptr = (u64 *)ring->rptr_cpu_addr; - - DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr); - return ((*rptr) >> 2); -} - -/** - * sdma_v5_2_ring_get_wptr - get the current write pointer - * - * @ring: amdgpu ring pointer - * - * Get the current wptr from the hardware (NAVI10+). - */ -static uint64_t sdma_v5_2_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - u64 wptr; - - if (ring->use_doorbell) { - /* XXX check if swapping is necessary on BE */ - wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr)); - DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr); - } else { - wptr = RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI)); - wptr = wptr << 32; - wptr |= RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR)); - DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr); - } - - return wptr >> 2; -} - -/** - * sdma_v5_2_ring_set_wptr - commit the write pointer - * - * @ring: amdgpu ring pointer - * - * Write the wptr back to the hardware (NAVI10+). - */ -static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - DRM_DEBUG("Setting write pointer\n"); - if (ring->use_doorbell) { - DRM_DEBUG("Using doorbell -- " - "wptr_offs == 0x%08x " - "lower_32_bits(ring->wptr << 2) == 0x%08x " - "upper_32_bits(ring->wptr << 2) == 0x%08x\n", - ring->wptr_offs, - lower_32_bits(ring->wptr << 2), - upper_32_bits(ring->wptr << 2)); - /* XXX check if swapping is necessary on BE */ - atomic64_set((atomic64_t *)ring->wptr_cpu_addr, - ring->wptr << 2); - DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n", - ring->doorbell_index, ring->wptr << 2); - WDOORBELL64(ring->doorbell_index, ring->wptr << 2); -<<<<<<< - /* SDMA seems to miss doorbells sometimes when powergating kicks in. - * Updating the wptr directly will wake it. This is only safe because - * we disallow gfxoff in begin_use() and then allow it again in end_use(). - */ - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); -======= - if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(5, 2, 1)) { - /* SDMA seems to miss doorbells sometimes when powergating kicks in. - * Updating the wptr directly will wake it. This is only safe because - * we disallow gfxoff in begin_use() and then allow it again in end_use(). - */ - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); - } ->>>>>>> - } else { - DRM_DEBUG("Not using doorbell -- " - "mmSDMA%i_GFX_RB_WPTR == 0x%08x " - "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", - ring->me, - lower_32_bits(ring->wptr << 2), - ring->me, - upper_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), - lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), - upper_32_bits(ring->wptr << 2)); - } -} - -static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) -{ - struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - int i; - - for (i = 0; i < count; i++) - if (sdma && sdma->burst_nop && (i == 0)) - amdgpu_ring_write(ring, ring->funcs->nop | - SDMA_PKT_NOP_HEADER_COUNT(count - 1)); - else - amdgpu_ring_write(ring, ring->funcs->nop); -} - -/** - * sdma_v5_2_ring_emit_ib - Schedule an IB on the DMA engine - * - * @ring: amdgpu ring pointer - * @job: job to retrieve vmid from - * @ib: IB object to schedule - * @flags: unused - * - * Schedule an IB in the DMA ring. - */ -static void sdma_v5_2_ring_emit_ib(struct amdgpu_ring *ring, - struct amdgpu_job *job, - struct amdgpu_ib *ib, - uint32_t flags) -{ - unsigned vmid = AMDGPU_JOB_GET_VMID(job); - uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid); - - /* An IB packet must end on a 8 DW boundary--the next dword - * must be on a 8-dword boundary. Our IB packet below is 6 - * dwords long, thus add x number of NOPs, such that, in - * modular arithmetic, - * wptr + 6 + x = 8k, k >= 0, which in C is, - * (wptr + 6 + x) % 8 = 0. - * The expression below, is a solution of x. - */ - sdma_v5_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7); - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | - SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf)); - /* base must be 32 byte aligned */ - amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0); - amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); - amdgpu_ring_write(ring, ib->length_dw); - amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr)); - amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr)); -} - -/** - * sdma_v5_2_ring_emit_mem_sync - flush the IB by graphics cache rinse - * - * @ring: amdgpu ring pointer - * - * flush the IB by graphics cache rinse. - */ -static void sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring *ring) -{ - uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | - SDMA_GCR_GLM_INV | SDMA_GCR_GL1_INV | - SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV | - SDMA_GCR_GLI_INV(1); - - /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) | - SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) | - SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16)); - amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) | - SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0)); -} - -/** - * sdma_v5_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring - * - * @ring: amdgpu ring pointer - * - * Emit an hdp flush packet on the requested DMA ring. - */ -static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - u32 ref_and_mask = 0; - const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; - - if (ring->me > 1) { - amdgpu_asic_flush_hdp(adev, ring); - } else { - ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */ - amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2); - amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2); - amdgpu_ring_write(ring, ref_and_mask); /* reference */ - amdgpu_ring_write(ring, ref_and_mask); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ - } -} - -/** - * sdma_v5_2_ring_emit_fence - emit a fence on the DMA ring - * - * @ring: amdgpu ring pointer - * @addr: address - * @seq: sequence number - * @flags: fence related flags - * - * Add a DMA fence packet to the ring to write - * the fence seq number and DMA trap packet to generate - * an interrupt if needed. - */ -static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, - unsigned flags) -{ - bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; - /* write the fence */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | - SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ - /* zero in first two bits */ - BUG_ON(addr & 0x3); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, lower_32_bits(seq)); - - /* optionally write high bits as well */ - if (write64bit) { - addr += 4; - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | - SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); - /* zero in first two bits */ - BUG_ON(addr & 0x3); - amdgpu_ring_write(ring, lower_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(addr)); - amdgpu_ring_write(ring, upper_32_bits(seq)); - } - - if ((flags & AMDGPU_FENCE_FLAG_INT)) { - uint32_t ctx = ring->is_mes_queue ? - (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0; - /* generate an interrupt */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP)); - amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx)); - } -} - - -/** - * sdma_v5_2_gfx_stop - stop the gfx async dma engines - * - * @adev: amdgpu_device pointer - * - * Stop the gfx async dma ring buffers. - */ -static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev) -{ - u32 rb_cntl, ib_cntl; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL)); - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl); - } -} - -/** - * sdma_v5_2_rlc_stop - stop the compute async dma engines - * - * @adev: amdgpu_device pointer - * - * Stop the compute async dma queues. - */ -static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev) -{ - /* XXX todo */ -} - -/** - * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch - * - * @adev: amdgpu_device pointer - * @enable: enable/disable the DMA MEs context switch. - * - * Halt or unhalt the async dma engines context switch. - */ -static void sdma_v5_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable) -{ - u32 f32_cntl, phase_quantum = 0; - int i; - - if (amdgpu_sdma_phase_quantum) { - unsigned value = amdgpu_sdma_phase_quantum; - unsigned unit = 0; - - while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> - SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) { - value = (value + 1) >> 1; - unit++; - } - if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> - SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) { - value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> - SDMA0_PHASE0_QUANTUM__VALUE__SHIFT); - unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> - SDMA0_PHASE0_QUANTUM__UNIT__SHIFT); - WARN_ONCE(1, - "clamping sdma_phase_quantum to %uK clock cycles\n", - value << unit); - } - phase_quantum = - value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT | - unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT; - } - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (enable && amdgpu_sdma_phase_quantum) { - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM), - phase_quantum); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM), - phase_quantum); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM), - phase_quantum); - } - - if (!amdgpu_sriov_vf(adev)) { - f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL)); - f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, - AUTO_CTXSW_ENABLE, enable ? 1 : 0); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl); - } - } - -} - -/** - * sdma_v5_2_enable - stop the async dma engines - * - * @adev: amdgpu_device pointer - * @enable: enable/disable the DMA MEs. - * - * Halt or unhalt the async dma engines. - */ -static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable) -{ - u32 f32_cntl; - int i; - - if (!enable) { - sdma_v5_2_gfx_stop(adev); - sdma_v5_2_rlc_stop(adev); - } - - if (!amdgpu_sriov_vf(adev)) { - for (i = 0; i < adev->sdma.num_instances; i++) { - f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL)); - f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl); - } - } -} - -/** - * sdma_v5_2_gfx_resume - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the gfx DMA ring buffers and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev) -{ - struct amdgpu_ring *ring; - u32 rb_cntl, ib_cntl; - u32 rb_bufsz; - u32 doorbell; - u32 doorbell_offset; - u32 temp; - u32 wptr_poll_cntl; - u64 wptr_gpu_addr; - int i, r; - - for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; - - if (!amdgpu_sriov_vf(adev)) - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0); - - /* Set ring buffer size in dwords */ - rb_bufsz = order_base_2(ring->ring_size / 4); - rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz); -#ifdef __BIG_ENDIAN - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1); - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, - RPTR_WRITEBACK_SWAP_ENABLE, 1); -#endif - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0); - - /* setup the wptr shadow polling */ - wptr_gpu_addr = ring->wptr_gpu_addr; - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO), - lower_32_bits(wptr_gpu_addr)); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI), - upper_32_bits(wptr_gpu_addr)); - wptr_poll_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, - mmSDMA0_GFX_RB_WPTR_POLL_CNTL)); - wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, - SDMA0_GFX_RB_WPTR_POLL_CNTL, - F32_POLL_ENABLE, 1); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), - wptr_poll_cntl); - - /* set the wb address whether it's enabled or not */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI), - upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO), - lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC); - - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1); - - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40); - - ring->wptr = 0; - - /* before programing wptr to a less value, need set minor_ptr_update first */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1); - - if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */ - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2)); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2)); - } - - doorbell = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL)); - doorbell_offset = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET)); - - if (ring->use_doorbell) { - doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1); - doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET, - OFFSET, ring->doorbell_index); - } else { - doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0); - } - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset); - - adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell, - ring->doorbell_index, - adev->doorbell_index.sdma_doorbell_range); - - if (amdgpu_sriov_vf(adev)) - sdma_v5_2_ring_set_wptr(ring); - - /* set minor_ptr_update to 0 after wptr programed */ - - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0); - - /* SRIOV VF has no control of any of registers below */ - if (!amdgpu_sriov_vf(adev)) { - /* set utc l1 enable flag always to 1 */ - temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1); - - /* enable MCBP */ - temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), temp); - - /* Set up RESP_MODE to non-copy addresses */ - temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3); - temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp); - - /* program default cache read and write policy */ - temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE)); - /* clean read policy and write policy bits */ - temp &= 0xFF0FFF; - temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) | - (CACHE_WRITE_POLICY_L2__DEFAULT << 14) | - SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp); - - /* unhalt engine */ - temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL)); - temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp); - } - - /* enable DMA RB */ - rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1); - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl); - - ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL)); - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1); -#ifdef __BIG_ENDIAN - ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1); -#endif - /* enable DMA IBs */ - WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl); - - if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */ - sdma_v5_2_ctx_switch_enable(adev, true); - sdma_v5_2_enable(adev, true); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - - return 0; -} - -/** - * sdma_v5_2_rlc_resume - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the compute DMA queues and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev) -{ - return 0; -} - -/** - * sdma_v5_2_load_microcode - load the sDMA ME ucode - * - * @adev: amdgpu_device pointer - * - * Loads the sDMA0/1/2/3 ucode. - * Returns 0 for success, -EINVAL if the ucode is not available. - */ -static int sdma_v5_2_load_microcode(struct amdgpu_device *adev) -{ - const struct sdma_firmware_header_v1_0 *hdr; - const __le32 *fw_data; - u32 fw_size; - int i, j; - - /* halt the MEs */ - sdma_v5_2_enable(adev, false); - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (!adev->sdma.instance[i].fw) - return -EINVAL; - - hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; - amdgpu_ucode_print_sdma_hdr(&hdr->header); - fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; - - fw_data = (const __le32 *) - (adev->sdma.instance[i].fw->data + - le32_to_cpu(hdr->header.ucode_array_offset_bytes)); - - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0); - - for (j = 0; j < fw_size; j++) { - if (amdgpu_emu_mode == 1 && j % 500 == 0) - msleep(1); - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++)); - } - - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version); - } - - return 0; -} - -static int sdma_v5_2_soft_reset(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - u32 grbm_soft_reset; - u32 tmp; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - grbm_soft_reset = REG_SET_FIELD(0, - GRBM_SOFT_RESET, SOFT_RESET_SDMA0, - 1); - grbm_soft_reset <<= i; - - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - tmp |= grbm_soft_reset; - DRM_DEBUG("GRBM_SOFT_RESET=0x%08X\n", tmp); - WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - - udelay(50); - - tmp &= ~grbm_soft_reset; - WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); - tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); - - udelay(50); - } - - return 0; -} - -/** - * sdma_v5_2_start - setup and start the async dma engines - * - * @adev: amdgpu_device pointer - * - * Set up the DMA engines and enable them. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_start(struct amdgpu_device *adev) -{ - int r = 0; - - if (amdgpu_sriov_vf(adev)) { - sdma_v5_2_ctx_switch_enable(adev, false); - sdma_v5_2_enable(adev, false); - - /* set RB registers */ - r = sdma_v5_2_gfx_resume(adev); - return r; - } - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { - r = sdma_v5_2_load_microcode(adev); - if (r) - return r; - - /* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */ - if (amdgpu_emu_mode == 1) - msleep(1000); - } - - sdma_v5_2_soft_reset(adev); - /* unhalt the MEs */ - sdma_v5_2_enable(adev, true); - /* enable sdma ring preemption */ - sdma_v5_2_ctx_switch_enable(adev, true); - - /* start the gfx rings and rlc compute queues */ - r = sdma_v5_2_gfx_resume(adev); - if (r) - return r; - r = sdma_v5_2_rlc_resume(adev); - - return r; -} - -static int sdma_v5_2_mqd_init(struct amdgpu_device *adev, void *mqd, - struct amdgpu_mqd_prop *prop) -{ - struct v10_sdma_mqd *m = mqd; - uint64_t wb_gpu_addr; - - m->sdmax_rlcx_rb_cntl = - order_base_2(prop->queue_size / 4) << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | - 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | - 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | - 1 << SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT; - - m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8); - m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8); - - m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0, - mmSDMA0_GFX_RB_WPTR_POLL_CNTL)); - - wb_gpu_addr = prop->wptr_gpu_addr; - m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr); - m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr); - - wb_gpu_addr = prop->rptr_gpu_addr; - m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr); - m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr); - - m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0, - mmSDMA0_GFX_IB_CNTL)); - - m->sdmax_rlcx_doorbell_offset = - prop->doorbell_index << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT; - - m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1); - - return 0; -} - -static void sdma_v5_2_set_mqd_funcs(struct amdgpu_device *adev) -{ - adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v10_sdma_mqd); - adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v5_2_mqd_init; -} - -/** - * sdma_v5_2_ring_test_ring - simple async dma engine test - * - * @ring: amdgpu_ring structure holding ring information - * - * Test the DMA engine by writing using it to write an - * value to memory. - * Returns 0 for success, error for failure. - */ -static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - unsigned i; - unsigned index; - int r; - u32 tmp; - u64 gpu_addr; - volatile uint32_t *cpu_ptr = NULL; - - tmp = 0xCAFEDEAD; - - if (ring->is_mes_queue) { - uint32_t offset = 0; - offset = amdgpu_mes_ctx_get_offs(ring, - AMDGPU_MES_CTX_PADDING_OFFS); - gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - *cpu_ptr = tmp; - } else { - r = amdgpu_device_wb_get(adev, &index); - if (r) { - dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); - return r; - } - - gpu_addr = adev->wb.gpu_addr + (index * 4); - adev->wb.wb[index] = cpu_to_le32(tmp); - } - - r = amdgpu_ring_alloc(ring, 20); - if (r) { - DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r); - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - return r; - } - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)); - amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); - amdgpu_ring_write(ring, upper_32_bits(gpu_addr)); - amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0)); - amdgpu_ring_write(ring, 0xDEADBEEF); - amdgpu_ring_commit(ring); - - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->is_mes_queue) - tmp = le32_to_cpu(*cpu_ptr); - else - tmp = le32_to_cpu(adev->wb.wb[index]); - if (tmp == 0xDEADBEEF) - break; - if (amdgpu_emu_mode == 1) - msleep(1); - else - udelay(1); - } - - if (i >= adev->usec_timeout) - r = -ETIMEDOUT; - - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - - return r; -} - -/** - * sdma_v5_2_ring_test_ib - test an IB on the DMA engine - * - * @ring: amdgpu_ring structure holding ring information - * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT - * - * Test a simple IB in the DMA ring. - * Returns 0 on success, error on failure. - */ -static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout) -{ - struct amdgpu_device *adev = ring->adev; - struct amdgpu_ib ib; - struct dma_fence *f = NULL; - unsigned index; - long r; - u32 tmp = 0; - u64 gpu_addr; - volatile uint32_t *cpu_ptr = NULL; - - tmp = 0xCAFEDEAD; - memset(&ib, 0, sizeof(ib)); - - if (ring->is_mes_queue) { - uint32_t offset = 0; - offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS); - ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - - offset = amdgpu_mes_ctx_get_offs(ring, - AMDGPU_MES_CTX_PADDING_OFFS); - gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); - cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); - *cpu_ptr = tmp; - } else { - r = amdgpu_device_wb_get(adev, &index); - if (r) { - dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); - return r; - } - - gpu_addr = adev->wb.gpu_addr + (index * 4); - adev->wb.wb[index] = cpu_to_le32(tmp); - - r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib); - if (r) { - DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); - goto err0; - } - } - - ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); - ib.ptr[1] = lower_32_bits(gpu_addr); - ib.ptr[2] = upper_32_bits(gpu_addr); - ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0); - ib.ptr[4] = 0xDEADBEEF; - ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); - ib.length_dw = 8; - - r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); - if (r) - goto err1; - - r = dma_fence_wait_timeout(f, false, timeout); - if (r == 0) { - DRM_ERROR("amdgpu: IB test timed out\n"); - r = -ETIMEDOUT; - goto err1; - } else if (r < 0) { - DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); - goto err1; - } - - if (ring->is_mes_queue) - tmp = le32_to_cpu(*cpu_ptr); - else - tmp = le32_to_cpu(adev->wb.wb[index]); - - if (tmp == 0xDEADBEEF) - r = 0; - else - r = -EINVAL; - -err1: - amdgpu_ib_free(adev, &ib, NULL); - dma_fence_put(f); -err0: - if (!ring->is_mes_queue) - amdgpu_device_wb_free(adev, index); - return r; -} - - -/** - * sdma_v5_2_vm_copy_pte - update PTEs by copying them from the GART - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @src: src addr to copy from - * @count: number of page entries to update - * - * Update PTEs by copying them from the GART using sDMA. - */ -static void sdma_v5_2_vm_copy_pte(struct amdgpu_ib *ib, - uint64_t pe, uint64_t src, - unsigned count) -{ - unsigned bytes = count * 8; - - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); - ib->ptr[ib->length_dw++] = bytes - 1; - ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ - ib->ptr[ib->length_dw++] = lower_32_bits(src); - ib->ptr[ib->length_dw++] = upper_32_bits(src); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - -} - -/** - * sdma_v5_2_vm_write_pte - update PTEs by writing them manually - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @value: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * - * Update PTEs by writing them manually using sDMA. - */ -static void sdma_v5_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, - uint64_t value, unsigned count, - uint32_t incr) -{ - unsigned ndw = count * 2; - - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - ib->ptr[ib->length_dw++] = ndw - 1; - for (; ndw > 0; ndw -= 2) { - ib->ptr[ib->length_dw++] = lower_32_bits(value); - ib->ptr[ib->length_dw++] = upper_32_bits(value); - value += incr; - } -} - -/** - * sdma_v5_2_vm_set_pte_pde - update the page tables using sDMA - * - * @ib: indirect buffer to fill with commands - * @pe: addr of the page entry - * @addr: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * @flags: access flags - * - * Update the page tables using sDMA. - */ -static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib, - uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint64_t flags) -{ - /* for physically contiguous pages (vram) */ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE); - ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */ - ib->ptr[ib->length_dw++] = upper_32_bits(pe); - ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ - ib->ptr[ib->length_dw++] = upper_32_bits(flags); - ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */ - ib->ptr[ib->length_dw++] = upper_32_bits(addr); - ib->ptr[ib->length_dw++] = incr; /* increment size */ - ib->ptr[ib->length_dw++] = 0; - ib->ptr[ib->length_dw++] = count - 1; /* number of entries */ -} - -/** - * sdma_v5_2_ring_pad_ib - pad the IB - * - * @ib: indirect buffer to fill with padding - * @ring: amdgpu_ring structure holding ring information - * - * Pad the IB with NOPs to a boundary multiple of 8. - */ -static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) -{ - struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); - u32 pad_count; - int i; - - pad_count = (-ib->length_dw) & 0x7; - for (i = 0; i < pad_count; i++) - if (sdma && sdma->burst_nop && (i == 0)) - ib->ptr[ib->length_dw++] = - SDMA_PKT_HEADER_OP(SDMA_OP_NOP) | - SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1); - else - ib->ptr[ib->length_dw++] = - SDMA_PKT_HEADER_OP(SDMA_OP_NOP); -} - - -/** - * sdma_v5_2_ring_emit_pipeline_sync - sync the pipeline - * - * @ring: amdgpu_ring pointer - * - * Make sure all previous operations are completed (CIK). - */ -static void sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring) -{ - uint32_t seq = ring->fence_drv.sync_seq; - uint64_t addr = ring->fence_drv.gpu_addr; - - /* wait for idle */ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */ - SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1)); - amdgpu_ring_write(ring, addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); - amdgpu_ring_write(ring, seq); /* reference */ - amdgpu_ring_write(ring, 0xffffffff); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ -} - - -/** - * sdma_v5_2_ring_emit_vm_flush - vm flush using sDMA - * - * @ring: amdgpu_ring pointer - * @vmid: vmid number to use - * @pd_addr: address - * - * Update the page table base and flush the VM TLB - * using sDMA. - */ -static void sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vmid, uint64_t pd_addr) -{ - amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); -} - -static void sdma_v5_2_ring_emit_wreg(struct amdgpu_ring *ring, - uint32_t reg, uint32_t val) -{ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | - SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); - amdgpu_ring_write(ring, reg); - amdgpu_ring_write(ring, val); -} - -static void sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | - SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | - SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */ - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, val); /* reference */ - amdgpu_ring_write(ring, mask); /* mask */ - amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | - SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); -} - -static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, - uint32_t reg0, uint32_t reg1, - uint32_t ref, uint32_t mask) -{ - amdgpu_ring_emit_wreg(ring, reg0, ref); - /* wait for a cycle to reset vm_inv_eng*_ack */ - amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0); - amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask); -} - -static int sdma_v5_2_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_sdma_init_microcode(adev, 0, true); - if (r) - return r; - - sdma_v5_2_set_ring_funcs(adev); - sdma_v5_2_set_buffer_funcs(adev); - sdma_v5_2_set_vm_pte_funcs(adev); - sdma_v5_2_set_irq_funcs(adev); - sdma_v5_2_set_mqd_funcs(adev); - - return 0; -} - -static unsigned sdma_v5_2_seq_to_irq_id(int seq_num) -{ - switch (seq_num) { - case 0: - return SOC15_IH_CLIENTID_SDMA0; - case 1: - return SOC15_IH_CLIENTID_SDMA1; - case 2: - return SOC15_IH_CLIENTID_SDMA2; - case 3: - return SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid; - default: - break; - } - return -EINVAL; -} - -static unsigned sdma_v5_2_seq_to_trap_id(int seq_num) -{ - switch (seq_num) { - case 0: - return SDMA0_5_0__SRCID__SDMA_TRAP; - case 1: - return SDMA1_5_0__SRCID__SDMA_TRAP; - case 2: - return SDMA2_5_0__SRCID__SDMA_TRAP; - case 3: - return SDMA3_5_0__SRCID__SDMA_TRAP; - default: - break; - } - return -EINVAL; -} - -static int sdma_v5_2_sw_init(void *handle) -{ - struct amdgpu_ring *ring; - int r, i; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - uint32_t *ptr; - - /* SDMA trap event */ - for (i = 0; i < adev->sdma.num_instances; i++) { - r = amdgpu_irq_add_id(adev, sdma_v5_2_seq_to_irq_id(i), - sdma_v5_2_seq_to_trap_id(i), - &adev->sdma.trap_irq); - if (r) - return r; - } - - for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; - ring->ring_obj = NULL; - ring->use_doorbell = true; - ring->me = i; - - DRM_INFO("use_doorbell being set to: [%s]\n", - ring->use_doorbell?"true":"false"); - - ring->doorbell_index = - (adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset - - ring->vm_hub = AMDGPU_GFXHUB(0); - sprintf(ring->name, "sdma%d", i); - r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq, - AMDGPU_SDMA_IRQ_INSTANCE0 + i, - AMDGPU_RING_PRIO_DEFAULT, NULL); - if (r) - return r; - } - - /* Allocate memory for SDMA IP Dump buffer */ - ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (ptr) - adev->sdma.ip_dump = ptr; - else - DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n"); - - return r; -} - -static int sdma_v5_2_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) - amdgpu_ring_fini(&adev->sdma.instance[i].ring); - - amdgpu_sdma_destroy_inst_ctx(adev, true); - - kfree(adev->sdma.ip_dump); - - return 0; -} - -static int sdma_v5_2_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_start(adev); -} - -static int sdma_v5_2_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (amdgpu_sriov_vf(adev)) - return 0; - - sdma_v5_2_ctx_switch_enable(adev, false); - sdma_v5_2_enable(adev, false); - - return 0; -} - -static int sdma_v5_2_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_hw_fini(adev); -} - -static int sdma_v5_2_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - return sdma_v5_2_hw_init(adev); -} - -static bool sdma_v5_2_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - u32 i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG)); - - if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK)) - return false; - } - - return true; -} - -static int sdma_v5_2_wait_for_idle(void *handle) -{ - unsigned i; - u32 sdma0, sdma1, sdma2, sdma3; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - for (i = 0; i < adev->usec_timeout; i++) { - sdma0 = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG)); - sdma1 = RREG32(sdma_v5_2_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG)); - sdma2 = RREG32(sdma_v5_2_get_reg_offset(adev, 2, mmSDMA0_STATUS_REG)); - sdma3 = RREG32(sdma_v5_2_get_reg_offset(adev, 3, mmSDMA0_STATUS_REG)); - - if (sdma0 & sdma1 & sdma2 & sdma3 & SDMA0_STATUS_REG__IDLE_MASK) - return 0; - udelay(1); - } - return -ETIMEDOUT; -} - -static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring) -{ - int i, r = 0; - struct amdgpu_device *adev = ring->adev; - u32 index = 0; - u64 sdma_gfx_preempt; - - amdgpu_sdma_get_index_from_ring(ring, &index); - sdma_gfx_preempt = - sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT); - - /* assert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, false); - - /* emit the trailing fence */ - ring->trail_seq += 1; - amdgpu_ring_alloc(ring, 10); - sdma_v5_2_ring_emit_fence(ring, ring->trail_fence_gpu_addr, - ring->trail_seq, 0); - amdgpu_ring_commit(ring); - - /* assert IB preemption */ - WREG32(sdma_gfx_preempt, 1); - - /* poll the trailing fence */ - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->trail_seq == - le32_to_cpu(*(ring->trail_fence_cpu_addr))) - break; - udelay(1); - } - - if (i >= adev->usec_timeout) { - r = -EINVAL; - DRM_ERROR("ring %d failed to be preempted\n", ring->idx); - } - - /* deassert IB preemption */ - WREG32(sdma_gfx_preempt, 0); - - /* deassert the preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, true); - return r; -} - -static int sdma_v5_2_set_trap_irq_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned type, - enum amdgpu_interrupt_state state) -{ - u32 sdma_cntl; - u32 reg_offset = sdma_v5_2_get_reg_offset(adev, type, mmSDMA0_CNTL); - - if (!amdgpu_sriov_vf(adev)) { - sdma_cntl = RREG32(reg_offset); - sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, - state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); - WREG32(reg_offset, sdma_cntl); - } - - return 0; -} - -static int sdma_v5_2_process_trap_irq(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t mes_queue_id = entry->src_data[0]; - - DRM_DEBUG("IH: SDMA trap\n"); - - if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) { - struct amdgpu_mes_queue *queue; - - mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK; - - spin_lock(&adev->mes.queue_id_lock); - queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id); - if (queue) { - DRM_DEBUG("process smda queue id = %d\n", mes_queue_id); - amdgpu_fence_process(queue->ring); - } - spin_unlock(&adev->mes.queue_id_lock); - return 0; - } - - switch (entry->client_id) { - case SOC15_IH_CLIENTID_SDMA0: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[0].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA1: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[1].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA2: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[2].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - case SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid: - switch (entry->ring_id) { - case 0: - amdgpu_fence_process(&adev->sdma.instance[3].ring); - break; - case 1: - /* XXX compute */ - break; - case 2: - /* XXX compute */ - break; - case 3: - /* XXX page queue*/ - break; - } - break; - } - return 0; -} - -static int sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - return 0; -} - -static bool sdma_v5_2_firmware_mgcg_support(struct amdgpu_device *adev, - int i) -{ - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 1): - if (adev->sdma.instance[i].fw_version < 70) - return false; - break; - case IP_VERSION(5, 2, 3): - if (adev->sdma.instance[i].fw_version < 47) - return false; - break; - case IP_VERSION(5, 2, 7): - if (adev->sdma.instance[i].fw_version < 9) - return false; - break; - default: - return true; - } - - return true; - -} - -static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t data, def; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - - if (!sdma_v5_2_firmware_mgcg_support(adev, i)) - adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG; - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { - /* Enable sdma clock gating */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL)); - data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK); - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data); - } else { - /* Disable sdma clock gating */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL)); - data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK | - SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK); - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data); - } - } -} - -static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev, - bool enable) -{ - uint32_t data, def; - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - if (adev->sdma.instance[i].fw_version < 70 && - amdgpu_ip_version(adev, SDMA0_HWIP, 0) == - IP_VERSION(5, 2, 1)) - adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS; - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) { - /* Enable sdma mem light sleep */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL)); - data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data); - - } else { - /* Disable sdma mem light sleep */ - def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL)); - data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; - if (def != data) - WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data); - - } - } -} - -static int sdma_v5_2_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (amdgpu_sriov_vf(adev)) - return 0; - - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 1): - case IP_VERSION(5, 2, 4): - case IP_VERSION(5, 2, 5): - case IP_VERSION(5, 2, 6): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 7): - sdma_v5_2_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE); - sdma_v5_2_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE); - break; - default: - break; - } - - return 0; -} - -static int sdma_v5_2_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -static void sdma_v5_2_get_clockgating_state(void *handle, u64 *flags) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int data; - - if (amdgpu_sriov_vf(adev)) - *flags = 0; - - /* AMD_CG_SUPPORT_SDMA_MGCG */ - data = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL)); - if (!(data & SDMA0_CLK_CTRL__CGCG_EN_OVERRIDE_MASK)) - *flags |= AMD_CG_SUPPORT_SDMA_MGCG; - - /* AMD_CG_SUPPORT_SDMA_LS */ - data = RREG32_KIQ(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL)); - if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK) - *flags |= AMD_CG_SUPPORT_SDMA_LS; -} - -static void sdma_v5_2_ring_begin_use(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - /* SDMA 5.2.3 (RMB) FW doesn't seem to properly - * disallow GFXOFF in some cases leading to - * hangs in SDMA. Disallow GFXOFF while SDMA is active. - * We can probably just limit this to 5.2.3, - * but it shouldn't hurt for other parts since - * this GFXOFF will be disallowed anyway when SDMA is - * active, this just makes it explicit. - * sdma_v5_2_ring_set_wptr() takes advantage of this - * to update the wptr because sometimes SDMA seems to miss - * doorbells when entering PG. If you remove this, update - * sdma_v5_2_ring_set_wptr() as well! - */ - amdgpu_gfx_off_ctrl(adev, false); -} - -static void sdma_v5_2_ring_end_use(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - /* SDMA 5.2.3 (RMB) FW doesn't seem to properly - * disallow GFXOFF in some cases leading to - * hangs in SDMA. Allow GFXOFF when SDMA is complete. - */ - amdgpu_gfx_off_ctrl(adev, true); -} - -static void sdma_v5_2_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - uint32_t instance_offset; - - if (!adev->sdma.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances); - for (i = 0; i < adev->sdma.num_instances; i++) { - instance_offset = i * reg_count; - drm_printf(p, "\nInstance:%d\n", i); - - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_5_2[j].reg_name, - adev->sdma.ip_dump[instance_offset + j]); - } -} - -static void sdma_v5_2_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t instance_offset; - uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2); - - if (!adev->sdma.ip_dump) - return; - - amdgpu_gfx_off_ctrl(adev, false); - for (i = 0; i < adev->sdma.num_instances; i++) { - instance_offset = i * reg_count; - for (j = 0; j < reg_count; j++) - adev->sdma.ip_dump[instance_offset + j] = - RREG32(sdma_v5_2_get_reg_offset(adev, i, - sdma_reg_list_5_2[j].reg_offset)); - } - amdgpu_gfx_off_ctrl(adev, true); -} - -static const struct amd_ip_funcs sdma_v5_2_ip_funcs = { - .name = "sdma_v5_2", - .early_init = sdma_v5_2_early_init, - .late_init = NULL, - .sw_init = sdma_v5_2_sw_init, - .sw_fini = sdma_v5_2_sw_fini, - .hw_init = sdma_v5_2_hw_init, - .hw_fini = sdma_v5_2_hw_fini, - .suspend = sdma_v5_2_suspend, - .resume = sdma_v5_2_resume, - .is_idle = sdma_v5_2_is_idle, - .wait_for_idle = sdma_v5_2_wait_for_idle, - .soft_reset = sdma_v5_2_soft_reset, - .set_clockgating_state = sdma_v5_2_set_clockgating_state, - .set_powergating_state = sdma_v5_2_set_powergating_state, - .get_clockgating_state = sdma_v5_2_get_clockgating_state, - .dump_ip_state = sdma_v5_2_dump_ip_state, - .print_ip_state = sdma_v5_2_print_ip_state, -}; - -static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = { - .type = AMDGPU_RING_TYPE_SDMA, - .align_mask = 0xf, - .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), - .support_64bit_ptrs = true, - .secure_submission_supported = true, - .get_rptr = sdma_v5_2_ring_get_rptr, - .get_wptr = sdma_v5_2_ring_get_wptr, - .set_wptr = sdma_v5_2_ring_set_wptr, - .emit_frame_size = - 5 + /* sdma_v5_2_ring_init_cond_exec */ - 6 + /* sdma_v5_2_ring_emit_hdp_flush */ - 3 + /* hdp_invalidate */ - 6 + /* sdma_v5_2_ring_emit_pipeline_sync */ - /* sdma_v5_2_ring_emit_vm_flush */ - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 + - 10 + 10 + 10, /* sdma_v5_2_ring_emit_fence x3 for user fence, vm fence */ - .emit_ib_size = 7 + 6, /* sdma_v5_2_ring_emit_ib */ - .emit_ib = sdma_v5_2_ring_emit_ib, - .emit_mem_sync = sdma_v5_2_ring_emit_mem_sync, - .emit_fence = sdma_v5_2_ring_emit_fence, - .emit_pipeline_sync = sdma_v5_2_ring_emit_pipeline_sync, - .emit_vm_flush = sdma_v5_2_ring_emit_vm_flush, - .emit_hdp_flush = sdma_v5_2_ring_emit_hdp_flush, - .test_ring = sdma_v5_2_ring_test_ring, - .test_ib = sdma_v5_2_ring_test_ib, - .insert_nop = sdma_v5_2_ring_insert_nop, - .pad_ib = sdma_v5_2_ring_pad_ib, - .begin_use = sdma_v5_2_ring_begin_use, - .end_use = sdma_v5_2_ring_end_use, - .emit_wreg = sdma_v5_2_ring_emit_wreg, - .emit_reg_wait = sdma_v5_2_ring_emit_reg_wait, - .emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait, - .init_cond_exec = sdma_v5_2_ring_init_cond_exec, - .preempt_ib = sdma_v5_2_ring_preempt_ib, -}; - -static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - adev->sdma.instance[i].ring.funcs = &sdma_v5_2_ring_funcs; - adev->sdma.instance[i].ring.me = i; - } -} - -static const struct amdgpu_irq_src_funcs sdma_v5_2_trap_irq_funcs = { - .set = sdma_v5_2_set_trap_irq_state, - .process = sdma_v5_2_process_trap_irq, -}; - -static const struct amdgpu_irq_src_funcs sdma_v5_2_illegal_inst_irq_funcs = { - .process = sdma_v5_2_process_illegal_inst_irq, -}; - -static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev) -{ - adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 + - adev->sdma.num_instances; - adev->sdma.trap_irq.funcs = &sdma_v5_2_trap_irq_funcs; - adev->sdma.illegal_inst_irq.funcs = &sdma_v5_2_illegal_inst_irq_funcs; -} - -/** - * sdma_v5_2_emit_copy_buffer - copy buffer using the sDMA engine - * - * @ib: indirect buffer to copy to - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @byte_count: number of bytes to xfer - * @copy_flags: copy flags for the buffers - * - * Copy GPU buffers using the DMA engine. - * Used by the amdgpu ttm implementation to move pages if - * registered as the asic copy callback. - */ -static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib, - uint64_t src_offset, - uint64_t dst_offset, - uint32_t byte_count, - uint32_t copy_flags) -{ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) | - SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0); - ib->ptr[ib->length_dw++] = byte_count - 1; - ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ - ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); - ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); -} - -/** - * sdma_v5_2_emit_fill_buffer - fill buffer using the sDMA engine - * - * @ib: indirect buffer to fill - * @src_data: value to write to buffer - * @dst_offset: dst GPU address - * @byte_count: number of bytes to xfer - * - * Fill GPU buffers using the DMA engine. - */ -static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib, - uint32_t src_data, - uint64_t dst_offset, - uint32_t byte_count) -{ - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL); - ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); - ib->ptr[ib->length_dw++] = src_data; - ib->ptr[ib->length_dw++] = byte_count - 1; -} - -static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { - .copy_max_bytes = 0x400000, - .copy_num_dw = 7, - .emit_copy_buffer = sdma_v5_2_emit_copy_buffer, - - .fill_max_bytes = 0x400000, - .fill_num_dw = 5, - .emit_fill_buffer = sdma_v5_2_emit_fill_buffer, -}; - -static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev) -{ - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } -} - -static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = { - .copy_pte_num_dw = 7, - .copy_pte = sdma_v5_2_vm_copy_pte, - .write_pte = sdma_v5_2_vm_write_pte, - .set_pte_pde = sdma_v5_2_vm_set_pte_pde, -}; - -static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev) -{ - unsigned i; - - if (adev->vm_manager.vm_pte_funcs == NULL) { - adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs; - for (i = 0; i < adev->sdma.num_instances; i++) { - adev->vm_manager.vm_pte_scheds[i] = - &adev->sdma.instance[i].ring.sched; - } - adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances; - } -} - -const struct amdgpu_ip_block_version sdma_v5_2_ip_block = { - .type = AMD_IP_BLOCK_TYPE_SDMA, - .major = 5, - .minor = 2, - .rev = 0, - .funcs = &sdma_v5_2_ip_funcs, -}; diff --git a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/postimage.2 b/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/postimage.2 deleted file mode 100644 index 0fda70336300..000000000000 --- a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/postimage.2 +++ /dev/null @@ -1,1927 +0,0 @@ -/* - * Copyright 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/firmware.h> -#include <drm/drm_drv.h> - -#include "amdgpu.h" -#include "amdgpu_vcn.h" -#include "amdgpu_pm.h" -#include "soc15.h" -#include "soc15d.h" -#include "soc15_hw_ip.h" -#include "vcn_v2_0.h" -#include "mmsch_v4_0_3.h" - -#include "vcn/vcn_4_0_3_offset.h" -#include "vcn/vcn_4_0_3_sh_mask.h" -#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" - -#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL -#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX -#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA -#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX - -#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 -#define VCN1_VID_SOC_ADDRESS_3_0 0x48300 - -static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) -}; - -#define NORMALIZE_VCN_REG_OFFSET(offset) \ - (offset & 0x1FFFF) - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state); -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, - int inst_idx, struct dpg_pause_state *new_state); -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect); -/** - * vcn_v4_0_3_early_init - set function pointers - * - * @handle: amdgpu_device pointer - * - * Set ring and irq function pointers - */ -static int vcn_v4_0_3_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* re-use enc ring as unified ring */ - adev->vcn.num_enc_rings = 1; - - vcn_v4_0_3_set_unified_ring_funcs(adev); - vcn_v4_0_3_set_irq_funcs(adev); - vcn_v4_0_3_set_ras_funcs(adev); - - return amdgpu_vcn_early_init(adev); -} - -/** - * vcn_v4_0_3_sw_init - sw init for VCN block - * - * @handle: amdgpu_device pointer - * - * Load firmware and sw initialization - */ -static int vcn_v4_0_3_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t *ptr; - - r = amdgpu_vcn_sw_init(adev); - if (r) - return r; - - amdgpu_vcn_setup_ucode(adev); - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - /* VCN DEC TRAP */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, - VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - vcn_inst = GET_INST(VCN, i); - - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->use_doorbell = true; - - if (!amdgpu_sriov_vf(adev)) - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst; - else - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 32 * vcn_inst; - - ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); - sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); - r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, - AMDGPU_RING_PRIO_DEFAULT, - &adev->vcn.inst[i].sched_score); - if (r) - return r; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); - fw_shared->sq.is_enabled = true; - - if (amdgpu_vcnfw_log) - amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); - } - - if (amdgpu_sriov_vf(adev)) { - r = amdgpu_virt_alloc_mm_table(adev); - if (r) - return r; - } - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) - adev->vcn.pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; - - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - r = amdgpu_vcn_ras_sw_init(adev); - if (r) { - dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); - return r; - } - } - - /* Allocate memory for VCN IP Dump buffer */ - ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (!ptr) { - DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); - adev->vcn.ip_dump = NULL; - } else { - adev->vcn.ip_dump = ptr; - } - - return 0; -} - -/** - * vcn_v4_0_3_sw_fini - sw fini for VCN block - * - * @handle: amdgpu_device pointer - * - * VCN suspend and free up sw allocation - */ -static int vcn_v4_0_3_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, r, idx; - - if (drm_dev_enter(&adev->ddev, &idx)) { - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = 0; - fw_shared->sq.is_enabled = cpu_to_le32(false); - } - drm_dev_exit(idx); - } - - if (amdgpu_sriov_vf(adev)) - amdgpu_virt_free_mm_table(adev); - - r = amdgpu_vcn_suspend(adev); - if (r) - return r; - - r = amdgpu_vcn_sw_fini(adev); - - kfree(adev->vcn.ip_dump); - - return r; -} - -/** - * vcn_v4_0_3_hw_init - start and test VCN block - * - * @handle: amdgpu_device pointer - * - * Initialize the hardware, boot up the VCPU and do some testing - */ -static int vcn_v4_0_3_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - - if (amdgpu_sriov_vf(adev)) { - r = vcn_v4_0_3_start_sriov(adev); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->wptr = 0; - ring->wptr_old = 0; - vcn_v4_0_3_unified_ring_set_wptr(ring); - ring->sched.ready = true; - } - } else { - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - ring = &adev->vcn.inst[i].ring_enc[0]; - - if (ring->use_doorbell) { - adev->nbio.funcs->vcn_doorbell_range( - adev, ring->use_doorbell, - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst, - adev->vcn.inst[i].aid_id); - - WREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL, - ring->doorbell_index - << VCN_RB1_DB_CTRL__OFFSET__SHIFT | - VCN_RB1_DB_CTRL__EN_MASK); - - /* Read DB_CTRL to flush the write DB_CTRL command. */ - RREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - } - - return r; -} - -/** - * vcn_v4_0_3_hw_fini - stop the hardware block - * - * @handle: amdgpu_device pointer - * - * Stop the VCN block, mark ring as not ready any more - */ -static int vcn_v4_0_3_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - cancel_delayed_work_sync(&adev->vcn.idle_work); - - if (adev->vcn.cur_state != AMD_PG_STATE_GATE) - vcn_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE); - - return 0; -} - -/** - * vcn_v4_0_3_suspend - suspend VCN block - * - * @handle: amdgpu_device pointer - * - * HW fini and suspend VCN block - */ -static int vcn_v4_0_3_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = vcn_v4_0_3_hw_fini(adev); - if (r) - return r; - - r = amdgpu_vcn_suspend(adev); - - return r; -} - -/** - * vcn_v4_0_3_resume - resume VCN block - * - * @handle: amdgpu_device pointer - * - * Resume firmware and hw init VCN block - */ -static int vcn_v4_0_3_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - r = vcn_v4_0_3_hw_init(adev); - - return r; -} - -/** - * vcn_v4_0_3_mc_resume - memory controller programming - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Let the VCN memory controller know it's offsets - */ -static void vcn_v4_0_3_mc_resume(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t offset, size, vcn_inst; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - vcn_inst = GET_INST(VCN, inst_idx); - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_lo)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_hi)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); - offset = 0; - } else { - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - offset = size; - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); - - /* cache window 1: stack */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, - AMDGPU_VCN_STACK_SIZE); - - /* cache window 2: context */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, - AMDGPU_VCN_CONTEXT_SIZE); - - /* non-cache window */ - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); - WREG32_SOC15( - VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); -} - -/** - * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Let the VCN memory controller know it's offsets with dpg mode - */ -static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - uint32_t offset, size; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_lo), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_hi), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } - offset = 0; - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - offset = size; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); - } - - if (!indirect) - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); - else - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); - - /* cache window 1: stack */ - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); - - /* cache window 2: context */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); - - /* non-cache window */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); - - /* VCN global tiling registers */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); -} - -/** - * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Disable clock gating for VCN block - */ -static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* VCN disable CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); - data &= ~(UVD_CGC_GATE__SYS_MASK - | UVD_CGC_GATE__MPEG2_MASK - | UVD_CGC_GATE__REGS_MASK - | UVD_CGC_GATE__RBC_MASK - | UVD_CGC_GATE__LMI_MC_MASK - | UVD_CGC_GATE__LMI_UMC_MASK - | UVD_CGC_GATE__MPC_MASK - | UVD_CGC_GATE__LBSI_MASK - | UVD_CGC_GATE__LRBBM_MASK - | UVD_CGC_GATE__WCB_MASK - | UVD_CGC_GATE__VCPU_MASK - | UVD_CGC_GATE__MMSCH_MASK); - - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK - | UVD_CGC_CTRL__MMSCH_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); - data |= (UVD_SUVD_CGC_GATE__SRE_MASK - | UVD_SUVD_CGC_GATE__SIT_MASK - | UVD_SUVD_CGC_GATE__SMP_MASK - | UVD_SUVD_CGC_GATE__SCM_MASK - | UVD_SUVD_CGC_GATE__SDB_MASK - | UVD_SUVD_CGC_GATE__SRE_H264_MASK - | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK - | UVD_SUVD_CGC_GATE__SIT_H264_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK - | UVD_SUVD_CGC_GATE__SCM_H264_MASK - | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK - | UVD_SUVD_CGC_GATE__SDB_H264_MASK - | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK - | UVD_SUVD_CGC_GATE__ENT_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK - | UVD_SUVD_CGC_GATE__SITE_MASK - | UVD_SUVD_CGC_GATE__SRE_VP9_MASK - | UVD_SUVD_CGC_GATE__SCM_VP9_MASK - | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK - | UVD_SUVD_CGC_GATE__SDB_VP9_MASK - | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode - * - * @adev: amdgpu_device pointer - * @sram_sel: sram select - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Disable clock gating for VCN block with dpg mode - */ -static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel, - int inst_idx, uint8_t indirect) -{ - uint32_t reg_data = 0; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - /* enable sw clock gating control */ - reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | - UVD_CGC_CTRL__MPEG2_MODE_MASK | - UVD_CGC_CTRL__REGS_MODE_MASK | - UVD_CGC_CTRL__RBC_MODE_MASK | - UVD_CGC_CTRL__LMI_MC_MODE_MASK | - UVD_CGC_CTRL__LMI_UMC_MODE_MASK | - UVD_CGC_CTRL__IDCT_MODE_MASK | - UVD_CGC_CTRL__MPRD_MODE_MASK | - UVD_CGC_CTRL__MPC_MODE_MASK | - UVD_CGC_CTRL__LBSI_MODE_MASK | - UVD_CGC_CTRL__LRBBM_MODE_MASK | - UVD_CGC_CTRL__WCB_MODE_MASK | - UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); - - /* turn off clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); - - /* turn on SUVD clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); - - /* turn on sw mode in UVD_SUVD_CGC_CTRL */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); -} - -/** - * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Enable clock gating for VCN block - */ -static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* enable VCN CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= (UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Start VCN block with dpg mode - */ -static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared = - adev->vcn.inst[inst_idx].fw_shared.cpu_addr; - struct amdgpu_ring *ring; - int vcn_inst; - uint32_t tmp; - - vcn_inst = GET_INST(VCN, inst_idx); - /* disable register anti-hang mechanism */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, - ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - /* enable dynamic power gating mode */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); - tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; - tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); - - if (indirect) { - DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", - inst_idx, adev->vcn.inst[inst_idx].aid_id); - adev->vcn.inst[inst_idx].dpg_sram_curr_addr = - (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; - /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ - WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, - adev->vcn.inst[inst_idx].aid_id, 0, true); - } - - /* enable clock gating */ - vcn_v4_0_3_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect); - - /* enable VCPU clock */ - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* disable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); - - /* setup regUVD_LMI_CTRL */ - tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__REQ_MODE_MASK | - UVD_LMI_CTRL__CRC_RESET_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | - (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | - 0x00100000L); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_CNTL), - 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXA0), - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXB0), - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUX), - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); - - vcn_v4_0_3_mc_resume_dpg_mode(adev, inst_idx, indirect); - - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* enable LMI MC and UMC channels */ - tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); - - vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); - - /* enable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); - - if (indirect) - amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); - - ring = &adev->vcn.inst[inst_idx].ring_enc[0]; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); - - /*resetting done, fw can check RB ring */ - fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); - - return 0; -} - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) -{ - int i, vcn_inst; - struct amdgpu_ring *ring_enc; - uint64_t cache_addr; - uint64_t rb_enc_addr; - uint64_t ctx_addr; - uint32_t param, resp, expected; - uint32_t offset, cache_size; - uint32_t tmp, timeout; - - struct amdgpu_mm_table *table = &adev->virt.mm_table; - uint32_t *table_loc; - uint32_t table_size; - uint32_t size, size_dw; - uint32_t init_status; - uint32_t enabled_vcn; - - struct mmsch_v4_0_cmd_direct_write - direct_wt = { {0} }; - struct mmsch_v4_0_cmd_direct_read_modify_write - direct_rd_mod_wt = { {0} }; - struct mmsch_v4_0_cmd_end end = { {0} }; - struct mmsch_v4_0_3_init_header header; - - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - volatile struct amdgpu_fw_shared_rb_setup *rb_setup; - - direct_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_WRITE; - direct_rd_mod_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; - end.cmd_header.command_type = MMSCH_COMMAND__END; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - vcn_inst = GET_INST(VCN, i); - - memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); - header.version = MMSCH_VERSION; - header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; - - table_loc = (uint32_t *)table->cpu_addr; - table_loc += header.total_size; - - table_size = 0; - - MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), - ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); - - cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); - - offset = 0; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), 0); - } else { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[i].gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[i].gpu_addr)); - offset = cache_size; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE0), - cache_size); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET1), 0); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE; - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET2), 0); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); - - fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; - rb_setup = &fw_shared->rb_setup; - - ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; - ring_enc->wptr = 0; - rb_enc_addr = ring_enc->gpu_addr; - - rb_setup->is_rb_enabled_flags |= RB_ENABLED; - rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); - rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); - rb_setup->rb_size = ring_enc->ring_size / 4; - fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); - MMSCH_V4_0_INSERT_END(); - - header.vcn0.init_status = 0; - header.vcn0.table_offset = header.total_size; - header.vcn0.table_size = table_size; - header.total_size += table_size; - - /* Send init table to mmsch */ - size = sizeof(struct mmsch_v4_0_3_init_header); - table_loc = (uint32_t *)table->cpu_addr; - memcpy((void *)table_loc, &header, size); - - ctx_addr = table->gpu_addr; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); - - tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); - tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; - tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); - - size = header.total_size; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); - - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); - - param = 0x00000001; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); - tmp = 0; - timeout = 1000; - resp = 0; - expected = MMSCH_VF_MAILBOX_RESP__OK; - while (resp != expected) { - resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); - if (resp != 0) - break; - - udelay(10); - tmp = tmp + 10; - if (tmp >= timeout) { - DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ - " waiting for regMMSCH_VF_MAILBOX_RESP "\ - "(expected=0x%08x, readback=0x%08x)\n", - tmp, expected, resp); - return -EBUSY; - } - } - - enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; - init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; - if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE - && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { - DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ - "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); - } - } - - return 0; -} - -/** - * vcn_v4_0_3_start - VCN start - * - * @adev: amdgpu_device pointer - * - * Start VCN block - */ -static int vcn_v4_0_3_start(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - struct amdgpu_ring *ring; - int i, j, k, r, vcn_inst; - uint32_t tmp; - - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, true); - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - r = vcn_v4_0_3_start_dpg_mode(adev, i, adev->vcn.indirect_sram); - continue; - } - - vcn_inst = GET_INST(VCN, i); - /* set VCN status busy */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | - UVD_STATUS__UVD_BUSY; - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); - - /*SW clock gating */ - vcn_v4_0_3_disable_clock_gating(adev, i); - - /* enable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__CLK_EN_MASK, - ~UVD_VCPU_CNTL__CLK_EN_MASK); - - /* disable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* enable LMI MC and UMC channels */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, - ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* setup regUVD_LMI_CTRL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, - tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); - - /* setup regUVD_MPC_CNTL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); - tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; - tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); - - /* setup UVD_MPC_SET_MUXA0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); - - /* setup UVD_MPC_SET_MUXB0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); - - /* setup UVD_MPC_SET_MUX */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); - - vcn_v4_0_3_mc_resume(adev, i); - - /* VCN global tiling registers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - - /* unblock VCPU register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - for (j = 0; j < 10; ++j) { - uint32_t status; - - for (k = 0; k < 100; ++k) { - status = RREG32_SOC15(VCN, vcn_inst, - regUVD_STATUS); - if (status & 2) - break; - mdelay(10); - } - r = 0; - if (status & 2) - break; - - DRM_DEV_ERROR(adev->dev, - "VCN decode not responding, trying to reset the VCPU!!!\n"); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - mdelay(10); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); - - mdelay(10); - r = -1; - } - - if (r) { - DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); - return r; - } - - /* enable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* clear the busy bit of VCN_STATUS */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, - ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); - - ring = &adev->vcn.inst[i].ring_enc[0]; - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - fw_shared->sq.queue_mode &= - cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); - - } - return 0; -} - -/** - * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * - * Stop VCN block with dpg mode - */ -static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t tmp; - int vcn_inst; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* Wait for power status to be 1 */ - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* wait for read ptr to be equal to write ptr */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); - - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* disable dynamic power gating mode */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, - ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); - return 0; -} - -/** - * vcn_v4_0_3_stop - VCN stop - * - * @adev: amdgpu_device pointer - * - * Stop VCN block - */ -static int vcn_v4_0_3_stop(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - int i, r = 0, vcn_inst; - uint32_t tmp; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - vcn_v4_0_3_stop_dpg_mode(adev, i); - continue; - } - - /* wait for vcn idle */ - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, - UVD_STATUS__IDLE, 0x7); - if (r) - goto Done; - - tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | - UVD_LMI_STATUS__READ_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* stall UMC channel */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); - tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); - tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | - UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* Unblock VCPU Register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), - UVD_RB_ARB_CTRL__VCPU_DIS_MASK, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - /* disable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~(UVD_VCPU_CNTL__CLK_EN_MASK)); - - /* reset LMI UMC/LMI/VCPU */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* clear VCN status */ - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); - - /* apply HW clock gating */ - vcn_v4_0_3_enable_clock_gating(adev, i); - } -Done: - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, false); - - return 0; -} - -/** - * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @new_state: pause state - * - * Pause dpg mode for VCN block - */ -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx, - struct dpg_pause_state *new_state) -{ - - return 0; -} - -/** - * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified read pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); -} - -/** - * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified write pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) - return *ring->wptr_cpu_addr; - else - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), - regUVD_RB_WPTR); -} - -static void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, mask); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned int vmid, uint64_t pd_addr) -{ - struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; - - pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); - - /* wait for reg writes */ - vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + - vmid * hub->ctx_addr_distance, - lower_32_bits(pd_addr), 0xffffffff); -} - -static void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - /* VCN engine access for HDP flush doesn't work when RRMT is enabled. - * This is a workaround to avoid any HDP flush through VCN ring. - */ -} - -/** - * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer - * - * @ring: amdgpu_ring pointer - * - * Commits the enc write pointer to the hardware - */ -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) { - *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); - WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); - } else { - WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, - lower_32_bits(ring->wptr)); - } -} - -static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { - .type = AMDGPU_RING_TYPE_VCN_ENC, - .align_mask = 0x3f, - .nop = VCN_ENC_CMD_NO_OP, - .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, - .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, - .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, - .emit_frame_size = - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + - 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ - 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ - 1, /* vcn_v2_0_enc_ring_insert_end */ - .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ - .emit_ib = vcn_v2_0_enc_ring_emit_ib, - .emit_fence = vcn_v2_0_enc_ring_emit_fence, - .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, - .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, - .test_ring = amdgpu_vcn_enc_ring_test_ring, - .test_ib = amdgpu_vcn_unified_ring_test_ib, - .insert_nop = amdgpu_ring_insert_nop, - .insert_end = vcn_v2_0_enc_ring_insert_end, - .pad_ib = amdgpu_ring_generic_pad_ib, - .begin_use = amdgpu_vcn_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, - .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, - .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, - .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, -}; - -/** - * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions - * - * @adev: amdgpu_device pointer - * - * Set unified ring functions - */ -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) -{ - int i, vcn_inst; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; - adev->vcn.inst[i].ring_enc[0].me = i; - vcn_inst = GET_INST(VCN, i); - adev->vcn.inst[i].aid_id = - vcn_inst / adev->vcn.num_inst_per_aid; - } -} - -/** - * vcn_v4_0_3_is_idle - check VCN block is idle - * - * @handle: amdgpu_device pointer - * - * Check whether VCN block is idle - */ -static bool vcn_v4_0_3_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 1; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == - UVD_STATUS__IDLE); - } - - return ret; -} - -/** - * vcn_v4_0_3_wait_for_idle - wait for VCN block idle - * - * @handle: amdgpu_device pointer - * - * Wait for VCN block idle - */ -static int vcn_v4_0_3_wait_for_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 0; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, - UVD_STATUS__IDLE, UVD_STATUS__IDLE); - if (ret) - return ret; - } - - return ret; -} - -/* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state - * - * @handle: amdgpu_device pointer - * @state: clock gating state - * - * Set VCN block clockgating state - */ -static int vcn_v4_0_3_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = state == AMD_CG_STATE_GATE; - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (enable) { - if (RREG32_SOC15(VCN, GET_INST(VCN, i), - regUVD_STATUS) != UVD_STATUS__IDLE) - return -EBUSY; - vcn_v4_0_3_enable_clock_gating(adev, i); - } else { - vcn_v4_0_3_disable_clock_gating(adev, i); - } - } - return 0; -} - -/** - * vcn_v4_0_3_set_powergating_state - set VCN block powergating state - * - * @handle: amdgpu_device pointer - * @state: power gating state - * - * Set VCN block powergating state - */ -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int ret; - - /* for SRIOV, guest should not control VCN Power-gating - * MMSCH FW should control Power-gating and clock-gating - * guest should avoid touching CGC and PG - */ - if (amdgpu_sriov_vf(adev)) { - adev->vcn.cur_state = AMD_PG_STATE_UNGATE; - return 0; - } - - if (state == adev->vcn.cur_state) - return 0; - - if (state == AMD_PG_STATE_GATE) - ret = vcn_v4_0_3_stop(adev); - else - ret = vcn_v4_0_3_start(adev); - - if (!ret) - adev->vcn.cur_state = state; - - return ret; -} - -/** - * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @type: interrupt types - * @state: interrupt states - * - * Set VCN block interrupt state - */ -static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned int type, - enum amdgpu_interrupt_state state) -{ - return 0; -} - -/** - * vcn_v4_0_3_process_interrupt - process VCN block interrupt - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @entry: interrupt entry from clients and sources - * - * Process VCN block interrupt - */ -static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t i, inst; - - i = node_id_to_phys_map[entry->node_id]; - - DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); - - for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) - if (adev->vcn.inst[inst].aid_id == i) - break; - - if (inst >= adev->vcn.num_vcn_inst) { - dev_WARN_ONCE(adev->dev, 1, - "Interrupt received for unknown VCN instance %d", - entry->node_id); - return 0; - } - - switch (entry->src_id) { - case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: - amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); - break; - default: - DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", - entry->src_id, entry->src_data[0]); - break; - } - - return 0; -} - -static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { - .set = vcn_v4_0_3_set_interrupt_state, - .process = vcn_v4_0_3_process_interrupt, -}; - -/** - * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions - * - * @adev: amdgpu_device pointer - * - * Set VCN block interrupt irq functions - */ -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst->irq.num_types++; - } - adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; -} - -static void vcn_v4_0_3_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t inst_off, is_powered; - - if (!adev->vcn.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) { - drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); - continue; - } - - inst_off = i * reg_count; - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) { - drm_printf(p, "\nActive Instance:VCN%d\n", i); - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, - adev->vcn.ip_dump[inst_off + j]); - } else { - drm_printf(p, "\nInactive Instance:VCN%d\n", i); - } - } -} - -static void vcn_v4_0_3_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - bool is_powered; - uint32_t inst_off, inst_id; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - - if (!adev->vcn.ip_dump) - return; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) - continue; - - inst_id = GET_INST(VCN, i); - inst_off = i * reg_count; - /* mmUVD_POWER_STATUS is always readable and is first element of the array */ - adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) - for (j = 1; j < reg_count; j++) - adev->vcn.ip_dump[inst_off + j] = - RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], - inst_id)); - } -} - -static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { - .name = "vcn_v4_0_3", - .early_init = vcn_v4_0_3_early_init, - .late_init = NULL, - .sw_init = vcn_v4_0_3_sw_init, - .sw_fini = vcn_v4_0_3_sw_fini, - .hw_init = vcn_v4_0_3_hw_init, - .hw_fini = vcn_v4_0_3_hw_fini, - .suspend = vcn_v4_0_3_suspend, - .resume = vcn_v4_0_3_resume, - .is_idle = vcn_v4_0_3_is_idle, - .wait_for_idle = vcn_v4_0_3_wait_for_idle, - .check_soft_reset = NULL, - .pre_soft_reset = NULL, - .soft_reset = NULL, - .post_soft_reset = NULL, - .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, - .set_powergating_state = vcn_v4_0_3_set_powergating_state, - .dump_ip_state = vcn_v4_0_3_dump_ip_state, - .print_ip_state = vcn_v4_0_3_print_ip_state, -}; - -const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { - .type = AMD_IP_BLOCK_TYPE_VCN, - .major = 4, - .minor = 0, - .rev = 3, - .funcs = &vcn_v4_0_3_ip_funcs, -}; - -static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, -}; - -static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst, - void *ras_err_status) -{ - struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; - - /* vcn v4_0_3 only support query uncorrectable errors */ - amdgpu_ras_inst_query_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - NULL, 0, GET_INST(VCN, vcn_inst), - AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, - &err_data->ue_count); -} - -static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, - void *ras_err_status) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); -} - -static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst) -{ - amdgpu_ras_inst_reset_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - GET_INST(VCN, vcn_inst)); -} - -static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_reset_ras_error_count(adev, i); -} - -static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { - .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, - .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, -}; - -static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { - .ras_block = { - .hw_ops = &vcn_v4_0_3_ras_hw_ops, - }, -}; - -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) -{ - adev->vcn.ras = &vcn_v4_0_3_ras; -} - -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect) -{ - uint32_t tmp; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) - return; - - tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), - tmp, 0, indirect); - - tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), - tmp, 0, indirect); - - tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), - tmp, 0, indirect); -} diff --git a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage b/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage deleted file mode 100644 index c55dabb2d37f..000000000000 --- a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage +++ /dev/null @@ -1,1930 +0,0 @@ -/* - * Copyright 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/firmware.h> -#include <drm/drm_drv.h> - -#include "amdgpu.h" -#include "amdgpu_vcn.h" -#include "amdgpu_pm.h" -#include "soc15.h" -#include "soc15d.h" -#include "soc15_hw_ip.h" -#include "vcn_v2_0.h" -#include "mmsch_v4_0_3.h" - -#include "vcn/vcn_4_0_3_offset.h" -#include "vcn/vcn_4_0_3_sh_mask.h" -#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" - -#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL -#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX -#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA -#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX - -#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 -#define VCN1_VID_SOC_ADDRESS_3_0 0x48300 - -<<<<<<< -======= -static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) -}; - ->>>>>>> -#define NORMALIZE_VCN_REG_OFFSET(offset) \ - (offset & 0x1FFFF) - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state); -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, - int inst_idx, struct dpg_pause_state *new_state); -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect); -/** - * vcn_v4_0_3_early_init - set function pointers - * - * @handle: amdgpu_device pointer - * - * Set ring and irq function pointers - */ -static int vcn_v4_0_3_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* re-use enc ring as unified ring */ - adev->vcn.num_enc_rings = 1; - - vcn_v4_0_3_set_unified_ring_funcs(adev); - vcn_v4_0_3_set_irq_funcs(adev); - vcn_v4_0_3_set_ras_funcs(adev); - - return amdgpu_vcn_early_init(adev); -} - -/** - * vcn_v4_0_3_sw_init - sw init for VCN block - * - * @handle: amdgpu_device pointer - * - * Load firmware and sw initialization - */ -static int vcn_v4_0_3_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t *ptr; - - r = amdgpu_vcn_sw_init(adev); - if (r) - return r; - - amdgpu_vcn_setup_ucode(adev); - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - /* VCN DEC TRAP */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, - VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - vcn_inst = GET_INST(VCN, i); - - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->use_doorbell = true; - - if (!amdgpu_sriov_vf(adev)) - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst; - else - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 32 * vcn_inst; - - ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); - sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); - r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, - AMDGPU_RING_PRIO_DEFAULT, - &adev->vcn.inst[i].sched_score); - if (r) - return r; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); - fw_shared->sq.is_enabled = true; - - if (amdgpu_vcnfw_log) - amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); - } - - if (amdgpu_sriov_vf(adev)) { - r = amdgpu_virt_alloc_mm_table(adev); - if (r) - return r; - } - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) - adev->vcn.pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; - - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - r = amdgpu_vcn_ras_sw_init(adev); - if (r) { - dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); - return r; - } - } - - /* Allocate memory for VCN IP Dump buffer */ - ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (!ptr) { - DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); - adev->vcn.ip_dump = NULL; - } else { - adev->vcn.ip_dump = ptr; - } - - return 0; -} - -/** - * vcn_v4_0_3_sw_fini - sw fini for VCN block - * - * @handle: amdgpu_device pointer - * - * VCN suspend and free up sw allocation - */ -static int vcn_v4_0_3_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, r, idx; - - if (drm_dev_enter(&adev->ddev, &idx)) { - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = 0; - fw_shared->sq.is_enabled = cpu_to_le32(false); - } - drm_dev_exit(idx); - } - - if (amdgpu_sriov_vf(adev)) - amdgpu_virt_free_mm_table(adev); - - r = amdgpu_vcn_suspend(adev); - if (r) - return r; - - r = amdgpu_vcn_sw_fini(adev); - - kfree(adev->vcn.ip_dump); - - return r; -} - -/** - * vcn_v4_0_3_hw_init - start and test VCN block - * - * @handle: amdgpu_device pointer - * - * Initialize the hardware, boot up the VCPU and do some testing - */ -static int vcn_v4_0_3_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - - if (amdgpu_sriov_vf(adev)) { - r = vcn_v4_0_3_start_sriov(adev); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->wptr = 0; - ring->wptr_old = 0; - vcn_v4_0_3_unified_ring_set_wptr(ring); - ring->sched.ready = true; - } - } else { - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - ring = &adev->vcn.inst[i].ring_enc[0]; - - if (ring->use_doorbell) { - adev->nbio.funcs->vcn_doorbell_range( - adev, ring->use_doorbell, - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst, - adev->vcn.inst[i].aid_id); - - WREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL, - ring->doorbell_index - << VCN_RB1_DB_CTRL__OFFSET__SHIFT | - VCN_RB1_DB_CTRL__EN_MASK); - - /* Read DB_CTRL to flush the write DB_CTRL command. */ - RREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - } - - return r; -} - -/** - * vcn_v4_0_3_hw_fini - stop the hardware block - * - * @handle: amdgpu_device pointer - * - * Stop the VCN block, mark ring as not ready any more - */ -static int vcn_v4_0_3_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - cancel_delayed_work_sync(&adev->vcn.idle_work); - - if (adev->vcn.cur_state != AMD_PG_STATE_GATE) - vcn_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE); - - return 0; -} - -/** - * vcn_v4_0_3_suspend - suspend VCN block - * - * @handle: amdgpu_device pointer - * - * HW fini and suspend VCN block - */ -static int vcn_v4_0_3_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = vcn_v4_0_3_hw_fini(adev); - if (r) - return r; - - r = amdgpu_vcn_suspend(adev); - - return r; -} - -/** - * vcn_v4_0_3_resume - resume VCN block - * - * @handle: amdgpu_device pointer - * - * Resume firmware and hw init VCN block - */ -static int vcn_v4_0_3_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - r = vcn_v4_0_3_hw_init(adev); - - return r; -} - -/** - * vcn_v4_0_3_mc_resume - memory controller programming - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Let the VCN memory controller know it's offsets - */ -static void vcn_v4_0_3_mc_resume(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t offset, size, vcn_inst; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - vcn_inst = GET_INST(VCN, inst_idx); - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_lo)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_hi)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); - offset = 0; - } else { - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - offset = size; - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); - - /* cache window 1: stack */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, - AMDGPU_VCN_STACK_SIZE); - - /* cache window 2: context */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, - AMDGPU_VCN_CONTEXT_SIZE); - - /* non-cache window */ - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); - WREG32_SOC15( - VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); -} - -/** - * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Let the VCN memory controller know it's offsets with dpg mode - */ -static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - uint32_t offset, size; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_lo), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_hi), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } - offset = 0; - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - offset = size; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); - } - - if (!indirect) - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); - else - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); - - /* cache window 1: stack */ - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); - - /* cache window 2: context */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); - - /* non-cache window */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); - - /* VCN global tiling registers */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); -} - -/** - * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Disable clock gating for VCN block - */ -static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* VCN disable CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); - data &= ~(UVD_CGC_GATE__SYS_MASK - | UVD_CGC_GATE__MPEG2_MASK - | UVD_CGC_GATE__REGS_MASK - | UVD_CGC_GATE__RBC_MASK - | UVD_CGC_GATE__LMI_MC_MASK - | UVD_CGC_GATE__LMI_UMC_MASK - | UVD_CGC_GATE__MPC_MASK - | UVD_CGC_GATE__LBSI_MASK - | UVD_CGC_GATE__LRBBM_MASK - | UVD_CGC_GATE__WCB_MASK - | UVD_CGC_GATE__VCPU_MASK - | UVD_CGC_GATE__MMSCH_MASK); - - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK - | UVD_CGC_CTRL__MMSCH_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); - data |= (UVD_SUVD_CGC_GATE__SRE_MASK - | UVD_SUVD_CGC_GATE__SIT_MASK - | UVD_SUVD_CGC_GATE__SMP_MASK - | UVD_SUVD_CGC_GATE__SCM_MASK - | UVD_SUVD_CGC_GATE__SDB_MASK - | UVD_SUVD_CGC_GATE__SRE_H264_MASK - | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK - | UVD_SUVD_CGC_GATE__SIT_H264_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK - | UVD_SUVD_CGC_GATE__SCM_H264_MASK - | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK - | UVD_SUVD_CGC_GATE__SDB_H264_MASK - | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK - | UVD_SUVD_CGC_GATE__ENT_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK - | UVD_SUVD_CGC_GATE__SITE_MASK - | UVD_SUVD_CGC_GATE__SRE_VP9_MASK - | UVD_SUVD_CGC_GATE__SCM_VP9_MASK - | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK - | UVD_SUVD_CGC_GATE__SDB_VP9_MASK - | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode - * - * @adev: amdgpu_device pointer - * @sram_sel: sram select - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Disable clock gating for VCN block with dpg mode - */ -static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel, - int inst_idx, uint8_t indirect) -{ - uint32_t reg_data = 0; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - /* enable sw clock gating control */ - reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | - UVD_CGC_CTRL__MPEG2_MODE_MASK | - UVD_CGC_CTRL__REGS_MODE_MASK | - UVD_CGC_CTRL__RBC_MODE_MASK | - UVD_CGC_CTRL__LMI_MC_MODE_MASK | - UVD_CGC_CTRL__LMI_UMC_MODE_MASK | - UVD_CGC_CTRL__IDCT_MODE_MASK | - UVD_CGC_CTRL__MPRD_MODE_MASK | - UVD_CGC_CTRL__MPC_MODE_MASK | - UVD_CGC_CTRL__LBSI_MODE_MASK | - UVD_CGC_CTRL__LRBBM_MODE_MASK | - UVD_CGC_CTRL__WCB_MODE_MASK | - UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); - - /* turn off clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); - - /* turn on SUVD clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); - - /* turn on sw mode in UVD_SUVD_CGC_CTRL */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); -} - -/** - * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Enable clock gating for VCN block - */ -static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* enable VCN CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= (UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Start VCN block with dpg mode - */ -static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared = - adev->vcn.inst[inst_idx].fw_shared.cpu_addr; - struct amdgpu_ring *ring; - int vcn_inst; - uint32_t tmp; - - vcn_inst = GET_INST(VCN, inst_idx); - /* disable register anti-hang mechanism */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, - ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - /* enable dynamic power gating mode */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); - tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; - tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); - - if (indirect) { - DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", - inst_idx, adev->vcn.inst[inst_idx].aid_id); - adev->vcn.inst[inst_idx].dpg_sram_curr_addr = - (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; - /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ - WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, - adev->vcn.inst[inst_idx].aid_id, 0, true); - } - - /* enable clock gating */ - vcn_v4_0_3_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect); - - /* enable VCPU clock */ - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* disable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); - - /* setup regUVD_LMI_CTRL */ - tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__REQ_MODE_MASK | - UVD_LMI_CTRL__CRC_RESET_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | - (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | - 0x00100000L); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_CNTL), - 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXA0), - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXB0), - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUX), - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); - - vcn_v4_0_3_mc_resume_dpg_mode(adev, inst_idx, indirect); - - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* enable LMI MC and UMC channels */ - tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); - - vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); - - /* enable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); - - if (indirect) - amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); - - ring = &adev->vcn.inst[inst_idx].ring_enc[0]; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); - - /*resetting done, fw can check RB ring */ - fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); - - return 0; -} - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) -{ - int i, vcn_inst; - struct amdgpu_ring *ring_enc; - uint64_t cache_addr; - uint64_t rb_enc_addr; - uint64_t ctx_addr; - uint32_t param, resp, expected; - uint32_t offset, cache_size; - uint32_t tmp, timeout; - - struct amdgpu_mm_table *table = &adev->virt.mm_table; - uint32_t *table_loc; - uint32_t table_size; - uint32_t size, size_dw; - uint32_t init_status; - uint32_t enabled_vcn; - - struct mmsch_v4_0_cmd_direct_write - direct_wt = { {0} }; - struct mmsch_v4_0_cmd_direct_read_modify_write - direct_rd_mod_wt = { {0} }; - struct mmsch_v4_0_cmd_end end = { {0} }; - struct mmsch_v4_0_3_init_header header; - - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - volatile struct amdgpu_fw_shared_rb_setup *rb_setup; - - direct_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_WRITE; - direct_rd_mod_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; - end.cmd_header.command_type = MMSCH_COMMAND__END; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - vcn_inst = GET_INST(VCN, i); - - memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); - header.version = MMSCH_VERSION; - header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; - - table_loc = (uint32_t *)table->cpu_addr; - table_loc += header.total_size; - - table_size = 0; - - MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), - ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); - - cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); - - offset = 0; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), 0); - } else { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[i].gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[i].gpu_addr)); - offset = cache_size; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE0), - cache_size); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET1), 0); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE; - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET2), 0); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); - - fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; - rb_setup = &fw_shared->rb_setup; - - ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; - ring_enc->wptr = 0; - rb_enc_addr = ring_enc->gpu_addr; - - rb_setup->is_rb_enabled_flags |= RB_ENABLED; - rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); - rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); - rb_setup->rb_size = ring_enc->ring_size / 4; - fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); - MMSCH_V4_0_INSERT_END(); - - header.vcn0.init_status = 0; - header.vcn0.table_offset = header.total_size; - header.vcn0.table_size = table_size; - header.total_size += table_size; - - /* Send init table to mmsch */ - size = sizeof(struct mmsch_v4_0_3_init_header); - table_loc = (uint32_t *)table->cpu_addr; - memcpy((void *)table_loc, &header, size); - - ctx_addr = table->gpu_addr; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); - - tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); - tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; - tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); - - size = header.total_size; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); - - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); - - param = 0x00000001; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); - tmp = 0; - timeout = 1000; - resp = 0; - expected = MMSCH_VF_MAILBOX_RESP__OK; - while (resp != expected) { - resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); - if (resp != 0) - break; - - udelay(10); - tmp = tmp + 10; - if (tmp >= timeout) { - DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ - " waiting for regMMSCH_VF_MAILBOX_RESP "\ - "(expected=0x%08x, readback=0x%08x)\n", - tmp, expected, resp); - return -EBUSY; - } - } - - enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; - init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; - if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE - && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { - DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ - "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); - } - } - - return 0; -} - -/** - * vcn_v4_0_3_start - VCN start - * - * @adev: amdgpu_device pointer - * - * Start VCN block - */ -static int vcn_v4_0_3_start(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - struct amdgpu_ring *ring; - int i, j, k, r, vcn_inst; - uint32_t tmp; - - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, true); - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - r = vcn_v4_0_3_start_dpg_mode(adev, i, adev->vcn.indirect_sram); - continue; - } - - vcn_inst = GET_INST(VCN, i); - /* set VCN status busy */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | - UVD_STATUS__UVD_BUSY; - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); - - /*SW clock gating */ - vcn_v4_0_3_disable_clock_gating(adev, i); - - /* enable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__CLK_EN_MASK, - ~UVD_VCPU_CNTL__CLK_EN_MASK); - - /* disable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* enable LMI MC and UMC channels */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, - ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* setup regUVD_LMI_CTRL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, - tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); - - /* setup regUVD_MPC_CNTL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); - tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; - tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); - - /* setup UVD_MPC_SET_MUXA0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); - - /* setup UVD_MPC_SET_MUXB0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); - - /* setup UVD_MPC_SET_MUX */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); - - vcn_v4_0_3_mc_resume(adev, i); - - /* VCN global tiling registers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - - /* unblock VCPU register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - for (j = 0; j < 10; ++j) { - uint32_t status; - - for (k = 0; k < 100; ++k) { - status = RREG32_SOC15(VCN, vcn_inst, - regUVD_STATUS); - if (status & 2) - break; - mdelay(10); - } - r = 0; - if (status & 2) - break; - - DRM_DEV_ERROR(adev->dev, - "VCN decode not responding, trying to reset the VCPU!!!\n"); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - mdelay(10); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); - - mdelay(10); - r = -1; - } - - if (r) { - DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); - return r; - } - - /* enable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* clear the busy bit of VCN_STATUS */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, - ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); - - ring = &adev->vcn.inst[i].ring_enc[0]; - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - fw_shared->sq.queue_mode &= - cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); - - } - return 0; -} - -/** - * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * - * Stop VCN block with dpg mode - */ -static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t tmp; - int vcn_inst; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* Wait for power status to be 1 */ - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* wait for read ptr to be equal to write ptr */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); - - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* disable dynamic power gating mode */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, - ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); - return 0; -} - -/** - * vcn_v4_0_3_stop - VCN stop - * - * @adev: amdgpu_device pointer - * - * Stop VCN block - */ -static int vcn_v4_0_3_stop(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - int i, r = 0, vcn_inst; - uint32_t tmp; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - vcn_v4_0_3_stop_dpg_mode(adev, i); - continue; - } - - /* wait for vcn idle */ - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, - UVD_STATUS__IDLE, 0x7); - if (r) - goto Done; - - tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | - UVD_LMI_STATUS__READ_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* stall UMC channel */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); - tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); - tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | - UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* Unblock VCPU Register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), - UVD_RB_ARB_CTRL__VCPU_DIS_MASK, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - /* disable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~(UVD_VCPU_CNTL__CLK_EN_MASK)); - - /* reset LMI UMC/LMI/VCPU */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* clear VCN status */ - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); - - /* apply HW clock gating */ - vcn_v4_0_3_enable_clock_gating(adev, i); - } -Done: - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, false); - - return 0; -} - -/** - * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @new_state: pause state - * - * Pause dpg mode for VCN block - */ -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx, - struct dpg_pause_state *new_state) -{ - - return 0; -} - -/** - * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified read pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); -} - -/** - * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified write pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) - return *ring->wptr_cpu_addr; - else - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), - regUVD_RB_WPTR); -} - -static void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, mask); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned int vmid, uint64_t pd_addr) -{ - struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; - - pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); - - /* wait for reg writes */ - vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + - vmid * hub->ctx_addr_distance, - lower_32_bits(pd_addr), 0xffffffff); -} - -static void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - /* VCN engine access for HDP flush doesn't work when RRMT is enabled. - * This is a workaround to avoid any HDP flush through VCN ring. - */ -} - -/** - * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer - * - * @ring: amdgpu_ring pointer - * - * Commits the enc write pointer to the hardware - */ -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) { - *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); - WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); - } else { - WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, - lower_32_bits(ring->wptr)); - } -} - -static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { - .type = AMDGPU_RING_TYPE_VCN_ENC, - .align_mask = 0x3f, - .nop = VCN_ENC_CMD_NO_OP, - .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, - .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, - .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, - .emit_frame_size = - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + - 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ - 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ - 1, /* vcn_v2_0_enc_ring_insert_end */ - .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ - .emit_ib = vcn_v2_0_enc_ring_emit_ib, - .emit_fence = vcn_v2_0_enc_ring_emit_fence, - .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, - .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, - .test_ring = amdgpu_vcn_enc_ring_test_ring, - .test_ib = amdgpu_vcn_unified_ring_test_ib, - .insert_nop = amdgpu_ring_insert_nop, - .insert_end = vcn_v2_0_enc_ring_insert_end, - .pad_ib = amdgpu_ring_generic_pad_ib, - .begin_use = amdgpu_vcn_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, - .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, - .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, - .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, -}; - -/** - * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions - * - * @adev: amdgpu_device pointer - * - * Set unified ring functions - */ -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) -{ - int i, vcn_inst; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; - adev->vcn.inst[i].ring_enc[0].me = i; - vcn_inst = GET_INST(VCN, i); - adev->vcn.inst[i].aid_id = - vcn_inst / adev->vcn.num_inst_per_aid; - } -} - -/** - * vcn_v4_0_3_is_idle - check VCN block is idle - * - * @handle: amdgpu_device pointer - * - * Check whether VCN block is idle - */ -static bool vcn_v4_0_3_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 1; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == - UVD_STATUS__IDLE); - } - - return ret; -} - -/** - * vcn_v4_0_3_wait_for_idle - wait for VCN block idle - * - * @handle: amdgpu_device pointer - * - * Wait for VCN block idle - */ -static int vcn_v4_0_3_wait_for_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 0; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, - UVD_STATUS__IDLE, UVD_STATUS__IDLE); - if (ret) - return ret; - } - - return ret; -} - -/* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state - * - * @handle: amdgpu_device pointer - * @state: clock gating state - * - * Set VCN block clockgating state - */ -static int vcn_v4_0_3_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = state == AMD_CG_STATE_GATE; - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (enable) { - if (RREG32_SOC15(VCN, GET_INST(VCN, i), - regUVD_STATUS) != UVD_STATUS__IDLE) - return -EBUSY; - vcn_v4_0_3_enable_clock_gating(adev, i); - } else { - vcn_v4_0_3_disable_clock_gating(adev, i); - } - } - return 0; -} - -/** - * vcn_v4_0_3_set_powergating_state - set VCN block powergating state - * - * @handle: amdgpu_device pointer - * @state: power gating state - * - * Set VCN block powergating state - */ -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int ret; - - /* for SRIOV, guest should not control VCN Power-gating - * MMSCH FW should control Power-gating and clock-gating - * guest should avoid touching CGC and PG - */ - if (amdgpu_sriov_vf(adev)) { - adev->vcn.cur_state = AMD_PG_STATE_UNGATE; - return 0; - } - - if (state == adev->vcn.cur_state) - return 0; - - if (state == AMD_PG_STATE_GATE) - ret = vcn_v4_0_3_stop(adev); - else - ret = vcn_v4_0_3_start(adev); - - if (!ret) - adev->vcn.cur_state = state; - - return ret; -} - -/** - * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @type: interrupt types - * @state: interrupt states - * - * Set VCN block interrupt state - */ -static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned int type, - enum amdgpu_interrupt_state state) -{ - return 0; -} - -/** - * vcn_v4_0_3_process_interrupt - process VCN block interrupt - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @entry: interrupt entry from clients and sources - * - * Process VCN block interrupt - */ -static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t i, inst; - - i = node_id_to_phys_map[entry->node_id]; - - DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); - - for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) - if (adev->vcn.inst[inst].aid_id == i) - break; - - if (inst >= adev->vcn.num_vcn_inst) { - dev_WARN_ONCE(adev->dev, 1, - "Interrupt received for unknown VCN instance %d", - entry->node_id); - return 0; - } - - switch (entry->src_id) { - case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: - amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); - break; - default: - DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", - entry->src_id, entry->src_data[0]); - break; - } - - return 0; -} - -static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { - .set = vcn_v4_0_3_set_interrupt_state, - .process = vcn_v4_0_3_process_interrupt, -}; - -/** - * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions - * - * @adev: amdgpu_device pointer - * - * Set VCN block interrupt irq functions - */ -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst->irq.num_types++; - } - adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; -} - -static void vcn_v4_0_3_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t inst_off, is_powered; - - if (!adev->vcn.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) { - drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); - continue; - } - - inst_off = i * reg_count; - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) { - drm_printf(p, "\nActive Instance:VCN%d\n", i); - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, - adev->vcn.ip_dump[inst_off + j]); - } else { - drm_printf(p, "\nInactive Instance:VCN%d\n", i); - } - } -} - -static void vcn_v4_0_3_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - bool is_powered; - uint32_t inst_off, inst_id; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - - if (!adev->vcn.ip_dump) - return; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) - continue; - - inst_id = GET_INST(VCN, i); - inst_off = i * reg_count; - /* mmUVD_POWER_STATUS is always readable and is first element of the array */ - adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) - for (j = 1; j < reg_count; j++) - adev->vcn.ip_dump[inst_off + j] = - RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], - inst_id)); - } -} - -static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { - .name = "vcn_v4_0_3", - .early_init = vcn_v4_0_3_early_init, - .late_init = NULL, - .sw_init = vcn_v4_0_3_sw_init, - .sw_fini = vcn_v4_0_3_sw_fini, - .hw_init = vcn_v4_0_3_hw_init, - .hw_fini = vcn_v4_0_3_hw_fini, - .suspend = vcn_v4_0_3_suspend, - .resume = vcn_v4_0_3_resume, - .is_idle = vcn_v4_0_3_is_idle, - .wait_for_idle = vcn_v4_0_3_wait_for_idle, - .check_soft_reset = NULL, - .pre_soft_reset = NULL, - .soft_reset = NULL, - .post_soft_reset = NULL, - .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, - .set_powergating_state = vcn_v4_0_3_set_powergating_state, - .dump_ip_state = vcn_v4_0_3_dump_ip_state, - .print_ip_state = vcn_v4_0_3_print_ip_state, -}; - -const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { - .type = AMD_IP_BLOCK_TYPE_VCN, - .major = 4, - .minor = 0, - .rev = 3, - .funcs = &vcn_v4_0_3_ip_funcs, -}; - -static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, -}; - -static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst, - void *ras_err_status) -{ - struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; - - /* vcn v4_0_3 only support query uncorrectable errors */ - amdgpu_ras_inst_query_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - NULL, 0, GET_INST(VCN, vcn_inst), - AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, - &err_data->ue_count); -} - -static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, - void *ras_err_status) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); -} - -static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst) -{ - amdgpu_ras_inst_reset_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - GET_INST(VCN, vcn_inst)); -} - -static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_reset_ras_error_count(adev, i); -} - -static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { - .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, - .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, -}; - -static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { - .ras_block = { - .hw_ops = &vcn_v4_0_3_ras_hw_ops, - }, -}; - -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) -{ - adev->vcn.ras = &vcn_v4_0_3_ras; -} - -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect) -{ - uint32_t tmp; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) - return; - - tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), - tmp, 0, indirect); - - tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), - tmp, 0, indirect); - - tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), - tmp, 0, indirect); -} diff --git a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.1 b/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.1 deleted file mode 100644 index c55dabb2d37f..000000000000 --- a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.1 +++ /dev/null @@ -1,1930 +0,0 @@ -/* - * Copyright 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/firmware.h> -#include <drm/drm_drv.h> - -#include "amdgpu.h" -#include "amdgpu_vcn.h" -#include "amdgpu_pm.h" -#include "soc15.h" -#include "soc15d.h" -#include "soc15_hw_ip.h" -#include "vcn_v2_0.h" -#include "mmsch_v4_0_3.h" - -#include "vcn/vcn_4_0_3_offset.h" -#include "vcn/vcn_4_0_3_sh_mask.h" -#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" - -#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL -#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX -#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA -#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX - -#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 -#define VCN1_VID_SOC_ADDRESS_3_0 0x48300 - -<<<<<<< -======= -static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) -}; - ->>>>>>> -#define NORMALIZE_VCN_REG_OFFSET(offset) \ - (offset & 0x1FFFF) - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state); -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, - int inst_idx, struct dpg_pause_state *new_state); -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect); -/** - * vcn_v4_0_3_early_init - set function pointers - * - * @handle: amdgpu_device pointer - * - * Set ring and irq function pointers - */ -static int vcn_v4_0_3_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* re-use enc ring as unified ring */ - adev->vcn.num_enc_rings = 1; - - vcn_v4_0_3_set_unified_ring_funcs(adev); - vcn_v4_0_3_set_irq_funcs(adev); - vcn_v4_0_3_set_ras_funcs(adev); - - return amdgpu_vcn_early_init(adev); -} - -/** - * vcn_v4_0_3_sw_init - sw init for VCN block - * - * @handle: amdgpu_device pointer - * - * Load firmware and sw initialization - */ -static int vcn_v4_0_3_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t *ptr; - - r = amdgpu_vcn_sw_init(adev); - if (r) - return r; - - amdgpu_vcn_setup_ucode(adev); - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - /* VCN DEC TRAP */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, - VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - vcn_inst = GET_INST(VCN, i); - - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->use_doorbell = true; - - if (!amdgpu_sriov_vf(adev)) - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst; - else - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 32 * vcn_inst; - - ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); - sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); - r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, - AMDGPU_RING_PRIO_DEFAULT, - &adev->vcn.inst[i].sched_score); - if (r) - return r; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); - fw_shared->sq.is_enabled = true; - - if (amdgpu_vcnfw_log) - amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); - } - - if (amdgpu_sriov_vf(adev)) { - r = amdgpu_virt_alloc_mm_table(adev); - if (r) - return r; - } - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) - adev->vcn.pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; - - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - r = amdgpu_vcn_ras_sw_init(adev); - if (r) { - dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); - return r; - } - } - - /* Allocate memory for VCN IP Dump buffer */ - ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (!ptr) { - DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); - adev->vcn.ip_dump = NULL; - } else { - adev->vcn.ip_dump = ptr; - } - - return 0; -} - -/** - * vcn_v4_0_3_sw_fini - sw fini for VCN block - * - * @handle: amdgpu_device pointer - * - * VCN suspend and free up sw allocation - */ -static int vcn_v4_0_3_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, r, idx; - - if (drm_dev_enter(&adev->ddev, &idx)) { - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = 0; - fw_shared->sq.is_enabled = cpu_to_le32(false); - } - drm_dev_exit(idx); - } - - if (amdgpu_sriov_vf(adev)) - amdgpu_virt_free_mm_table(adev); - - r = amdgpu_vcn_suspend(adev); - if (r) - return r; - - r = amdgpu_vcn_sw_fini(adev); - - kfree(adev->vcn.ip_dump); - - return r; -} - -/** - * vcn_v4_0_3_hw_init - start and test VCN block - * - * @handle: amdgpu_device pointer - * - * Initialize the hardware, boot up the VCPU and do some testing - */ -static int vcn_v4_0_3_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - - if (amdgpu_sriov_vf(adev)) { - r = vcn_v4_0_3_start_sriov(adev); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->wptr = 0; - ring->wptr_old = 0; - vcn_v4_0_3_unified_ring_set_wptr(ring); - ring->sched.ready = true; - } - } else { - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - ring = &adev->vcn.inst[i].ring_enc[0]; - - if (ring->use_doorbell) { - adev->nbio.funcs->vcn_doorbell_range( - adev, ring->use_doorbell, - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst, - adev->vcn.inst[i].aid_id); - - WREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL, - ring->doorbell_index - << VCN_RB1_DB_CTRL__OFFSET__SHIFT | - VCN_RB1_DB_CTRL__EN_MASK); - - /* Read DB_CTRL to flush the write DB_CTRL command. */ - RREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - } - - return r; -} - -/** - * vcn_v4_0_3_hw_fini - stop the hardware block - * - * @handle: amdgpu_device pointer - * - * Stop the VCN block, mark ring as not ready any more - */ -static int vcn_v4_0_3_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - cancel_delayed_work_sync(&adev->vcn.idle_work); - - if (adev->vcn.cur_state != AMD_PG_STATE_GATE) - vcn_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE); - - return 0; -} - -/** - * vcn_v4_0_3_suspend - suspend VCN block - * - * @handle: amdgpu_device pointer - * - * HW fini and suspend VCN block - */ -static int vcn_v4_0_3_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = vcn_v4_0_3_hw_fini(adev); - if (r) - return r; - - r = amdgpu_vcn_suspend(adev); - - return r; -} - -/** - * vcn_v4_0_3_resume - resume VCN block - * - * @handle: amdgpu_device pointer - * - * Resume firmware and hw init VCN block - */ -static int vcn_v4_0_3_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - r = vcn_v4_0_3_hw_init(adev); - - return r; -} - -/** - * vcn_v4_0_3_mc_resume - memory controller programming - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Let the VCN memory controller know it's offsets - */ -static void vcn_v4_0_3_mc_resume(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t offset, size, vcn_inst; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - vcn_inst = GET_INST(VCN, inst_idx); - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_lo)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_hi)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); - offset = 0; - } else { - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - offset = size; - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); - - /* cache window 1: stack */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, - AMDGPU_VCN_STACK_SIZE); - - /* cache window 2: context */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, - AMDGPU_VCN_CONTEXT_SIZE); - - /* non-cache window */ - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); - WREG32_SOC15( - VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); -} - -/** - * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Let the VCN memory controller know it's offsets with dpg mode - */ -static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - uint32_t offset, size; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_lo), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_hi), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } - offset = 0; - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - offset = size; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); - } - - if (!indirect) - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); - else - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); - - /* cache window 1: stack */ - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); - - /* cache window 2: context */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); - - /* non-cache window */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); - - /* VCN global tiling registers */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); -} - -/** - * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Disable clock gating for VCN block - */ -static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* VCN disable CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); - data &= ~(UVD_CGC_GATE__SYS_MASK - | UVD_CGC_GATE__MPEG2_MASK - | UVD_CGC_GATE__REGS_MASK - | UVD_CGC_GATE__RBC_MASK - | UVD_CGC_GATE__LMI_MC_MASK - | UVD_CGC_GATE__LMI_UMC_MASK - | UVD_CGC_GATE__MPC_MASK - | UVD_CGC_GATE__LBSI_MASK - | UVD_CGC_GATE__LRBBM_MASK - | UVD_CGC_GATE__WCB_MASK - | UVD_CGC_GATE__VCPU_MASK - | UVD_CGC_GATE__MMSCH_MASK); - - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK - | UVD_CGC_CTRL__MMSCH_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); - data |= (UVD_SUVD_CGC_GATE__SRE_MASK - | UVD_SUVD_CGC_GATE__SIT_MASK - | UVD_SUVD_CGC_GATE__SMP_MASK - | UVD_SUVD_CGC_GATE__SCM_MASK - | UVD_SUVD_CGC_GATE__SDB_MASK - | UVD_SUVD_CGC_GATE__SRE_H264_MASK - | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK - | UVD_SUVD_CGC_GATE__SIT_H264_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK - | UVD_SUVD_CGC_GATE__SCM_H264_MASK - | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK - | UVD_SUVD_CGC_GATE__SDB_H264_MASK - | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK - | UVD_SUVD_CGC_GATE__ENT_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK - | UVD_SUVD_CGC_GATE__SITE_MASK - | UVD_SUVD_CGC_GATE__SRE_VP9_MASK - | UVD_SUVD_CGC_GATE__SCM_VP9_MASK - | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK - | UVD_SUVD_CGC_GATE__SDB_VP9_MASK - | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode - * - * @adev: amdgpu_device pointer - * @sram_sel: sram select - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Disable clock gating for VCN block with dpg mode - */ -static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel, - int inst_idx, uint8_t indirect) -{ - uint32_t reg_data = 0; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - /* enable sw clock gating control */ - reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | - UVD_CGC_CTRL__MPEG2_MODE_MASK | - UVD_CGC_CTRL__REGS_MODE_MASK | - UVD_CGC_CTRL__RBC_MODE_MASK | - UVD_CGC_CTRL__LMI_MC_MODE_MASK | - UVD_CGC_CTRL__LMI_UMC_MODE_MASK | - UVD_CGC_CTRL__IDCT_MODE_MASK | - UVD_CGC_CTRL__MPRD_MODE_MASK | - UVD_CGC_CTRL__MPC_MODE_MASK | - UVD_CGC_CTRL__LBSI_MODE_MASK | - UVD_CGC_CTRL__LRBBM_MODE_MASK | - UVD_CGC_CTRL__WCB_MODE_MASK | - UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); - - /* turn off clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); - - /* turn on SUVD clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); - - /* turn on sw mode in UVD_SUVD_CGC_CTRL */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); -} - -/** - * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Enable clock gating for VCN block - */ -static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* enable VCN CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= (UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Start VCN block with dpg mode - */ -static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared = - adev->vcn.inst[inst_idx].fw_shared.cpu_addr; - struct amdgpu_ring *ring; - int vcn_inst; - uint32_t tmp; - - vcn_inst = GET_INST(VCN, inst_idx); - /* disable register anti-hang mechanism */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, - ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - /* enable dynamic power gating mode */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); - tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; - tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); - - if (indirect) { - DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", - inst_idx, adev->vcn.inst[inst_idx].aid_id); - adev->vcn.inst[inst_idx].dpg_sram_curr_addr = - (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; - /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ - WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, - adev->vcn.inst[inst_idx].aid_id, 0, true); - } - - /* enable clock gating */ - vcn_v4_0_3_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect); - - /* enable VCPU clock */ - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* disable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); - - /* setup regUVD_LMI_CTRL */ - tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__REQ_MODE_MASK | - UVD_LMI_CTRL__CRC_RESET_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | - (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | - 0x00100000L); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_CNTL), - 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXA0), - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXB0), - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUX), - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); - - vcn_v4_0_3_mc_resume_dpg_mode(adev, inst_idx, indirect); - - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* enable LMI MC and UMC channels */ - tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); - - vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); - - /* enable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); - - if (indirect) - amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); - - ring = &adev->vcn.inst[inst_idx].ring_enc[0]; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); - - /*resetting done, fw can check RB ring */ - fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); - - return 0; -} - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) -{ - int i, vcn_inst; - struct amdgpu_ring *ring_enc; - uint64_t cache_addr; - uint64_t rb_enc_addr; - uint64_t ctx_addr; - uint32_t param, resp, expected; - uint32_t offset, cache_size; - uint32_t tmp, timeout; - - struct amdgpu_mm_table *table = &adev->virt.mm_table; - uint32_t *table_loc; - uint32_t table_size; - uint32_t size, size_dw; - uint32_t init_status; - uint32_t enabled_vcn; - - struct mmsch_v4_0_cmd_direct_write - direct_wt = { {0} }; - struct mmsch_v4_0_cmd_direct_read_modify_write - direct_rd_mod_wt = { {0} }; - struct mmsch_v4_0_cmd_end end = { {0} }; - struct mmsch_v4_0_3_init_header header; - - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - volatile struct amdgpu_fw_shared_rb_setup *rb_setup; - - direct_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_WRITE; - direct_rd_mod_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; - end.cmd_header.command_type = MMSCH_COMMAND__END; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - vcn_inst = GET_INST(VCN, i); - - memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); - header.version = MMSCH_VERSION; - header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; - - table_loc = (uint32_t *)table->cpu_addr; - table_loc += header.total_size; - - table_size = 0; - - MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), - ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); - - cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); - - offset = 0; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), 0); - } else { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[i].gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[i].gpu_addr)); - offset = cache_size; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE0), - cache_size); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET1), 0); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE; - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET2), 0); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); - - fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; - rb_setup = &fw_shared->rb_setup; - - ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; - ring_enc->wptr = 0; - rb_enc_addr = ring_enc->gpu_addr; - - rb_setup->is_rb_enabled_flags |= RB_ENABLED; - rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); - rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); - rb_setup->rb_size = ring_enc->ring_size / 4; - fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); - MMSCH_V4_0_INSERT_END(); - - header.vcn0.init_status = 0; - header.vcn0.table_offset = header.total_size; - header.vcn0.table_size = table_size; - header.total_size += table_size; - - /* Send init table to mmsch */ - size = sizeof(struct mmsch_v4_0_3_init_header); - table_loc = (uint32_t *)table->cpu_addr; - memcpy((void *)table_loc, &header, size); - - ctx_addr = table->gpu_addr; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); - - tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); - tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; - tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); - - size = header.total_size; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); - - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); - - param = 0x00000001; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); - tmp = 0; - timeout = 1000; - resp = 0; - expected = MMSCH_VF_MAILBOX_RESP__OK; - while (resp != expected) { - resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); - if (resp != 0) - break; - - udelay(10); - tmp = tmp + 10; - if (tmp >= timeout) { - DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ - " waiting for regMMSCH_VF_MAILBOX_RESP "\ - "(expected=0x%08x, readback=0x%08x)\n", - tmp, expected, resp); - return -EBUSY; - } - } - - enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; - init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; - if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE - && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { - DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ - "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); - } - } - - return 0; -} - -/** - * vcn_v4_0_3_start - VCN start - * - * @adev: amdgpu_device pointer - * - * Start VCN block - */ -static int vcn_v4_0_3_start(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - struct amdgpu_ring *ring; - int i, j, k, r, vcn_inst; - uint32_t tmp; - - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, true); - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - r = vcn_v4_0_3_start_dpg_mode(adev, i, adev->vcn.indirect_sram); - continue; - } - - vcn_inst = GET_INST(VCN, i); - /* set VCN status busy */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | - UVD_STATUS__UVD_BUSY; - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); - - /*SW clock gating */ - vcn_v4_0_3_disable_clock_gating(adev, i); - - /* enable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__CLK_EN_MASK, - ~UVD_VCPU_CNTL__CLK_EN_MASK); - - /* disable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* enable LMI MC and UMC channels */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, - ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* setup regUVD_LMI_CTRL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, - tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); - - /* setup regUVD_MPC_CNTL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); - tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; - tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); - - /* setup UVD_MPC_SET_MUXA0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); - - /* setup UVD_MPC_SET_MUXB0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); - - /* setup UVD_MPC_SET_MUX */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); - - vcn_v4_0_3_mc_resume(adev, i); - - /* VCN global tiling registers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - - /* unblock VCPU register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - for (j = 0; j < 10; ++j) { - uint32_t status; - - for (k = 0; k < 100; ++k) { - status = RREG32_SOC15(VCN, vcn_inst, - regUVD_STATUS); - if (status & 2) - break; - mdelay(10); - } - r = 0; - if (status & 2) - break; - - DRM_DEV_ERROR(adev->dev, - "VCN decode not responding, trying to reset the VCPU!!!\n"); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - mdelay(10); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); - - mdelay(10); - r = -1; - } - - if (r) { - DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); - return r; - } - - /* enable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* clear the busy bit of VCN_STATUS */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, - ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); - - ring = &adev->vcn.inst[i].ring_enc[0]; - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - fw_shared->sq.queue_mode &= - cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); - - } - return 0; -} - -/** - * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * - * Stop VCN block with dpg mode - */ -static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t tmp; - int vcn_inst; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* Wait for power status to be 1 */ - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* wait for read ptr to be equal to write ptr */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); - - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* disable dynamic power gating mode */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, - ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); - return 0; -} - -/** - * vcn_v4_0_3_stop - VCN stop - * - * @adev: amdgpu_device pointer - * - * Stop VCN block - */ -static int vcn_v4_0_3_stop(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - int i, r = 0, vcn_inst; - uint32_t tmp; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - vcn_v4_0_3_stop_dpg_mode(adev, i); - continue; - } - - /* wait for vcn idle */ - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, - UVD_STATUS__IDLE, 0x7); - if (r) - goto Done; - - tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | - UVD_LMI_STATUS__READ_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* stall UMC channel */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); - tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); - tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | - UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* Unblock VCPU Register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), - UVD_RB_ARB_CTRL__VCPU_DIS_MASK, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - /* disable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~(UVD_VCPU_CNTL__CLK_EN_MASK)); - - /* reset LMI UMC/LMI/VCPU */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* clear VCN status */ - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); - - /* apply HW clock gating */ - vcn_v4_0_3_enable_clock_gating(adev, i); - } -Done: - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, false); - - return 0; -} - -/** - * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @new_state: pause state - * - * Pause dpg mode for VCN block - */ -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx, - struct dpg_pause_state *new_state) -{ - - return 0; -} - -/** - * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified read pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); -} - -/** - * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified write pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) - return *ring->wptr_cpu_addr; - else - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), - regUVD_RB_WPTR); -} - -static void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, mask); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned int vmid, uint64_t pd_addr) -{ - struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; - - pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); - - /* wait for reg writes */ - vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + - vmid * hub->ctx_addr_distance, - lower_32_bits(pd_addr), 0xffffffff); -} - -static void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - /* VCN engine access for HDP flush doesn't work when RRMT is enabled. - * This is a workaround to avoid any HDP flush through VCN ring. - */ -} - -/** - * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer - * - * @ring: amdgpu_ring pointer - * - * Commits the enc write pointer to the hardware - */ -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) { - *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); - WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); - } else { - WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, - lower_32_bits(ring->wptr)); - } -} - -static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { - .type = AMDGPU_RING_TYPE_VCN_ENC, - .align_mask = 0x3f, - .nop = VCN_ENC_CMD_NO_OP, - .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, - .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, - .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, - .emit_frame_size = - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + - 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ - 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ - 1, /* vcn_v2_0_enc_ring_insert_end */ - .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ - .emit_ib = vcn_v2_0_enc_ring_emit_ib, - .emit_fence = vcn_v2_0_enc_ring_emit_fence, - .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, - .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, - .test_ring = amdgpu_vcn_enc_ring_test_ring, - .test_ib = amdgpu_vcn_unified_ring_test_ib, - .insert_nop = amdgpu_ring_insert_nop, - .insert_end = vcn_v2_0_enc_ring_insert_end, - .pad_ib = amdgpu_ring_generic_pad_ib, - .begin_use = amdgpu_vcn_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, - .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, - .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, - .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, -}; - -/** - * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions - * - * @adev: amdgpu_device pointer - * - * Set unified ring functions - */ -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) -{ - int i, vcn_inst; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; - adev->vcn.inst[i].ring_enc[0].me = i; - vcn_inst = GET_INST(VCN, i); - adev->vcn.inst[i].aid_id = - vcn_inst / adev->vcn.num_inst_per_aid; - } -} - -/** - * vcn_v4_0_3_is_idle - check VCN block is idle - * - * @handle: amdgpu_device pointer - * - * Check whether VCN block is idle - */ -static bool vcn_v4_0_3_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 1; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == - UVD_STATUS__IDLE); - } - - return ret; -} - -/** - * vcn_v4_0_3_wait_for_idle - wait for VCN block idle - * - * @handle: amdgpu_device pointer - * - * Wait for VCN block idle - */ -static int vcn_v4_0_3_wait_for_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 0; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, - UVD_STATUS__IDLE, UVD_STATUS__IDLE); - if (ret) - return ret; - } - - return ret; -} - -/* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state - * - * @handle: amdgpu_device pointer - * @state: clock gating state - * - * Set VCN block clockgating state - */ -static int vcn_v4_0_3_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = state == AMD_CG_STATE_GATE; - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (enable) { - if (RREG32_SOC15(VCN, GET_INST(VCN, i), - regUVD_STATUS) != UVD_STATUS__IDLE) - return -EBUSY; - vcn_v4_0_3_enable_clock_gating(adev, i); - } else { - vcn_v4_0_3_disable_clock_gating(adev, i); - } - } - return 0; -} - -/** - * vcn_v4_0_3_set_powergating_state - set VCN block powergating state - * - * @handle: amdgpu_device pointer - * @state: power gating state - * - * Set VCN block powergating state - */ -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int ret; - - /* for SRIOV, guest should not control VCN Power-gating - * MMSCH FW should control Power-gating and clock-gating - * guest should avoid touching CGC and PG - */ - if (amdgpu_sriov_vf(adev)) { - adev->vcn.cur_state = AMD_PG_STATE_UNGATE; - return 0; - } - - if (state == adev->vcn.cur_state) - return 0; - - if (state == AMD_PG_STATE_GATE) - ret = vcn_v4_0_3_stop(adev); - else - ret = vcn_v4_0_3_start(adev); - - if (!ret) - adev->vcn.cur_state = state; - - return ret; -} - -/** - * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @type: interrupt types - * @state: interrupt states - * - * Set VCN block interrupt state - */ -static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned int type, - enum amdgpu_interrupt_state state) -{ - return 0; -} - -/** - * vcn_v4_0_3_process_interrupt - process VCN block interrupt - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @entry: interrupt entry from clients and sources - * - * Process VCN block interrupt - */ -static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t i, inst; - - i = node_id_to_phys_map[entry->node_id]; - - DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); - - for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) - if (adev->vcn.inst[inst].aid_id == i) - break; - - if (inst >= adev->vcn.num_vcn_inst) { - dev_WARN_ONCE(adev->dev, 1, - "Interrupt received for unknown VCN instance %d", - entry->node_id); - return 0; - } - - switch (entry->src_id) { - case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: - amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); - break; - default: - DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", - entry->src_id, entry->src_data[0]); - break; - } - - return 0; -} - -static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { - .set = vcn_v4_0_3_set_interrupt_state, - .process = vcn_v4_0_3_process_interrupt, -}; - -/** - * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions - * - * @adev: amdgpu_device pointer - * - * Set VCN block interrupt irq functions - */ -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst->irq.num_types++; - } - adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; -} - -static void vcn_v4_0_3_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t inst_off, is_powered; - - if (!adev->vcn.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) { - drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); - continue; - } - - inst_off = i * reg_count; - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) { - drm_printf(p, "\nActive Instance:VCN%d\n", i); - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, - adev->vcn.ip_dump[inst_off + j]); - } else { - drm_printf(p, "\nInactive Instance:VCN%d\n", i); - } - } -} - -static void vcn_v4_0_3_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - bool is_powered; - uint32_t inst_off, inst_id; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - - if (!adev->vcn.ip_dump) - return; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) - continue; - - inst_id = GET_INST(VCN, i); - inst_off = i * reg_count; - /* mmUVD_POWER_STATUS is always readable and is first element of the array */ - adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) - for (j = 1; j < reg_count; j++) - adev->vcn.ip_dump[inst_off + j] = - RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], - inst_id)); - } -} - -static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { - .name = "vcn_v4_0_3", - .early_init = vcn_v4_0_3_early_init, - .late_init = NULL, - .sw_init = vcn_v4_0_3_sw_init, - .sw_fini = vcn_v4_0_3_sw_fini, - .hw_init = vcn_v4_0_3_hw_init, - .hw_fini = vcn_v4_0_3_hw_fini, - .suspend = vcn_v4_0_3_suspend, - .resume = vcn_v4_0_3_resume, - .is_idle = vcn_v4_0_3_is_idle, - .wait_for_idle = vcn_v4_0_3_wait_for_idle, - .check_soft_reset = NULL, - .pre_soft_reset = NULL, - .soft_reset = NULL, - .post_soft_reset = NULL, - .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, - .set_powergating_state = vcn_v4_0_3_set_powergating_state, - .dump_ip_state = vcn_v4_0_3_dump_ip_state, - .print_ip_state = vcn_v4_0_3_print_ip_state, -}; - -const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { - .type = AMD_IP_BLOCK_TYPE_VCN, - .major = 4, - .minor = 0, - .rev = 3, - .funcs = &vcn_v4_0_3_ip_funcs, -}; - -static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, -}; - -static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst, - void *ras_err_status) -{ - struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; - - /* vcn v4_0_3 only support query uncorrectable errors */ - amdgpu_ras_inst_query_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - NULL, 0, GET_INST(VCN, vcn_inst), - AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, - &err_data->ue_count); -} - -static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, - void *ras_err_status) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); -} - -static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst) -{ - amdgpu_ras_inst_reset_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - GET_INST(VCN, vcn_inst)); -} - -static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_reset_ras_error_count(adev, i); -} - -static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { - .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, - .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, -}; - -static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { - .ras_block = { - .hw_ops = &vcn_v4_0_3_ras_hw_ops, - }, -}; - -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) -{ - adev->vcn.ras = &vcn_v4_0_3_ras; -} - -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect) -{ - uint32_t tmp; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) - return; - - tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), - tmp, 0, indirect); - - tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), - tmp, 0, indirect); - - tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), - tmp, 0, indirect); -} diff --git a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.2 b/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.2 deleted file mode 100644 index c55dabb2d37f..000000000000 --- a/rr-cache/feb901dea4bef5048493327b0af8cf88d7a586fb/preimage.2 +++ /dev/null @@ -1,1930 +0,0 @@ -/* - * Copyright 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/firmware.h> -#include <drm/drm_drv.h> - -#include "amdgpu.h" -#include "amdgpu_vcn.h" -#include "amdgpu_pm.h" -#include "soc15.h" -#include "soc15d.h" -#include "soc15_hw_ip.h" -#include "vcn_v2_0.h" -#include "mmsch_v4_0_3.h" - -#include "vcn/vcn_4_0_3_offset.h" -#include "vcn/vcn_4_0_3_sh_mask.h" -#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" - -#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL -#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX -#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA -#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX - -#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 -#define VCN1_VID_SOC_ADDRESS_3_0 0x48300 - -<<<<<<< -======= -static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), - SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) -}; - ->>>>>>> -#define NORMALIZE_VCN_REG_OFFSET(offset) \ - (offset & 0x1FFFF) - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state); -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, - int inst_idx, struct dpg_pause_state *new_state); -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect); -/** - * vcn_v4_0_3_early_init - set function pointers - * - * @handle: amdgpu_device pointer - * - * Set ring and irq function pointers - */ -static int vcn_v4_0_3_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* re-use enc ring as unified ring */ - adev->vcn.num_enc_rings = 1; - - vcn_v4_0_3_set_unified_ring_funcs(adev); - vcn_v4_0_3_set_irq_funcs(adev); - vcn_v4_0_3_set_ras_funcs(adev); - - return amdgpu_vcn_early_init(adev); -} - -/** - * vcn_v4_0_3_sw_init - sw init for VCN block - * - * @handle: amdgpu_device pointer - * - * Load firmware and sw initialization - */ -static int vcn_v4_0_3_sw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t *ptr; - - r = amdgpu_vcn_sw_init(adev); - if (r) - return r; - - amdgpu_vcn_setup_ucode(adev); - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - /* VCN DEC TRAP */ - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, - VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - vcn_inst = GET_INST(VCN, i); - - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->use_doorbell = true; - - if (!amdgpu_sriov_vf(adev)) - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst; - else - ring->doorbell_index = - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 32 * vcn_inst; - - ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); - sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); - r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, - AMDGPU_RING_PRIO_DEFAULT, - &adev->vcn.inst[i].sched_score); - if (r) - return r; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); - fw_shared->sq.is_enabled = true; - - if (amdgpu_vcnfw_log) - amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); - } - - if (amdgpu_sriov_vf(adev)) { - r = amdgpu_virt_alloc_mm_table(adev); - if (r) - return r; - } - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) - adev->vcn.pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; - - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - r = amdgpu_vcn_ras_sw_init(adev); - if (r) { - dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); - return r; - } - } - - /* Allocate memory for VCN IP Dump buffer */ - ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); - if (!ptr) { - DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); - adev->vcn.ip_dump = NULL; - } else { - adev->vcn.ip_dump = ptr; - } - - return 0; -} - -/** - * vcn_v4_0_3_sw_fini - sw fini for VCN block - * - * @handle: amdgpu_device pointer - * - * VCN suspend and free up sw allocation - */ -static int vcn_v4_0_3_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, r, idx; - - if (drm_dev_enter(&adev->ddev, &idx)) { - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->present_flag_0 = 0; - fw_shared->sq.is_enabled = cpu_to_le32(false); - } - drm_dev_exit(idx); - } - - if (amdgpu_sriov_vf(adev)) - amdgpu_virt_free_mm_table(adev); - - r = amdgpu_vcn_suspend(adev); - if (r) - return r; - - r = amdgpu_vcn_sw_fini(adev); - - kfree(adev->vcn.ip_dump); - - return r; -} - -/** - * vcn_v4_0_3_hw_init - start and test VCN block - * - * @handle: amdgpu_device pointer - * - * Initialize the hardware, boot up the VCPU and do some testing - */ -static int vcn_v4_0_3_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct amdgpu_ring *ring; - int i, r, vcn_inst; - - if (amdgpu_sriov_vf(adev)) { - r = vcn_v4_0_3_start_sriov(adev); - if (r) - return r; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ring = &adev->vcn.inst[i].ring_enc[0]; - ring->wptr = 0; - ring->wptr_old = 0; - vcn_v4_0_3_unified_ring_set_wptr(ring); - ring->sched.ready = true; - } - } else { - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - ring = &adev->vcn.inst[i].ring_enc[0]; - - if (ring->use_doorbell) { - adev->nbio.funcs->vcn_doorbell_range( - adev, ring->use_doorbell, - (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + - 9 * vcn_inst, - adev->vcn.inst[i].aid_id); - - WREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL, - ring->doorbell_index - << VCN_RB1_DB_CTRL__OFFSET__SHIFT | - VCN_RB1_DB_CTRL__EN_MASK); - - /* Read DB_CTRL to flush the write DB_CTRL command. */ - RREG32_SOC15( - VCN, GET_INST(VCN, ring->me), - regVCN_RB1_DB_CTRL); - } - - r = amdgpu_ring_test_helper(ring); - if (r) - return r; - } - } - - return r; -} - -/** - * vcn_v4_0_3_hw_fini - stop the hardware block - * - * @handle: amdgpu_device pointer - * - * Stop the VCN block, mark ring as not ready any more - */ -static int vcn_v4_0_3_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - cancel_delayed_work_sync(&adev->vcn.idle_work); - - if (adev->vcn.cur_state != AMD_PG_STATE_GATE) - vcn_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE); - - return 0; -} - -/** - * vcn_v4_0_3_suspend - suspend VCN block - * - * @handle: amdgpu_device pointer - * - * HW fini and suspend VCN block - */ -static int vcn_v4_0_3_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = vcn_v4_0_3_hw_fini(adev); - if (r) - return r; - - r = amdgpu_vcn_suspend(adev); - - return r; -} - -/** - * vcn_v4_0_3_resume - resume VCN block - * - * @handle: amdgpu_device pointer - * - * Resume firmware and hw init VCN block - */ -static int vcn_v4_0_3_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_vcn_resume(adev); - if (r) - return r; - - r = vcn_v4_0_3_hw_init(adev); - - return r; -} - -/** - * vcn_v4_0_3_mc_resume - memory controller programming - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Let the VCN memory controller know it's offsets - */ -static void vcn_v4_0_3_mc_resume(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t offset, size, vcn_inst; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - vcn_inst = GET_INST(VCN, inst_idx); - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_lo)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] - .tmr_mc_addr_hi)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); - offset = 0; - } else { - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); - offset = size; - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); - - /* cache window 1: stack */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, - AMDGPU_VCN_STACK_SIZE); - - /* cache window 2: context */ - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, - AMDGPU_VCN_CONTEXT_SIZE); - - /* non-cache window */ - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15( - VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); - WREG32_SOC15( - VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); -} - -/** - * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Let the VCN memory controller know it's offsets with dpg mode - */ -static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - uint32_t offset, size; - const struct common_firmware_header *hdr; - - hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data; - size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); - - /* cache window 0: fw */ - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_lo), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + - inst_idx].tmr_mc_addr_hi), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); - } - offset = 0; - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); - offset = size; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); - } - - if (!indirect) - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); - else - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); - - /* cache window 1: stack */ - if (!indirect) { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } else { - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); - } - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); - - /* cache window 2: context */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); - - /* non-cache window */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); - - /* VCN global tiling registers */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); -} - -/** - * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Disable clock gating for VCN block - */ -static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* VCN disable CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); - data &= ~(UVD_CGC_GATE__SYS_MASK - | UVD_CGC_GATE__MPEG2_MASK - | UVD_CGC_GATE__REGS_MASK - | UVD_CGC_GATE__RBC_MASK - | UVD_CGC_GATE__LMI_MC_MASK - | UVD_CGC_GATE__LMI_UMC_MASK - | UVD_CGC_GATE__MPC_MASK - | UVD_CGC_GATE__LBSI_MASK - | UVD_CGC_GATE__LRBBM_MASK - | UVD_CGC_GATE__WCB_MASK - | UVD_CGC_GATE__VCPU_MASK - | UVD_CGC_GATE__MMSCH_MASK); - - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK - | UVD_CGC_CTRL__MMSCH_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); - data |= (UVD_SUVD_CGC_GATE__SRE_MASK - | UVD_SUVD_CGC_GATE__SIT_MASK - | UVD_SUVD_CGC_GATE__SMP_MASK - | UVD_SUVD_CGC_GATE__SCM_MASK - | UVD_SUVD_CGC_GATE__SDB_MASK - | UVD_SUVD_CGC_GATE__SRE_H264_MASK - | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK - | UVD_SUVD_CGC_GATE__SIT_H264_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK - | UVD_SUVD_CGC_GATE__SCM_H264_MASK - | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK - | UVD_SUVD_CGC_GATE__SDB_H264_MASK - | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK - | UVD_SUVD_CGC_GATE__ENT_MASK - | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK - | UVD_SUVD_CGC_GATE__SITE_MASK - | UVD_SUVD_CGC_GATE__SRE_VP9_MASK - | UVD_SUVD_CGC_GATE__SCM_VP9_MASK - | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK - | UVD_SUVD_CGC_GATE__SDB_VP9_MASK - | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode - * - * @adev: amdgpu_device pointer - * @sram_sel: sram select - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Disable clock gating for VCN block with dpg mode - */ -static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel, - int inst_idx, uint8_t indirect) -{ - uint32_t reg_data = 0; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - /* enable sw clock gating control */ - reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | - UVD_CGC_CTRL__MPEG2_MODE_MASK | - UVD_CGC_CTRL__REGS_MODE_MASK | - UVD_CGC_CTRL__RBC_MODE_MASK | - UVD_CGC_CTRL__LMI_MC_MODE_MASK | - UVD_CGC_CTRL__LMI_UMC_MODE_MASK | - UVD_CGC_CTRL__IDCT_MODE_MASK | - UVD_CGC_CTRL__MPRD_MODE_MASK | - UVD_CGC_CTRL__MPC_MODE_MASK | - UVD_CGC_CTRL__LBSI_MODE_MASK | - UVD_CGC_CTRL__LRBBM_MODE_MASK | - UVD_CGC_CTRL__WCB_MODE_MASK | - UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); - - /* turn off clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); - - /* turn on SUVD clock gating */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); - - /* turn on sw mode in UVD_SUVD_CGC_CTRL */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); -} - -/** - * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number - * - * Enable clock gating for VCN block - */ -static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t data; - int vcn_inst; - - if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) - return; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* enable VCN CGC */ - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; - data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; - data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); - data |= (UVD_CGC_CTRL__SYS_MODE_MASK - | UVD_CGC_CTRL__MPEG2_MODE_MASK - | UVD_CGC_CTRL__REGS_MODE_MASK - | UVD_CGC_CTRL__RBC_MODE_MASK - | UVD_CGC_CTRL__LMI_MC_MODE_MASK - | UVD_CGC_CTRL__LMI_UMC_MODE_MASK - | UVD_CGC_CTRL__MPC_MODE_MASK - | UVD_CGC_CTRL__LBSI_MODE_MASK - | UVD_CGC_CTRL__LRBBM_MODE_MASK - | UVD_CGC_CTRL__WCB_MODE_MASK - | UVD_CGC_CTRL__VCPU_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); - - data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); - data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK - | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK - | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK - | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK - | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK - | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK - | UVD_SUVD_CGC_CTRL__IME_MODE_MASK - | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); - WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); -} - -/** - * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @indirect: indirectly write sram - * - * Start VCN block with dpg mode - */ -static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared = - adev->vcn.inst[inst_idx].fw_shared.cpu_addr; - struct amdgpu_ring *ring; - int vcn_inst; - uint32_t tmp; - - vcn_inst = GET_INST(VCN, inst_idx); - /* disable register anti-hang mechanism */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, - ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - /* enable dynamic power gating mode */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); - tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; - tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); - - if (indirect) { - DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", - inst_idx, adev->vcn.inst[inst_idx].aid_id); - adev->vcn.inst[inst_idx].dpg_sram_curr_addr = - (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; - /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ - WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, - adev->vcn.inst[inst_idx].aid_id, 0, true); - } - - /* enable clock gating */ - vcn_v4_0_3_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect); - - /* enable VCPU clock */ - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* disable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); - - /* setup regUVD_LMI_CTRL */ - tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__REQ_MODE_MASK | - UVD_LMI_CTRL__CRC_RESET_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | - (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | - 0x00100000L); - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_CNTL), - 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXA0), - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUXB0), - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); - - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MPC_SET_MUX), - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); - - vcn_v4_0_3_mc_resume_dpg_mode(adev, inst_idx, indirect); - - tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); - tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); - - /* enable LMI MC and UMC channels */ - tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); - - vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); - - /* enable master interrupt */ - WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( - VCN, 0, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); - - if (indirect) - amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); - - ring = &adev->vcn.inst[inst_idx].ring_enc[0]; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); - - /*resetting done, fw can check RB ring */ - fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); - - return 0; -} - -static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) -{ - int i, vcn_inst; - struct amdgpu_ring *ring_enc; - uint64_t cache_addr; - uint64_t rb_enc_addr; - uint64_t ctx_addr; - uint32_t param, resp, expected; - uint32_t offset, cache_size; - uint32_t tmp, timeout; - - struct amdgpu_mm_table *table = &adev->virt.mm_table; - uint32_t *table_loc; - uint32_t table_size; - uint32_t size, size_dw; - uint32_t init_status; - uint32_t enabled_vcn; - - struct mmsch_v4_0_cmd_direct_write - direct_wt = { {0} }; - struct mmsch_v4_0_cmd_direct_read_modify_write - direct_rd_mod_wt = { {0} }; - struct mmsch_v4_0_cmd_end end = { {0} }; - struct mmsch_v4_0_3_init_header header; - - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - volatile struct amdgpu_fw_shared_rb_setup *rb_setup; - - direct_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_WRITE; - direct_rd_mod_wt.cmd_header.command_type = - MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; - end.cmd_header.command_type = MMSCH_COMMAND__END; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - vcn_inst = GET_INST(VCN, i); - - memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); - header.version = MMSCH_VERSION; - header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; - - table_loc = (uint32_t *)table->cpu_addr; - table_loc += header.total_size; - - table_size = 0; - - MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), - ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); - - cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); - - offset = 0; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), 0); - } else { - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[i].gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[i].gpu_addr)); - offset = cache_size; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET0), - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); - } - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE0), - cache_size); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET1), 0); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); - - cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + - AMDGPU_VCN_STACK_SIZE; - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_OFFSET2), 0); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); - - fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; - rb_setup = &fw_shared->rb_setup; - - ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; - ring_enc->wptr = 0; - rb_enc_addr = ring_enc->gpu_addr; - - rb_setup->is_rb_enabled_flags |= RB_ENABLED; - rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); - rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); - rb_setup->rb_size = ring_enc->ring_size / 4; - fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); - - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), - lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), - upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); - MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, - regUVD_VCPU_NONCACHE_SIZE0), - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); - MMSCH_V4_0_INSERT_END(); - - header.vcn0.init_status = 0; - header.vcn0.table_offset = header.total_size; - header.vcn0.table_size = table_size; - header.total_size += table_size; - - /* Send init table to mmsch */ - size = sizeof(struct mmsch_v4_0_3_init_header); - table_loc = (uint32_t *)table->cpu_addr; - memcpy((void *)table_loc, &header, size); - - ctx_addr = table->gpu_addr; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); - - tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); - tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; - tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); - - size = header.total_size; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); - - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); - - param = 0x00000001; - WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); - tmp = 0; - timeout = 1000; - resp = 0; - expected = MMSCH_VF_MAILBOX_RESP__OK; - while (resp != expected) { - resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); - if (resp != 0) - break; - - udelay(10); - tmp = tmp + 10; - if (tmp >= timeout) { - DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ - " waiting for regMMSCH_VF_MAILBOX_RESP "\ - "(expected=0x%08x, readback=0x%08x)\n", - tmp, expected, resp); - return -EBUSY; - } - } - - enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; - init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; - if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE - && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { - DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ - "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); - } - } - - return 0; -} - -/** - * vcn_v4_0_3_start - VCN start - * - * @adev: amdgpu_device pointer - * - * Start VCN block - */ -static int vcn_v4_0_3_start(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - struct amdgpu_ring *ring; - int i, j, k, r, vcn_inst; - uint32_t tmp; - - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, true); - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - r = vcn_v4_0_3_start_dpg_mode(adev, i, adev->vcn.indirect_sram); - continue; - } - - vcn_inst = GET_INST(VCN, i); - /* set VCN status busy */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | - UVD_STATUS__UVD_BUSY; - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); - - /*SW clock gating */ - vcn_v4_0_3_disable_clock_gating(adev, i); - - /* enable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__CLK_EN_MASK, - ~UVD_VCPU_CNTL__CLK_EN_MASK); - - /* disable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* enable LMI MC and UMC channels */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, - ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* setup regUVD_LMI_CTRL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, - tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | - UVD_LMI_CTRL__MASK_MC_URGENT_MASK | - UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | - UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); - - /* setup regUVD_MPC_CNTL */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); - tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; - tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); - - /* setup UVD_MPC_SET_MUXA0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, - ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); - - /* setup UVD_MPC_SET_MUXB0 */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, - ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | - (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | - (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); - - /* setup UVD_MPC_SET_MUX */ - WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, - ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | - (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | - (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); - - vcn_v4_0_3_mc_resume(adev, i); - - /* VCN global tiling registers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, - adev->gfx.config.gb_addr_config); - - /* unblock VCPU register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - for (j = 0; j < 10; ++j) { - uint32_t status; - - for (k = 0; k < 100; ++k) { - status = RREG32_SOC15(VCN, vcn_inst, - regUVD_STATUS); - if (status & 2) - break; - mdelay(10); - } - r = 0; - if (status & 2) - break; - - DRM_DEV_ERROR(adev->dev, - "VCN decode not responding, trying to reset the VCPU!!!\n"); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - mdelay(10); - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, - regUVD_VCPU_CNTL), - 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); - - mdelay(10); - r = -1; - } - - if (r) { - DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); - return r; - } - - /* enable master interrupt */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), - UVD_MASTINT_EN__VCPU_EN_MASK, - ~UVD_MASTINT_EN__VCPU_EN_MASK); - - /* clear the busy bit of VCN_STATUS */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, - ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); - - ring = &adev->vcn.inst[i].ring_enc[0]; - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - - /* program the RB_BASE for ring buffer */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, - lower_32_bits(ring->gpu_addr)); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, - upper_32_bits(ring->gpu_addr)); - - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, - ring->ring_size / sizeof(uint32_t)); - - /* resetting ring, fw should not check RB ring */ - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - /* Initialize the ring buffer's read and write pointers */ - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); - WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); - - tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); - tmp |= VCN_RB_ENABLE__RB_EN_MASK; - WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); - - ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - fw_shared->sq.queue_mode &= - cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); - - } - return 0; -} - -/** - * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * - * Stop VCN block with dpg mode - */ -static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx) -{ - uint32_t tmp; - int vcn_inst; - - vcn_inst = GET_INST(VCN, inst_idx); - - /* Wait for power status to be 1 */ - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* wait for read ptr to be equal to write ptr */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); - - SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); - - /* disable dynamic power gating mode */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, - ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); - return 0; -} - -/** - * vcn_v4_0_3_stop - VCN stop - * - * @adev: amdgpu_device pointer - * - * Stop VCN block - */ -static int vcn_v4_0_3_stop(struct amdgpu_device *adev) -{ - volatile struct amdgpu_vcn4_fw_shared *fw_shared; - int i, r = 0, vcn_inst; - uint32_t tmp; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - vcn_inst = GET_INST(VCN, i); - - fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; - fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; - - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { - vcn_v4_0_3_stop_dpg_mode(adev, i); - continue; - } - - /* wait for vcn idle */ - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, - UVD_STATUS__IDLE, 0x7); - if (r) - goto Done; - - tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | - UVD_LMI_STATUS__READ_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_MASK | - UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* stall UMC channel */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); - tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); - tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | - UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; - r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, - tmp); - if (r) - goto Done; - - /* Unblock VCPU Register access */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), - UVD_RB_ARB_CTRL__VCPU_DIS_MASK, - ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); - - /* release VCPU reset to boot */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), - UVD_VCPU_CNTL__BLK_RST_MASK, - ~UVD_VCPU_CNTL__BLK_RST_MASK); - - /* disable VCPU clock */ - WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, - ~(UVD_VCPU_CNTL__CLK_EN_MASK)); - - /* reset LMI UMC/LMI/VCPU */ - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); - tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; - WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); - - /* clear VCN status */ - WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); - - /* apply HW clock gating */ - vcn_v4_0_3_enable_clock_gating(adev, i); - } -Done: - if (adev->pm.dpm_enabled) - amdgpu_dpm_enable_uvd(adev, false); - - return 0; -} - -/** - * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode - * - * @adev: amdgpu_device pointer - * @inst_idx: instance number index - * @new_state: pause state - * - * Pause dpg mode for VCN block - */ -static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx, - struct dpg_pause_state *new_state) -{ - - return 0; -} - -/** - * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified read pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); -} - -/** - * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer - * - * @ring: amdgpu_ring pointer - * - * Returns the current hardware unified write pointer - */ -static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) - return *ring->wptr_cpu_addr; - else - return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), - regUVD_RB_WPTR); -} - -static void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, - uint32_t val, uint32_t mask) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, mask); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) -{ - /* For VF, only local offsets should be used */ - if (amdgpu_sriov_vf(ring->adev)) - reg = NORMALIZE_VCN_REG_OFFSET(reg); - - amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); - amdgpu_ring_write(ring, reg << 2); - amdgpu_ring_write(ring, val); -} - -static void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned int vmid, uint64_t pd_addr) -{ - struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; - - pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); - - /* wait for reg writes */ - vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + - vmid * hub->ctx_addr_distance, - lower_32_bits(pd_addr), 0xffffffff); -} - -static void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) -{ - /* VCN engine access for HDP flush doesn't work when RRMT is enabled. - * This is a workaround to avoid any HDP flush through VCN ring. - */ -} - -/** - * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer - * - * @ring: amdgpu_ring pointer - * - * Commits the enc write pointer to the hardware - */ -static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - - if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) - DRM_ERROR("wrong ring id is identified in %s", __func__); - - if (ring->use_doorbell) { - *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); - WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); - } else { - WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, - lower_32_bits(ring->wptr)); - } -} - -static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { - .type = AMDGPU_RING_TYPE_VCN_ENC, - .align_mask = 0x3f, - .nop = VCN_ENC_CMD_NO_OP, - .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, - .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, - .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, - .emit_frame_size = - SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + - SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + - 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ - 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ - 1, /* vcn_v2_0_enc_ring_insert_end */ - .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ - .emit_ib = vcn_v2_0_enc_ring_emit_ib, - .emit_fence = vcn_v2_0_enc_ring_emit_fence, - .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, - .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, - .test_ring = amdgpu_vcn_enc_ring_test_ring, - .test_ib = amdgpu_vcn_unified_ring_test_ib, - .insert_nop = amdgpu_ring_insert_nop, - .insert_end = vcn_v2_0_enc_ring_insert_end, - .pad_ib = amdgpu_ring_generic_pad_ib, - .begin_use = amdgpu_vcn_ring_begin_use, - .end_use = amdgpu_vcn_ring_end_use, - .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, - .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, - .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, -}; - -/** - * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions - * - * @adev: amdgpu_device pointer - * - * Set unified ring functions - */ -static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) -{ - int i, vcn_inst; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; - adev->vcn.inst[i].ring_enc[0].me = i; - vcn_inst = GET_INST(VCN, i); - adev->vcn.inst[i].aid_id = - vcn_inst / adev->vcn.num_inst_per_aid; - } -} - -/** - * vcn_v4_0_3_is_idle - check VCN block is idle - * - * @handle: amdgpu_device pointer - * - * Check whether VCN block is idle - */ -static bool vcn_v4_0_3_is_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 1; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == - UVD_STATUS__IDLE); - } - - return ret; -} - -/** - * vcn_v4_0_3_wait_for_idle - wait for VCN block idle - * - * @handle: amdgpu_device pointer - * - * Wait for VCN block idle - */ -static int vcn_v4_0_3_wait_for_idle(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, ret = 0; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, - UVD_STATUS__IDLE, UVD_STATUS__IDLE); - if (ret) - return ret; - } - - return ret; -} - -/* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state - * - * @handle: amdgpu_device pointer - * @state: clock gating state - * - * Set VCN block clockgating state - */ -static int vcn_v4_0_3_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = state == AMD_CG_STATE_GATE; - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - if (enable) { - if (RREG32_SOC15(VCN, GET_INST(VCN, i), - regUVD_STATUS) != UVD_STATUS__IDLE) - return -EBUSY; - vcn_v4_0_3_enable_clock_gating(adev, i); - } else { - vcn_v4_0_3_disable_clock_gating(adev, i); - } - } - return 0; -} - -/** - * vcn_v4_0_3_set_powergating_state - set VCN block powergating state - * - * @handle: amdgpu_device pointer - * @state: power gating state - * - * Set VCN block powergating state - */ -static int vcn_v4_0_3_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int ret; - - /* for SRIOV, guest should not control VCN Power-gating - * MMSCH FW should control Power-gating and clock-gating - * guest should avoid touching CGC and PG - */ - if (amdgpu_sriov_vf(adev)) { - adev->vcn.cur_state = AMD_PG_STATE_UNGATE; - return 0; - } - - if (state == adev->vcn.cur_state) - return 0; - - if (state == AMD_PG_STATE_GATE) - ret = vcn_v4_0_3_stop(adev); - else - ret = vcn_v4_0_3_start(adev); - - if (!ret) - adev->vcn.cur_state = state; - - return ret; -} - -/** - * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @type: interrupt types - * @state: interrupt states - * - * Set VCN block interrupt state - */ -static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned int type, - enum amdgpu_interrupt_state state) -{ - return 0; -} - -/** - * vcn_v4_0_3_process_interrupt - process VCN block interrupt - * - * @adev: amdgpu_device pointer - * @source: interrupt sources - * @entry: interrupt entry from clients and sources - * - * Process VCN block interrupt - */ -static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - uint32_t i, inst; - - i = node_id_to_phys_map[entry->node_id]; - - DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); - - for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) - if (adev->vcn.inst[inst].aid_id == i) - break; - - if (inst >= adev->vcn.num_vcn_inst) { - dev_WARN_ONCE(adev->dev, 1, - "Interrupt received for unknown VCN instance %d", - entry->node_id); - return 0; - } - - switch (entry->src_id) { - case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: - amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); - break; - default: - DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", - entry->src_id, entry->src_data[0]); - break; - } - - return 0; -} - -static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { - .set = vcn_v4_0_3_set_interrupt_state, - .process = vcn_v4_0_3_process_interrupt, -}; - -/** - * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions - * - * @adev: amdgpu_device pointer - * - * Set VCN block interrupt irq functions - */ -static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { - adev->vcn.inst->irq.num_types++; - } - adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; -} - -static void vcn_v4_0_3_print_ip_state(void *handle, struct drm_printer *p) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - uint32_t inst_off, is_powered; - - if (!adev->vcn.ip_dump) - return; - - drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) { - drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); - continue; - } - - inst_off = i * reg_count; - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) { - drm_printf(p, "\nActive Instance:VCN%d\n", i); - for (j = 0; j < reg_count; j++) - drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, - adev->vcn.ip_dump[inst_off + j]); - } else { - drm_printf(p, "\nInactive Instance:VCN%d\n", i); - } - } -} - -static void vcn_v4_0_3_dump_ip_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i, j; - bool is_powered; - uint32_t inst_off, inst_id; - uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); - - if (!adev->vcn.ip_dump) - return; - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - if (adev->vcn.harvest_config & (1 << i)) - continue; - - inst_id = GET_INST(VCN, i); - inst_off = i * reg_count; - /* mmUVD_POWER_STATUS is always readable and is first element of the array */ - adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); - is_powered = (adev->vcn.ip_dump[inst_off] & - UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; - - if (is_powered) - for (j = 1; j < reg_count; j++) - adev->vcn.ip_dump[inst_off + j] = - RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], - inst_id)); - } -} - -static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { - .name = "vcn_v4_0_3", - .early_init = vcn_v4_0_3_early_init, - .late_init = NULL, - .sw_init = vcn_v4_0_3_sw_init, - .sw_fini = vcn_v4_0_3_sw_fini, - .hw_init = vcn_v4_0_3_hw_init, - .hw_fini = vcn_v4_0_3_hw_fini, - .suspend = vcn_v4_0_3_suspend, - .resume = vcn_v4_0_3_resume, - .is_idle = vcn_v4_0_3_is_idle, - .wait_for_idle = vcn_v4_0_3_wait_for_idle, - .check_soft_reset = NULL, - .pre_soft_reset = NULL, - .soft_reset = NULL, - .post_soft_reset = NULL, - .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, - .set_powergating_state = vcn_v4_0_3_set_powergating_state, - .dump_ip_state = vcn_v4_0_3_dump_ip_state, - .print_ip_state = vcn_v4_0_3_print_ip_state, -}; - -const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { - .type = AMD_IP_BLOCK_TYPE_VCN, - .major = 4, - .minor = 0, - .rev = 3, - .funcs = &vcn_v4_0_3_ip_funcs, -}; - -static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, - {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), - 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, -}; - -static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst, - void *ras_err_status) -{ - struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; - - /* vcn v4_0_3 only support query uncorrectable errors */ - amdgpu_ras_inst_query_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - NULL, 0, GET_INST(VCN, vcn_inst), - AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, - &err_data->ue_count); -} - -static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, - void *ras_err_status) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); -} - -static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, - uint32_t vcn_inst) -{ - amdgpu_ras_inst_reset_ras_error_count(adev, - vcn_v4_0_3_ue_reg_list, - ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), - GET_INST(VCN, vcn_inst)); -} - -static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) -{ - uint32_t i; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { - dev_warn(adev->dev, "VCN RAS is not supported\n"); - return; - } - - for (i = 0; i < adev->vcn.num_vcn_inst; i++) - vcn_v4_0_3_inst_reset_ras_error_count(adev, i); -} - -static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { - .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, - .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, -}; - -static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { - .ras_block = { - .hw_ops = &vcn_v4_0_3_ras_hw_ops, - }, -}; - -static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) -{ - adev->vcn.ras = &vcn_v4_0_3_ras; -} - -static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, - int inst_idx, bool indirect) -{ - uint32_t tmp; - - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) - return; - - tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | - VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), - tmp, 0, indirect); - - tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), - tmp, 0, indirect); - - tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; - WREG32_SOC15_DPG_MODE(inst_idx, - SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), - tmp, 0, indirect); -} |