diff options
Diffstat (limited to 'drivers')
32 files changed, 449 insertions, 248 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index f50e3ba4d7a5..589939631ed4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -29,6 +29,7 @@ #include <linux/mm.h> #include <linux/kthread.h> #include <linux/workqueue.h> +#include <linux/mmu_notifier.h> #include <kgd_kfd_interface.h> #include <drm/ttm/ttm_execbuf_util.h> #include "amdgpu_sync.h" @@ -65,6 +66,7 @@ struct kgd_mem { struct mutex lock; struct amdgpu_bo *bo; struct dma_buf *dmabuf; + struct hmm_range *range; struct list_head attachments; /* protected by amdkfd_process_info.lock */ struct ttm_validate_buffer validate_list; @@ -75,7 +77,7 @@ struct kgd_mem { uint32_t alloc_flags; - atomic_t invalid; + uint32_t invalid; struct amdkfd_process_info *process_info; struct amdgpu_sync sync; @@ -131,7 +133,8 @@ struct amdkfd_process_info { struct amdgpu_amdkfd_fence *eviction_fence; /* MMU-notifier related fields */ - atomic_t evicted_bos; + struct mutex notifier_lock; + uint32_t evicted_bos; struct delayed_work restore_userptr_work; struct pid *pid; bool block_mmu_notifications; @@ -180,7 +183,8 @@ int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data); bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); +int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, + unsigned long cur_seq, struct kgd_mem *mem); #else static inline bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) @@ -201,7 +205,8 @@ int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) } static inline -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) +int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, + unsigned long cur_seq, struct kgd_mem *mem) { return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 8782916e64a0..0a854bb8b47e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -964,7 +964,9 @@ static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr, * later stage when it is scheduled by another ioctl called by * CRIU master process for the target pid for restore. */ - atomic_inc(&mem->invalid); + mutex_lock(&process_info->notifier_lock); + mem->invalid++; + mutex_unlock(&process_info->notifier_lock); mutex_unlock(&process_info->lock); return 0; } @@ -1301,6 +1303,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, return -ENOMEM; mutex_init(&info->lock); + mutex_init(&info->notifier_lock); INIT_LIST_HEAD(&info->vm_list_head); INIT_LIST_HEAD(&info->kfd_bo_list); INIT_LIST_HEAD(&info->userptr_valid_list); @@ -1317,7 +1320,6 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, } info->pid = get_task_pid(current->group_leader, PIDTYPE_PID); - atomic_set(&info->evicted_bos, 0); INIT_DELAYED_WORK(&info->restore_userptr_work, amdgpu_amdkfd_restore_userptr_worker); @@ -1372,6 +1374,7 @@ reserve_pd_fail: put_pid(info->pid); create_evict_fence_fail: mutex_destroy(&info->lock); + mutex_destroy(&info->notifier_lock); kfree(info); } return ret; @@ -1496,6 +1499,7 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, cancel_delayed_work_sync(&process_info->restore_userptr_work); put_pid(process_info->pid); mutex_destroy(&process_info->lock); + mutex_destroy(&process_info->notifier_lock); kfree(process_info); } } @@ -1548,7 +1552,9 @@ int amdgpu_amdkfd_criu_resume(void *p) mutex_lock(&pinfo->lock); pr_debug("scheduling work\n"); - atomic_inc(&pinfo->evicted_bos); + mutex_lock(&pinfo->notifier_lock); + pinfo->evicted_bos++; + mutex_unlock(&pinfo->notifier_lock); if (!READ_ONCE(pinfo->block_mmu_notifications)) { ret = -EINVAL; goto out_unlock; @@ -1773,8 +1779,13 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( list_del(&bo_list_entry->head); mutex_unlock(&process_info->lock); - /* No more MMU notifiers */ - amdgpu_hmm_unregister(mem->bo); + /* Cleanup user pages and MMU notifiers */ + if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { + amdgpu_hmm_unregister(mem->bo); + mutex_lock(&process_info->notifier_lock); + amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range); + mutex_unlock(&process_info->notifier_lock); + } ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); if (unlikely(ret)) @@ -1864,6 +1875,16 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( */ mutex_lock(&mem->process_info->lock); + /* Lock notifier lock. If we find an invalid userptr BO, we can be + * sure that the MMU notifier is no longer running + * concurrently and the queues are actually stopped + */ + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { + mutex_lock(&mem->process_info->notifier_lock); + is_invalid_userptr = !!mem->invalid; + mutex_unlock(&mem->process_info->notifier_lock); + } + mutex_lock(&mem->lock); domain = mem->domain; @@ -2241,34 +2262,38 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev, * * Runs in MMU notifier, may be in RECLAIM_FS context. This means it * cannot do any memory allocations, and cannot take any locks that - * are held elsewhere while allocating memory. Therefore this is as - * simple as possible, using atomic counters. + * are held elsewhere while allocating memory. * * It doesn't do anything to the BO itself. The real work happens in * restore, where we get updated page addresses. This function only * ensures that GPU access to the BO is stopped. */ -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, - struct mm_struct *mm) +int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, + unsigned long cur_seq, struct kgd_mem *mem) { struct amdkfd_process_info *process_info = mem->process_info; - int evicted_bos; int r = 0; - /* Do not process MMU notifications until stage-4 IOCTL is received */ + /* Do not process MMU notifications during CRIU restore until + * KFD_CRIU_OP_RESUME IOCTL is received + */ if (READ_ONCE(process_info->block_mmu_notifications)) return 0; - atomic_inc(&mem->invalid); - evicted_bos = atomic_inc_return(&process_info->evicted_bos); - if (evicted_bos == 1) { + mutex_lock(&process_info->notifier_lock); + mmu_interval_set_seq(mni, cur_seq); + + mem->invalid++; + if (++process_info->evicted_bos == 1) { /* First eviction, stop the queues */ - r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_USERPTR); + r = kgd2kfd_quiesce_mm(mni->mm, + KFD_QUEUE_EVICTION_TRIGGER_USERPTR); if (r) pr_err("Failed to quiesce KFD\n"); schedule_delayed_work(&process_info->restore_userptr_work, msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); } + mutex_unlock(&process_info->notifier_lock); return r; } @@ -2285,54 +2310,58 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info, struct kgd_mem *mem, *tmp_mem; struct amdgpu_bo *bo; struct ttm_operation_ctx ctx = { false, false }; - int invalid, ret; + uint32_t invalid; + int ret = 0; - /* Move all invalidated BOs to the userptr_inval_list and - * release their user pages by migration to the CPU domain - */ + mutex_lock(&process_info->notifier_lock); + + /* Move all invalidated BOs to the userptr_inval_list */ list_for_each_entry_safe(mem, tmp_mem, &process_info->userptr_valid_list, - validate_list.head) { - if (!atomic_read(&mem->invalid)) - continue; /* BO is still valid */ - - bo = mem->bo; - - if (amdgpu_bo_reserve(bo, true)) - return -EAGAIN; - amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); - ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); - amdgpu_bo_unreserve(bo); - if (ret) { - pr_err("%s: Failed to invalidate userptr BO\n", - __func__); - return -EAGAIN; - } - - list_move_tail(&mem->validate_list.head, - &process_info->userptr_inval_list); - } - - if (list_empty(&process_info->userptr_inval_list)) - return 0; /* All evicted userptr BOs were freed */ + validate_list.head) + if (mem->invalid) + list_move_tail(&mem->validate_list.head, + &process_info->userptr_inval_list); /* Go through userptr_inval_list and update any invalid user_pages */ list_for_each_entry(mem, &process_info->userptr_inval_list, validate_list.head) { - struct hmm_range *range; - - invalid = atomic_read(&mem->invalid); + invalid = mem->invalid; if (!invalid) /* BO hasn't been invalidated since the last - * revalidation attempt. Keep its BO list. + * revalidation attempt. Keep its page list. */ continue; bo = mem->bo; + amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range); + mem->range = NULL; + + /* BO reservations and getting user pages (hmm_range_fault) + * must happen outside the notifier lock + */ + mutex_unlock(&process_info->notifier_lock); + + /* Move the BO to system (CPU) domain if necessary to unmap + * and free the SG table + */ + if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) { + if (amdgpu_bo_reserve(bo, true)) + return -EAGAIN; + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + amdgpu_bo_unreserve(bo); + if (ret) { + pr_err("%s: Failed to invalidate userptr BO\n", + __func__); + return -EAGAIN; + } + } + /* Get updated user pages */ ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, - &range); + &mem->range); if (ret) { pr_debug("Failed %d to get user pages\n", ret); @@ -2345,30 +2374,32 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info, */ if (ret != -EFAULT) return ret; - } else { - /* - * FIXME: Cannot ignore the return code, must hold - * notifier_lock - */ - amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range); + ret = 0; } + mutex_lock(&process_info->notifier_lock); + /* Mark the BO as valid unless it was invalidated * again concurrently. */ - if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) - return -EAGAIN; + if (mem->invalid != invalid) { + ret = -EAGAIN; + goto unlock_out; + } + mem->invalid = 0; } - return 0; +unlock_out: + mutex_unlock(&process_info->notifier_lock); + + return ret; } /* Validate invalid userptr BOs * - * Validates BOs on the userptr_inval_list, and moves them back to the - * userptr_valid_list. Also updates GPUVM page tables with new page - * addresses and waits for the page table updates to complete. + * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables + * with new page addresses and waits for the page table updates to complete. */ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) { @@ -2439,9 +2470,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) } } - list_move_tail(&mem->validate_list.head, - &process_info->userptr_valid_list); - /* Update mapping. If the BO was not validated * (because we couldn't get user pages), this will * clear the page table entries, which will result in @@ -2457,7 +2485,9 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) if (ret) { pr_err("%s: update PTE failed\n", __func__); /* make sure this gets validated again */ - atomic_inc(&mem->invalid); + mutex_lock(&process_info->notifier_lock); + mem->invalid++; + mutex_unlock(&process_info->notifier_lock); goto unreserve_out; } } @@ -2477,6 +2507,36 @@ out_no_mem: return ret; } +/* Confirm that all user pages are valid while holding the notifier lock + * + * Moves valid BOs from the userptr_inval_list back to userptr_val_list. + */ +static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info) +{ + struct kgd_mem *mem, *tmp_mem; + int ret = 0; + + list_for_each_entry_safe(mem, tmp_mem, + &process_info->userptr_inval_list, + validate_list.head) { + bool valid = amdgpu_ttm_tt_get_user_pages_done( + mem->bo->tbo.ttm, mem->range); + + mem->range = NULL; + if (!valid) { + WARN(!mem->invalid, "Invalid BO not marked invalid"); + ret = -EAGAIN; + continue; + } + WARN(mem->invalid, "Valid BO is marked invalid"); + + list_move_tail(&mem->validate_list.head, + &process_info->userptr_valid_list); + } + + return ret; +} + /* Worker callback to restore evicted userptr BOs * * Tries to update and validate all userptr BOs. If successful and no @@ -2491,9 +2551,11 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) restore_userptr_work); struct task_struct *usertask; struct mm_struct *mm; - int evicted_bos; + uint32_t evicted_bos; - evicted_bos = atomic_read(&process_info->evicted_bos); + mutex_lock(&process_info->notifier_lock); + evicted_bos = process_info->evicted_bos; + mutex_unlock(&process_info->notifier_lock); if (!evicted_bos) return; @@ -2516,9 +2578,6 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) * and we can just restart the queues. */ if (!list_empty(&process_info->userptr_inval_list)) { - if (atomic_read(&process_info->evicted_bos) != evicted_bos) - goto unlock_out; /* Concurrent eviction, try again */ - if (validate_invalid_user_pages(process_info)) goto unlock_out; } @@ -2527,10 +2586,17 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) * be a first eviction that calls quiesce_mm. The eviction * reference counting inside KFD will handle this case. */ - if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) != - evicted_bos) - goto unlock_out; - evicted_bos = 0; + mutex_lock(&process_info->notifier_lock); + if (process_info->evicted_bos != evicted_bos) + goto unlock_notifier_out; + + if (confirm_valid_user_pages_locked(process_info)) { + WARN(1, "User pages unexpectedly invalid"); + goto unlock_notifier_out; + } + + process_info->evicted_bos = evicted_bos = 0; + if (kgd2kfd_resume_mm(mm)) { pr_err("%s: Failed to resume KFD\n", __func__); /* No recovery from this failure. Probably the CP is @@ -2538,6 +2604,8 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) */ } +unlock_notifier_out: + mutex_unlock(&process_info->notifier_lock); unlock_out: mutex_unlock(&process_info->lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index cfa411c12072..64660a41d53c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4112,6 +4112,11 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) adev->in_suspend = true; + /* Evict the majority of BOs before grabbing the full access */ + r = amdgpu_device_evict_resources(adev); + if (r) + return r; + if (amdgpu_sriov_vf(adev)) { amdgpu_virt_fini_data_exchange(adev); r = amdgpu_virt_request_full_gpu(adev, false); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7383272c6a3a..b4f2d61ea0d5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2039,6 +2039,15 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, "See modparam exp_hw_support\n"); return -ENODEV; } + /* differentiate between P10 and P11 asics with the same DID */ + if (pdev->device == 0x67FF && + (pdev->revision == 0xE3 || + pdev->revision == 0xE7 || + pdev->revision == 0xF3 || + pdev->revision == 0xF7)) { + flags &= ~AMD_ASIC_MASK; + flags |= CHIP_POLARIS10; + } /* Due to hardware bugs, S/G Display on raven requires a 1:1 IOMMU mapping, * however, SME requires an indirect IOMMU mapping because the encryption @@ -2108,12 +2117,12 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, ddev); - ret = amdgpu_driver_load_kms(adev, ent->driver_data); + ret = amdgpu_driver_load_kms(adev, flags); if (ret) goto err_pci; retry_init: - ret = drm_dev_register(ddev, ent->driver_data); + ret = drm_dev_register(ddev, flags); if (ret == -EAGAIN && ++retry <= 3) { DRM_INFO("retry init %d\n", retry); /* Don't request EX mode too frequently which is attacking */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c index 2c38ac7bc643..4620c4712ce3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c @@ -64,7 +64,8 @@ static bool is_fru_eeprom_supported(struct amdgpu_device *adev, u32 *fru_addr) sizeof(atom_ctx->vbios_version)) || strnstr(atom_ctx->vbios_version, "D163", sizeof(atom_ctx->vbios_version))) { - *fru_addr = FRU_EEPROM_MADDR_6; + if (fru_addr) + *fru_addr = FRU_EEPROM_MADDR_6; return true; } else { return false; @@ -83,7 +84,8 @@ static bool is_fru_eeprom_supported(struct amdgpu_device *adev, u32 *fru_addr) sizeof(atom_ctx->vbios_version))) { return false; } else { - *fru_addr = FRU_EEPROM_MADDR_6; + if (fru_addr) + *fru_addr = FRU_EEPROM_MADDR_6; return true; } } else { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 62e98f1ad770..a0780a4e3e61 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -113,7 +113,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, bp.resv = resv; bp.preferred_domain = initial_domain; bp.flags = flags; - bp.domain = initial_domain | AMDGPU_GEM_DOMAIN_CPU; + bp.domain = initial_domain; bp.bo_ptr_size = sizeof(struct amdgpu_bo); r = amdgpu_bo_create_user(adev, &bp, &ubo); @@ -332,10 +332,20 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, } initial_domain = (u32)(0xffffffff & args->in.domains); +retry: r = amdgpu_gem_object_create(adev, size, args->in.alignment, - initial_domain, flags, ttm_bo_type_device, - resv, &gobj); + initial_domain, + flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) { + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + goto retry; + } + + if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { + initial_domain |= AMDGPU_GEM_DOMAIN_GTT; + goto retry; + } DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c index 65715cb395d8..2dadcfe43d03 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -105,17 +105,11 @@ static bool amdgpu_hmm_invalidate_hsa(struct mmu_interval_notifier *mni, unsigned long cur_seq) { struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); - struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); if (!mmu_notifier_range_blockable(range)) return false; - mutex_lock(&adev->notifier_lock); - - mmu_interval_set_seq(mni, cur_seq); - - amdgpu_amdkfd_evict_userptr(bo->kfd_bo, bo->notifier.mm); - mutex_unlock(&adev->notifier_lock); + amdgpu_amdkfd_evict_userptr(mni, cur_seq, bo->kfd_bo); return true; } @@ -244,9 +238,9 @@ out_free_range: return r; } -int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range) +bool amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range) { - int r; + bool r; r = mmu_interval_read_retry(hmm_range->notifier, hmm_range->notifier_seq); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h index 13ed94d3b01b..e2edcd010ccc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h @@ -29,12 +29,13 @@ #include <linux/rwsem.h> #include <linux/workqueue.h> #include <linux/interval_tree.h> +#include <linux/mmu_notifier.h> int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, uint64_t start, uint64_t npages, bool readonly, void *owner, struct page **pages, struct hmm_range **phmm_range); -int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range); +bool amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range); #if defined(CONFIG_HMM_MIRROR) int amdgpu_hmm_register(struct amdgpu_bo *bo, unsigned long addr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 2a9a2593dc18..fcb711a11a5b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -165,6 +165,26 @@ bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, atomic_read(&adev->gpu_reset_counter); } +/* Check if we need to switch to another set of resources */ +static bool amdgpu_vmid_gds_switch_needed(struct amdgpu_vmid *id, + struct amdgpu_job *job) +{ + return id->gds_base != job->gds_base || + id->gds_size != job->gds_size || + id->gws_base != job->gws_base || + id->gws_size != job->gws_size || + id->oa_base != job->oa_base || + id->oa_size != job->oa_size; +} + +/* Check if the id is compatible with the job */ +static bool amdgpu_vmid_compatible(struct amdgpu_vmid *id, + struct amdgpu_job *job) +{ + return id->pd_gpu_addr == job->vm_pd_addr && + !amdgpu_vmid_gds_switch_needed(id, job); +} + /** * amdgpu_vmid_grab_idle - grab idle VMID * @@ -258,14 +278,15 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, { struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; uint64_t fence_context = adev->fence_context + ring->idx; bool needs_flush = vm->use_cpu_for_update; uint64_t updates = amdgpu_vm_tlb_seq(vm); int r; - *id = vm->reserved_vmid[vmhub]; + *id = id_mgr->reserved; if ((*id)->owner != vm->immediate.fence_context || - (*id)->pd_gpu_addr != job->vm_pd_addr || + !amdgpu_vmid_compatible(*id, job) || (*id)->flushed_updates < updates || !(*id)->last_flush || ((*id)->last_flush->context != fence_context && @@ -294,8 +315,8 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, if (r) return r; - (*id)->flushed_updates = updates; job->vm_needs_flush = needs_flush; + job->spm_update_needed = true; return 0; } @@ -333,7 +354,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, if ((*id)->owner != vm->immediate.fence_context) continue; - if ((*id)->pd_gpu_addr != job->vm_pd_addr) + if (!amdgpu_vmid_compatible(*id, job)) continue; if (!(*id)->last_flush || @@ -355,7 +376,6 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, if (r) return r; - (*id)->flushed_updates = updates; job->vm_needs_flush |= needs_flush; return 0; } @@ -408,22 +428,30 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring, if (r) goto error; - id->flushed_updates = amdgpu_vm_tlb_seq(vm); job->vm_needs_flush = true; } list_move_tail(&id->list, &id_mgr->ids_lru); } - id->pd_gpu_addr = job->vm_pd_addr; - id->owner = vm->immediate.fence_context; - + job->gds_switch_needed = amdgpu_vmid_gds_switch_needed(id, job); if (job->vm_needs_flush) { + id->flushed_updates = amdgpu_vm_tlb_seq(vm); dma_fence_put(id->last_flush); id->last_flush = NULL; } job->vmid = id - id_mgr->ids; job->pasid = vm->pasid; + + id->gds_base = job->gds_base; + id->gds_size = job->gds_size; + id->gws_base = job->gws_base; + id->gws_size = job->gws_size; + id->oa_base = job->oa_base; + id->oa_size = job->oa_size; + id->pd_gpu_addr = job->vm_pd_addr; + id->owner = vm->immediate.fence_context; + trace_amdgpu_vm_grab_id(vm, ring, job); error: @@ -435,31 +463,27 @@ int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned vmhub) { - struct amdgpu_vmid_mgr *id_mgr; - struct amdgpu_vmid *idle; - int r = 0; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; - id_mgr = &adev->vm_manager.id_mgr[vmhub]; mutex_lock(&id_mgr->lock); if (vm->reserved_vmid[vmhub]) goto unlock; - if (atomic_inc_return(&id_mgr->reserved_vmid_num) > - AMDGPU_VM_MAX_RESERVED_VMID) { - DRM_ERROR("Over limitation of reserved vmid\n"); - atomic_dec(&id_mgr->reserved_vmid_num); - r = -EINVAL; - goto unlock; + + ++id_mgr->reserved_use_count; + if (!id_mgr->reserved) { + struct amdgpu_vmid *id; + + id = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, + list); + /* Remove from normal round robin handling */ + list_del_init(&id->list); + id_mgr->reserved = id; } - /* Select the first entry VMID */ - idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, list); - list_del_init(&idle->list); - vm->reserved_vmid[vmhub] = idle; - mutex_unlock(&id_mgr->lock); + vm->reserved_vmid[vmhub] = true; - return 0; unlock: mutex_unlock(&id_mgr->lock); - return r; + return 0; } void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, @@ -469,12 +493,12 @@ void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; mutex_lock(&id_mgr->lock); - if (vm->reserved_vmid[vmhub]) { - list_add(&vm->reserved_vmid[vmhub]->list, - &id_mgr->ids_lru); - vm->reserved_vmid[vmhub] = NULL; - atomic_dec(&id_mgr->reserved_vmid_num); + if (vm->reserved_vmid[vmhub] && + !--id_mgr->reserved_use_count) { + /* give the reserved ID back to normal round robin */ + list_add(&id_mgr->reserved->list, &id_mgr->ids_lru); } + vm->reserved_vmid[vmhub] = false; mutex_unlock(&id_mgr->lock); } @@ -541,7 +565,7 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) mutex_init(&id_mgr->lock); INIT_LIST_HEAD(&id_mgr->ids_lru); - atomic_set(&id_mgr->reserved_vmid_num, 0); + id_mgr->reserved_use_count = 0; /* manage only VMIDs not used by KFD */ id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index 57efe61dceed..d1cc09b45da4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -67,7 +67,8 @@ struct amdgpu_vmid_mgr { unsigned num_ids; struct list_head ids_lru; struct amdgpu_vmid ids[AMDGPU_NUM_VMID]; - atomic_t reserved_vmid_num; + struct amdgpu_vmid *reserved; + unsigned int reserved_use_count; }; int amdgpu_pasid_alloc(unsigned int bits); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index a372802ea4e0..52f2e313ea17 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -53,6 +53,8 @@ struct amdgpu_job { uint32_t preamble_status; uint32_t preemption_status; bool vm_needs_flush; + bool gds_switch_needed; + bool spm_update_needed; uint64_t vm_pd_addr; unsigned vmid; unsigned pasid; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 919bbea2e3ac..4e684c2afc70 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -346,17 +346,16 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev, * @adev: amdgpu device object * @offset: offset of the BO * @size: size of the BO - * @domain: where to place it * @bo_ptr: used to initialize BOs in structures * @cpu_addr: optional CPU address mapping * - * Creates a kernel BO at a specific offset in the address space of the domain. + * Creates a kernel BO at a specific offset in VRAM. * * Returns: * 0 on success, negative error code otherwise. */ int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, - uint64_t offset, uint64_t size, uint32_t domain, + uint64_t offset, uint64_t size, struct amdgpu_bo **bo_ptr, void **cpu_addr) { struct ttm_operation_ctx ctx = { false, false }; @@ -366,8 +365,9 @@ int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, offset &= PAGE_MASK; size = ALIGN(size, PAGE_SIZE); - r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, domain, bo_ptr, - NULL, cpu_addr); + r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, + cpu_addr); if (r) return r; @@ -422,6 +422,8 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, if (*bo == NULL) return; + WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); + if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { if (cpu_addr) amdgpu_bo_kunmap(*bo); @@ -446,27 +448,24 @@ static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, /* * If GTT is part of requested domains the check must succeed to - * allow fall back to GTT + * allow fall back to GTT. */ if (domain & AMDGPU_GEM_DOMAIN_GTT) { man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); - if (size < man->size) + if (man && size < man->size) return true; - else - goto fail; - } - - if (domain & AMDGPU_GEM_DOMAIN_VRAM) { + else if (!man) + WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); + goto fail; + } else if (domain & AMDGPU_GEM_DOMAIN_VRAM) { man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); - if (size < man->size) + if (man && size < man->size) return true; - else - goto fail; + goto fail; } - /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */ return true; @@ -581,7 +580,11 @@ int amdgpu_bo_create(struct amdgpu_device *adev, bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; bo->tbo.bdev = &adev->mman.bdev; - amdgpu_bo_placement_from_domain(bo, bp->domain); + if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | + AMDGPU_GEM_DOMAIN_GDS)) + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); + else + amdgpu_bo_placement_from_domain(bo, bp->domain); if (bp->type == ttm_bo_type_kernel) bo->tbo.priority = 1; @@ -1506,7 +1509,8 @@ u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, uint32_t domain) { - if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { + if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && + ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { domain = AMDGPU_GEM_DOMAIN_VRAM; if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) domain = AMDGPU_GEM_DOMAIN_GTT; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 147b79c10cbb..93207badf83f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -284,7 +284,7 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev, u32 domain, struct amdgpu_bo **bo_ptr, u64 *gpu_addr, void **cpu_addr); int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, - uint64_t offset, uint64_t size, uint32_t domain, + uint64_t offset, uint64_t size, struct amdgpu_bo **bo_ptr, void **cpu_addr); int amdgpu_bo_create_user(struct amdgpu_device *adev, struct amdgpu_bo_param *bp, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index b4236572eae1..55e0284b2bdd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -695,8 +695,19 @@ out_unlock: return r; } +/* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations + */ +void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm, + struct hmm_range *range) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt && gtt->userptr && range) + amdgpu_hmm_range_get_pages_done(range); +} + /* - * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change + * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change * Check if the pages backing this ttm range have been invalidated * * Returns: true if pages are still valid @@ -714,10 +725,6 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm, WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); - /* - * FIXME: Must always hold notifier_lock for this, and must - * not ignore the return code. - */ return !amdgpu_hmm_range_get_pages_done(range); } #endif @@ -1569,7 +1576,6 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) return amdgpu_bo_create_kernel_at(adev, adev->mman.fw_vram_usage_start_offset, adev->mman.fw_vram_usage_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.fw_vram_usage_reserved_bo, &adev->mman.fw_vram_usage_va); } @@ -1595,7 +1601,6 @@ static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev) return amdgpu_bo_create_kernel_at(adev, adev->mman.drv_vram_usage_start_offset, adev->mman.drv_vram_usage_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.drv_vram_usage_reserved_bo, &adev->mman.drv_vram_usage_va); } @@ -1676,7 +1681,6 @@ static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) ret = amdgpu_bo_create_kernel_at(adev, ctx->c2p_train_data_offset, ctx->train_data_size, - AMDGPU_GEM_DOMAIN_VRAM, &ctx->c2p_bo, NULL); if (ret) { @@ -1690,7 +1694,6 @@ static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) ret = amdgpu_bo_create_kernel_at(adev, adev->gmc.real_vram_size - adev->mman.discovery_tmr_size, adev->mman.discovery_tmr_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.discovery_memory, NULL); if (ret) { @@ -1791,21 +1794,18 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) * avoid display artifacts while transitioning between pre-OS * and driver. */ r = amdgpu_bo_create_kernel_at(adev, 0, adev->mman.stolen_vga_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.stolen_vga_memory, NULL); if (r) return r; r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, adev->mman.stolen_extended_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.stolen_extended_memory, NULL); if (r) return r; r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_reserved_offset, adev->mman.stolen_reserved_size, - AMDGPU_GEM_DOMAIN_VRAM, &adev->mman.stolen_reserved_memory, NULL); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index b4d8ba2789f3..e2cd5894afc9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -159,6 +159,8 @@ uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type); #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages, struct hmm_range **range); +void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm, + struct hmm_range *range); bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm, struct hmm_range *range); #else @@ -168,6 +170,10 @@ static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, { return -EPERM; } +static inline void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm, + struct hmm_range *range) +{ +} static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm, struct hmm_range *range) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 15544f262ec1..2994b9db196f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -395,7 +395,6 @@ static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) */ if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GPU_PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, &bo, NULL)) DRM_DEBUG("RAS WARN: reserve vram for retired page %llx fail\n", bp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index c05cff979004..dc379dc22c77 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -484,25 +484,20 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; - struct amdgpu_vmid *id; - bool gds_switch_needed; - bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; if (job->vmid == 0) return false; - id = &id_mgr->ids[job->vmid]; - gds_switch_needed = ring->funcs->emit_gds_switch && ( - id->gds_base != job->gds_base || - id->gds_size != job->gds_size || - id->gws_base != job->gws_base || - id->gws_size != job->gws_size || - id->oa_base != job->oa_base || - id->oa_size != job->oa_size); - - if (amdgpu_vmid_had_gpu_reset(adev, id)) + + if (job->vm_needs_flush || ring->has_compute_vm_bug) + return true; + + if (ring->funcs->emit_gds_switch && job->gds_switch_needed) + return true; + + if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid])) return true; - return vm_flush_needed || gds_switch_needed; + return false; } /** @@ -524,27 +519,20 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; - bool gds_switch_needed = ring->funcs->emit_gds_switch && ( - id->gds_base != job->gds_base || - id->gds_size != job->gds_size || - id->gws_base != job->gws_base || - id->gws_size != job->gws_size || - id->oa_base != job->oa_base || - id->oa_size != job->oa_size); + bool spm_update_needed = job->spm_update_needed; + bool gds_switch_needed = ring->funcs->emit_gds_switch && + job->gds_switch_needed; bool vm_flush_needed = job->vm_needs_flush; struct dma_fence *fence = NULL; bool pasid_mapping_needed = false; unsigned patch_offset = 0; - bool update_spm_vmid_needed = (job->vm && (job->vm->reserved_vmid[vmhub] != NULL)); int r; - if (update_spm_vmid_needed && adev->gfx.rlc.funcs->update_spm_vmid) - adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid); - if (amdgpu_vmid_had_gpu_reset(adev, id)) { gds_switch_needed = true; vm_flush_needed = true; pasid_mapping_needed = true; + spm_update_needed = true; } mutex_lock(&id_mgr->lock); @@ -577,6 +565,17 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, if (pasid_mapping_needed) amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); + if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid) + adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid); + + if (!ring->is_mes_queue && ring->funcs->emit_gds_switch && + gds_switch_needed) { + amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, + job->gds_size, job->gws_base, + job->gws_size, job->oa_base, + job->oa_size); + } + if (vm_flush_needed || pasid_mapping_needed) { r = amdgpu_fence_emit(ring, &fence, NULL, 0); if (r) @@ -601,20 +600,6 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, } dma_fence_put(fence); - if (!ring->is_mes_queue && ring->funcs->emit_gds_switch && - gds_switch_needed) { - id->gds_base = job->gds_base; - id->gds_size = job->gds_size; - id->gws_base = job->gws_base; - id->gws_size = job->gws_size; - id->oa_base = job->oa_base; - id->oa_size = job->oa_size; - amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, - job->gds_size, job->gws_base, - job->gws_size, job->oa_base, - job->oa_size); - } - if (ring->funcs->patch_cond_exec) amdgpu_ring_patch_cond_exec(ring, patch_offset); @@ -2383,7 +2368,6 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) union drm_amdgpu_vm *args = data; struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; - long timeout = msecs_to_jiffies(2000); int r; switch (args->in.op) { @@ -2395,21 +2379,6 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) return r; break; case AMDGPU_VM_OP_UNRESERVE_VMID: - if (amdgpu_sriov_runtime(adev)) - timeout = 8 * timeout; - - /* Wait vm idle to make sure the vmid set in SPM_VMID is - * not referenced anymore. - */ - r = amdgpu_bo_reserve(fpriv->vm.root.bo, true); - if (r) - return r; - - r = amdgpu_vm_wait_idle(&fpriv->vm, timeout); - if (r < 0) - return r; - - amdgpu_bo_unreserve(fpriv->vm.root.bo); amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 6546e786bf00..094bb4807303 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -119,9 +119,6 @@ struct amdgpu_bo_vm; /* Reserve 2MB at top/bottom of address space for kernel use */ #define AMDGPU_VA_RESERVED_SIZE (2ULL << 20) -/* max vmids dedicated for process */ -#define AMDGPU_VM_MAX_RESERVED_VMID 1 - /* See vm_update_mode */ #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0) #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1) @@ -298,8 +295,7 @@ struct amdgpu_vm { struct dma_fence *last_unlocked; unsigned int pasid; - /* dedicated to vm */ - struct amdgpu_vmid *reserved_vmid[AMDGPU_MAX_VMHUBS]; + bool reserved_vmid[AMDGPU_MAX_VMHUBS]; /* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */ bool use_cpu_for_update; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 50386eb2eec8..08d6cf79fb15 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -1185,6 +1185,8 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping, uint64_t *flags) { + struct amdgpu_bo *bo = mapping->bo_va->base.bo; + *flags &= ~AMDGPU_PTE_EXECUTABLE; *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; @@ -1196,7 +1198,7 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, *flags &= ~AMDGPU_PTE_VALID; } - if (mapping->bo_va->base.bo) + if (bo && bo->tbo.resource) gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo, mapping, flags); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 77277d90b6e2..50c783e19f5a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1503,6 +1503,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) case IP_VERSION(3, 0, 1): case IP_VERSION(3, 1, 2): case IP_VERSION(3, 1, 3): + case IP_VERSION(3, 1, 4): case IP_VERSION(3, 1, 5): case IP_VERSION(3, 1, 6): init_data.flags.gpu_vm_support = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c index 0f746bb4e500..d51f1ce02874 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c @@ -55,7 +55,7 @@ void hubbub1_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A); s = &wm->sets[1]; s->wm_set = 1; @@ -65,7 +65,7 @@ void hubbub1_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B); s = &wm->sets[2]; s->wm_set = 2; @@ -75,7 +75,7 @@ void hubbub1_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C); s = &wm->sets[3]; s->wm_set = 3; @@ -85,7 +85,7 @@ void hubbub1_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D); } void hubbub1_allow_self_refresh_control(struct hubbub *hubbub, bool allow) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index c8ec11839b4d..fe2023f18b7d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -159,7 +159,7 @@ static void dcn10_log_hubbub_state(struct dc *dc, DTN_INFO_MICRO_SEC(s->pte_meta_urgent); DTN_INFO_MICRO_SEC(s->sr_enter); DTN_INFO_MICRO_SEC(s->sr_exit); - DTN_INFO_MICRO_SEC(s->dram_clk_chanage); + DTN_INFO_MICRO_SEC(s->dram_clk_change); DTN_INFO("\n"); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c index e8b6065fffad..a0f8e31d2adc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c @@ -83,7 +83,7 @@ static unsigned int dcn10_get_hubbub_state(struct dc *dc, char *pBuf, unsigned i memset(&wm, 0, sizeof(struct dcn_hubbub_wm)); dc->res_pool->hubbub->funcs->wm_read_state(dc->res_pool->hubbub, &wm); - chars_printed = snprintf_count(pBuf, remaining_buffer, "wm_set_index,data_urgent,pte_meta_urgent,sr_enter,sr_exit,dram_clk_chanage\n"); + chars_printed = snprintf_count(pBuf, remaining_buffer, "wm_set_index,data_urgent,pte_meta_urgent,sr_enter,sr_exit,dram_clk_change\n"); remaining_buffer -= chars_printed; pBuf += chars_printed; @@ -98,7 +98,7 @@ static unsigned int dcn10_get_hubbub_state(struct dc *dc, char *pBuf, unsigned i (s->pte_meta_urgent * frac) / ref_clk_mhz / frac, (s->pte_meta_urgent * frac) / ref_clk_mhz % frac, (s->sr_enter * frac) / ref_clk_mhz / frac, (s->sr_enter * frac) / ref_clk_mhz % frac, (s->sr_exit * frac) / ref_clk_mhz / frac, (s->sr_exit * frac) / ref_clk_mhz % frac, - (s->dram_clk_chanage * frac) / ref_clk_mhz / frac, (s->dram_clk_chanage * frac) / ref_clk_mhz % frac); + (s->dram_clk_change * frac) / ref_clk_mhz / frac, (s->dram_clk_change * frac) / ref_clk_mhz % frac); remaining_buffer -= chars_printed; pBuf += chars_printed; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c index aacb1fb5c73e..24bd93219936 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c @@ -500,7 +500,7 @@ void hubbub2_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A); s = &wm->sets[1]; s->wm_set = 1; @@ -511,7 +511,7 @@ void hubbub2_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B); s = &wm->sets[2]; s->wm_set = 2; @@ -522,7 +522,7 @@ void hubbub2_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C); s = &wm->sets[3]; s->wm_set = 3; @@ -533,7 +533,7 @@ void hubbub2_wm_read_state(struct hubbub *hubbub, s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D); s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D); } - s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D); + s->dram_clk_change = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D); } void hubbub2_get_dchub_ref_freq(struct hubbub *hubbub, diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c index c5e200d09038..aeb0e0d9b70a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c @@ -635,7 +635,7 @@ void hubbub21_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, &s->sr_exit); REG_GET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, - DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, &s->dram_clk_chanage); + DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, &s->dram_clk_change); s = &wm->sets[1]; s->wm_set = 1; @@ -649,7 +649,7 @@ void hubbub21_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, &s->sr_exit); REG_GET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, - DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, &s->dram_clk_chanage); + DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, &s->dram_clk_change); s = &wm->sets[2]; s->wm_set = 2; @@ -663,7 +663,7 @@ void hubbub21_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, &s->sr_exit); REG_GET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, - DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, &s->dram_clk_chanage); + DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, &s->dram_clk_change); s = &wm->sets[3]; s->wm_set = 3; @@ -677,7 +677,7 @@ void hubbub21_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, &s->sr_exit); REG_GET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, - DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, &s->dram_clk_chanage); + DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, &s->dram_clk_change); } static void hubbub21_apply_DEDCN21_147_wa(struct hubbub *hubbub) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c index 5947c2cb0f30..9501403a48a9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c @@ -865,7 +865,7 @@ static void hubbub32_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, &s->sr_exit); REG_GET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, - DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, &s->dram_clk_chanage); + DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, &s->dram_clk_change); REG_GET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, &s->usr_retrain); @@ -885,7 +885,7 @@ static void hubbub32_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, &s->sr_exit); REG_GET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, - DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, &s->dram_clk_chanage); + DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, &s->dram_clk_change); REG_GET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, &s->usr_retrain); @@ -905,7 +905,7 @@ static void hubbub32_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, &s->sr_exit); REG_GET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, - DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, &s->dram_clk_chanage); + DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, &s->dram_clk_change); REG_GET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, &s->usr_retrain); @@ -925,7 +925,7 @@ static void hubbub32_wm_read_state(struct hubbub *hubbub, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, &s->sr_exit); REG_GET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, - DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, &s->dram_clk_chanage); + DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, &s->dram_clk_change); REG_GET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, &s->usr_retrain); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h index f2e1fcb668fb..5b0265c0df61 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h @@ -46,7 +46,7 @@ struct dcn_hubbub_wm_set { uint32_t pte_meta_urgent; uint32_t sr_enter; uint32_t sr_exit; - uint32_t dram_clk_chanage; + uint32_t dram_clk_change; uint32_t usr_retrain; uint32_t fclk_pstate_change; }; diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h index a4e3425b1027..4180c71d930f 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h @@ -241,7 +241,8 @@ __SMU_DUMMY_MAP(GetGfxOffEntryCount), \ __SMU_DUMMY_MAP(LogGfxOffResidency), \ __SMU_DUMMY_MAP(SetNumBadMemoryPagesRetired), \ - __SMU_DUMMY_MAP(SetBadMemoryPagesRetiredFlagsPerChannel), + __SMU_DUMMY_MAP(SetBadMemoryPagesRetiredFlagsPerChannel), \ + __SMU_DUMMY_MAP(AllowGpo), #undef __SMU_DUMMY_MAP #define __SMU_DUMMY_MAP(type) SMU_MSG_##type diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h index 865d6358918d..ea29ac6a80e6 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h @@ -272,6 +272,9 @@ int smu_v13_0_init_pptable_microcode(struct smu_context *smu); int smu_v13_0_run_btc(struct smu_context *smu); +int smu_v13_0_gpo_control(struct smu_context *smu, + bool enablement); + int smu_v13_0_deep_sleep_control(struct smu_context *smu, bool enablement); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index f5e90e0a99df..e3a80ac987df 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -2180,6 +2180,21 @@ int smu_v13_0_run_btc(struct smu_context *smu) return res; } +int smu_v13_0_gpo_control(struct smu_context *smu, + bool enablement) +{ + int res; + + res = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_AllowGpo, + enablement ? 1 : 0, + NULL); + if (res) + dev_err(smu->adev->dev, "SetGpoAllow %d failed!\n", enablement); + + return res; +} + int smu_v13_0_deep_sleep_control(struct smu_context *smu, bool enablement) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 87d7c66e49ef..713fb6ad39f6 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -144,6 +144,7 @@ static struct cmn2asic_msg_mapping smu_v13_0_0_message_map[SMU_MSG_MAX_COUNT] = MSG_MAP(SetNumBadMemoryPagesRetired, PPSMC_MSG_SetNumBadMemoryPagesRetired, 0), MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel, PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel, 0), + MSG_MAP(AllowGpo, PPSMC_MSG_SetGpoAllow, 0), }; static struct cmn2asic_mapping smu_v13_0_0_clk_map[SMU_CLK_COUNT] = { @@ -210,6 +211,8 @@ static struct cmn2asic_mapping smu_v13_0_0_feature_mask_map[SMU_FEATURE_COUNT] = FEA_MAP(MEM_TEMP_READ), FEA_MAP(ATHUB_MMHUB_PG), FEA_MAP(SOC_PCC), + [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, + [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, }; static struct cmn2asic_mapping smu_v13_0_0_table_map[SMU_TABLE_COUNT] = { @@ -540,6 +543,23 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu) dpm_table); if (ret) return ret; + + /* + * Update the reported maximum shader clock to the value + * which can be guarded to be achieved on all cards. This + * is aligned with Window setting. And considering that value + * might be not the peak frequency the card can achieve, it + * is normal some real-time clock frequency can overtake this + * labelled maximum clock frequency(for example in pp_dpm_sclk + * sysfs output). + */ + if (skutable->DriverReportedClocks.GameClockAc && + (dpm_table->dpm_levels[dpm_table->count - 1].value > + skutable->DriverReportedClocks.GameClockAc)) { + dpm_table->dpm_levels[dpm_table->count - 1].value = + skutable->DriverReportedClocks.GameClockAc; + dpm_table->max = skutable->DriverReportedClocks.GameClockAc; + } } else { dpm_table->count = 1; dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; @@ -802,6 +822,57 @@ static int smu_v13_0_0_get_smu_metrics_data(struct smu_context *smu, return ret; } +static int smu_v13_0_0_get_dpm_ultimate_freq(struct smu_context *smu, + enum smu_clk_type clk_type, + uint32_t *min, + uint32_t *max) +{ + struct smu_13_0_dpm_context *dpm_context = + smu->smu_dpm.dpm_context; + struct smu_13_0_dpm_table *dpm_table; + + switch (clk_type) { + case SMU_MCLK: + case SMU_UCLK: + /* uclk dpm table */ + dpm_table = &dpm_context->dpm_tables.uclk_table; + break; + case SMU_GFXCLK: + case SMU_SCLK: + /* gfxclk dpm table */ + dpm_table = &dpm_context->dpm_tables.gfx_table; + break; + case SMU_SOCCLK: + /* socclk dpm table */ + dpm_table = &dpm_context->dpm_tables.soc_table; + break; + case SMU_FCLK: + /* fclk dpm table */ + dpm_table = &dpm_context->dpm_tables.fclk_table; + break; + case SMU_VCLK: + case SMU_VCLK1: + /* vclk dpm table */ + dpm_table = &dpm_context->dpm_tables.vclk_table; + break; + case SMU_DCLK: + case SMU_DCLK1: + /* dclk dpm table */ + dpm_table = &dpm_context->dpm_tables.dclk_table; + break; + default: + dev_err(smu->adev->dev, "Unsupported clock type!\n"); + return -EINVAL; + } + + if (min) + *min = dpm_table->min; + if (max) + *max = dpm_table->max; + + return 0; +} + static int smu_v13_0_0_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, @@ -1304,9 +1375,17 @@ static int smu_v13_0_0_populate_umd_state_clk(struct smu_context *smu) &dpm_context->dpm_tables.fclk_table; struct smu_umd_pstate_table *pstate_table = &smu->pstate_table; + struct smu_table_context *table_context = &smu->smu_table; + PPTable_t *pptable = table_context->driver_pptable; + DriverReportedClocks_t driver_clocks = + pptable->SkuTable.DriverReportedClocks; pstate_table->gfxclk_pstate.min = gfx_table->min; - pstate_table->gfxclk_pstate.peak = gfx_table->max; + if (driver_clocks.GameClockAc && + (driver_clocks.GameClockAc < gfx_table->max)) + pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc; + else + pstate_table->gfxclk_pstate.peak = gfx_table->max; pstate_table->uclk_pstate.min = mem_table->min; pstate_table->uclk_pstate.peak = mem_table->max; @@ -1323,12 +1402,12 @@ static int smu_v13_0_0_populate_umd_state_clk(struct smu_context *smu) pstate_table->fclk_pstate.min = fclk_table->min; pstate_table->fclk_pstate.peak = fclk_table->max; - /* - * For now, just use the mininum clock frequency. - * TODO: update them when the real pstate settings available - */ - pstate_table->gfxclk_pstate.standard = gfx_table->min; - pstate_table->uclk_pstate.standard = mem_table->min; + if (driver_clocks.BaseClockAc && + driver_clocks.BaseClockAc < gfx_table->max) + pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc; + else + pstate_table->gfxclk_pstate.standard = gfx_table->max; + pstate_table->uclk_pstate.standard = mem_table->max; pstate_table->socclk_pstate.standard = soc_table->min; pstate_table->vclk_pstate.standard = vclk_table->min; pstate_table->dclk_pstate.standard = dclk_table->min; @@ -1899,7 +1978,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .get_enabled_mask = smu_cmn_get_enabled_mask, .dpm_set_vcn_enable = smu_v13_0_set_vcn_enable, .dpm_set_jpeg_enable = smu_v13_0_set_jpeg_enable, - .get_dpm_ultimate_freq = smu_v13_0_get_dpm_ultimate_freq, + .get_dpm_ultimate_freq = smu_v13_0_0_get_dpm_ultimate_freq, .get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values, .read_sensor = smu_v13_0_0_read_sensor, .feature_is_enabled = smu_cmn_feature_is_enabled, @@ -1947,6 +2026,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .set_df_cstate = smu_v13_0_0_set_df_cstate, .send_hbm_bad_pages_num = smu_v13_0_0_smu_send_bad_mem_page_num, .send_hbm_bad_channel_flag = smu_v13_0_0_send_bad_mem_channel_flag, + .gpo_control = smu_v13_0_gpo_control, }; void smu_v13_0_0_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index c3c9ef523e59..ab1c004606be 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -123,6 +123,7 @@ static struct cmn2asic_msg_mapping smu_v13_0_7_message_map[SMU_MSG_MAX_COUNT] = MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), + MSG_MAP(AllowGpo, PPSMC_MSG_SetGpoAllow, 0), }; static struct cmn2asic_mapping smu_v13_0_7_clk_map[SMU_CLK_COUNT] = { @@ -189,6 +190,8 @@ static struct cmn2asic_mapping smu_v13_0_7_feature_mask_map[SMU_FEATURE_COUNT] = FEA_MAP(MEM_TEMP_READ), FEA_MAP(ATHUB_MMHUB_PG), FEA_MAP(SOC_PCC), + [SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, + [SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT}, }; static struct cmn2asic_mapping smu_v13_0_7_table_map[SMU_TABLE_COUNT] = { @@ -1688,6 +1691,7 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = { .mode1_reset = smu_v13_0_mode1_reset, .set_mp1_state = smu_v13_0_7_set_mp1_state, .set_df_cstate = smu_v13_0_7_set_df_cstate, + .gpo_control = smu_v13_0_gpo_control, }; void smu_v13_0_7_set_ppt_funcs(struct smu_context *smu) |