diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu')
88 files changed, 1905 insertions, 8557 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index f76bcb9c45e4..466da5954a68 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -57,7 +57,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ # add asic specific block amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ - ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o + dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index bcef6ea4bcf9..8d0d7f3dd5fb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -411,6 +411,8 @@ struct amdgpu_fpriv { struct amdgpu_ctx_mgr ctx_mgr; }; +int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); + int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned size, struct amdgpu_ib *ib); void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib, @@ -542,6 +544,11 @@ struct amdgpu_asic_funcs { bool (*need_full_reset)(struct amdgpu_device *adev); /* initialize doorbell layout for specific asic*/ void (*init_doorbell_index)(struct amdgpu_device *adev); + /* PCIe bandwidth usage */ + void (*get_pcie_usage)(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1); + /* do we need to reset the asic at init time (e.g., kexec) */ + bool (*need_reset_on_init)(struct amdgpu_device *adev); }; /* @@ -634,7 +641,7 @@ struct amdgpu_nbio_funcs { void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring); u32 (*get_memsize)(struct amdgpu_device *adev); void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance, - bool use_doorbell, int doorbell_index); + bool use_doorbell, int doorbell_index, int doorbell_size); void (*enable_doorbell_aperture)(struct amdgpu_device *adev, bool enable); void (*enable_doorbell_selfring_aperture)(struct amdgpu_device *adev, @@ -1042,6 +1049,8 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r)) #define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev)) #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev)) +#define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1))) +#define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev)) /* Common functions */ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 2dfaf158ef07..fe1d7368c1e6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -28,8 +28,6 @@ #include <linux/module.h> #include <linux/dma-buf.h> -const struct kgd2kfd_calls *kgd2kfd; - static const unsigned int compute_vmid_bitmap = 0xFF00; /* Total memory size in system memory and all GPU VRAM. Used to @@ -47,12 +45,9 @@ int amdgpu_amdkfd_init(void) amdgpu_amdkfd_total_mem_size *= si.mem_unit; #ifdef CONFIG_HSA_AMD - ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd); - if (ret) - kgd2kfd = NULL; + ret = kgd2kfd_init(); amdgpu_amdkfd_gpuvm_init_mem_limits(); #else - kgd2kfd = NULL; ret = -ENOENT; #endif @@ -61,17 +56,13 @@ int amdgpu_amdkfd_init(void) void amdgpu_amdkfd_fini(void) { - if (kgd2kfd) - kgd2kfd->exit(); + kgd2kfd_exit(); } void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) { const struct kfd2kgd_calls *kfd2kgd; - if (!kgd2kfd) - return; - switch (adev->asic_type) { #ifdef CONFIG_DRM_AMDGPU_CIK case CHIP_KAVERI: @@ -98,8 +89,8 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) return; } - adev->kfd.dev = kgd2kfd->probe((struct kgd_dev *)adev, - adev->pdev, kfd2kgd); + adev->kfd.dev = kgd2kfd_probe((struct kgd_dev *)adev, + adev->pdev, kfd2kgd); if (adev->kfd.dev) amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; @@ -140,7 +131,7 @@ static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) { - int i, n; + int i; int last_valid_bit; if (adev->kfd.dev) { @@ -151,7 +142,9 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) .gpuvm_size = min(adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GMC_HOLE_START), - .drm_render_minor = adev->ddev->render->index + .drm_render_minor = adev->ddev->render->index, + .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, + }; /* this is going to have a few of the MSBs set that we need to @@ -181,44 +174,29 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) &gpu_resources.doorbell_aperture_size, &gpu_resources.doorbell_start_offset); - if (adev->asic_type < CHIP_VEGA10) { - kgd2kfd->device_init(adev->kfd.dev, &gpu_resources); - return; - } - - n = (adev->asic_type < CHIP_VEGA20) ? 2 : 8; - - for (i = 0; i < n; i += 2) { - /* On SOC15 the BIF is involved in routing - * doorbells using the low 12 bits of the - * address. Communicate the assignments to - * KFD. KFD uses two doorbell pages per - * process in case of 64-bit doorbells so we - * can use each doorbell assignment twice. - */ - gpu_resources.sdma_doorbell[0][i] = - adev->doorbell_index.sdma_engine0 + (i >> 1); - gpu_resources.sdma_doorbell[0][i+1] = - adev->doorbell_index.sdma_engine0 + 0x200 + (i >> 1); - gpu_resources.sdma_doorbell[1][i] = - adev->doorbell_index.sdma_engine1 + (i >> 1); - gpu_resources.sdma_doorbell[1][i+1] = - adev->doorbell_index.sdma_engine1 + 0x200 + (i >> 1); - } - /* Doorbells 0x0e0-0ff and 0x2e0-2ff are reserved for - * SDMA, IH and VCN. So don't use them for the CP. + /* Since SOC15, BIF starts to statically use the + * lower 12 bits of doorbell addresses for routing + * based on settings in registers like + * SDMA0_DOORBELL_RANGE etc.. + * In order to route a doorbell to CP engine, the lower + * 12 bits of its address has to be outside the range + * set for SDMA, VCN, and IH blocks. */ - gpu_resources.reserved_doorbell_mask = 0x1e0; - gpu_resources.reserved_doorbell_val = 0x0e0; + if (adev->asic_type >= CHIP_VEGA10) { + gpu_resources.non_cp_doorbells_start = + adev->doorbell_index.first_non_cp; + gpu_resources.non_cp_doorbells_end = + adev->doorbell_index.last_non_cp; + } - kgd2kfd->device_init(adev->kfd.dev, &gpu_resources); + kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); } } void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) { if (adev->kfd.dev) { - kgd2kfd->device_exit(adev->kfd.dev); + kgd2kfd_device_exit(adev->kfd.dev); adev->kfd.dev = NULL; } } @@ -227,13 +205,13 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, const void *ih_ring_entry) { if (adev->kfd.dev) - kgd2kfd->interrupt(adev->kfd.dev, ih_ring_entry); + kgd2kfd_interrupt(adev->kfd.dev, ih_ring_entry); } void amdgpu_amdkfd_suspend(struct amdgpu_device *adev) { if (adev->kfd.dev) - kgd2kfd->suspend(adev->kfd.dev); + kgd2kfd_suspend(adev->kfd.dev); } int amdgpu_amdkfd_resume(struct amdgpu_device *adev) @@ -241,7 +219,7 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev) int r = 0; if (adev->kfd.dev) - r = kgd2kfd->resume(adev->kfd.dev); + r = kgd2kfd_resume(adev->kfd.dev); return r; } @@ -251,7 +229,7 @@ int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev) int r = 0; if (adev->kfd.dev) - r = kgd2kfd->pre_reset(adev->kfd.dev); + r = kgd2kfd_pre_reset(adev->kfd.dev); return r; } @@ -261,7 +239,7 @@ int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev) int r = 0; if (adev->kfd.dev) - r = kgd2kfd->post_reset(adev->kfd.dev); + r = kgd2kfd_post_reset(adev->kfd.dev); return r; } @@ -619,4 +597,47 @@ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void) { return NULL; } + +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + const struct kfd2kgd_calls *f2g) +{ + return NULL; +} + +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources) +{ + return false; +} + +void kgd2kfd_device_exit(struct kfd_dev *kfd) +{ +} + +void kgd2kfd_exit(void) +{ +} + +void kgd2kfd_suspend(struct kfd_dev *kfd) +{ +} + +int kgd2kfd_resume(struct kfd_dev *kfd) +{ + return 0; +} + +int kgd2kfd_pre_reset(struct kfd_dev *kfd) +{ + return 0; +} + +int kgd2kfd_post_reset(struct kfd_dev *kfd) +{ + return 0; +} + +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) +{ +} #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 70429f7aa9a8..0b31a1859023 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -33,7 +33,6 @@ #include "amdgpu_sync.h" #include "amdgpu_vm.h" -extern const struct kgd2kfd_calls *kgd2kfd; extern uint64_t amdgpu_amdkfd_total_mem_size; struct amdgpu_device; @@ -214,4 +213,22 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, void amdgpu_amdkfd_gpuvm_init_mem_limits(void); void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); +/* KGD2KFD callbacks */ +int kgd2kfd_init(void); +void kgd2kfd_exit(void); +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + const struct kfd2kgd_calls *f2g); +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources); +void kgd2kfd_device_exit(struct kfd_dev *kfd); +void kgd2kfd_suspend(struct kfd_dev *kfd); +int kgd2kfd_resume(struct kfd_dev *kfd); +int kgd2kfd_pre_reset(struct kfd_dev *kfd); +int kgd2kfd_post_reset(struct kfd_dev *kfd); +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); +int kgd2kfd_quiesce_mm(struct mm_struct *mm); +int kgd2kfd_resume_mm(struct mm_struct *mm); +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, + struct dma_fence *fence); + #endif /* AMDGPU_AMDKFD_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c index 574c1181ae9a..3107b9575929 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c @@ -122,7 +122,7 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f) if (dma_fence_is_signaled(f)) return true; - if (!kgd2kfd->schedule_evict_and_restore_process(fence->mm, f)) + if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f)) return true; return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index be1ab43473c6..1921dec3df7a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -204,38 +204,25 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) } -/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence(s) from BO's +/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO - * @ef: [IN] If ef is specified, then this eviction fence is removed if it + * @ef: [IN] This eviction fence is removed if it * is present in the shared list. - * @ef_list: [OUT] Returns list of eviction fences. These fences are removed - * from BO's reservation object shared list. - * @ef_count: [OUT] Number of fences in ef_list. * - * NOTE: If called with ef_list, then amdgpu_amdkfd_add_eviction_fence must be - * called to restore the eviction fences and to avoid memory leak. This is - * useful for shared BOs. * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, - struct amdgpu_amdkfd_fence *ef, - struct amdgpu_amdkfd_fence ***ef_list, - unsigned int *ef_count) + struct amdgpu_amdkfd_fence *ef) { struct reservation_object *resv = bo->tbo.resv; struct reservation_object_list *old, *new; unsigned int i, j, k; - if (!ef && !ef_list) + if (!ef) return -EINVAL; - if (ef_list) { - *ef_list = NULL; - *ef_count = 0; - } - old = reservation_object_get_list(resv); if (!old) return 0; @@ -254,8 +241,7 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, f = rcu_dereference_protected(old->shared[i], reservation_object_held(resv)); - if ((ef && f->context == ef->base.context) || - (!ef && to_amdgpu_amdkfd_fence(f))) + if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); @@ -263,21 +249,6 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, new->shared_max = old->shared_max; new->shared_count = k; - if (!ef) { - unsigned int count = old->shared_count - j; - - /* Alloc memory for count number of eviction fence pointers. - * Fill the ef_list array and ef_count - */ - *ef_list = kcalloc(count, sizeof(**ef_list), GFP_KERNEL); - *ef_count = count; - - if (!*ef_list) { - kfree(new); - return -ENOMEM; - } - } - /* Install the new fence list, seqcount provides the barriers */ preempt_disable(); write_seqcount_begin(&resv->seq); @@ -291,46 +262,13 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, f = rcu_dereference_protected(new->shared[i], reservation_object_held(resv)); - if (!ef) - (*ef_list)[k++] = to_amdgpu_amdkfd_fence(f); - else - dma_fence_put(f); + dma_fence_put(f); } kfree_rcu(old, rcu); return 0; } -/* amdgpu_amdkfd_add_eviction_fence - Adds eviction fence(s) back into BO's - * reservation object. - * - * @bo: [IN] Add eviction fences to this BO - * @ef_list: [IN] List of eviction fences to be added - * @ef_count: [IN] Number of fences in ef_list. - * - * NOTE: Must call amdgpu_amdkfd_remove_eviction_fence before calling this - * function. - */ -static void amdgpu_amdkfd_add_eviction_fence(struct amdgpu_bo *bo, - struct amdgpu_amdkfd_fence **ef_list, - unsigned int ef_count) -{ - int i; - - if (!ef_list || !ef_count) - return; - - for (i = 0; i < ef_count; i++) { - amdgpu_bo_fence(bo, &ef_list[i]->base, true); - /* Re-adding the fence takes an additional reference. Drop that - * reference. - */ - dma_fence_put(&ef_list[i]->base); - } - - kfree(ef_list); -} - static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { @@ -346,18 +284,8 @@ static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; - if (wait) { - struct amdgpu_amdkfd_fence **ef_list; - unsigned int ef_count; - - ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL, &ef_list, - &ef_count); - if (ret) - goto validate_fail; - - ttm_bo_wait(&bo->tbo, false, false); - amdgpu_amdkfd_add_eviction_fence(bo, ef_list, ef_count); - } + if (wait) + amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; @@ -444,7 +372,6 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, { int ret; struct kfd_bo_va_list *bo_va_entry; - struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; @@ -484,14 +411,8 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, *p_bo_va_entry = bo_va_entry; /* Allocate new page tables if needed and validate - * them. Clearing of new page tables and validate need to wait - * on move fences. We don't want that to trigger the eviction - * fence, so remove it temporarily. + * them. */ - amdgpu_amdkfd_remove_eviction_fence(pd, - vm->process_info->eviction_fence, - NULL, NULL); - ret = amdgpu_vm_alloc_pts(adev, vm, va, amdgpu_bo_size(bo)); if (ret) { pr_err("Failed to allocate pts, err=%d\n", ret); @@ -504,13 +425,9 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, goto err_alloc_pts; } - /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); - return 0; err_alloc_pts: - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); list_del(&bo_va_entry->bo_list); err_vmadd: @@ -809,24 +726,11 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev, { struct amdgpu_bo_va *bo_va = entry->bo_va; struct amdgpu_vm *vm = bo_va->base.vm; - struct amdgpu_bo *pd = vm->root.base.bo; - /* Remove eviction fence from PD (and thereby from PTs too as - * they share the resv. object). Otherwise during PT update - * job (see amdgpu_vm_bo_update_mapping), eviction fence would - * get added to job->sync object and job execution would - * trigger the eviction fence. - */ - amdgpu_amdkfd_remove_eviction_fence(pd, - vm->process_info->eviction_fence, - NULL, NULL); amdgpu_vm_bo_unmap(adev, bo_va, entry->va); amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); - /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); - amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false); return 0; @@ -1002,7 +906,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, pr_err("validate_pt_pd_bos() failed\n"); goto validate_pd_fail; } - ret = ttm_bo_wait(&vm->root.base.bo->tbo, false, false); + amdgpu_bo_sync_wait(vm->root.base.bo, AMDGPU_FENCE_OWNER_KFD, false); if (ret) goto wait_pd_fail; amdgpu_bo_fence(vm->root.base.bo, @@ -1389,8 +1293,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( * attached */ amdgpu_amdkfd_remove_eviction_fence(mem->bo, - process_info->eviction_fence, - NULL, NULL); + process_info->eviction_fence); pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, mem->va + bo_size * (1 + mem->aql_queue)); @@ -1617,8 +1520,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( if (mem->mapped_to_gpu_memory == 0 && !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count) amdgpu_amdkfd_remove_eviction_fence(mem->bo, - process_info->eviction_fence, - NULL, NULL); + process_info->eviction_fence); unreserve_out: unreserve_bo_and_vms(&ctx, false, false); @@ -1679,7 +1581,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, } amdgpu_amdkfd_remove_eviction_fence( - bo, mem->process_info->eviction_fence, NULL, NULL); + bo, mem->process_info->eviction_fence); list_del_init(&mem->validate_list.head); if (size) @@ -1790,7 +1692,7 @@ int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, evicted_bos = atomic_inc_return(&process_info->evicted_bos); if (evicted_bos == 1) { /* First eviction, stop the queues */ - r = kgd2kfd->quiesce_mm(mm); + r = kgd2kfd_quiesce_mm(mm); if (r) pr_err("Failed to quiesce KFD\n"); schedule_delayed_work(&process_info->restore_userptr_work, @@ -1945,16 +1847,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) amdgpu_sync_create(&sync); - /* Avoid triggering eviction fences when unmapping invalid - * userptr BOs (waits for all fences, doesn't use - * FENCE_OWNER_VM) - */ - list_for_each_entry(peer_vm, &process_info->vm_list_head, - vm_list_node) - amdgpu_amdkfd_remove_eviction_fence(peer_vm->root.base.bo, - process_info->eviction_fence, - NULL, NULL); - ret = process_validate_vms(process_info); if (ret) goto unreserve_out; @@ -2015,10 +1907,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) ret = process_update_pds(process_info, &sync); unreserve_out: - list_for_each_entry(peer_vm, &process_info->vm_list_head, - vm_list_node) - amdgpu_bo_fence(peer_vm->root.base.bo, - &process_info->eviction_fence->base, true); ttm_eu_backoff_reservation(&ticket, &resv_list); amdgpu_sync_wait(&sync, false); amdgpu_sync_free(&sync); @@ -2082,7 +1970,7 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) evicted_bos) goto unlock_out; evicted_bos = 0; - if (kgd2kfd->resume_mm(mm)) { + if (kgd2kfd_resume_mm(mm)) { pr_err("%s: Failed to resume KFD\n", __func__); /* No recovery from this failure. Probably the CP is * hanging. No point trying again. diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index 69ad6ec0a4f3..bf04c12bd324 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -25,8 +25,8 @@ */ #include <drm/drmP.h> #include <drm/drm_edid.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_fb_helper.h> +#include <drm/drm_probe_helper.h> #include <drm/amdgpu_drm.h> #include "amdgpu.h" #include "atom.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 1c49b8266d69..52a5e4fdc95b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -214,6 +214,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: + case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: break; default: @@ -1090,6 +1091,15 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); + + if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { + struct drm_sched_fence *s_fence = to_drm_sched_fence(fence); + struct dma_fence *old = fence; + + fence = dma_fence_get(&s_fence->scheduled); + dma_fence_put(old); + } + if (IS_ERR(fence)) { r = PTR_ERR(fence); amdgpu_ctx_put(ctx); @@ -1177,7 +1187,8 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, chunk = &p->chunks[i]; - if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES) { + if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES || + chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { r = amdgpu_cs_process_fence_dep(p, chunk); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index d85184b5b35c..7b526593eb77 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -124,6 +124,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; struct drm_sched_rq *rqs[AMDGPU_MAX_RINGS]; unsigned num_rings; + unsigned num_rqs = 0; switch (i) { case AMDGPU_HW_IP_GFX: @@ -166,12 +167,16 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, break; } - for (j = 0; j < num_rings; ++j) - rqs[j] = &rings[j]->sched.sched_rq[priority]; + for (j = 0; j < num_rings; ++j) { + if (!rings[j]->adev) + continue; + + rqs[num_rqs++] = &rings[j]->sched.sched_rq[priority]; + } for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) r = drm_sched_entity_init(&ctx->entities[i][j].entity, - rqs, num_rings, &ctx->guilty); + rqs, num_rqs, &ctx->guilty); if (r) goto error_cleanup_entities; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index dd9a4fb9ce39..4ae3ff9a1d4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -158,9 +158,6 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, while (size) { uint32_t value; - if (*pos > adev->rmmio_size) - goto end; - if (read) { value = RREG32(*pos >> 2); r = put_user(value, (uint32_t *)buf); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 7ff3a28fc903..4f8fb4ecde34 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -30,8 +30,8 @@ #include <linux/console.h> #include <linux/slab.h> #include <drm/drmP.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_atomic_helper.h> +#include <drm/drm_probe_helper.h> #include <drm/amdgpu_drm.h> #include <linux/vgaarb.h> #include <linux/vga_switcheroo.h> @@ -1645,7 +1645,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) { DRM_ERROR("sw_init of IP block <%s> failed %d\n", adev->ip_blocks[i].version->funcs->name, r); - return r; + goto init_failed; } adev->ip_blocks[i].status.sw = true; @@ -1654,17 +1654,17 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) r = amdgpu_device_vram_scratch_init(adev); if (r) { DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); - return r; + goto init_failed; } r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); if (r) { DRM_ERROR("hw_init %d failed %d\n", i, r); - return r; + goto init_failed; } r = amdgpu_device_wb_init(adev); if (r) { DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); - return r; + goto init_failed; } adev->ip_blocks[i].status.hw = true; @@ -1675,7 +1675,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) AMDGPU_CSA_SIZE); if (r) { DRM_ERROR("allocate CSA failed %d\n", r); - return r; + goto init_failed; } } } @@ -1683,30 +1683,32 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ if (r) - return r; + goto init_failed; r = amdgpu_device_ip_hw_init_phase1(adev); if (r) - return r; + goto init_failed; r = amdgpu_device_fw_loading(adev); if (r) - return r; + goto init_failed; r = amdgpu_device_ip_hw_init_phase2(adev); if (r) - return r; + goto init_failed; if (adev->gmc.xgmi.num_physical_nodes > 1) amdgpu_xgmi_add_device(adev); amdgpu_amdkfd_device_init(adev); +init_failed: if (amdgpu_sriov_vf(adev)) { - amdgpu_virt_init_data_exchange(adev); + if (!r) + amdgpu_virt_init_data_exchange(adev); amdgpu_virt_release_full_gpu(adev, true); } - return 0; + return r; } /** @@ -2133,7 +2135,7 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) continue; r = block->version->funcs->hw_init(adev); - DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); if (r) return r; } @@ -2167,7 +2169,7 @@ static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) continue; r = block->version->funcs->hw_init(adev); - DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); if (r) return r; } @@ -2548,6 +2550,17 @@ int amdgpu_device_init(struct amdgpu_device *adev, /* detect if we are with an SRIOV vbios */ amdgpu_device_detect_sriov_bios(adev); + /* check if we need to reset the asic + * E.g., driver was not cleanly unloaded previously, etc. + */ + if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { + r = amdgpu_asic_reset(adev); + if (r) { + dev_err(adev->dev, "asic reset on init failed\n"); + goto failed; + } + } + /* Post card if necessary */ if (amdgpu_device_need_post(adev)) { if (!adev->bios) { @@ -2612,6 +2625,8 @@ fence_driver_init: } dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); + if (amdgpu_virt_request_full_gpu(adev, false)) + amdgpu_virt_release_full_gpu(adev, false); goto failed; } @@ -2707,7 +2722,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized){ if (!amdgpu_device_has_dc_support(adev)) - drm_crtc_force_disable_all(adev->ddev); + drm_helper_force_disable_all(adev->ddev); else drm_atomic_helper_shutdown(adev->ddev); } @@ -3298,17 +3313,15 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); - - if (job && job->base.sched != &ring->sched) - continue; - - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + drm_sched_stop(&ring->sched); /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); } + if(job) + drm_sched_increase_karma(&job->base); + if (!amdgpu_sriov_vf(adev)) { @@ -3454,14 +3467,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { @@ -3521,9 +3530,9 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, * by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive = amdgpu_get_xgmi_hive(adev); + hive = amdgpu_get_xgmi_hive(adev, 0); if (hive && adev->gmc.xgmi.num_physical_nodes > 1 && - !mutex_trylock(&hive->hive_lock)) + !mutex_trylock(&hive->reset_lock)) return 0; /* Start with adev pre asic reset first for soft reset check.*/ @@ -3602,13 +3611,45 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ } if (hive && adev->gmc.xgmi.num_physical_nodes > 1) - mutex_unlock(&hive->hive_lock); + mutex_unlock(&hive->reset_lock); if (r) dev_info(adev->dev, "GPU reset end with ret = %d\n", r); return r; } +static void amdgpu_device_get_min_pci_speed_width(struct amdgpu_device *adev, + enum pci_bus_speed *speed, + enum pcie_link_width *width) +{ + struct pci_dev *pdev = adev->pdev; + enum pci_bus_speed cur_speed; + enum pcie_link_width cur_width; + + *speed = PCI_SPEED_UNKNOWN; + *width = PCIE_LNK_WIDTH_UNKNOWN; + + while (pdev) { + cur_speed = pcie_get_speed_cap(pdev); + cur_width = pcie_get_width_cap(pdev); + + if (cur_speed != PCI_SPEED_UNKNOWN) { + if (*speed == PCI_SPEED_UNKNOWN) + *speed = cur_speed; + else if (cur_speed < *speed) + *speed = cur_speed; + } + + if (cur_width != PCIE_LNK_WIDTH_UNKNOWN) { + if (*width == PCIE_LNK_WIDTH_UNKNOWN) + *width = cur_width; + else if (cur_width < *width) + *width = cur_width; + } + pdev = pci_upstream_bridge(pdev); + } +} + /** * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot * @@ -3621,8 +3662,8 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) { struct pci_dev *pdev; - enum pci_bus_speed speed_cap; - enum pcie_link_width link_width; + enum pci_bus_speed speed_cap, platform_speed_cap; + enum pcie_link_width platform_link_width; if (amdgpu_pcie_gen_cap) adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; @@ -3639,6 +3680,12 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) return; } + if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) + return; + + amdgpu_device_get_min_pci_speed_width(adev, &platform_speed_cap, + &platform_link_width); + if (adev->pm.pcie_gen_mask == 0) { /* asic caps */ pdev = adev->pdev; @@ -3664,22 +3711,20 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; } /* platform caps */ - pdev = adev->ddev->pdev->bus->self; - speed_cap = pcie_get_speed_cap(pdev); - if (speed_cap == PCI_SPEED_UNKNOWN) { + if (platform_speed_cap == PCI_SPEED_UNKNOWN) { adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); } else { - if (speed_cap == PCIE_SPEED_16_0GT) + if (platform_speed_cap == PCIE_SPEED_16_0GT) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); - else if (speed_cap == PCIE_SPEED_8_0GT) + else if (platform_speed_cap == PCIE_SPEED_8_0GT) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); - else if (speed_cap == PCIE_SPEED_5_0GT) + else if (platform_speed_cap == PCIE_SPEED_5_0GT) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); else @@ -3688,12 +3733,10 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) } } if (adev->pm.pcie_mlw_mask == 0) { - pdev = adev->ddev->pdev->bus->self; - link_width = pcie_get_width_cap(pdev); - if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { + if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; } else { - switch (link_width) { + switch (platform_link_width) { case PCIE_LNK_X32: adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h index be620b29f4aa..68959b923f89 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h @@ -51,14 +51,7 @@ struct amdgpu_doorbell_index { uint32_t userqueue_start; uint32_t userqueue_end; uint32_t gfx_ring0; - uint32_t sdma_engine0; - uint32_t sdma_engine1; - uint32_t sdma_engine2; - uint32_t sdma_engine3; - uint32_t sdma_engine4; - uint32_t sdma_engine5; - uint32_t sdma_engine6; - uint32_t sdma_engine7; + uint32_t sdma_engine[8]; uint32_t ih; union { struct { @@ -78,7 +71,11 @@ struct amdgpu_doorbell_index { uint32_t vce_ring6_7; } uvd_vce; }; + uint32_t first_non_cp; + uint32_t last_non_cp; uint32_t max_assignment; + /* Per engine SDMA doorbell size in dword */ + uint32_t sdma_doorbell_range; }; typedef enum _AMDGPU_DOORBELL_ASSIGNMENT @@ -148,6 +145,10 @@ typedef enum _AMDGPU_VEGA20_DOORBELL_ASSIGNMENT AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3 = 0x18D, AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5 = 0x18E, AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7 = 0x18F, + + AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0, + AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7, + AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT = 0x18F, AMDGPU_VEGA20_DOORBELL_INVALID = 0xFFFF } AMDGPU_VEGA20_DOORBELL_ASSIGNMENT; @@ -227,6 +228,9 @@ typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT AMDGPU_DOORBELL64_VCE_RING4_5 = 0xFE, AMDGPU_DOORBELL64_VCE_RING6_7 = 0xFF, + AMDGPU_DOORBELL64_FIRST_NON_CP = AMDGPU_DOORBELL64_sDMA_ENGINE0, + AMDGPU_DOORBELL64_LAST_NON_CP = AMDGPU_DOORBELL64_VCE_RING6_7, + AMDGPU_DOORBELL64_MAX_ASSIGNMENT = 0xFF, AMDGPU_DOORBELL64_INVALID = 0xFFFF } AMDGPU_DOORBELL64_ASSIGNMENT; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c index 1c4595562f8f..344967df3137 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c @@ -184,61 +184,6 @@ u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev) return vrefresh; } -void amdgpu_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, - u32 *p, u32 *u) -{ - u32 b_c = 0; - u32 i_c; - u32 tmp; - - i_c = (i * r_c) / 100; - tmp = i_c >> p_b; - - while (tmp) { - b_c++; - tmp >>= 1; - } - - *u = (b_c + 1) / 2; - *p = i_c / (1 << (2 * (*u))); -} - -int amdgpu_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th) -{ - u32 k, a, ah, al; - u32 t1; - - if ((fl == 0) || (fh == 0) || (fl > fh)) - return -EINVAL; - - k = (100 * fh) / fl; - t1 = (t * (k - 100)); - a = (1000 * (100 * h + t1)) / (10000 + (t1 / 100)); - a = (a + 5) / 10; - ah = ((a * t) + 5000) / 10000; - al = a - ah; - - *th = t - ah; - *tl = t + al; - - return 0; -} - -bool amdgpu_is_uvd_state(u32 class, u32 class2) -{ - if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) - return true; - if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) - return true; - return false; -} - bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor) { switch (sensor) { @@ -949,39 +894,6 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, return AMDGPU_PCIE_GEN1; } -u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev, - u16 asic_lanes, - u16 default_lanes) -{ - switch (asic_lanes) { - case 0: - default: - return default_lanes; - case 1: - return 1; - case 2: - return 2; - case 4: - return 4; - case 8: - return 8; - case 12: - return 12; - case 16: - return 16; - } -} - -u8 amdgpu_encode_pci_lane_width(u32 lanes) -{ - u8 encoded_lanes[] = { 0, 1, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6 }; - - if (lanes > 16) - return 0; - - return encoded_lanes[lanes]; -} - struct amd_vce_state* amdgpu_get_vce_clock_state(void *handle, u32 idx) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h index f972cd156795..e871e022c129 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h @@ -364,6 +364,14 @@ enum amdgpu_pcie_gen { ((adev)->powerplay.pp_funcs->enable_mgpu_fan_boost(\ (adev)->powerplay.pp_handle)) +#define amdgpu_dpm_get_ppfeature_status(adev, buf) \ + ((adev)->powerplay.pp_funcs->get_ppfeature_status(\ + (adev)->powerplay.pp_handle, (buf))) + +#define amdgpu_dpm_set_ppfeature_status(adev, ppfeatures) \ + ((adev)->powerplay.pp_funcs->set_ppfeature_status(\ + (adev)->powerplay.pp_handle, (ppfeatures))) + struct amdgpu_dpm { struct amdgpu_ps *ps; /* number of valid power states */ @@ -478,10 +486,6 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev); u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev); void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev); -bool amdgpu_is_uvd_state(u32 class, u32 class2); -void amdgpu_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, - u32 *p, u32 *u); -int amdgpu_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th); bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor); @@ -497,11 +501,6 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, enum amdgpu_pcie_gen asic_gen, enum amdgpu_pcie_gen default_gen); -u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev, - u16 asic_lanes, - u16 default_lanes); -u8 amdgpu_encode_pci_lane_width(u32 lanes); - struct amd_vce_state* amdgpu_get_vce_clock_state(void *handle, u32 idx); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index c806f984bcc5..7419ea8a388b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -32,7 +32,7 @@ #include <linux/module.h> #include <linux/pm_runtime.h> #include <linux/vga_switcheroo.h> -#include <drm/drm_crtc_helper.h> +#include <drm/drm_probe_helper.h> #include "amdgpu.h" #include "amdgpu_irq.h" @@ -71,9 +71,12 @@ * - 3.25.0 - Add support for sensor query info (stable pstate sclk/mclk). * - 3.26.0 - GFX9: Process AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE. * - 3.27.0 - Add new chunk to to AMDGPU_CS to enable BO_LIST creation. + * - 3.28.0 - Add AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES + * - 3.29.0 - Add AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID + * - 3.30.0 - Add AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE. */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 27 +#define KMS_DRIVER_MINOR 30 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit = 0; @@ -1176,6 +1179,22 @@ static const struct file_operations amdgpu_driver_kms_fops = { #endif }; +int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv) +{ + struct drm_file *file; + + if (!filp) + return -EINVAL; + + if (filp->f_op != &amdgpu_driver_kms_fops) { + return -EINVAL; + } + + file = filp->private_data; + *fpriv = file->driver_priv; + return 0; +} + static bool amdgpu_get_crtc_scanout_position(struct drm_device *dev, unsigned int pipe, bool in_vblank_irq, int *vpos, int *hpos, @@ -1189,7 +1208,7 @@ amdgpu_get_crtc_scanout_position(struct drm_device *dev, unsigned int pipe, static struct drm_driver kms_driver = { .driver_features = DRIVER_USE_AGP | DRIVER_ATOMIC | - DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | + DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_MODESET | DRIVER_SYNCOBJ, .load = amdgpu_driver_load_kms, .open = amdgpu_driver_open_kms, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h index ecbcefe49a98..f89f5734d985 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h @@ -37,6 +37,8 @@ struct amdgpu_gds { struct amdgpu_gds_asic_info mem; struct amdgpu_gds_asic_info gws; struct amdgpu_gds_asic_info oa; + uint32_t gds_compute_max_wave_id; + /* At present, GDS, GWS and OA resources for gfx (graphics) * is always pre-allocated and available for graphics operation. * Such resource is shared between all gfx clients. diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index f4f00217546e..d21dd2f369da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -54,10 +54,6 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, memset(&bp, 0, sizeof(bp)); *obj = NULL; - /* At least align on page size */ - if (alignment < PAGE_SIZE) { - alignment = PAGE_SIZE; - } bp.size = size; bp.byte_align = alignment; @@ -244,9 +240,6 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, return -EINVAL; } flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS; - /* GDS allocations must be DW aligned */ - if (args->in.domains & AMDGPU_GEM_DOMAIN_GDS) - size = ALIGN(size, 4); } if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index c48207b377bc..0b8ef2d27d6b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -202,12 +202,12 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, amdgpu_asic_flush_hdp(adev, ring); } + if (need_ctx_switch) + status |= AMDGPU_HAVE_CTX_SWITCH; + skip_preamble = ring->current_ctx == fence_ctx; if (job && ring->funcs->emit_cntxcntl) { - if (need_ctx_switch) - status |= AMDGPU_HAVE_CTX_SWITCH; status |= job->preamble_status; - amdgpu_ring_emit_cntxcntl(ring, status); } @@ -221,8 +221,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, !amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */ continue; - amdgpu_ring_emit_ib(ring, job, ib, need_ctx_switch); - need_ctx_switch = false; + amdgpu_ring_emit_ib(ring, job, ib, status); + status &= ~AMDGPU_HAVE_CTX_SWITCH; } if (ring->funcs->emit_tmz) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index 8af67f649660..1c50be3ab8a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -52,6 +52,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, ih->use_bus_addr = use_bus_addr; if (use_bus_addr) { + dma_addr_t dma_addr; + if (ih->ring) return 0; @@ -59,21 +61,26 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, * add them to the end of the ring allocation. */ ih->ring = dma_alloc_coherent(adev->dev, ih->ring_size + 8, - &ih->rb_dma_addr, GFP_KERNEL); + &dma_addr, GFP_KERNEL); if (ih->ring == NULL) return -ENOMEM; memset((void *)ih->ring, 0, ih->ring_size + 8); - ih->wptr_offs = (ih->ring_size / 4) + 0; - ih->rptr_offs = (ih->ring_size / 4) + 1; + ih->gpu_addr = dma_addr; + ih->wptr_addr = dma_addr + ih->ring_size; + ih->wptr_cpu = &ih->ring[ih->ring_size / 4]; + ih->rptr_addr = dma_addr + ih->ring_size + 4; + ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1]; } else { - r = amdgpu_device_wb_get(adev, &ih->wptr_offs); + unsigned wptr_offs, rptr_offs; + + r = amdgpu_device_wb_get(adev, &wptr_offs); if (r) return r; - r = amdgpu_device_wb_get(adev, &ih->rptr_offs); + r = amdgpu_device_wb_get(adev, &rptr_offs); if (r) { - amdgpu_device_wb_free(adev, ih->wptr_offs); + amdgpu_device_wb_free(adev, wptr_offs); return r; } @@ -82,10 +89,15 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, &ih->ring_obj, &ih->gpu_addr, (void **)&ih->ring); if (r) { - amdgpu_device_wb_free(adev, ih->rptr_offs); - amdgpu_device_wb_free(adev, ih->wptr_offs); + amdgpu_device_wb_free(adev, rptr_offs); + amdgpu_device_wb_free(adev, wptr_offs); return r; } + + ih->wptr_addr = adev->wb.gpu_addr + wptr_offs * 4; + ih->wptr_cpu = &adev->wb.wb[wptr_offs]; + ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4; + ih->rptr_cpu = &adev->wb.wb[rptr_offs]; } return 0; } @@ -109,13 +121,13 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) * add them to the end of the ring allocation. */ dma_free_coherent(adev->dev, ih->ring_size + 8, - (void *)ih->ring, ih->rb_dma_addr); + (void *)ih->ring, ih->gpu_addr); ih->ring = NULL; } else { amdgpu_bo_free_kernel(&ih->ring_obj, &ih->gpu_addr, (void **)&ih->ring); - amdgpu_device_wb_free(adev, ih->wptr_offs); - amdgpu_device_wb_free(adev, ih->rptr_offs); + amdgpu_device_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4); + amdgpu_device_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4); } } @@ -128,16 +140,14 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) * Interrupt hander (VI), walk the IH ring. * Returns irq process return code. */ -int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, - void (*callback)(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih)) +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { u32 wptr; if (!ih->enabled || adev->shutdown) return IRQ_NONE; - wptr = amdgpu_ih_get_wptr(adev); + wptr = amdgpu_ih_get_wptr(adev, ih); restart_ih: /* is somebody else already processing irqs? */ @@ -150,15 +160,15 @@ restart_ih: rmb(); while (ih->rptr != wptr) { - callback(adev, ih); + amdgpu_irq_dispatch(adev, ih); ih->rptr &= ih->ptr_mask; } - amdgpu_ih_set_rptr(adev); + amdgpu_ih_set_rptr(adev, ih); atomic_set(&ih->lock, 0); /* make sure wptr hasn't changed while processing */ - wptr = amdgpu_ih_get_wptr(adev); + wptr = amdgpu_ih_get_wptr(adev, ih); if (wptr != ih->rptr) goto restart_ih; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index f877bb78d10a..113a1ba13d4a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -31,40 +31,44 @@ struct amdgpu_iv_entry; * R6xx+ IH ring */ struct amdgpu_ih_ring { - struct amdgpu_bo *ring_obj; - volatile uint32_t *ring; - unsigned rptr; unsigned ring_size; - uint64_t gpu_addr; uint32_t ptr_mask; - atomic_t lock; - bool enabled; - unsigned wptr_offs; - unsigned rptr_offs; u32 doorbell_index; bool use_doorbell; bool use_bus_addr; - dma_addr_t rb_dma_addr; /* only used when use_bus_addr = true */ + + struct amdgpu_bo *ring_obj; + volatile uint32_t *ring; + uint64_t gpu_addr; + + uint64_t wptr_addr; + volatile uint32_t *wptr_cpu; + + uint64_t rptr_addr; + volatile uint32_t *rptr_cpu; + + bool enabled; + unsigned rptr; + atomic_t lock; }; /* provided by the ih block */ struct amdgpu_ih_funcs { /* ring read/write ptr handling, called from interrupt context */ - u32 (*get_wptr)(struct amdgpu_device *adev); - void (*decode_iv)(struct amdgpu_device *adev, + u32 (*get_wptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); + void (*decode_iv)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, struct amdgpu_iv_entry *entry); - void (*set_rptr)(struct amdgpu_device *adev); + void (*set_rptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); }; -#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev)) -#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv)) -#define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev)) +#define amdgpu_ih_get_wptr(adev, ih) (adev)->irq.ih_funcs->get_wptr((adev), (ih)) +#define amdgpu_ih_decode_iv(adev, iv) \ + (adev)->irq.ih_funcs->decode_iv((adev), (ih), (iv)) +#define amdgpu_ih_set_rptr(adev, ih) (adev)->irq.ih_funcs->set_rptr((adev), (ih)) int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, unsigned ring_size, bool use_bus_addr); void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); -int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, - void (*callback)(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih)); +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index b7968f426862..af4c3b1af322 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -131,27 +131,6 @@ void amdgpu_irq_disable_all(struct amdgpu_device *adev) } /** - * amdgpu_irq_callback - callback from the IH ring - * - * @adev: amdgpu device pointer - * @ih: amdgpu ih ring - * - * Callback from IH ring processing to handle the entry at the current position - * and advance the read pointer. - */ -static void amdgpu_irq_callback(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih) -{ - u32 ring_index = ih->rptr >> 2; - struct amdgpu_iv_entry entry; - - entry.iv_entry = (const uint32_t *)&ih->ring[ring_index]; - amdgpu_ih_decode_iv(adev, &entry); - - amdgpu_irq_dispatch(adev, &entry); -} - -/** * amdgpu_irq_handler - IRQ handler * * @irq: IRQ number (unused) @@ -168,13 +147,43 @@ irqreturn_t amdgpu_irq_handler(int irq, void *arg) struct amdgpu_device *adev = dev->dev_private; irqreturn_t ret; - ret = amdgpu_ih_process(adev, &adev->irq.ih, amdgpu_irq_callback); + ret = amdgpu_ih_process(adev, &adev->irq.ih); if (ret == IRQ_HANDLED) pm_runtime_mark_last_busy(dev->dev); return ret; } /** + * amdgpu_irq_handle_ih1 - kick of processing for IH1 + * + * @work: work structure in struct amdgpu_irq + * + * Kick of processing IH ring 1. + */ +static void amdgpu_irq_handle_ih1(struct work_struct *work) +{ + struct amdgpu_device *adev = container_of(work, struct amdgpu_device, + irq.ih1_work); + + amdgpu_ih_process(adev, &adev->irq.ih1); +} + +/** + * amdgpu_irq_handle_ih2 - kick of processing for IH2 + * + * @work: work structure in struct amdgpu_irq + * + * Kick of processing IH ring 2. + */ +static void amdgpu_irq_handle_ih2(struct work_struct *work) +{ + struct amdgpu_device *adev = container_of(work, struct amdgpu_device, + irq.ih2_work); + + amdgpu_ih_process(adev, &adev->irq.ih2); +} + +/** * amdgpu_msi_ok - check whether MSI functionality is enabled * * @adev: amdgpu device pointer (unused) @@ -238,6 +247,9 @@ int amdgpu_irq_init(struct amdgpu_device *adev) amdgpu_hotplug_work_func); } + INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1); + INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2); + adev->irq.installed = true; r = drm_irq_install(adev->ddev, adev->ddev->pdev->irq); if (r) { @@ -359,15 +371,22 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev, * Dispatches IRQ to IP blocks. */ void amdgpu_irq_dispatch(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih) { - unsigned client_id = entry->client_id; - unsigned src_id = entry->src_id; + u32 ring_index = ih->rptr >> 2; + struct amdgpu_iv_entry entry; + unsigned client_id, src_id; struct amdgpu_irq_src *src; bool handled = false; int r; - trace_amdgpu_iv(entry); + entry.iv_entry = (const uint32_t *)&ih->ring[ring_index]; + amdgpu_ih_decode_iv(adev, &entry); + + trace_amdgpu_iv(ih - &adev->irq.ih, &entry); + + client_id = entry.client_id; + src_id = entry.src_id; if (client_id >= AMDGPU_IRQ_CLIENTID_MAX) { DRM_DEBUG("Invalid client_id in IV: %d\n", client_id); @@ -383,7 +402,7 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, client_id, src_id); } else if ((src = adev->irq.client[client_id].sources[src_id])) { - r = src->funcs->process(adev, src, entry); + r = src->funcs->process(adev, src, &entry); if (r < 0) DRM_ERROR("error processing interrupt (%d)\n", r); else if (r) @@ -395,7 +414,7 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, /* Send it to amdkfd as well if it isn't already handled */ if (!handled) - amdgpu_amdkfd_interrupt(adev, entry->iv_entry); + amdgpu_amdkfd_interrupt(adev, entry.iv_entry); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index f6ce171cb8aa..c718e94a55c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -87,9 +87,11 @@ struct amdgpu_irq { /* status, etc. */ bool msi_enabled; /* msi enabled */ - /* interrupt ring */ - struct amdgpu_ih_ring ih; - const struct amdgpu_ih_funcs *ih_funcs; + /* interrupt rings */ + struct amdgpu_ih_ring ih, ih1, ih2; + const struct amdgpu_ih_funcs *ih_funcs; + struct work_struct ih1_work, ih2_work; + struct amdgpu_irq_src self_irq; /* gen irq stuff */ struct irq_domain *domain; /* GPU irq controller domain */ @@ -106,7 +108,7 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev, unsigned client_id, unsigned src_id, struct amdgpu_irq_src *source); void amdgpu_irq_dispatch(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry); + struct amdgpu_ih_ring *ih); int amdgpu_irq_update(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type); int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 5dc349173e4f..e860412043bb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -207,7 +207,7 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) if (!r) { acpi_status = amdgpu_acpi_init(adev); if (acpi_status) - dev_dbg(&dev->pdev->dev, + dev_dbg(&dev->pdev->dev, "Error during ACPI methods call\n"); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index 3aa42c64484a..889e443eeee7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -38,6 +38,7 @@ #include <drm/drm_crtc_helper.h> #include <drm/drm_fb_helper.h> #include <drm/drm_plane_helper.h> +#include <drm/drm_probe_helper.h> #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> #include <linux/hrtimer.h> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 728e15e5d68a..ec9e45004bff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -426,12 +426,20 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, size_t acc_size; int r; - page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; - if (bp->domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | - AMDGPU_GEM_DOMAIN_OA)) + /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ + if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { + /* GWS and OA don't need any alignment. */ + page_align = bp->byte_align; size <<= PAGE_SHIFT; - else + } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { + /* Both size and alignment must be a multiple of 4. */ + page_align = ALIGN(bp->byte_align, 4); + size = ALIGN(size, 4) << PAGE_SHIFT; + } else { + /* Memory should be aligned at least to a page size. */ + page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; size = ALIGN(size, PAGE_SIZE); + } if (!amdgpu_bo_validate_size(adev, size, bp->domain)) return -ENOMEM; @@ -1277,6 +1285,30 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, } /** + * amdgpu_sync_wait_resv - Wait for BO reservation fences + * + * @bo: buffer object + * @owner: fence owner + * @intr: Whether the wait is interruptible + * + * Returns: + * 0 on success, errno otherwise. + */ +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_sync sync; + int r; + + amdgpu_sync_create(&sync); + amdgpu_sync_resv(adev, &sync, bo->tbo.resv, owner, false); + r = amdgpu_sync_wait(&sync, intr); + amdgpu_sync_free(&sync); + + return r; +} + +/** * amdgpu_bo_gpu_offset - return GPU offset of bo * @bo: amdgpu object for which we query the offset * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 9291c2f837e9..220a6a7b1bc1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -266,6 +266,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, bool shared); +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr); u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); int amdgpu_bo_validate(struct amdgpu_bo *bo); int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c index 0ed41a9d2d77..a7adb7b6bd98 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c @@ -626,11 +626,71 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, } /** - * DOC: pp_dpm_sclk pp_dpm_mclk pp_dpm_pcie + * DOC: ppfeatures + * + * The amdgpu driver provides a sysfs API for adjusting what powerplay + * features to be enabled. The file ppfeatures is used for this. And + * this is only available for Vega10 and later dGPUs. + * + * Reading back the file will show you the followings: + * - Current ppfeature masks + * - List of the all supported powerplay features with their naming, + * bitmasks and enablement status('Y'/'N' means "enabled"/"disabled"). + * + * To manually enable or disable a specific feature, just set or clear + * the corresponding bit from original ppfeature masks and input the + * new ppfeature masks. + */ +static ssize_t amdgpu_set_ppfeature_status(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint64_t featuremask; + int ret; + + ret = kstrtou64(buf, 0, &featuremask); + if (ret) + return -EINVAL; + + pr_debug("featuremask = 0x%llx\n", featuremask); + + if (adev->powerplay.pp_funcs->set_ppfeature_status) { + ret = amdgpu_dpm_set_ppfeature_status(adev, featuremask); + if (ret) + return -EINVAL; + } + + return count; +} + +static ssize_t amdgpu_get_ppfeature_status(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->powerplay.pp_funcs->get_ppfeature_status) + return amdgpu_dpm_get_ppfeature_status(adev, buf); + + return snprintf(buf, PAGE_SIZE, "\n"); +} + +/** + * DOC: pp_dpm_sclk pp_dpm_mclk pp_dpm_socclk pp_dpm_fclk pp_dpm_dcefclk + * pp_dpm_pcie * * The amdgpu driver provides a sysfs API for adjusting what power levels * are enabled for a given power state. The files pp_dpm_sclk, pp_dpm_mclk, - * and pp_dpm_pcie are used for this. + * pp_dpm_socclk, pp_dpm_fclk, pp_dpm_dcefclk and pp_dpm_pcie are used for + * this. + * + * pp_dpm_socclk and pp_dpm_dcefclk interfaces are only available for + * Vega10 and later ASICs. + * pp_dpm_fclk interface is only available for Vega20 and later ASICs. * * Reading back the files will show you the available power levels within * the power state and the clock information for those levels. @@ -640,6 +700,8 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, * Secondly,Enter a new value for each level by inputing a string that * contains " echo xx xx xx > pp_dpm_sclk/mclk/pcie" * E.g., echo 4 5 6 to > pp_dpm_sclk will enable sclk levels 4, 5, and 6. + * + * NOTE: change to the dcefclk max dpm level is not supported now */ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, @@ -750,6 +812,114 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, return count; } +static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_SOCCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_SOCCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_FCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_FCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_DCEFCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_DCEFCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev, struct device_attribute *attr, char *buf) @@ -990,6 +1160,31 @@ static ssize_t amdgpu_get_busy_percent(struct device *dev, return snprintf(buf, PAGE_SIZE, "%d\n", value); } +/** + * DOC: pcie_bw + * + * The amdgpu driver provides a sysfs API for estimating how much data + * has been received and sent by the GPU in the last second through PCIe. + * The file pcie_bw is used for this. + * The Perf counters count the number of received and sent messages and return + * those values, as well as the maximum payload size of a PCIe packet (mps). + * Note that it is not possible to easily and quickly obtain the size of each + * packet transmitted, so we output the max payload size (mps) to allow for + * quick estimation of the PCIe bandwidth usage + */ +static ssize_t amdgpu_get_pcie_bw(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint64_t count0, count1; + + amdgpu_asic_get_pcie_usage(adev, &count0, &count1); + return snprintf(buf, PAGE_SIZE, "%llu %llu %i\n", + count0, count1, pcie_get_mps(adev->pdev)); +} + static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, amdgpu_get_dpm_state, amdgpu_set_dpm_state); static DEVICE_ATTR(power_dpm_force_performance_level, S_IRUGO | S_IWUSR, amdgpu_get_dpm_forced_performance_level, @@ -1008,6 +1203,15 @@ static DEVICE_ATTR(pp_dpm_sclk, S_IRUGO | S_IWUSR, static DEVICE_ATTR(pp_dpm_mclk, S_IRUGO | S_IWUSR, amdgpu_get_pp_dpm_mclk, amdgpu_set_pp_dpm_mclk); +static DEVICE_ATTR(pp_dpm_socclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_socclk, + amdgpu_set_pp_dpm_socclk); +static DEVICE_ATTR(pp_dpm_fclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_fclk, + amdgpu_set_pp_dpm_fclk); +static DEVICE_ATTR(pp_dpm_dcefclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_dcefclk, + amdgpu_set_pp_dpm_dcefclk); static DEVICE_ATTR(pp_dpm_pcie, S_IRUGO | S_IWUSR, amdgpu_get_pp_dpm_pcie, amdgpu_set_pp_dpm_pcie); @@ -1025,6 +1229,10 @@ static DEVICE_ATTR(pp_od_clk_voltage, S_IRUGO | S_IWUSR, amdgpu_set_pp_od_clk_voltage); static DEVICE_ATTR(gpu_busy_percent, S_IRUGO, amdgpu_get_busy_percent, NULL); +static DEVICE_ATTR(pcie_bw, S_IRUGO, amdgpu_get_pcie_bw, NULL); +static DEVICE_ATTR(ppfeatures, S_IRUGO | S_IWUSR, + amdgpu_get_ppfeature_status, + amdgpu_set_ppfeature_status); static ssize_t amdgpu_hwmon_show_temp(struct device *dev, struct device_attribute *attr, @@ -1516,6 +1724,75 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, return count; } +static ssize_t amdgpu_hwmon_show_sclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + uint32_t sclk; + int r, size = sizeof(sclk); + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* sanity check PP is enabled */ + if (!(adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->read_sensor)) + return -EINVAL; + + /* get the sclk */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, + (void *)&sclk, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", sclk * 10 * 1000); +} + +static ssize_t amdgpu_hwmon_show_sclk_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "sclk\n"); +} + +static ssize_t amdgpu_hwmon_show_mclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + uint32_t mclk; + int r, size = sizeof(mclk); + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* sanity check PP is enabled */ + if (!(adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->read_sensor)) + return -EINVAL; + + /* get the sclk */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, + (void *)&mclk, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", mclk * 10 * 1000); +} + +static ssize_t amdgpu_hwmon_show_mclk_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "mclk\n"); +} /** * DOC: hwmon @@ -1532,6 +1809,10 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, * * - GPU fan * + * - GPU gfx/compute engine clock + * + * - GPU memory clock (dGPU only) + * * hwmon interfaces for GPU temperature: * * - temp1_input: the on die GPU temperature in millidegrees Celsius @@ -1576,6 +1857,12 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, * * - fan[1-*]_enable: Enable or disable the sensors.1: Enable 0: Disable * + * hwmon interfaces for GPU clocks: + * + * - freq1_input: the gfx/compute clock in hertz + * + * - freq2_input: the memory clock in hertz + * * You can use hwmon tools like sensors to view this information on your system. * */ @@ -1600,6 +1887,10 @@ static SENSOR_DEVICE_ATTR(power1_average, S_IRUGO, amdgpu_hwmon_show_power_avg, static SENSOR_DEVICE_ATTR(power1_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 0); +static SENSOR_DEVICE_ATTR(freq1_input, S_IRUGO, amdgpu_hwmon_show_sclk, NULL, 0); +static SENSOR_DEVICE_ATTR(freq1_label, S_IRUGO, amdgpu_hwmon_show_sclk_label, NULL, 0); +static SENSOR_DEVICE_ATTR(freq2_input, S_IRUGO, amdgpu_hwmon_show_mclk, NULL, 0); +static SENSOR_DEVICE_ATTR(freq2_label, S_IRUGO, amdgpu_hwmon_show_mclk_label, NULL, 0); static struct attribute *hwmon_attributes[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, @@ -1622,6 +1913,10 @@ static struct attribute *hwmon_attributes[] = { &sensor_dev_attr_power1_cap_max.dev_attr.attr, &sensor_dev_attr_power1_cap_min.dev_attr.attr, &sensor_dev_attr_power1_cap.dev_attr.attr, + &sensor_dev_attr_freq1_input.dev_attr.attr, + &sensor_dev_attr_freq1_label.dev_attr.attr, + &sensor_dev_attr_freq2_input.dev_attr.attr, + &sensor_dev_attr_freq2_label.dev_attr.attr, NULL }; @@ -1713,6 +2008,12 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, attr == &sensor_dev_attr_in1_label.dev_attr.attr)) return 0; + /* no mclk on APUs */ + if ((adev->flags & AMD_IS_APU) && + (attr == &sensor_dev_attr_freq2_input.dev_attr.attr || + attr == &sensor_dev_attr_freq2_label.dev_attr.attr)) + return 0; + return effective_mode; } @@ -2071,6 +2372,25 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) DRM_ERROR("failed to create device file pp_dpm_mclk\n"); return ret; } + if (adev->asic_type >= CHIP_VEGA10) { + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_socclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_socclk\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_dcefclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_dcefclk\n"); + return ret; + } + } + if (adev->asic_type >= CHIP_VEGA20) { + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_fclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_fclk\n"); + return ret; + } + } ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie); if (ret) { DRM_ERROR("failed to create device file pp_dpm_pcie\n"); @@ -2109,12 +2429,31 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) "gpu_busy_level\n"); return ret; } + /* PCIe Perf counters won't work on APU nodes */ + if (!(adev->flags & AMD_IS_APU)) { + ret = device_create_file(adev->dev, &dev_attr_pcie_bw); + if (ret) { + DRM_ERROR("failed to create device file pcie_bw\n"); + return ret; + } + } ret = amdgpu_debugfs_pm_init(adev); if (ret) { DRM_ERROR("Failed to register debugfs file for dpm!\n"); return ret; } + if ((adev->asic_type >= CHIP_VEGA10) && + !(adev->flags & AMD_IS_APU)) { + ret = device_create_file(adev->dev, + &dev_attr_ppfeatures); + if (ret) { + DRM_ERROR("failed to create device file " + "ppfeatures\n"); + return ret; + } + } + adev->pm.sysfs_initialized = true; return 0; @@ -2139,7 +2478,13 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); + if (adev->asic_type >= CHIP_VEGA10) { + device_remove_file(adev->dev, &dev_attr_pp_dpm_socclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_dcefclk); + } device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie); + if (adev->asic_type >= CHIP_VEGA20) + device_remove_file(adev->dev, &dev_attr_pp_dpm_fclk); device_remove_file(adev->dev, &dev_attr_pp_sclk_od); device_remove_file(adev->dev, &dev_attr_pp_mclk_od); device_remove_file(adev->dev, @@ -2148,6 +2493,11 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) device_remove_file(adev->dev, &dev_attr_pp_od_clk_voltage); device_remove_file(adev->dev, &dev_attr_gpu_busy_percent); + if (!(adev->flags & AMD_IS_APU)) + device_remove_file(adev->dev, &dev_attr_pcie_bw); + if ((adev->asic_type >= CHIP_VEGA10) && + !(adev->flags & AMD_IS_APU)) + device_remove_file(adev->dev, &dev_attr_ppfeatures); } void amdgpu_pm_compute_clocks(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 3a9b48b227ac..3091488cd8cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -67,9 +67,6 @@ static int psp_sw_init(void *handle) psp->adev = adev; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - ret = psp_init_microcode(psp); if (ret) { DRM_ERROR("Failed to load psp firmware!\n"); @@ -83,9 +80,6 @@ static int psp_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - release_firmware(adev->psp.sos_fw); adev->psp.sos_fw = NULL; release_firmware(adev->psp.asd_fw); @@ -142,13 +136,24 @@ psp_cmd_submit_buf(struct psp_context *psp, while (*((unsigned int *)psp->fence_buf) != index) msleep(1); - /* the status field must be 0 after FW is loaded */ - if (ucode && psp->cmd_buf_mem->resp.status) { - DRM_ERROR("failed loading with status (%d) and ucode id (%d)\n", - psp->cmd_buf_mem->resp.status, ucode->ucode_id); - return -EINVAL; + /* In some cases, psp response status is not 0 even there is no + * problem while the command is submitted. Some version of PSP FW + * doesn't write 0 to that field. + * So here we would like to only print a warning instead of an error + * during psp initialization to avoid breaking hw_init and it doesn't + * return -EINVAL. + */ + if (psp->cmd_buf_mem->resp.status) { + if (ucode) + DRM_WARN("failed to load ucode id (%d) ", + ucode->ucode_id); + DRM_WARN("psp command failed and response status is (%d)\n", + psp->cmd_buf_mem->resp.status); } + /* get xGMI session id from response buffer */ + cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id; + if (ucode) { ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo; ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi; @@ -500,6 +505,98 @@ static int psp_hw_start(struct psp_context *psp) return 0; } +static int psp_get_fw_type(struct amdgpu_firmware_info *ucode, + enum psp_gfx_fw_type *type) +{ + switch (ucode->ucode_id) { + case AMDGPU_UCODE_ID_SDMA0: + *type = GFX_FW_TYPE_SDMA0; + break; + case AMDGPU_UCODE_ID_SDMA1: + *type = GFX_FW_TYPE_SDMA1; + break; + case AMDGPU_UCODE_ID_CP_CE: + *type = GFX_FW_TYPE_CP_CE; + break; + case AMDGPU_UCODE_ID_CP_PFP: + *type = GFX_FW_TYPE_CP_PFP; + break; + case AMDGPU_UCODE_ID_CP_ME: + *type = GFX_FW_TYPE_CP_ME; + break; + case AMDGPU_UCODE_ID_CP_MEC1: + *type = GFX_FW_TYPE_CP_MEC; + break; + case AMDGPU_UCODE_ID_CP_MEC1_JT: + *type = GFX_FW_TYPE_CP_MEC_ME1; + break; + case AMDGPU_UCODE_ID_CP_MEC2: + *type = GFX_FW_TYPE_CP_MEC; + break; + case AMDGPU_UCODE_ID_CP_MEC2_JT: + *type = GFX_FW_TYPE_CP_MEC_ME2; + break; + case AMDGPU_UCODE_ID_RLC_G: + *type = GFX_FW_TYPE_RLC_G; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM; + break; + case AMDGPU_UCODE_ID_SMC: + *type = GFX_FW_TYPE_SMU; + break; + case AMDGPU_UCODE_ID_UVD: + *type = GFX_FW_TYPE_UVD; + break; + case AMDGPU_UCODE_ID_UVD1: + *type = GFX_FW_TYPE_UVD1; + break; + case AMDGPU_UCODE_ID_VCE: + *type = GFX_FW_TYPE_VCE; + break; + case AMDGPU_UCODE_ID_VCN: + *type = GFX_FW_TYPE_VCN; + break; + case AMDGPU_UCODE_ID_DMCU_ERAM: + *type = GFX_FW_TYPE_DMCU_ERAM; + break; + case AMDGPU_UCODE_ID_DMCU_INTV: + *type = GFX_FW_TYPE_DMCU_ISR; + break; + case AMDGPU_UCODE_ID_MAXIMUM: + default: + return -EINVAL; + } + + return 0; +} + +static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, + struct psp_gfx_cmd_resp *cmd) +{ + int ret; + uint64_t fw_mem_mc_addr = ucode->mc_addr; + + memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); + + cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; + cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); + cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); + cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; + + ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); + if (ret) + DRM_ERROR("Unknown firmware type\n"); + + return ret; +} + static int psp_np_fw_load(struct psp_context *psp) { int i, ret; @@ -521,7 +618,7 @@ static int psp_np_fw_load(struct psp_context *psp) /*skip ucode loading in SRIOV VF */ continue; - ret = psp_prep_cmd_buf(ucode, psp->cmd); + ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd); if (ret) return ret; @@ -546,7 +643,7 @@ static int psp_load_fw(struct amdgpu_device *adev) struct psp_context *psp = &adev->psp; if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) { - psp_ring_destroy(psp, PSP_RING_TYPE__KM); + psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */ goto skip_memalloc; } @@ -623,10 +720,6 @@ static int psp_hw_init(void *handle) int ret; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - mutex_lock(&adev->firmware.mutex); /* * This sequence is just used on hw_init only once, no need on @@ -656,9 +749,6 @@ static int psp_hw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - if (adev->gmc.xgmi.num_physical_nodes > 1 && psp->xgmi_context.initialized == 1) psp_xgmi_terminate(psp); @@ -687,9 +777,6 @@ static int psp_suspend(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - if (adev->gmc.xgmi.num_physical_nodes > 1 && psp->xgmi_context.initialized == 1) { ret = psp_xgmi_terminate(psp); @@ -714,9 +801,6 @@ static int psp_resume(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) - return 0; - DRM_INFO("PSP is resuming...\n"); mutex_lock(&adev->firmware.mutex); @@ -752,11 +836,6 @@ static bool psp_check_fw_loading_status(struct amdgpu_device *adev, { struct amdgpu_firmware_info *ucode = NULL; - if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { - DRM_INFO("firmware is not loaded by PSP\n"); - return true; - } - if (!adev->firmware.fw_size) return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 3ee573b4016e..2ef98cc755d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -65,8 +65,6 @@ struct psp_funcs int (*init_microcode)(struct psp_context *psp); int (*bootloader_load_sysdrv)(struct psp_context *psp); int (*bootloader_load_sos)(struct psp_context *psp); - int (*prep_cmd_buf)(struct amdgpu_firmware_info *ucode, - struct psp_gfx_cmd_resp *cmd); int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type); int (*ring_create)(struct psp_context *psp, enum psp_ring_type ring_type); @@ -176,7 +174,6 @@ struct psp_xgmi_topology_info { struct psp_xgmi_node_info nodes[AMDGPU_XGMI_MAX_CONNECTED_NODES]; }; -#define psp_prep_cmd_buf(ucode, type) (psp)->funcs->prep_cmd_buf((ucode), (type)) #define psp_ring_init(psp, type) (psp)->funcs->ring_init((psp), (type)) #define psp_ring_create(psp, type) (psp)->funcs->ring_create((psp), (type)) #define psp_ring_stop(psp, type) (psp)->funcs->ring_stop((psp), (type)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index d87e828a084b..d7fae2676269 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -131,7 +131,7 @@ struct amdgpu_ring_funcs { void (*emit_ib)(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch); + uint32_t flags); void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr, uint64_t seq, unsigned flags); void (*emit_pipeline_sync)(struct amdgpu_ring *ring); @@ -229,7 +229,7 @@ struct amdgpu_ring { #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r)) #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) -#define amdgpu_ring_emit_ib(r, job, ib, c) ((r)->funcs->emit_ib((r), (job), (ib), (c))) +#define amdgpu_ring_emit_ib(r, job, ib, flags) ((r)->funcs->emit_ib((r), (job), (ib), (flags))) #define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r)) #define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr)) #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index 12f2bf97611f..bfaf5c6323be 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -388,7 +388,7 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, soffset, eoffset, eoffset - soffset); if (i->fence) - seq_printf(m, " protected by 0x%08x on context %llu", + seq_printf(m, " protected by 0x%016llx on context %llu", i->fence->seqno, i->fence->context); seq_printf(m, "\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 1cafe8d83a4d..0767a93e4d91 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -54,16 +54,20 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, enum drm_sched_priority priority) { struct file *filp = fget(fd); - struct drm_file *file; struct amdgpu_fpriv *fpriv; struct amdgpu_ctx *ctx; uint32_t id; + int r; if (!filp) return -EINVAL; - file = filp->private_data; - fpriv = file->driver_priv; + r = amdgpu_file_to_fpriv(filp, &fpriv); + if (r) { + fput(filp); + return r; + } + idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id) amdgpu_ctx_priority_override(ctx, priority); @@ -72,6 +76,39 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, return 0; } +static int amdgpu_sched_context_priority_override(struct amdgpu_device *adev, + int fd, + unsigned ctx_id, + enum drm_sched_priority priority) +{ + struct file *filp = fget(fd); + struct amdgpu_fpriv *fpriv; + struct amdgpu_ctx *ctx; + int r; + + if (!filp) + return -EINVAL; + + r = amdgpu_file_to_fpriv(filp, &fpriv); + if (r) { + fput(filp); + return r; + } + + ctx = amdgpu_ctx_get(fpriv, ctx_id); + + if (!ctx) { + fput(filp); + return -EINVAL; + } + + amdgpu_ctx_priority_override(ctx, priority); + amdgpu_ctx_put(ctx); + fput(filp); + + return 0; +} + int amdgpu_sched_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { @@ -81,7 +118,7 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, int r; priority = amdgpu_to_sched_priority(args->in.priority); - if (args->in.flags || priority == DRM_SCHED_PRIORITY_INVALID) + if (priority == DRM_SCHED_PRIORITY_INVALID) return -EINVAL; switch (args->in.op) { @@ -90,6 +127,12 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, args->in.fd, priority); break; + case AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE: + r = amdgpu_sched_context_priority_override(adev, + args->in.fd, + args->in.ctx_id, + priority); + break; default: DRM_ERROR("Invalid sched op specified: %d\n", args->in.op); r = -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 626abca770a0..d3ca2424b5fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -76,9 +76,10 @@ TRACE_EVENT(amdgpu_mm_wreg, ); TRACE_EVENT(amdgpu_iv, - TP_PROTO(struct amdgpu_iv_entry *iv), - TP_ARGS(iv), + TP_PROTO(unsigned ih, struct amdgpu_iv_entry *iv), + TP_ARGS(ih, iv), TP_STRUCT__entry( + __field(unsigned, ih) __field(unsigned, client_id) __field(unsigned, src_id) __field(unsigned, ring_id) @@ -90,6 +91,7 @@ TRACE_EVENT(amdgpu_iv, __array(unsigned, src_data, 4) ), TP_fast_assign( + __entry->ih = ih; __entry->client_id = iv->client_id; __entry->src_id = iv->src_id; __entry->ring_id = iv->ring_id; @@ -103,8 +105,9 @@ TRACE_EVENT(amdgpu_iv, __entry->src_data[2] = iv->src_data[2]; __entry->src_data[3] = iv->src_data[3]; ), - TP_printk("client_id:%u src_id:%u ring:%u vmid:%u timestamp: %llu pasid:%u src_data: %08x %08x %08x %08x", - __entry->client_id, __entry->src_id, + TP_printk("ih:%u client_id:%u src_id:%u ring:%u vmid:%u " + "timestamp: %llu pasid:%u src_data: %08x %08x %08x %08x", + __entry->ih, __entry->client_id, __entry->src_id, __entry->ring_id, __entry->vmid, __entry->timestamp, __entry->pasid, __entry->src_data[0], __entry->src_data[1], diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index c91ec3101d00..73e71e61dc99 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1546,7 +1546,8 @@ static struct ttm_bo_driver amdgpu_bo_driver = { .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, .io_mem_free = &amdgpu_ttm_io_mem_free, .io_mem_pfn = amdgpu_ttm_io_mem_pfn, - .access_memory = &amdgpu_ttm_access_memory + .access_memory = &amdgpu_ttm_access_memory, + .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify }; /* @@ -1755,7 +1756,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) } r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS, + 4, AMDGPU_GEM_DOMAIN_GDS, &adev->gds.gds_gfx_bo, NULL, NULL); if (r) return r; @@ -1768,7 +1769,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) } r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS, + 1, AMDGPU_GEM_DOMAIN_GWS, &adev->gds.gws_gfx_bo, NULL, NULL); if (r) return r; @@ -1781,7 +1782,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) } r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA, + 1, AMDGPU_GEM_DOMAIN_OA, &adev->gds.oa_gfx_bo, NULL, NULL); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 98a1b2ce2b9d..c021b114c8a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -1035,7 +1035,7 @@ out: void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { amdgpu_ring_write(ring, VCE_CMD_IB); amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h index 50293652af14..30ea54dd9117 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h @@ -66,7 +66,7 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp); int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx); int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx); void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, - struct amdgpu_ib *ib, bool ctx_switch); + struct amdgpu_ib *ib, uint32_t flags); void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned flags); int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 698bcb8ce61d..ead851413c0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -107,14 +107,6 @@ struct amdgpu_pte_update_params { * DMA addresses to use for mapping, used during VM update by CPU */ dma_addr_t *pages_addr; - - /** - * @kptr: - * - * Kernel pointer of PD/PT BO that needs to be updated, - * used during VM update by CPU - */ - void *kptr; }; /** @@ -623,6 +615,28 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, list_add(&entry->tv.head, validated); } +void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo) +{ + struct amdgpu_bo *abo; + struct amdgpu_vm_bo_base *bo_base; + + if (!amdgpu_bo_is_amdgpu_bo(bo)) + return; + + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) + return; + + abo = ttm_to_amdgpu_bo(bo); + if (!abo->parent) + return; + for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) { + struct amdgpu_vm *vm = bo_base->vm; + + if (abo->tbo.resv == vm->root.base.bo->tbo.resv) + vm->bulk_moveable = false; + } + +} /** * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU * @@ -686,8 +700,6 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_bo_base *bo_base, *tmp; int r = 0; - vm->bulk_moveable &= list_empty(&vm->evicted); - list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { struct amdgpu_bo *bo = bo_base->bo; @@ -801,15 +813,22 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, addr += ats_entries * 8; } - if (entries) + if (entries) { + uint64_t value = 0; + + /* Workaround for fault priority problem on GMC9 */ + if (level == AMDGPU_VM_PTB && adev->asic_type >= CHIP_VEGA10) + value = AMDGPU_PTE_EXECUTABLE; + amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0, - entries, 0, 0); + entries, 0, value); + } amdgpu_ring_pad_ib(ring, &job->ibs[0]); WARN_ON(job->ibs[0].length_dw > 64); r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv, - AMDGPU_FENCE_OWNER_UNDEFINED, false); + AMDGPU_FENCE_OWNER_KFD, false); if (r) goto error_free; @@ -1313,31 +1332,6 @@ static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params, } } - -/** - * amdgpu_vm_wait_pd - Wait for PT BOs to be free. - * - * @adev: amdgpu_device pointer - * @vm: related vm - * @owner: fence owner - * - * Returns: - * 0 on success, errno otherwise. - */ -static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm, - void *owner) -{ - struct amdgpu_sync sync; - int r; - - amdgpu_sync_create(&sync); - amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false); - r = amdgpu_sync_wait(&sync, true); - amdgpu_sync_free(&sync); - - return r; -} - /** * amdgpu_vm_update_func - helper to call update function * @@ -1432,7 +1426,8 @@ restart: params.adev = adev; if (vm->use_cpu_for_update) { - r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM); + r = amdgpu_bo_sync_wait(vm->root.base.bo, + AMDGPU_FENCE_OWNER_VM, true); if (unlikely(r)) return r; @@ -1505,20 +1500,27 @@ error: } /** - * amdgpu_vm_update_huge - figure out parameters for PTE updates + * amdgpu_vm_update_flags - figure out flags for PTE updates * * Make sure to set the right flags for the PTEs at the desired level. */ -static void amdgpu_vm_update_huge(struct amdgpu_pte_update_params *params, - struct amdgpu_bo *bo, unsigned level, - uint64_t pe, uint64_t addr, - unsigned count, uint32_t incr, - uint64_t flags) +static void amdgpu_vm_update_flags(struct amdgpu_pte_update_params *params, + struct amdgpu_bo *bo, unsigned level, + uint64_t pe, uint64_t addr, + unsigned count, uint32_t incr, + uint64_t flags) { if (level != AMDGPU_VM_PTB) { flags |= AMDGPU_PDE_PTE; amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); + + } else if (params->adev->asic_type >= CHIP_VEGA10 && + !(flags & AMDGPU_PTE_VALID) && + !(flags & AMDGPU_PTE_PRT)) { + + /* Workaround for fault priority problem on GMC9 */ + flags |= AMDGPU_PTE_EXECUTABLE; } amdgpu_vm_update_func(params, bo, pe, addr, count, incr, flags); @@ -1675,9 +1677,9 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params, uint64_t upd_end = min(entry_end, frag_end); unsigned nptes = (upd_end - frag_start) >> shift; - amdgpu_vm_update_huge(params, pt, cursor.level, - pe_start, dst, nptes, incr, - flags | AMDGPU_PTE_FRAG(frag)); + amdgpu_vm_update_flags(params, pt, cursor.level, + pe_start, dst, nptes, incr, + flags | AMDGPU_PTE_FRAG(frag)); pe_start += nptes * 8; dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift; @@ -1746,22 +1748,29 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, params.adev = adev; params.vm = vm; - /* sync to everything on unmapping */ + /* sync to everything except eviction fences on unmapping */ if (!(flags & AMDGPU_PTE_VALID)) - owner = AMDGPU_FENCE_OWNER_UNDEFINED; + owner = AMDGPU_FENCE_OWNER_KFD; if (vm->use_cpu_for_update) { /* params.src is used as flag to indicate system Memory */ if (pages_addr) params.src = ~0; - /* Wait for PT BOs to be free. PTs share the same resv. object + /* Wait for PT BOs to be idle. PTs share the same resv. object * as the root PD BO */ - r = amdgpu_vm_wait_pd(adev, vm, owner); + r = amdgpu_bo_sync_wait(vm->root.base.bo, owner, true); if (unlikely(r)) return r; + /* Wait for any BO move to be completed */ + if (exclusive) { + r = dma_fence_wait(exclusive, true); + if (unlikely(r)) + return r; + } + params.func = amdgpu_vm_cpu_set_ptes; params.pages_addr = pages_addr; return amdgpu_vm_update_ptes(¶ms, start, last + 1, @@ -1775,13 +1784,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, /* * reserve space for two commands every (1 << BLOCK_SIZE) * entries or 2k dwords (whatever is smaller) - * - * The second command is for the shadow pagetables. */ + ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1); + + /* The second command is for the shadow pagetables. */ if (vm->root.base.bo->shadow) - ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2; - else - ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1); + ncmds *= 2; /* padding, etc. */ ndw = 64; @@ -1800,10 +1808,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, ndw += ncmds * 10; /* extra commands for begin/end fragments */ + ncmds = 2 * adev->vm_manager.fragment_size; if (vm->root.base.bo->shadow) - ndw += 2 * 10 * adev->vm_manager.fragment_size * 2; - else - ndw += 2 * 10 * adev->vm_manager.fragment_size; + ncmds *= 2; + + ndw += 10 * ncmds; params.func = amdgpu_vm_do_set_ptes; } @@ -3005,7 +3014,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, } DRM_DEBUG_DRIVER("VM update mode is %s\n", vm->use_cpu_for_update ? "CPU" : "SDMA"); - WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)), + WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), "CPU update of VM recommended only for large BAR system\n"); vm->last_update = NULL; @@ -3135,7 +3144,7 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, uns vm->pte_support_ats = pte_support_ats; DRM_DEBUG_DRIVER("VM update mode is %s\n", vm->use_cpu_for_update ? "CPU" : "SDMA"); - WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)), + WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), "CPU update of VM recommended only for large BAR system\n"); if (vm->pasid) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index e8dcfd59fc93..81ff8177f092 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -363,4 +363,6 @@ int amdgpu_vm_add_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key) void amdgpu_vm_clear_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key); +void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index 8a8bc60cb6b4..407dd16cc35c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -40,26 +40,40 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive) return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock) { int i; struct amdgpu_hive_info *tmp; if (!adev->gmc.xgmi.hive_id) return NULL; + + mutex_lock(&xgmi_mutex); + for (i = 0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; - if (tmp->hive_id == adev->gmc.xgmi.hive_id) + if (tmp->hive_id == adev->gmc.xgmi.hive_id) { + if (lock) + mutex_lock(&tmp->hive_lock); + mutex_unlock(&xgmi_mutex); return tmp; + } } - if (i >= AMDGPU_MAX_XGMI_HIVE) + if (i >= AMDGPU_MAX_XGMI_HIVE) { + mutex_unlock(&xgmi_mutex); return NULL; + } /* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id; INIT_LIST_HEAD(&tmp->device_list); mutex_init(&tmp->hive_lock); + mutex_init(&tmp->reset_lock); + if (lock) + mutex_lock(&tmp->hive_lock); + + mutex_unlock(&xgmi_mutex); return tmp; } @@ -77,10 +91,6 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev "XGMI: Set topology failure on device %llx, hive %llx, ret %d", adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id, ret); - else - dev_info(adev->dev, "XGMI: Set topology for node %d, hive 0x%llx.\n", - adev->gmc.xgmi.physical_node_id, - adev->gmc.xgmi.hive_id); return ret; } @@ -111,10 +121,14 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) return ret; } - mutex_lock(&xgmi_mutex); - hive = amdgpu_get_xgmi_hive(adev); - if (!hive) + hive = amdgpu_get_xgmi_hive(adev, 1); + if (!hive) { + ret = -EINVAL; + dev_err(adev->dev, + "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n", + adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id); goto exit; + } hive_topology = &hive->topology_info; @@ -142,8 +156,11 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) break; } + dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n", + adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id); + + mutex_unlock(&hive->hive_lock); exit: - mutex_unlock(&xgmi_mutex); return ret; } @@ -154,15 +171,14 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev) if (!adev->gmc.xgmi.supported) return; - mutex_lock(&xgmi_mutex); - - hive = amdgpu_get_xgmi_hive(adev); + hive = amdgpu_get_xgmi_hive(adev, 1); if (!hive) - goto exit; + return; - if (!(hive->number_devices--)) + if (!(hive->number_devices--)) { mutex_destroy(&hive->hive_lock); - -exit: - mutex_unlock(&xgmi_mutex); + mutex_destroy(&hive->reset_lock); + } else { + mutex_unlock(&hive->hive_lock); + } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h index 6151eb9c8ad3..14bc60664159 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h @@ -29,10 +29,11 @@ struct amdgpu_hive_info { struct list_head device_list; struct psp_xgmi_topology_info topology_info; int number_devices; - struct mutex hive_lock; + struct mutex hive_lock, + reset_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock); int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev); int amdgpu_xgmi_add_device(struct amdgpu_device *adev); void amdgpu_xgmi_remove_device(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c index e9934de1b9cf..dd30f4e61a8c 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.c +++ b/drivers/gpu/drm/amd/amdgpu/atom.c @@ -27,6 +27,8 @@ #include <linux/slab.h> #include <asm/unaligned.h> +#include <drm/drm_util.h> + #define ATOM_DEBUG #include "atom.h" diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c deleted file mode 100644 index 86e14c754dd4..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ /dev/null @@ -1,6844 +0,0 @@ -/* - * Copyright 2013 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <linux/firmware.h> -#include <drm/drmP.h> -#include "amdgpu.h" -#include "amdgpu_pm.h" -#include "amdgpu_ucode.h" -#include "cikd.h" -#include "amdgpu_dpm.h" -#include "ci_dpm.h" -#include "gfx_v7_0.h" -#include "atom.h" -#include "amd_pcie.h" -#include <linux/seq_file.h> - -#include "smu/smu_7_0_1_d.h" -#include "smu/smu_7_0_1_sh_mask.h" - -#include "dce/dce_8_0_d.h" -#include "dce/dce_8_0_sh_mask.h" - -#include "bif/bif_4_1_d.h" -#include "bif/bif_4_1_sh_mask.h" - -#include "gca/gfx_7_2_d.h" -#include "gca/gfx_7_2_sh_mask.h" - -#include "gmc/gmc_7_1_d.h" -#include "gmc/gmc_7_1_sh_mask.h" - -MODULE_FIRMWARE("amdgpu/bonaire_smc.bin"); -MODULE_FIRMWARE("amdgpu/bonaire_k_smc.bin"); -MODULE_FIRMWARE("amdgpu/hawaii_smc.bin"); -MODULE_FIRMWARE("amdgpu/hawaii_k_smc.bin"); - -#define MC_CG_ARB_FREQ_F0 0x0a -#define MC_CG_ARB_FREQ_F1 0x0b -#define MC_CG_ARB_FREQ_F2 0x0c -#define MC_CG_ARB_FREQ_F3 0x0d - -#define SMC_RAM_END 0x40000 - -#define VOLTAGE_SCALE 4 -#define VOLTAGE_VID_OFFSET_SCALE1 625 -#define VOLTAGE_VID_OFFSET_SCALE2 100 - -static const struct amd_pm_funcs ci_dpm_funcs; - -static const struct ci_pt_defaults defaults_hawaii_xt = -{ - 1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000, - { 0x2E, 0x00, 0x00, 0x88, 0x00, 0x00, 0x72, 0x60, 0x51, 0xA7, 0x79, 0x6B, 0x90, 0xBD, 0x79 }, - { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 } -}; - -static const struct ci_pt_defaults defaults_hawaii_pro = -{ - 1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062, - { 0x2E, 0x00, 0x00, 0x88, 0x00, 0x00, 0x72, 0x60, 0x51, 0xA7, 0x79, 0x6B, 0x90, 0xBD, 0x79 }, - { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 } -}; - -static const struct ci_pt_defaults defaults_bonaire_xt = -{ - 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000, - { 0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8, 0xC9, 0xC9, 0x2F, 0x4D, 0x61 }, - { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } -}; - -#if 0 -static const struct ci_pt_defaults defaults_bonaire_pro = -{ - 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x65062, - { 0x8C, 0x23F, 0x244, 0xA6, 0x83, 0x85, 0x86, 0x86, 0x83, 0xDB, 0xDB, 0xDA, 0x67, 0x60, 0x5F }, - { 0x187, 0x193, 0x193, 0x1C7, 0x1D1, 0x1D1, 0x210, 0x219, 0x219, 0x266, 0x26C, 0x26C, 0x2C9, 0x2CB, 0x2CB } -}; -#endif - -static const struct ci_pt_defaults defaults_saturn_xt = -{ - 1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x70000, - { 0x8C, 0x247, 0x249, 0xA6, 0x80, 0x81, 0x8B, 0x89, 0x86, 0xC9, 0xCA, 0xC9, 0x4D, 0x4D, 0x4D }, - { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 } -}; - -#if 0 -static const struct ci_pt_defaults defaults_saturn_pro = -{ - 1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x30000, - { 0x96, 0x21D, 0x23B, 0xA1, 0x85, 0x87, 0x83, 0x84, 0x81, 0xE6, 0xE6, 0xE6, 0x71, 0x6A, 0x6A }, - { 0x193, 0x19E, 0x19E, 0x1D2, 0x1DC, 0x1DC, 0x21A, 0x223, 0x223, 0x26E, 0x27E, 0x274, 0x2CF, 0x2D2, 0x2D2 } -}; -#endif - -static const struct ci_pt_config_reg didt_config_ci[] = -{ - { 0x10, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x10, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x10, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x10, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x11, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x11, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x11, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x11, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x12, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x12, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x12, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x12, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x2, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x2, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x2, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x1, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x1, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x0, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x30, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x30, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x30, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x30, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x31, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x31, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x31, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x31, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x32, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x32, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x32, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x32, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x22, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x22, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x22, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x21, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x21, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x20, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x50, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x50, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x50, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x50, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x51, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x51, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x51, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x51, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x52, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x52, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x52, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x52, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x42, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x42, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x42, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x41, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x41, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x40, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x70, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x70, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x70, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x70, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x71, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x71, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x71, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x71, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x72, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x72, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x72, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x72, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x62, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x62, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x62, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x61, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x61, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, - { 0x60, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, - { 0xFFFFFFFF } -}; - -static u8 ci_get_memory_module_index(struct amdgpu_device *adev) -{ - return (u8) ((RREG32(mmBIOS_SCRATCH_4) >> 16) & 0xff); -} - -#define MC_CG_ARB_FREQ_F0 0x0a -#define MC_CG_ARB_FREQ_F1 0x0b -#define MC_CG_ARB_FREQ_F2 0x0c -#define MC_CG_ARB_FREQ_F3 0x0d - -static int ci_copy_and_switch_arb_sets(struct amdgpu_device *adev, - u32 arb_freq_src, u32 arb_freq_dest) -{ - u32 mc_arb_dram_timing; - u32 mc_arb_dram_timing2; - u32 burst_time; - u32 mc_cg_config; - - switch (arb_freq_src) { - case MC_CG_ARB_FREQ_F0: - mc_arb_dram_timing = RREG32(mmMC_ARB_DRAM_TIMING); - mc_arb_dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2); - burst_time = (RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE0_MASK) >> - MC_ARB_BURST_TIME__STATE0__SHIFT; - break; - case MC_CG_ARB_FREQ_F1: - mc_arb_dram_timing = RREG32(mmMC_ARB_DRAM_TIMING_1); - mc_arb_dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2_1); - burst_time = (RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE1_MASK) >> - MC_ARB_BURST_TIME__STATE1__SHIFT; - break; - default: - return -EINVAL; - } - - switch (arb_freq_dest) { - case MC_CG_ARB_FREQ_F0: - WREG32(mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing); - WREG32(mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2); - WREG32_P(mmMC_ARB_BURST_TIME, (burst_time << MC_ARB_BURST_TIME__STATE0__SHIFT), - ~MC_ARB_BURST_TIME__STATE0_MASK); - break; - case MC_CG_ARB_FREQ_F1: - WREG32(mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing); - WREG32(mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2); - WREG32_P(mmMC_ARB_BURST_TIME, (burst_time << MC_ARB_BURST_TIME__STATE1__SHIFT), - ~MC_ARB_BURST_TIME__STATE1_MASK); - break; - default: - return -EINVAL; - } - - mc_cg_config = RREG32(mmMC_CG_CONFIG) | 0x0000000F; - WREG32(mmMC_CG_CONFIG, mc_cg_config); - WREG32_P(mmMC_ARB_CG, (arb_freq_dest) << MC_ARB_CG__CG_ARB_REQ__SHIFT, - ~MC_ARB_CG__CG_ARB_REQ_MASK); - - return 0; -} - -static u8 ci_get_ddr3_mclk_frequency_ratio(u32 memory_clock) -{ - u8 mc_para_index; - - if (memory_clock < 10000) - mc_para_index = 0; - else if (memory_clock >= 80000) - mc_para_index = 0x0f; - else - mc_para_index = (u8)((memory_clock - 10000) / 5000 + 1); - return mc_para_index; -} - -static u8 ci_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode) -{ - u8 mc_para_index; - - if (strobe_mode) { - if (memory_clock < 12500) - mc_para_index = 0x00; - else if (memory_clock > 47500) - mc_para_index = 0x0f; - else - mc_para_index = (u8)((memory_clock - 10000) / 2500); - } else { - if (memory_clock < 65000) - mc_para_index = 0x00; - else if (memory_clock > 135000) - mc_para_index = 0x0f; - else - mc_para_index = (u8)((memory_clock - 60000) / 5000); - } - return mc_para_index; -} - -static void ci_trim_voltage_table_to_fit_state_table(struct amdgpu_device *adev, - u32 max_voltage_steps, - struct atom_voltage_table *voltage_table) -{ - unsigned int i, diff; - - if (voltage_table->count <= max_voltage_steps) - return; - - diff = voltage_table->count - max_voltage_steps; - - for (i = 0; i < max_voltage_steps; i++) - voltage_table->entries[i] = voltage_table->entries[i + diff]; - - voltage_table->count = max_voltage_steps; -} - -static int ci_get_std_voltage_value_sidd(struct amdgpu_device *adev, - struct atom_voltage_table_entry *voltage_table, - u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd); -static int ci_set_power_limit(struct amdgpu_device *adev, u32 n); -static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev, - u32 target_tdp); -static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate); -static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev); - -static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev, - PPSMC_Msg msg, u32 parameter); -static void ci_thermal_start_smc_fan_control(struct amdgpu_device *adev); -static void ci_fan_ctrl_set_default_mode(struct amdgpu_device *adev); - -static struct ci_power_info *ci_get_pi(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = adev->pm.dpm.priv; - - return pi; -} - -static struct ci_ps *ci_get_ps(struct amdgpu_ps *rps) -{ - struct ci_ps *ps = rps->ps_priv; - - return ps; -} - -static void ci_initialize_powertune_defaults(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - switch (adev->pdev->device) { - case 0x6649: - case 0x6650: - case 0x6651: - case 0x6658: - case 0x665C: - case 0x665D: - default: - pi->powertune_defaults = &defaults_bonaire_xt; - break; - case 0x6640: - case 0x6641: - case 0x6646: - case 0x6647: - pi->powertune_defaults = &defaults_saturn_xt; - break; - case 0x67B8: - case 0x67B0: - pi->powertune_defaults = &defaults_hawaii_xt; - break; - case 0x67BA: - case 0x67B1: - pi->powertune_defaults = &defaults_hawaii_pro; - break; - case 0x67A0: - case 0x67A1: - case 0x67A2: - case 0x67A8: - case 0x67A9: - case 0x67AA: - case 0x67B9: - case 0x67BE: - pi->powertune_defaults = &defaults_bonaire_xt; - break; - } - - pi->dte_tj_offset = 0; - - pi->caps_power_containment = true; - pi->caps_cac = false; - pi->caps_sq_ramping = false; - pi->caps_db_ramping = false; - pi->caps_td_ramping = false; - pi->caps_tcp_ramping = false; - - if (pi->caps_power_containment) { - pi->caps_cac = true; - if (adev->asic_type == CHIP_HAWAII) - pi->enable_bapm_feature = false; - else - pi->enable_bapm_feature = true; - pi->enable_tdc_limit_feature = true; - pi->enable_pkg_pwr_tracking_feature = true; - } -} - -static u8 ci_convert_to_vid(u16 vddc) -{ - return (6200 - (vddc * VOLTAGE_SCALE)) / 25; -} - -static int ci_populate_bapm_vddc_vid_sidd(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd; - u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd; - u8 *hi2_vid = pi->smc_powertune_table.BapmVddCVidHiSidd2; - u32 i; - - if (adev->pm.dpm.dyn_state.cac_leakage_table.entries == NULL) - return -EINVAL; - if (adev->pm.dpm.dyn_state.cac_leakage_table.count > 8) - return -EINVAL; - if (adev->pm.dpm.dyn_state.cac_leakage_table.count != - adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count) - return -EINVAL; - - for (i = 0; i < adev->pm.dpm.dyn_state.cac_leakage_table.count; i++) { - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { - lo_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1); - hi_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2); - hi2_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3); - } else { - lo_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc); - hi_vid[i] = ci_convert_to_vid((u16)adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage); - } - } - return 0; -} - -static int ci_populate_vddc_vid(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u8 *vid = pi->smc_powertune_table.VddCVid; - u32 i; - - if (pi->vddc_voltage_table.count > 8) - return -EINVAL; - - for (i = 0; i < pi->vddc_voltage_table.count; i++) - vid[i] = ci_convert_to_vid(pi->vddc_voltage_table.entries[i].value); - - return 0; -} - -static int ci_populate_svi_load_line(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; - - pi->smc_powertune_table.SviLoadLineEn = pt_defaults->svi_load_line_en; - pi->smc_powertune_table.SviLoadLineVddC = pt_defaults->svi_load_line_vddc; - pi->smc_powertune_table.SviLoadLineTrimVddC = 3; - pi->smc_powertune_table.SviLoadLineOffsetVddC = 0; - - return 0; -} - -static int ci_populate_tdc_limit(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; - u16 tdc_limit; - - tdc_limit = adev->pm.dpm.dyn_state.cac_tdp_table->tdc * 256; - pi->smc_powertune_table.TDC_VDDC_PkgLimit = cpu_to_be16(tdc_limit); - pi->smc_powertune_table.TDC_VDDC_ThrottleReleaseLimitPerc = - pt_defaults->tdc_vddc_throttle_release_limit_perc; - pi->smc_powertune_table.TDC_MAWt = pt_defaults->tdc_mawt; - - return 0; -} - -static int ci_populate_dw8(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; - int ret; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, PmFuseTable) + - offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl), - (u32 *)&pi->smc_powertune_table.TdcWaterfallCtl, - pi->sram_end); - if (ret) - return -EINVAL; - else - pi->smc_powertune_table.TdcWaterfallCtl = pt_defaults->tdc_waterfall_ctl; - - return 0; -} - -static int ci_populate_fuzzy_fan(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if ((adev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) || - (adev->pm.dpm.fan.fan_output_sensitivity == 0)) - adev->pm.dpm.fan.fan_output_sensitivity = - adev->pm.dpm.fan.default_fan_output_sensitivity; - - pi->smc_powertune_table.FuzzyFan_PwmSetDelta = - cpu_to_be16(adev->pm.dpm.fan.fan_output_sensitivity); - - return 0; -} - -static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd; - u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd; - int i, min, max; - - min = max = hi_vid[0]; - for (i = 0; i < 8; i++) { - if (0 != hi_vid[i]) { - if (min > hi_vid[i]) - min = hi_vid[i]; - if (max < hi_vid[i]) - max = hi_vid[i]; - } - - if (0 != lo_vid[i]) { - if (min > lo_vid[i]) - min = lo_vid[i]; - if (max < lo_vid[i]) - max = lo_vid[i]; - } - } - - if ((min == 0) || (max == 0)) - return -EINVAL; - pi->smc_powertune_table.GnbLPMLMaxVid = (u8)max; - pi->smc_powertune_table.GnbLPMLMinVid = (u8)min; - - return 0; -} - -static int ci_populate_bapm_vddc_base_leakage_sidd(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u16 hi_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd; - u16 lo_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd; - struct amdgpu_cac_tdp_table *cac_tdp_table = - adev->pm.dpm.dyn_state.cac_tdp_table; - - hi_sidd = cac_tdp_table->high_cac_leakage / 100 * 256; - lo_sidd = cac_tdp_table->low_cac_leakage / 100 * 256; - - pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd = cpu_to_be16(hi_sidd); - pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd = cpu_to_be16(lo_sidd); - - return 0; -} - -static int ci_populate_bapm_parameters_in_dpm_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; - SMU7_Discrete_DpmTable *dpm_table = &pi->smc_state_table; - struct amdgpu_cac_tdp_table *cac_tdp_table = - adev->pm.dpm.dyn_state.cac_tdp_table; - struct amdgpu_ppm_table *ppm = adev->pm.dpm.dyn_state.ppm_table; - int i, j, k; - const u16 *def1; - const u16 *def2; - - dpm_table->DefaultTdp = cac_tdp_table->tdp * 256; - dpm_table->TargetTdp = cac_tdp_table->configurable_tdp * 256; - - dpm_table->DTETjOffset = (u8)pi->dte_tj_offset; - dpm_table->GpuTjMax = - (u8)(pi->thermal_temp_setting.temperature_high / 1000); - dpm_table->GpuTjHyst = 8; - - dpm_table->DTEAmbientTempBase = pt_defaults->dte_ambient_temp_base; - - if (ppm) { - dpm_table->PPM_PkgPwrLimit = cpu_to_be16((u16)ppm->dgpu_tdp * 256 / 1000); - dpm_table->PPM_TemperatureLimit = cpu_to_be16((u16)ppm->tj_max * 256); - } else { - dpm_table->PPM_PkgPwrLimit = cpu_to_be16(0); - dpm_table->PPM_TemperatureLimit = cpu_to_be16(0); - } - - dpm_table->BAPM_TEMP_GRADIENT = cpu_to_be32(pt_defaults->bapm_temp_gradient); - def1 = pt_defaults->bapmti_r; - def2 = pt_defaults->bapmti_rc; - - for (i = 0; i < SMU7_DTE_ITERATIONS; i++) { - for (j = 0; j < SMU7_DTE_SOURCES; j++) { - for (k = 0; k < SMU7_DTE_SINKS; k++) { - dpm_table->BAPMTI_R[i][j][k] = cpu_to_be16(*def1); - dpm_table->BAPMTI_RC[i][j][k] = cpu_to_be16(*def2); - def1++; - def2++; - } - } - } - - return 0; -} - -static int ci_populate_pm_base(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 pm_fuse_table_offset; - int ret; - - if (pi->caps_power_containment) { - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, PmFuseTable), - &pm_fuse_table_offset, pi->sram_end); - if (ret) - return ret; - ret = ci_populate_bapm_vddc_vid_sidd(adev); - if (ret) - return ret; - ret = ci_populate_vddc_vid(adev); - if (ret) - return ret; - ret = ci_populate_svi_load_line(adev); - if (ret) - return ret; - ret = ci_populate_tdc_limit(adev); - if (ret) - return ret; - ret = ci_populate_dw8(adev); - if (ret) - return ret; - ret = ci_populate_fuzzy_fan(adev); - if (ret) - return ret; - ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(adev); - if (ret) - return ret; - ret = ci_populate_bapm_vddc_base_leakage_sidd(adev); - if (ret) - return ret; - ret = amdgpu_ci_copy_bytes_to_smc(adev, pm_fuse_table_offset, - (u8 *)&pi->smc_powertune_table, - sizeof(SMU7_Discrete_PmFuses), pi->sram_end); - if (ret) - return ret; - } - - return 0; -} - -static void ci_do_enable_didt(struct amdgpu_device *adev, const bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 data; - - if (pi->caps_sq_ramping) { - data = RREG32_DIDT(ixDIDT_SQ_CTRL0); - if (enable) - data |= DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK; - else - data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK; - WREG32_DIDT(ixDIDT_SQ_CTRL0, data); - } - - if (pi->caps_db_ramping) { - data = RREG32_DIDT(ixDIDT_DB_CTRL0); - if (enable) - data |= DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK; - else - data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK; - WREG32_DIDT(ixDIDT_DB_CTRL0, data); - } - - if (pi->caps_td_ramping) { - data = RREG32_DIDT(ixDIDT_TD_CTRL0); - if (enable) - data |= DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK; - else - data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK; - WREG32_DIDT(ixDIDT_TD_CTRL0, data); - } - - if (pi->caps_tcp_ramping) { - data = RREG32_DIDT(ixDIDT_TCP_CTRL0); - if (enable) - data |= DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK; - else - data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK; - WREG32_DIDT(ixDIDT_TCP_CTRL0, data); - } -} - -static int ci_program_pt_config_registers(struct amdgpu_device *adev, - const struct ci_pt_config_reg *cac_config_regs) -{ - const struct ci_pt_config_reg *config_regs = cac_config_regs; - u32 data; - u32 cache = 0; - - if (config_regs == NULL) - return -EINVAL; - - while (config_regs->offset != 0xFFFFFFFF) { - if (config_regs->type == CISLANDS_CONFIGREG_CACHE) { - cache |= ((config_regs->value << config_regs->shift) & config_regs->mask); - } else { - switch (config_regs->type) { - case CISLANDS_CONFIGREG_SMC_IND: - data = RREG32_SMC(config_regs->offset); - break; - case CISLANDS_CONFIGREG_DIDT_IND: - data = RREG32_DIDT(config_regs->offset); - break; - default: - data = RREG32(config_regs->offset); - break; - } - - data &= ~config_regs->mask; - data |= ((config_regs->value << config_regs->shift) & config_regs->mask); - data |= cache; - - switch (config_regs->type) { - case CISLANDS_CONFIGREG_SMC_IND: - WREG32_SMC(config_regs->offset, data); - break; - case CISLANDS_CONFIGREG_DIDT_IND: - WREG32_DIDT(config_regs->offset, data); - break; - default: - WREG32(config_regs->offset, data); - break; - } - cache = 0; - } - config_regs++; - } - return 0; -} - -static int ci_enable_didt(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - - if (pi->caps_sq_ramping || pi->caps_db_ramping || - pi->caps_td_ramping || pi->caps_tcp_ramping) { - amdgpu_gfx_rlc_enter_safe_mode(adev); - - if (enable) { - ret = ci_program_pt_config_registers(adev, didt_config_ci); - if (ret) { - amdgpu_gfx_rlc_exit_safe_mode(adev); - return ret; - } - } - - ci_do_enable_didt(adev, enable); - - amdgpu_gfx_rlc_exit_safe_mode(adev); - } - - return 0; -} - -static int ci_enable_power_containment(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - int ret = 0; - - if (enable) { - pi->power_containment_features = 0; - if (pi->caps_power_containment) { - if (pi->enable_bapm_feature) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableDTE); - if (smc_result != PPSMC_Result_OK) - ret = -EINVAL; - else - pi->power_containment_features |= POWERCONTAINMENT_FEATURE_BAPM; - } - - if (pi->enable_tdc_limit_feature) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_TDCLimitEnable); - if (smc_result != PPSMC_Result_OK) - ret = -EINVAL; - else - pi->power_containment_features |= POWERCONTAINMENT_FEATURE_TDCLimit; - } - - if (pi->enable_pkg_pwr_tracking_feature) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PkgPwrLimitEnable); - if (smc_result != PPSMC_Result_OK) { - ret = -EINVAL; - } else { - struct amdgpu_cac_tdp_table *cac_tdp_table = - adev->pm.dpm.dyn_state.cac_tdp_table; - u32 default_pwr_limit = - (u32)(cac_tdp_table->maximum_power_delivery_limit * 256); - - pi->power_containment_features |= POWERCONTAINMENT_FEATURE_PkgPwrLimit; - - ci_set_power_limit(adev, default_pwr_limit); - } - } - } - } else { - if (pi->caps_power_containment && pi->power_containment_features) { - if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_TDCLimit) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_TDCLimitDisable); - - if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_BAPM) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableDTE); - - if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PkgPwrLimitDisable); - pi->power_containment_features = 0; - } - } - - return ret; -} - -static int ci_enable_smc_cac(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - int ret = 0; - - if (pi->caps_cac) { - if (enable) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableCac); - if (smc_result != PPSMC_Result_OK) { - ret = -EINVAL; - pi->cac_enabled = false; - } else { - pi->cac_enabled = true; - } - } else if (pi->cac_enabled) { - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableCac); - pi->cac_enabled = false; - } - } - - return ret; -} - -static int ci_enable_thermal_based_sclk_dpm(struct amdgpu_device *adev, - bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result = PPSMC_Result_OK; - - if (pi->thermal_sclk_dpm_enabled) { - if (enable) - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_ENABLE_THERMAL_DPM); - else - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DISABLE_THERMAL_DPM); - } - - if (smc_result == PPSMC_Result_OK) - return 0; - else - return -EINVAL; -} - -static int ci_power_control_set_level(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_cac_tdp_table *cac_tdp_table = - adev->pm.dpm.dyn_state.cac_tdp_table; - s32 adjust_percent; - s32 target_tdp; - int ret = 0; - bool adjust_polarity = false; /* ??? */ - - if (pi->caps_power_containment) { - adjust_percent = adjust_polarity ? - adev->pm.dpm.tdp_adjustment : (-1 * adev->pm.dpm.tdp_adjustment); - target_tdp = ((100 + adjust_percent) * - (s32)cac_tdp_table->configurable_tdp) / 100; - - ret = ci_set_overdrive_target_tdp(adev, (u32)target_tdp); - } - - return ret; -} - -static void ci_dpm_powergate_uvd(void *handle, bool gate) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - - pi->uvd_power_gated = gate; - - if (gate) { - /* stop the UVD block */ - amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, - AMD_PG_STATE_GATE); - ci_update_uvd_dpm(adev, gate); - } else { - amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, - AMD_PG_STATE_UNGATE); - ci_update_uvd_dpm(adev, gate); - } -} - -static bool ci_dpm_vblank_too_short(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - u32 vblank_time = amdgpu_dpm_get_vblank_time(adev); - u32 switch_limit = adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 300; - - /* disable mclk switching if the refresh is >120Hz, even if the - * blanking period would allow it - */ - if (amdgpu_dpm_get_vrefresh(adev) > 120) - return true; - - if (vblank_time < switch_limit) - return true; - else - return false; - -} - -static void ci_apply_state_adjust_rules(struct amdgpu_device *adev, - struct amdgpu_ps *rps) -{ - struct ci_ps *ps = ci_get_ps(rps); - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_clock_and_voltage_limits *max_limits; - bool disable_mclk_switching; - u32 sclk, mclk; - int i; - - if (rps->vce_active) { - rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk; - rps->ecclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].ecclk; - } else { - rps->evclk = 0; - rps->ecclk = 0; - } - - if ((adev->pm.dpm.new_active_crtc_count > 1) || - ci_dpm_vblank_too_short(adev)) - disable_mclk_switching = true; - else - disable_mclk_switching = false; - - if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) - pi->battery_state = true; - else - pi->battery_state = false; - - if (adev->pm.ac_power) - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - - if (adev->pm.ac_power == false) { - for (i = 0; i < ps->performance_level_count; i++) { - if (ps->performance_levels[i].mclk > max_limits->mclk) - ps->performance_levels[i].mclk = max_limits->mclk; - if (ps->performance_levels[i].sclk > max_limits->sclk) - ps->performance_levels[i].sclk = max_limits->sclk; - } - } - - /* XXX validate the min clocks required for display */ - - if (disable_mclk_switching) { - mclk = ps->performance_levels[ps->performance_level_count - 1].mclk; - sclk = ps->performance_levels[0].sclk; - } else { - mclk = ps->performance_levels[0].mclk; - sclk = ps->performance_levels[0].sclk; - } - - if (adev->pm.pm_display_cfg.min_core_set_clock > sclk) - sclk = adev->pm.pm_display_cfg.min_core_set_clock; - - if (adev->pm.pm_display_cfg.min_mem_set_clock > mclk) - mclk = adev->pm.pm_display_cfg.min_mem_set_clock; - - if (rps->vce_active) { - if (sclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk) - sclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk; - if (mclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk) - mclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk; - } - - ps->performance_levels[0].sclk = sclk; - ps->performance_levels[0].mclk = mclk; - - if (ps->performance_levels[1].sclk < ps->performance_levels[0].sclk) - ps->performance_levels[1].sclk = ps->performance_levels[0].sclk; - - if (disable_mclk_switching) { - if (ps->performance_levels[0].mclk < ps->performance_levels[1].mclk) - ps->performance_levels[0].mclk = ps->performance_levels[1].mclk; - } else { - if (ps->performance_levels[1].mclk < ps->performance_levels[0].mclk) - ps->performance_levels[1].mclk = ps->performance_levels[0].mclk; - } -} - -static int ci_thermal_set_temperature_range(struct amdgpu_device *adev, - int min_temp, int max_temp) -{ - int low_temp = 0 * 1000; - int high_temp = 255 * 1000; - u32 tmp; - - if (low_temp < min_temp) - low_temp = min_temp; - if (high_temp > max_temp) - high_temp = max_temp; - if (high_temp < low_temp) { - DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); - return -EINVAL; - } - - tmp = RREG32_SMC(ixCG_THERMAL_INT); - tmp &= ~(CG_THERMAL_INT__DIG_THERM_INTH_MASK | CG_THERMAL_INT__DIG_THERM_INTL_MASK); - tmp |= ((high_temp / 1000) << CG_THERMAL_INT__DIG_THERM_INTH__SHIFT) | - ((low_temp / 1000)) << CG_THERMAL_INT__DIG_THERM_INTL__SHIFT; - WREG32_SMC(ixCG_THERMAL_INT, tmp); - -#if 0 - /* XXX: need to figure out how to handle this properly */ - tmp = RREG32_SMC(ixCG_THERMAL_CTRL); - tmp &= DIG_THERM_DPM_MASK; - tmp |= DIG_THERM_DPM(high_temp / 1000); - WREG32_SMC(ixCG_THERMAL_CTRL, tmp); -#endif - - adev->pm.dpm.thermal.min_temp = low_temp; - adev->pm.dpm.thermal.max_temp = high_temp; - return 0; -} - -static int ci_thermal_enable_alert(struct amdgpu_device *adev, - bool enable) -{ - u32 thermal_int = RREG32_SMC(ixCG_THERMAL_INT); - PPSMC_Result result; - - if (enable) { - thermal_int &= ~(CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK | - CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK); - WREG32_SMC(ixCG_THERMAL_INT, thermal_int); - result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Thermal_Cntl_Enable); - if (result != PPSMC_Result_OK) { - DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); - return -EINVAL; - } - } else { - thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK | - CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK; - WREG32_SMC(ixCG_THERMAL_INT, thermal_int); - result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Thermal_Cntl_Disable); - if (result != PPSMC_Result_OK) { - DRM_DEBUG_KMS("Could not disable thermal interrupts.\n"); - return -EINVAL; - } - } - - return 0; -} - -static void ci_fan_ctrl_set_static_mode(struct amdgpu_device *adev, u32 mode) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - - if (pi->fan_ctrl_is_in_default_mode) { - tmp = (RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__FDO_PWM_MODE_MASK) - >> CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT; - pi->fan_ctrl_default_mode = tmp; - tmp = (RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__TMIN_MASK) - >> CG_FDO_CTRL2__TMIN__SHIFT; - pi->t_min = tmp; - pi->fan_ctrl_is_in_default_mode = false; - } - - tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TMIN_MASK; - tmp |= 0 << CG_FDO_CTRL2__TMIN__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL2, tmp); - - tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__FDO_PWM_MODE_MASK; - tmp |= mode << CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL2, tmp); -} - -static int ci_thermal_setup_fan_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE }; - u32 duty100; - u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2; - u16 fdo_min, slope1, slope2; - u32 reference_clock, tmp; - int ret; - u64 tmp64; - - if (!pi->fan_table_start) { - adev->pm.dpm.fan.ucode_fan_control = false; - return 0; - } - - duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK) - >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT; - - if (duty100 == 0) { - adev->pm.dpm.fan.ucode_fan_control = false; - return 0; - } - - tmp64 = (u64)adev->pm.dpm.fan.pwm_min * duty100; - do_div(tmp64, 10000); - fdo_min = (u16)tmp64; - - t_diff1 = adev->pm.dpm.fan.t_med - adev->pm.dpm.fan.t_min; - t_diff2 = adev->pm.dpm.fan.t_high - adev->pm.dpm.fan.t_med; - - pwm_diff1 = adev->pm.dpm.fan.pwm_med - adev->pm.dpm.fan.pwm_min; - pwm_diff2 = adev->pm.dpm.fan.pwm_high - adev->pm.dpm.fan.pwm_med; - - slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100); - slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100); - - fan_table.TempMin = cpu_to_be16((50 + adev->pm.dpm.fan.t_min) / 100); - fan_table.TempMed = cpu_to_be16((50 + adev->pm.dpm.fan.t_med) / 100); - fan_table.TempMax = cpu_to_be16((50 + adev->pm.dpm.fan.t_max) / 100); - - fan_table.Slope1 = cpu_to_be16(slope1); - fan_table.Slope2 = cpu_to_be16(slope2); - - fan_table.FdoMin = cpu_to_be16(fdo_min); - - fan_table.HystDown = cpu_to_be16(adev->pm.dpm.fan.t_hyst); - - fan_table.HystUp = cpu_to_be16(1); - - fan_table.HystSlope = cpu_to_be16(1); - - fan_table.TempRespLim = cpu_to_be16(5); - - reference_clock = amdgpu_asic_get_xclk(adev); - - fan_table.RefreshPeriod = cpu_to_be32((adev->pm.dpm.fan.cycle_delay * - reference_clock) / 1600); - - fan_table.FdoMax = cpu_to_be16((u16)duty100); - - tmp = (RREG32_SMC(ixCG_MULT_THERMAL_CTRL) & CG_MULT_THERMAL_CTRL__TEMP_SEL_MASK) - >> CG_MULT_THERMAL_CTRL__TEMP_SEL__SHIFT; - fan_table.TempSrc = (uint8_t)tmp; - - ret = amdgpu_ci_copy_bytes_to_smc(adev, - pi->fan_table_start, - (u8 *)(&fan_table), - sizeof(fan_table), - pi->sram_end); - - if (ret) { - DRM_ERROR("Failed to load fan table to the SMC."); - adev->pm.dpm.fan.ucode_fan_control = false; - } - - return 0; -} - -static int ci_fan_ctrl_start_smc_fan_control(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result ret; - - if (pi->caps_od_fuzzy_fan_control_support) { - ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_StartFanControl, - FAN_CONTROL_FUZZY); - if (ret != PPSMC_Result_OK) - return -EINVAL; - ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_SetFanPwmMax, - adev->pm.dpm.fan.default_max_fan_pwm); - if (ret != PPSMC_Result_OK) - return -EINVAL; - } else { - ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_StartFanControl, - FAN_CONTROL_TABLE); - if (ret != PPSMC_Result_OK) - return -EINVAL; - } - - pi->fan_is_controlled_by_smc = true; - return 0; -} - - -static int ci_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev) -{ - PPSMC_Result ret; - struct ci_power_info *pi = ci_get_pi(adev); - - ret = amdgpu_ci_send_msg_to_smc(adev, PPSMC_StopFanControl); - if (ret == PPSMC_Result_OK) { - pi->fan_is_controlled_by_smc = false; - return 0; - } else { - return -EINVAL; - } -} - -static int ci_dpm_get_fan_speed_percent(void *handle, - u32 *speed) -{ - u32 duty, duty100; - u64 tmp64; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev->pm.no_fan) - return -ENOENT; - - duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK) - >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT; - duty = (RREG32_SMC(ixCG_THERMAL_STATUS) & CG_THERMAL_STATUS__FDO_PWM_DUTY_MASK) - >> CG_THERMAL_STATUS__FDO_PWM_DUTY__SHIFT; - - if (duty100 == 0) - return -EINVAL; - - tmp64 = (u64)duty * 100; - do_div(tmp64, duty100); - *speed = (u32)tmp64; - - if (*speed > 100) - *speed = 100; - - return 0; -} - -static int ci_dpm_set_fan_speed_percent(void *handle, - u32 speed) -{ - u32 tmp; - u32 duty, duty100; - u64 tmp64; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - - if (adev->pm.no_fan) - return -ENOENT; - - if (pi->fan_is_controlled_by_smc) - return -EINVAL; - - if (speed > 100) - return -EINVAL; - - duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK) - >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT; - - if (duty100 == 0) - return -EINVAL; - - tmp64 = (u64)speed * duty100; - do_div(tmp64, 100); - duty = (u32)tmp64; - - tmp = RREG32_SMC(ixCG_FDO_CTRL0) & ~CG_FDO_CTRL0__FDO_STATIC_DUTY_MASK; - tmp |= duty << CG_FDO_CTRL0__FDO_STATIC_DUTY__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL0, tmp); - - return 0; -} - -static void ci_dpm_set_fan_control_mode(void *handle, u32 mode) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - switch (mode) { - case AMD_FAN_CTRL_NONE: - if (adev->pm.dpm.fan.ucode_fan_control) - ci_fan_ctrl_stop_smc_fan_control(adev); - ci_dpm_set_fan_speed_percent(adev, 100); - break; - case AMD_FAN_CTRL_MANUAL: - if (adev->pm.dpm.fan.ucode_fan_control) - ci_fan_ctrl_stop_smc_fan_control(adev); - break; - case AMD_FAN_CTRL_AUTO: - if (adev->pm.dpm.fan.ucode_fan_control) - ci_thermal_start_smc_fan_control(adev); - break; - default: - break; - } -} - -static u32 ci_dpm_get_fan_control_mode(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - - if (pi->fan_is_controlled_by_smc) - return AMD_FAN_CTRL_AUTO; - else - return AMD_FAN_CTRL_MANUAL; -} - -#if 0 -static int ci_fan_ctrl_get_fan_speed_rpm(struct amdgpu_device *adev, - u32 *speed) -{ - u32 tach_period; - u32 xclk = amdgpu_asic_get_xclk(adev); - - if (adev->pm.no_fan) - return -ENOENT; - - if (adev->pm.fan_pulses_per_revolution == 0) - return -ENOENT; - - tach_period = (RREG32_SMC(ixCG_TACH_STATUS) & CG_TACH_STATUS__TACH_PERIOD_MASK) - >> CG_TACH_STATUS__TACH_PERIOD__SHIFT; - if (tach_period == 0) - return -ENOENT; - - *speed = 60 * xclk * 10000 / tach_period; - - return 0; -} - -static int ci_fan_ctrl_set_fan_speed_rpm(struct amdgpu_device *adev, - u32 speed) -{ - u32 tach_period, tmp; - u32 xclk = amdgpu_asic_get_xclk(adev); - - if (adev->pm.no_fan) - return -ENOENT; - - if (adev->pm.fan_pulses_per_revolution == 0) - return -ENOENT; - - if ((speed < adev->pm.fan_min_rpm) || - (speed > adev->pm.fan_max_rpm)) - return -EINVAL; - - if (adev->pm.dpm.fan.ucode_fan_control) - ci_fan_ctrl_stop_smc_fan_control(adev); - - tach_period = 60 * xclk * 10000 / (8 * speed); - tmp = RREG32_SMC(ixCG_TACH_CTRL) & ~CG_TACH_CTRL__TARGET_PERIOD_MASK; - tmp |= tach_period << CG_TACH_CTRL__TARGET_PERIOD__SHIFT; - WREG32_SMC(CG_TACH_CTRL, tmp); - - ci_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC_RPM); - - return 0; -} -#endif - -static void ci_fan_ctrl_set_default_mode(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - - if (!pi->fan_ctrl_is_in_default_mode) { - tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__FDO_PWM_MODE_MASK; - tmp |= pi->fan_ctrl_default_mode << CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL2, tmp); - - tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TMIN_MASK; - tmp |= pi->t_min << CG_FDO_CTRL2__TMIN__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL2, tmp); - pi->fan_ctrl_is_in_default_mode = true; - } -} - -static void ci_thermal_start_smc_fan_control(struct amdgpu_device *adev) -{ - if (adev->pm.dpm.fan.ucode_fan_control) { - ci_fan_ctrl_start_smc_fan_control(adev); - ci_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC); - } -} - -static void ci_thermal_initialize(struct amdgpu_device *adev) -{ - u32 tmp; - - if (adev->pm.fan_pulses_per_revolution) { - tmp = RREG32_SMC(ixCG_TACH_CTRL) & ~CG_TACH_CTRL__EDGE_PER_REV_MASK; - tmp |= (adev->pm.fan_pulses_per_revolution - 1) - << CG_TACH_CTRL__EDGE_PER_REV__SHIFT; - WREG32_SMC(ixCG_TACH_CTRL, tmp); - } - - tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TACH_PWM_RESP_RATE_MASK; - tmp |= 0x28 << CG_FDO_CTRL2__TACH_PWM_RESP_RATE__SHIFT; - WREG32_SMC(ixCG_FDO_CTRL2, tmp); -} - -static int ci_thermal_start_thermal_controller(struct amdgpu_device *adev) -{ - int ret; - - ci_thermal_initialize(adev); - ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN, CISLANDS_TEMP_RANGE_MAX); - if (ret) - return ret; - ret = ci_thermal_enable_alert(adev, true); - if (ret) - return ret; - if (adev->pm.dpm.fan.ucode_fan_control) { - ret = ci_thermal_setup_fan_table(adev); - if (ret) - return ret; - ci_thermal_start_smc_fan_control(adev); - } - - return 0; -} - -static void ci_thermal_stop_thermal_controller(struct amdgpu_device *adev) -{ - if (!adev->pm.no_fan) - ci_fan_ctrl_set_default_mode(adev); -} - -static int ci_read_smc_soft_register(struct amdgpu_device *adev, - u16 reg_offset, u32 *value) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - return amdgpu_ci_read_smc_sram_dword(adev, - pi->soft_regs_start + reg_offset, - value, pi->sram_end); -} - -static int ci_write_smc_soft_register(struct amdgpu_device *adev, - u16 reg_offset, u32 value) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - return amdgpu_ci_write_smc_sram_dword(adev, - pi->soft_regs_start + reg_offset, - value, pi->sram_end); -} - -static void ci_init_fps_limits(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - SMU7_Discrete_DpmTable *table = &pi->smc_state_table; - - if (pi->caps_fps) { - u16 tmp; - - tmp = 45; - table->FpsHighT = cpu_to_be16(tmp); - - tmp = 30; - table->FpsLowT = cpu_to_be16(tmp); - } -} - -static int ci_update_sclk_t(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret = 0; - u32 low_sclk_interrupt_t = 0; - - if (pi->caps_sclk_throttle_low_notification) { - low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t); - - ret = amdgpu_ci_copy_bytes_to_smc(adev, - pi->dpm_table_start + - offsetof(SMU7_Discrete_DpmTable, LowSclkInterruptT), - (u8 *)&low_sclk_interrupt_t, - sizeof(u32), pi->sram_end); - - } - - return ret; -} - -static void ci_get_leakage_voltages(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u16 leakage_id, virtual_voltage_id; - u16 vddc, vddci; - int i; - - pi->vddc_leakage.count = 0; - pi->vddci_leakage.count = 0; - - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { - for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) { - virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i; - if (amdgpu_atombios_get_voltage_evv(adev, virtual_voltage_id, &vddc) != 0) - continue; - if (vddc != 0 && vddc != virtual_voltage_id) { - pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc; - pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id; - pi->vddc_leakage.count++; - } - } - } else if (amdgpu_atombios_get_leakage_id_from_vbios(adev, &leakage_id) == 0) { - for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) { - virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i; - if (amdgpu_atombios_get_leakage_vddc_based_on_leakage_params(adev, &vddc, &vddci, - virtual_voltage_id, - leakage_id) == 0) { - if (vddc != 0 && vddc != virtual_voltage_id) { - pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc; - pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id; - pi->vddc_leakage.count++; - } - if (vddci != 0 && vddci != virtual_voltage_id) { - pi->vddci_leakage.actual_voltage[pi->vddci_leakage.count] = vddci; - pi->vddci_leakage.leakage_id[pi->vddci_leakage.count] = virtual_voltage_id; - pi->vddci_leakage.count++; - } - } - } - } -} - -static void ci_set_dpm_event_sources(struct amdgpu_device *adev, u32 sources) -{ - struct ci_power_info *pi = ci_get_pi(adev); - bool want_thermal_protection; - enum amdgpu_dpm_event_src dpm_event_src; - u32 tmp; - - switch (sources) { - case 0: - default: - want_thermal_protection = false; - break; - case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL): - want_thermal_protection = true; - dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGITAL; - break; - case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL): - want_thermal_protection = true; - dpm_event_src = AMDGPU_DPM_EVENT_SRC_EXTERNAL; - break; - case ((1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL) | - (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL)): - want_thermal_protection = true; - dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL; - break; - } - - if (want_thermal_protection) { -#if 0 - /* XXX: need to figure out how to handle this properly */ - tmp = RREG32_SMC(ixCG_THERMAL_CTRL); - tmp &= DPM_EVENT_SRC_MASK; - tmp |= DPM_EVENT_SRC(dpm_event_src); - WREG32_SMC(ixCG_THERMAL_CTRL, tmp); -#endif - - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - if (pi->thermal_protection) - tmp &= ~GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK; - else - tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - } else { - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - } -} - -static void ci_enable_auto_throttle_source(struct amdgpu_device *adev, - enum amdgpu_dpm_auto_throttle_src source, - bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (enable) { - if (!(pi->active_auto_throttle_sources & (1 << source))) { - pi->active_auto_throttle_sources |= 1 << source; - ci_set_dpm_event_sources(adev, pi->active_auto_throttle_sources); - } - } else { - if (pi->active_auto_throttle_sources & (1 << source)) { - pi->active_auto_throttle_sources &= ~(1 << source); - ci_set_dpm_event_sources(adev, pi->active_auto_throttle_sources); - } - } -} - -static void ci_enable_vr_hot_gpio_interrupt(struct amdgpu_device *adev) -{ - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableVRHotGPIOInterrupt); -} - -static int ci_unfreeze_sclk_mclk_dpm(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - - if (!pi->need_update_smu7_dpm_table) - return 0; - - if ((!pi->sclk_dpm_key_disabled) && - (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_SCLKDPM_UnfreezeLevel); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - if ((!pi->mclk_dpm_key_disabled) && - (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_UnfreezeLevel); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - pi->need_update_smu7_dpm_table = 0; - return 0; -} - -static int ci_enable_sclk_mclk_dpm(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - - if (enable) { - if (!pi->sclk_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DPM_Enable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - if (!pi->mclk_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_Enable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - - WREG32_P(mmMC_SEQ_CNTL_3, MC_SEQ_CNTL_3__CAC_EN_MASK, - ~MC_SEQ_CNTL_3__CAC_EN_MASK); - - WREG32_SMC(ixLCAC_MC0_CNTL, 0x05); - WREG32_SMC(ixLCAC_MC1_CNTL, 0x05); - WREG32_SMC(ixLCAC_CPL_CNTL, 0x100005); - - udelay(10); - - WREG32_SMC(ixLCAC_MC0_CNTL, 0x400005); - WREG32_SMC(ixLCAC_MC1_CNTL, 0x400005); - WREG32_SMC(ixLCAC_CPL_CNTL, 0x500005); - } - } else { - if (!pi->sclk_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DPM_Disable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - if (!pi->mclk_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_Disable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - } - - return 0; -} - -static int ci_start_dpm(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - int ret; - u32 tmp; - - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - tmp |= GENERAL_PWRMGT__GLOBAL_PWRMGT_EN_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - - tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL); - tmp |= SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN_MASK; - WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp); - - ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, VoltageChangeTimeout), 0x1000); - - WREG32_P(mmBIF_LNCNT_RESET, 0, ~BIF_LNCNT_RESET__RESET_LNCNT_EN_MASK); - - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Voltage_Cntl_Enable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - - ret = ci_enable_sclk_mclk_dpm(adev, true); - if (ret) - return ret; - - if (!pi->pcie_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PCIeDPM_Enable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_freeze_sclk_mclk_dpm(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - - if (!pi->need_update_smu7_dpm_table) - return 0; - - if ((!pi->sclk_dpm_key_disabled) && - (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_SCLKDPM_FreezeLevel); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - if ((!pi->mclk_dpm_key_disabled) && - (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_FreezeLevel); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_stop_dpm(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result smc_result; - int ret; - u32 tmp; - - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - tmp &= ~GENERAL_PWRMGT__GLOBAL_PWRMGT_EN_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - - tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL); - tmp &= ~SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN_MASK; - WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp); - - if (!pi->pcie_dpm_key_disabled) { - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PCIeDPM_Disable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - ret = ci_enable_sclk_mclk_dpm(adev, false); - if (ret) - return ret; - - smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Voltage_Cntl_Disable); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - - return 0; -} - -static void ci_enable_sclk_control(struct amdgpu_device *adev, bool enable) -{ - u32 tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL); - - if (enable) - tmp &= ~SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF_MASK; - else - tmp |= SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF_MASK; - WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp); -} - -#if 0 -static int ci_notify_hw_of_power_source(struct amdgpu_device *adev, - bool ac_power) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_cac_tdp_table *cac_tdp_table = - adev->pm.dpm.dyn_state.cac_tdp_table; - u32 power_limit; - - if (ac_power) - power_limit = (u32)(cac_tdp_table->maximum_power_delivery_limit * 256); - else - power_limit = (u32)(cac_tdp_table->battery_power_limit * 256); - - ci_set_power_limit(adev, power_limit); - - if (pi->caps_automatic_dc_transition) { - if (ac_power) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_RunningOnAC); - else - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Remove_DC_Clamp); - } - - return 0; -} -#endif - -static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev, - PPSMC_Msg msg, u32 parameter) -{ - WREG32(mmSMC_MSG_ARG_0, parameter); - return amdgpu_ci_send_msg_to_smc(adev, msg); -} - -static PPSMC_Result amdgpu_ci_send_msg_to_smc_return_parameter(struct amdgpu_device *adev, - PPSMC_Msg msg, u32 *parameter) -{ - PPSMC_Result smc_result; - - smc_result = amdgpu_ci_send_msg_to_smc(adev, msg); - - if ((smc_result == PPSMC_Result_OK) && parameter) - *parameter = RREG32(mmSMC_MSG_ARG_0); - - return smc_result; -} - -static int ci_dpm_force_state_sclk(struct amdgpu_device *adev, u32 n) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (!pi->sclk_dpm_key_disabled) { - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SCLKDPM_SetEnabledMask, 1 << n); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_dpm_force_state_mclk(struct amdgpu_device *adev, u32 n) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (!pi->mclk_dpm_key_disabled) { - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_MCLKDPM_SetEnabledMask, 1 << n); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_dpm_force_state_pcie(struct amdgpu_device *adev, u32 n) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (!pi->pcie_dpm_key_disabled) { - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_PCIeDPM_ForceLevel, n); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_set_power_limit(struct amdgpu_device *adev, u32 n) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) { - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_PkgPwrSetLimit, n); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - - return 0; -} - -static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev, - u32 target_tdp) -{ - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_OverDriveSetTargetTdp, target_tdp); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - return 0; -} - -#if 0 -static int ci_set_boot_state(struct amdgpu_device *adev) -{ - return ci_enable_sclk_mclk_dpm(adev, false); -} -#endif - -static u32 ci_get_average_sclk_freq(struct amdgpu_device *adev) -{ - u32 sclk_freq; - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_return_parameter(adev, - PPSMC_MSG_API_GetSclkFrequency, - &sclk_freq); - if (smc_result != PPSMC_Result_OK) - sclk_freq = 0; - - return sclk_freq; -} - -static u32 ci_get_average_mclk_freq(struct amdgpu_device *adev) -{ - u32 mclk_freq; - PPSMC_Result smc_result = - amdgpu_ci_send_msg_to_smc_return_parameter(adev, - PPSMC_MSG_API_GetMclkFrequency, - &mclk_freq); - if (smc_result != PPSMC_Result_OK) - mclk_freq = 0; - - return mclk_freq; -} - -static void ci_dpm_start_smc(struct amdgpu_device *adev) -{ - int i; - - amdgpu_ci_program_jump_on_start(adev); - amdgpu_ci_start_smc_clock(adev); - amdgpu_ci_start_smc(adev); - for (i = 0; i < adev->usec_timeout; i++) { - if (RREG32_SMC(ixFIRMWARE_FLAGS) & FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) - break; - } -} - -static void ci_dpm_stop_smc(struct amdgpu_device *adev) -{ - amdgpu_ci_reset_smc(adev); - amdgpu_ci_stop_smc_clock(adev); -} - -static int ci_process_firmware_header(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - int ret; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, DpmTable), - &tmp, pi->sram_end); - if (ret) - return ret; - - pi->dpm_table_start = tmp; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, SoftRegisters), - &tmp, pi->sram_end); - if (ret) - return ret; - - pi->soft_regs_start = tmp; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, mcRegisterTable), - &tmp, pi->sram_end); - if (ret) - return ret; - - pi->mc_reg_table_start = tmp; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, FanTable), - &tmp, pi->sram_end); - if (ret) - return ret; - - pi->fan_table_start = tmp; - - ret = amdgpu_ci_read_smc_sram_dword(adev, - SMU7_FIRMWARE_HEADER_LOCATION + - offsetof(SMU7_Firmware_Header, mcArbDramTimingTable), - &tmp, pi->sram_end); - if (ret) - return ret; - - pi->arb_table_start = tmp; - - return 0; -} - -static void ci_read_clock_registers(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - pi->clock_registers.cg_spll_func_cntl = - RREG32_SMC(ixCG_SPLL_FUNC_CNTL); - pi->clock_registers.cg_spll_func_cntl_2 = - RREG32_SMC(ixCG_SPLL_FUNC_CNTL_2); - pi->clock_registers.cg_spll_func_cntl_3 = - RREG32_SMC(ixCG_SPLL_FUNC_CNTL_3); - pi->clock_registers.cg_spll_func_cntl_4 = - RREG32_SMC(ixCG_SPLL_FUNC_CNTL_4); - pi->clock_registers.cg_spll_spread_spectrum = - RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM); - pi->clock_registers.cg_spll_spread_spectrum_2 = - RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM_2); - pi->clock_registers.dll_cntl = RREG32(mmDLL_CNTL); - pi->clock_registers.mclk_pwrmgt_cntl = RREG32(mmMCLK_PWRMGT_CNTL); - pi->clock_registers.mpll_ad_func_cntl = RREG32(mmMPLL_AD_FUNC_CNTL); - pi->clock_registers.mpll_dq_func_cntl = RREG32(mmMPLL_DQ_FUNC_CNTL); - pi->clock_registers.mpll_func_cntl = RREG32(mmMPLL_FUNC_CNTL); - pi->clock_registers.mpll_func_cntl_1 = RREG32(mmMPLL_FUNC_CNTL_1); - pi->clock_registers.mpll_func_cntl_2 = RREG32(mmMPLL_FUNC_CNTL_2); - pi->clock_registers.mpll_ss1 = RREG32(mmMPLL_SS1); - pi->clock_registers.mpll_ss2 = RREG32(mmMPLL_SS2); -} - -static void ci_init_sclk_t(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - pi->low_sclk_interrupt_t = 0; -} - -static void ci_enable_thermal_protection(struct amdgpu_device *adev, - bool enable) -{ - u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT); - - if (enable) - tmp &= ~GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK; - else - tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); -} - -static void ci_enable_acpi_power_management(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT); - - tmp |= GENERAL_PWRMGT__STATIC_PM_EN_MASK; - - WREG32_SMC(ixGENERAL_PWRMGT, tmp); -} - -#if 0 -static int ci_enter_ulp_state(struct amdgpu_device *adev) -{ - - WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower); - - udelay(25000); - - return 0; -} - -static int ci_exit_ulp_state(struct amdgpu_device *adev) -{ - int i; - - WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower); - - udelay(7000); - - for (i = 0; i < adev->usec_timeout; i++) { - if (RREG32(mmSMC_RESP_0) == 1) - break; - udelay(1000); - } - - return 0; -} -#endif - -static int ci_notify_smc_display_change(struct amdgpu_device *adev, - bool has_display) -{ - PPSMC_Msg msg = has_display ? PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay; - - return (amdgpu_ci_send_msg_to_smc(adev, msg) == PPSMC_Result_OK) ? 0 : -EINVAL; -} - -static int ci_enable_ds_master_switch(struct amdgpu_device *adev, - bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (enable) { - if (pi->caps_sclk_ds) { - if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_ON) != PPSMC_Result_OK) - return -EINVAL; - } else { - if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK) - return -EINVAL; - } - } else { - if (pi->caps_sclk_ds) { - if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK) - return -EINVAL; - } - } - - return 0; -} - -static void ci_program_display_gap(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixCG_DISPLAY_GAP_CNTL); - u32 pre_vbi_time_in_us; - u32 frame_time_in_us; - u32 ref_clock = adev->clock.spll.reference_freq; - u32 refresh_rate = amdgpu_dpm_get_vrefresh(adev); - u32 vblank_time = amdgpu_dpm_get_vblank_time(adev); - - tmp &= ~CG_DISPLAY_GAP_CNTL__DISP_GAP_MASK; - if (adev->pm.dpm.new_active_crtc_count > 0) - tmp |= (AMDGPU_PM_DISPLAY_GAP_VBLANK_OR_WM << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT); - else - tmp |= (AMDGPU_PM_DISPLAY_GAP_IGNORE << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT); - WREG32_SMC(ixCG_DISPLAY_GAP_CNTL, tmp); - - if (refresh_rate == 0) - refresh_rate = 60; - if (vblank_time == 0xffffffff) - vblank_time = 500; - frame_time_in_us = 1000000 / refresh_rate; - pre_vbi_time_in_us = - frame_time_in_us - 200 - vblank_time; - tmp = pre_vbi_time_in_us * (ref_clock / 100); - - WREG32_SMC(ixCG_DISPLAY_GAP_CNTL2, tmp); - ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, PreVBlankGap), 0x64); - ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us)); - - - ci_notify_smc_display_change(adev, (adev->pm.dpm.new_active_crtc_count == 1)); - -} - -static void ci_enable_spread_spectrum(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - - if (enable) { - if (pi->caps_sclk_ss_support) { - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - tmp |= GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - } - } else { - tmp = RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM); - tmp &= ~CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK; - WREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM, tmp); - - tmp = RREG32_SMC(ixGENERAL_PWRMGT); - tmp &= ~GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); - } -} - -static void ci_program_sstp(struct amdgpu_device *adev) -{ - WREG32_SMC(ixCG_STATIC_SCREEN_PARAMETER, - ((CISLANDS_SSTU_DFLT << CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD_UNIT__SHIFT) | - (CISLANDS_SST_DFLT << CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD__SHIFT))); -} - -static void ci_enable_display_gap(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixCG_DISPLAY_GAP_CNTL); - - tmp &= ~(CG_DISPLAY_GAP_CNTL__DISP_GAP_MASK | - CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG_MASK); - tmp |= ((AMDGPU_PM_DISPLAY_GAP_IGNORE << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT) | - (AMDGPU_PM_DISPLAY_GAP_VBLANK << CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG__SHIFT)); - - WREG32_SMC(ixCG_DISPLAY_GAP_CNTL, tmp); -} - -static void ci_program_vc(struct amdgpu_device *adev) -{ - u32 tmp; - - tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL); - tmp &= ~(SCLK_PWRMGT_CNTL__RESET_SCLK_CNT_MASK | SCLK_PWRMGT_CNTL__RESET_BUSY_CNT_MASK); - WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp); - - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, CISLANDS_VRC_DFLT0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_1, CISLANDS_VRC_DFLT1); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_2, CISLANDS_VRC_DFLT2); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_3, CISLANDS_VRC_DFLT3); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_4, CISLANDS_VRC_DFLT4); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_5, CISLANDS_VRC_DFLT5); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_6, CISLANDS_VRC_DFLT6); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_7, CISLANDS_VRC_DFLT7); -} - -static void ci_clear_vc(struct amdgpu_device *adev) -{ - u32 tmp; - - tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL); - tmp |= (SCLK_PWRMGT_CNTL__RESET_SCLK_CNT_MASK | SCLK_PWRMGT_CNTL__RESET_BUSY_CNT_MASK); - WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp); - - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_1, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_2, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_3, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_4, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_5, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_6, 0); - WREG32_SMC(ixCG_FREQ_TRAN_VOTING_7, 0); -} - -static int ci_upload_firmware(struct amdgpu_device *adev) -{ - int i, ret; - - if (amdgpu_ci_is_smc_running(adev)) { - DRM_INFO("smc is running, no need to load smc firmware\n"); - return 0; - } - - for (i = 0; i < adev->usec_timeout; i++) { - if (RREG32_SMC(ixRCU_UC_EVENTS) & RCU_UC_EVENTS__boot_seq_done_MASK) - break; - } - WREG32_SMC(ixSMC_SYSCON_MISC_CNTL, 1); - - amdgpu_ci_stop_smc_clock(adev); - amdgpu_ci_reset_smc(adev); - - ret = amdgpu_ci_load_smc_ucode(adev, SMC_RAM_END); - - return ret; - -} - -static int ci_get_svi2_voltage_table(struct amdgpu_device *adev, - struct amdgpu_clock_voltage_dependency_table *voltage_dependency_table, - struct atom_voltage_table *voltage_table) -{ - u32 i; - - if (voltage_dependency_table == NULL) - return -EINVAL; - - voltage_table->mask_low = 0; - voltage_table->phase_delay = 0; - - voltage_table->count = voltage_dependency_table->count; - for (i = 0; i < voltage_table->count; i++) { - voltage_table->entries[i].value = voltage_dependency_table->entries[i].v; - voltage_table->entries[i].smio_low = 0; - } - - return 0; -} - -static int ci_construct_voltage_tables(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - - if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { - ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDC, - VOLTAGE_OBJ_GPIO_LUT, - &pi->vddc_voltage_table); - if (ret) - return ret; - } else if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { - ret = ci_get_svi2_voltage_table(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk, - &pi->vddc_voltage_table); - if (ret) - return ret; - } - - if (pi->vddc_voltage_table.count > SMU7_MAX_LEVELS_VDDC) - ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_VDDC, - &pi->vddc_voltage_table); - - if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { - ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDCI, - VOLTAGE_OBJ_GPIO_LUT, - &pi->vddci_voltage_table); - if (ret) - return ret; - } else if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { - ret = ci_get_svi2_voltage_table(adev, - &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk, - &pi->vddci_voltage_table); - if (ret) - return ret; - } - - if (pi->vddci_voltage_table.count > SMU7_MAX_LEVELS_VDDCI) - ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_VDDCI, - &pi->vddci_voltage_table); - - if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { - ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_MVDDC, - VOLTAGE_OBJ_GPIO_LUT, - &pi->mvdd_voltage_table); - if (ret) - return ret; - } else if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { - ret = ci_get_svi2_voltage_table(adev, - &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, - &pi->mvdd_voltage_table); - if (ret) - return ret; - } - - if (pi->mvdd_voltage_table.count > SMU7_MAX_LEVELS_MVDD) - ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_MVDD, - &pi->mvdd_voltage_table); - - return 0; -} - -static void ci_populate_smc_voltage_table(struct amdgpu_device *adev, - struct atom_voltage_table_entry *voltage_table, - SMU7_Discrete_VoltageLevel *smc_voltage_table) -{ - int ret; - - ret = ci_get_std_voltage_value_sidd(adev, voltage_table, - &smc_voltage_table->StdVoltageHiSidd, - &smc_voltage_table->StdVoltageLoSidd); - - if (ret) { - smc_voltage_table->StdVoltageHiSidd = voltage_table->value * VOLTAGE_SCALE; - smc_voltage_table->StdVoltageLoSidd = voltage_table->value * VOLTAGE_SCALE; - } - - smc_voltage_table->Voltage = cpu_to_be16(voltage_table->value * VOLTAGE_SCALE); - smc_voltage_table->StdVoltageHiSidd = - cpu_to_be16(smc_voltage_table->StdVoltageHiSidd); - smc_voltage_table->StdVoltageLoSidd = - cpu_to_be16(smc_voltage_table->StdVoltageLoSidd); -} - -static int ci_populate_smc_vddc_table(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - unsigned int count; - - table->VddcLevelCount = pi->vddc_voltage_table.count; - for (count = 0; count < table->VddcLevelCount; count++) { - ci_populate_smc_voltage_table(adev, - &pi->vddc_voltage_table.entries[count], - &table->VddcLevel[count]); - - if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) - table->VddcLevel[count].Smio |= - pi->vddc_voltage_table.entries[count].smio_low; - else - table->VddcLevel[count].Smio = 0; - } - table->VddcLevelCount = cpu_to_be32(table->VddcLevelCount); - - return 0; -} - -static int ci_populate_smc_vddci_table(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - unsigned int count; - struct ci_power_info *pi = ci_get_pi(adev); - - table->VddciLevelCount = pi->vddci_voltage_table.count; - for (count = 0; count < table->VddciLevelCount; count++) { - ci_populate_smc_voltage_table(adev, - &pi->vddci_voltage_table.entries[count], - &table->VddciLevel[count]); - - if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) - table->VddciLevel[count].Smio |= - pi->vddci_voltage_table.entries[count].smio_low; - else - table->VddciLevel[count].Smio = 0; - } - table->VddciLevelCount = cpu_to_be32(table->VddciLevelCount); - - return 0; -} - -static int ci_populate_smc_mvdd_table(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - unsigned int count; - - table->MvddLevelCount = pi->mvdd_voltage_table.count; - for (count = 0; count < table->MvddLevelCount; count++) { - ci_populate_smc_voltage_table(adev, - &pi->mvdd_voltage_table.entries[count], - &table->MvddLevel[count]); - - if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) - table->MvddLevel[count].Smio |= - pi->mvdd_voltage_table.entries[count].smio_low; - else - table->MvddLevel[count].Smio = 0; - } - table->MvddLevelCount = cpu_to_be32(table->MvddLevelCount); - - return 0; -} - -static int ci_populate_smc_voltage_tables(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - int ret; - - ret = ci_populate_smc_vddc_table(adev, table); - if (ret) - return ret; - - ret = ci_populate_smc_vddci_table(adev, table); - if (ret) - return ret; - - ret = ci_populate_smc_mvdd_table(adev, table); - if (ret) - return ret; - - return 0; -} - -static int ci_populate_mvdd_value(struct amdgpu_device *adev, u32 mclk, - SMU7_Discrete_VoltageLevel *voltage) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 i = 0; - - if (pi->mvdd_control != CISLANDS_VOLTAGE_CONTROL_NONE) { - for (i = 0; i < adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count; i++) { - if (mclk <= adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries[i].clk) { - voltage->Voltage = pi->mvdd_voltage_table.entries[i].value; - break; - } - } - - if (i >= adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count) - return -EINVAL; - } - - return -EINVAL; -} - -static int ci_get_std_voltage_value_sidd(struct amdgpu_device *adev, - struct atom_voltage_table_entry *voltage_table, - u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd) -{ - u16 v_index, idx; - bool voltage_found = false; - *std_voltage_hi_sidd = voltage_table->value * VOLTAGE_SCALE; - *std_voltage_lo_sidd = voltage_table->value * VOLTAGE_SCALE; - - if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL) - return -EINVAL; - - if (adev->pm.dpm.dyn_state.cac_leakage_table.entries) { - for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { - if (voltage_table->value == - adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { - voltage_found = true; - if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count) - idx = v_index; - else - idx = adev->pm.dpm.dyn_state.cac_leakage_table.count - 1; - *std_voltage_lo_sidd = - adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE; - *std_voltage_hi_sidd = - adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE; - break; - } - } - - if (!voltage_found) { - for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { - if (voltage_table->value <= - adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { - voltage_found = true; - if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count) - idx = v_index; - else - idx = adev->pm.dpm.dyn_state.cac_leakage_table.count - 1; - *std_voltage_lo_sidd = - adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE; - *std_voltage_hi_sidd = - adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE; - break; - } - } - } - } - - return 0; -} - -static void ci_populate_phase_value_based_on_sclk(struct amdgpu_device *adev, - const struct amdgpu_phase_shedding_limits_table *limits, - u32 sclk, - u32 *phase_shedding) -{ - unsigned int i; - - *phase_shedding = 1; - - for (i = 0; i < limits->count; i++) { - if (sclk < limits->entries[i].sclk) { - *phase_shedding = i; - break; - } - } -} - -static void ci_populate_phase_value_based_on_mclk(struct amdgpu_device *adev, - const struct amdgpu_phase_shedding_limits_table *limits, - u32 mclk, - u32 *phase_shedding) -{ - unsigned int i; - - *phase_shedding = 1; - - for (i = 0; i < limits->count; i++) { - if (mclk < limits->entries[i].mclk) { - *phase_shedding = i; - break; - } - } -} - -static int ci_init_arb_table_index(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - int ret; - - ret = amdgpu_ci_read_smc_sram_dword(adev, pi->arb_table_start, - &tmp, pi->sram_end); - if (ret) - return ret; - - tmp &= 0x00FFFFFF; - tmp |= MC_CG_ARB_FREQ_F1 << 24; - - return amdgpu_ci_write_smc_sram_dword(adev, pi->arb_table_start, - tmp, pi->sram_end); -} - -static int ci_get_dependency_volt_by_clk(struct amdgpu_device *adev, - struct amdgpu_clock_voltage_dependency_table *allowed_clock_voltage_table, - u32 clock, u32 *voltage) -{ - u32 i = 0; - - if (allowed_clock_voltage_table->count == 0) - return -EINVAL; - - for (i = 0; i < allowed_clock_voltage_table->count; i++) { - if (allowed_clock_voltage_table->entries[i].clk >= clock) { - *voltage = allowed_clock_voltage_table->entries[i].v; - return 0; - } - } - - *voltage = allowed_clock_voltage_table->entries[i-1].v; - - return 0; -} - -static u8 ci_get_sleep_divider_id_from_clock(u32 sclk, u32 min_sclk_in_sr) -{ - u32 i; - u32 tmp; - u32 min = max(min_sclk_in_sr, (u32)CISLAND_MINIMUM_ENGINE_CLOCK); - - if (sclk < min) - return 0; - - for (i = CISLAND_MAX_DEEPSLEEP_DIVIDER_ID; ; i--) { - tmp = sclk >> i; - if (tmp >= min || i == 0) - break; - } - - return (u8)i; -} - -static int ci_initial_switch_from_arb_f0_to_f1(struct amdgpu_device *adev) -{ - return ci_copy_and_switch_arb_sets(adev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); -} - -static int ci_reset_to_default(struct amdgpu_device *adev) -{ - return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ? - 0 : -EINVAL; -} - -static int ci_force_switch_to_arb_f0(struct amdgpu_device *adev) -{ - u32 tmp; - - tmp = (RREG32_SMC(ixSMC_SCRATCH9) & 0x0000ff00) >> 8; - - if (tmp == MC_CG_ARB_FREQ_F0) - return 0; - - return ci_copy_and_switch_arb_sets(adev, tmp, MC_CG_ARB_FREQ_F0); -} - -static void ci_register_patching_mc_arb(struct amdgpu_device *adev, - const u32 engine_clock, - const u32 memory_clock, - u32 *dram_timimg2) -{ - bool patch; - u32 tmp, tmp2; - - tmp = RREG32(mmMC_SEQ_MISC0); - patch = ((tmp & 0x0000f00) == 0x300) ? true : false; - - if (patch && - ((adev->pdev->device == 0x67B0) || - (adev->pdev->device == 0x67B1))) { - if ((memory_clock > 100000) && (memory_clock <= 125000)) { - tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff; - *dram_timimg2 &= ~0x00ff0000; - *dram_timimg2 |= tmp2 << 16; - } else if ((memory_clock > 125000) && (memory_clock <= 137500)) { - tmp2 = (((0x36 * engine_clock) / 137500) - 1) & 0xff; - *dram_timimg2 &= ~0x00ff0000; - *dram_timimg2 |= tmp2 << 16; - } - } -} - -static int ci_populate_memory_timing_parameters(struct amdgpu_device *adev, - u32 sclk, - u32 mclk, - SMU7_Discrete_MCArbDramTimingTableEntry *arb_regs) -{ - u32 dram_timing; - u32 dram_timing2; - u32 burst_time; - - amdgpu_atombios_set_engine_dram_timings(adev, sclk, mclk); - - dram_timing = RREG32(mmMC_ARB_DRAM_TIMING); - dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2); - burst_time = RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE0_MASK; - - ci_register_patching_mc_arb(adev, sclk, mclk, &dram_timing2); - - arb_regs->McArbDramTiming = cpu_to_be32(dram_timing); - arb_regs->McArbDramTiming2 = cpu_to_be32(dram_timing2); - arb_regs->McArbBurstTime = (u8)burst_time; - - return 0; -} - -static int ci_do_program_memory_timing_parameters(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - SMU7_Discrete_MCArbDramTimingTable arb_regs; - u32 i, j; - int ret = 0; - - memset(&arb_regs, 0, sizeof(SMU7_Discrete_MCArbDramTimingTable)); - - for (i = 0; i < pi->dpm_table.sclk_table.count; i++) { - for (j = 0; j < pi->dpm_table.mclk_table.count; j++) { - ret = ci_populate_memory_timing_parameters(adev, - pi->dpm_table.sclk_table.dpm_levels[i].value, - pi->dpm_table.mclk_table.dpm_levels[j].value, - &arb_regs.entries[i][j]); - if (ret) - break; - } - } - - if (ret == 0) - ret = amdgpu_ci_copy_bytes_to_smc(adev, - pi->arb_table_start, - (u8 *)&arb_regs, - sizeof(SMU7_Discrete_MCArbDramTimingTable), - pi->sram_end); - - return ret; -} - -static int ci_program_memory_timing_parameters(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (pi->need_update_smu7_dpm_table == 0) - return 0; - - return ci_do_program_memory_timing_parameters(adev); -} - -static void ci_populate_smc_initial_state(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_boot_state) -{ - struct ci_ps *boot_state = ci_get_ps(amdgpu_boot_state); - struct ci_power_info *pi = ci_get_pi(adev); - u32 level = 0; - - for (level = 0; level < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; level++) { - if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[level].clk >= - boot_state->performance_levels[0].sclk) { - pi->smc_state_table.GraphicsBootLevel = level; - break; - } - } - - for (level = 0; level < adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.count; level++) { - if (adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries[level].clk >= - boot_state->performance_levels[0].mclk) { - pi->smc_state_table.MemoryBootLevel = level; - break; - } - } -} - -static u32 ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table *dpm_table) -{ - u32 i; - u32 mask_value = 0; - - for (i = dpm_table->count; i > 0; i--) { - mask_value = mask_value << 1; - if (dpm_table->dpm_levels[i-1].enabled) - mask_value |= 0x1; - else - mask_value &= 0xFFFFFFFE; - } - - return mask_value; -} - -static void ci_populate_smc_link_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_dpm_table *dpm_table = &pi->dpm_table; - u32 i; - - for (i = 0; i < dpm_table->pcie_speed_table.count; i++) { - table->LinkLevel[i].PcieGenSpeed = - (u8)dpm_table->pcie_speed_table.dpm_levels[i].value; - table->LinkLevel[i].PcieLaneCount = - amdgpu_encode_pci_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1); - table->LinkLevel[i].EnabledForActivity = 1; - table->LinkLevel[i].DownT = cpu_to_be32(5); - table->LinkLevel[i].UpT = cpu_to_be32(30); - } - - pi->smc_state_table.LinkLevelCount = (u8)dpm_table->pcie_speed_table.count; - pi->dpm_level_enable_mask.pcie_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table); -} - -static int ci_populate_smc_uvd_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - u32 count; - struct atom_clock_dividers dividers; - int ret = -EINVAL; - - table->UvdLevelCount = - adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count; - - for (count = 0; count < table->UvdLevelCount; count++) { - table->UvdLevel[count].VclkFrequency = - adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].vclk; - table->UvdLevel[count].DclkFrequency = - adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].dclk; - table->UvdLevel[count].MinVddc = - adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; - table->UvdLevel[count].MinVddcPhases = 1; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, - table->UvdLevel[count].VclkFrequency, false, ÷rs); - if (ret) - return ret; - - table->UvdLevel[count].VclkDivider = (u8)dividers.post_divider; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, - table->UvdLevel[count].DclkFrequency, false, ÷rs); - if (ret) - return ret; - - table->UvdLevel[count].DclkDivider = (u8)dividers.post_divider; - - table->UvdLevel[count].VclkFrequency = cpu_to_be32(table->UvdLevel[count].VclkFrequency); - table->UvdLevel[count].DclkFrequency = cpu_to_be32(table->UvdLevel[count].DclkFrequency); - table->UvdLevel[count].MinVddc = cpu_to_be16(table->UvdLevel[count].MinVddc); - } - - return ret; -} - -static int ci_populate_smc_vce_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - u32 count; - struct atom_clock_dividers dividers; - int ret = -EINVAL; - - table->VceLevelCount = - adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count; - - for (count = 0; count < table->VceLevelCount; count++) { - table->VceLevel[count].Frequency = - adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].evclk; - table->VceLevel[count].MinVoltage = - (u16)adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; - table->VceLevel[count].MinPhases = 1; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, - table->VceLevel[count].Frequency, false, ÷rs); - if (ret) - return ret; - - table->VceLevel[count].Divider = (u8)dividers.post_divider; - - table->VceLevel[count].Frequency = cpu_to_be32(table->VceLevel[count].Frequency); - table->VceLevel[count].MinVoltage = cpu_to_be16(table->VceLevel[count].MinVoltage); - } - - return ret; - -} - -static int ci_populate_smc_acp_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - u32 count; - struct atom_clock_dividers dividers; - int ret = -EINVAL; - - table->AcpLevelCount = (u8) - (adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count); - - for (count = 0; count < table->AcpLevelCount; count++) { - table->AcpLevel[count].Frequency = - adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].clk; - table->AcpLevel[count].MinVoltage = - adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].v; - table->AcpLevel[count].MinPhases = 1; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, - table->AcpLevel[count].Frequency, false, ÷rs); - if (ret) - return ret; - - table->AcpLevel[count].Divider = (u8)dividers.post_divider; - - table->AcpLevel[count].Frequency = cpu_to_be32(table->AcpLevel[count].Frequency); - table->AcpLevel[count].MinVoltage = cpu_to_be16(table->AcpLevel[count].MinVoltage); - } - - return ret; -} - -static int ci_populate_smc_samu_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - u32 count; - struct atom_clock_dividers dividers; - int ret = -EINVAL; - - table->SamuLevelCount = - adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count; - - for (count = 0; count < table->SamuLevelCount; count++) { - table->SamuLevel[count].Frequency = - adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].clk; - table->SamuLevel[count].MinVoltage = - adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; - table->SamuLevel[count].MinPhases = 1; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, - table->SamuLevel[count].Frequency, false, ÷rs); - if (ret) - return ret; - - table->SamuLevel[count].Divider = (u8)dividers.post_divider; - - table->SamuLevel[count].Frequency = cpu_to_be32(table->SamuLevel[count].Frequency); - table->SamuLevel[count].MinVoltage = cpu_to_be16(table->SamuLevel[count].MinVoltage); - } - - return ret; -} - -static int ci_calculate_mclk_params(struct amdgpu_device *adev, - u32 memory_clock, - SMU7_Discrete_MemoryLevel *mclk, - bool strobe_mode, - bool dll_state_on) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 dll_cntl = pi->clock_registers.dll_cntl; - u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl; - u32 mpll_ad_func_cntl = pi->clock_registers.mpll_ad_func_cntl; - u32 mpll_dq_func_cntl = pi->clock_registers.mpll_dq_func_cntl; - u32 mpll_func_cntl = pi->clock_registers.mpll_func_cntl; - u32 mpll_func_cntl_1 = pi->clock_registers.mpll_func_cntl_1; - u32 mpll_func_cntl_2 = pi->clock_registers.mpll_func_cntl_2; - u32 mpll_ss1 = pi->clock_registers.mpll_ss1; - u32 mpll_ss2 = pi->clock_registers.mpll_ss2; - struct atom_mpll_param mpll_param; - int ret; - - ret = amdgpu_atombios_get_memory_pll_dividers(adev, memory_clock, strobe_mode, &mpll_param); - if (ret) - return ret; - - mpll_func_cntl &= ~MPLL_FUNC_CNTL__BWCTRL_MASK; - mpll_func_cntl |= (mpll_param.bwcntl << MPLL_FUNC_CNTL__BWCTRL__SHIFT); - - mpll_func_cntl_1 &= ~(MPLL_FUNC_CNTL_1__CLKF_MASK | MPLL_FUNC_CNTL_1__CLKFRAC_MASK | - MPLL_FUNC_CNTL_1__VCO_MODE_MASK); - mpll_func_cntl_1 |= (mpll_param.clkf) << MPLL_FUNC_CNTL_1__CLKF__SHIFT | - (mpll_param.clkfrac << MPLL_FUNC_CNTL_1__CLKFRAC__SHIFT) | - (mpll_param.vco_mode << MPLL_FUNC_CNTL_1__VCO_MODE__SHIFT); - - mpll_ad_func_cntl &= ~MPLL_AD_FUNC_CNTL__YCLK_POST_DIV_MASK; - mpll_ad_func_cntl |= (mpll_param.post_div << MPLL_AD_FUNC_CNTL__YCLK_POST_DIV__SHIFT); - - if (adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) { - mpll_dq_func_cntl &= ~(MPLL_DQ_FUNC_CNTL__YCLK_SEL_MASK | - MPLL_AD_FUNC_CNTL__YCLK_POST_DIV_MASK); - mpll_dq_func_cntl |= (mpll_param.yclk_sel << MPLL_DQ_FUNC_CNTL__YCLK_SEL__SHIFT) | - (mpll_param.post_div << MPLL_AD_FUNC_CNTL__YCLK_POST_DIV__SHIFT); - } - - if (pi->caps_mclk_ss_support) { - struct amdgpu_atom_ss ss; - u32 freq_nom; - u32 tmp; - u32 reference_clock = adev->clock.mpll.reference_freq; - - if (mpll_param.qdr == 1) - freq_nom = memory_clock * 4 * (1 << mpll_param.post_div); - else - freq_nom = memory_clock * 2 * (1 << mpll_param.post_div); - - tmp = (freq_nom / reference_clock); - tmp = tmp * tmp; - if (amdgpu_atombios_get_asic_ss_info(adev, &ss, - ASIC_INTERNAL_MEMORY_SS, freq_nom)) { - u32 clks = reference_clock * 5 / ss.rate; - u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom); - - mpll_ss1 &= ~MPLL_SS1__CLKV_MASK; - mpll_ss1 |= (clkv << MPLL_SS1__CLKV__SHIFT); - - mpll_ss2 &= ~MPLL_SS2__CLKS_MASK; - mpll_ss2 |= (clks << MPLL_SS2__CLKS__SHIFT); - } - } - - mclk_pwrmgt_cntl &= ~MCLK_PWRMGT_CNTL__DLL_SPEED_MASK; - mclk_pwrmgt_cntl |= (mpll_param.dll_speed << MCLK_PWRMGT_CNTL__DLL_SPEED__SHIFT); - - if (dll_state_on) - mclk_pwrmgt_cntl |= MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK | - MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK; - else - mclk_pwrmgt_cntl &= ~(MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK | - MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK); - - mclk->MclkFrequency = memory_clock; - mclk->MpllFuncCntl = mpll_func_cntl; - mclk->MpllFuncCntl_1 = mpll_func_cntl_1; - mclk->MpllFuncCntl_2 = mpll_func_cntl_2; - mclk->MpllAdFuncCntl = mpll_ad_func_cntl; - mclk->MpllDqFuncCntl = mpll_dq_func_cntl; - mclk->MclkPwrmgtCntl = mclk_pwrmgt_cntl; - mclk->DllCntl = dll_cntl; - mclk->MpllSs1 = mpll_ss1; - mclk->MpllSs2 = mpll_ss2; - - return 0; -} - -static int ci_populate_single_memory_level(struct amdgpu_device *adev, - u32 memory_clock, - SMU7_Discrete_MemoryLevel *memory_level) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - bool dll_state_on; - - if (adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries) { - ret = ci_get_dependency_volt_by_clk(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk, - memory_clock, &memory_level->MinVddc); - if (ret) - return ret; - } - - if (adev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries) { - ret = ci_get_dependency_volt_by_clk(adev, - &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk, - memory_clock, &memory_level->MinVddci); - if (ret) - return ret; - } - - if (adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries) { - ret = ci_get_dependency_volt_by_clk(adev, - &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, - memory_clock, &memory_level->MinMvdd); - if (ret) - return ret; - } - - memory_level->MinVddcPhases = 1; - - if (pi->vddc_phase_shed_control) - ci_populate_phase_value_based_on_mclk(adev, - &adev->pm.dpm.dyn_state.phase_shedding_limits_table, - memory_clock, - &memory_level->MinVddcPhases); - - memory_level->EnabledForActivity = 1; - memory_level->EnabledForThrottle = 1; - memory_level->UpH = 0; - memory_level->DownH = 100; - memory_level->VoltageDownH = 0; - memory_level->ActivityLevel = (u16)pi->mclk_activity_target; - - memory_level->StutterEnable = false; - memory_level->StrobeEnable = false; - memory_level->EdcReadEnable = false; - memory_level->EdcWriteEnable = false; - memory_level->RttEnable = false; - - memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; - - if (pi->mclk_stutter_mode_threshold && - (memory_clock <= pi->mclk_stutter_mode_threshold) && - (!pi->uvd_enabled) && - (RREG32(mmDPG_PIPE_STUTTER_CONTROL) & DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK) && - (adev->pm.dpm.new_active_crtc_count <= 2)) - memory_level->StutterEnable = true; - - if (pi->mclk_strobe_mode_threshold && - (memory_clock <= pi->mclk_strobe_mode_threshold)) - memory_level->StrobeEnable = 1; - - if (adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) { - memory_level->StrobeRatio = - ci_get_mclk_frequency_ratio(memory_clock, memory_level->StrobeEnable); - if (pi->mclk_edc_enable_threshold && - (memory_clock > pi->mclk_edc_enable_threshold)) - memory_level->EdcReadEnable = true; - - if (pi->mclk_edc_wr_enable_threshold && - (memory_clock > pi->mclk_edc_wr_enable_threshold)) - memory_level->EdcWriteEnable = true; - - if (memory_level->StrobeEnable) { - if (ci_get_mclk_frequency_ratio(memory_clock, true) >= - ((RREG32(mmMC_SEQ_MISC7) >> 16) & 0xf)) - dll_state_on = ((RREG32(mmMC_SEQ_MISC5) >> 1) & 0x1) ? true : false; - else - dll_state_on = ((RREG32(mmMC_SEQ_MISC6) >> 1) & 0x1) ? true : false; - } else { - dll_state_on = pi->dll_default_on; - } - } else { - memory_level->StrobeRatio = ci_get_ddr3_mclk_frequency_ratio(memory_clock); - dll_state_on = ((RREG32(mmMC_SEQ_MISC5) >> 1) & 0x1) ? true : false; - } - - ret = ci_calculate_mclk_params(adev, memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on); - if (ret) - return ret; - - memory_level->MinVddc = cpu_to_be32(memory_level->MinVddc * VOLTAGE_SCALE); - memory_level->MinVddcPhases = cpu_to_be32(memory_level->MinVddcPhases); - memory_level->MinVddci = cpu_to_be32(memory_level->MinVddci * VOLTAGE_SCALE); - memory_level->MinMvdd = cpu_to_be32(memory_level->MinMvdd * VOLTAGE_SCALE); - - memory_level->MclkFrequency = cpu_to_be32(memory_level->MclkFrequency); - memory_level->ActivityLevel = cpu_to_be16(memory_level->ActivityLevel); - memory_level->MpllFuncCntl = cpu_to_be32(memory_level->MpllFuncCntl); - memory_level->MpllFuncCntl_1 = cpu_to_be32(memory_level->MpllFuncCntl_1); - memory_level->MpllFuncCntl_2 = cpu_to_be32(memory_level->MpllFuncCntl_2); - memory_level->MpllAdFuncCntl = cpu_to_be32(memory_level->MpllAdFuncCntl); - memory_level->MpllDqFuncCntl = cpu_to_be32(memory_level->MpllDqFuncCntl); - memory_level->MclkPwrmgtCntl = cpu_to_be32(memory_level->MclkPwrmgtCntl); - memory_level->DllCntl = cpu_to_be32(memory_level->DllCntl); - memory_level->MpllSs1 = cpu_to_be32(memory_level->MpllSs1); - memory_level->MpllSs2 = cpu_to_be32(memory_level->MpllSs2); - - return 0; -} - -static int ci_populate_smc_acpi_level(struct amdgpu_device *adev, - SMU7_Discrete_DpmTable *table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct atom_clock_dividers dividers; - SMU7_Discrete_VoltageLevel voltage_level; - u32 spll_func_cntl = pi->clock_registers.cg_spll_func_cntl; - u32 spll_func_cntl_2 = pi->clock_registers.cg_spll_func_cntl_2; - u32 dll_cntl = pi->clock_registers.dll_cntl; - u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl; - int ret; - - table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC; - - if (pi->acpi_vddc) - table->ACPILevel.MinVddc = cpu_to_be32(pi->acpi_vddc * VOLTAGE_SCALE); - else - table->ACPILevel.MinVddc = cpu_to_be32(pi->min_vddc_in_pp_table * VOLTAGE_SCALE); - - table->ACPILevel.MinVddcPhases = pi->vddc_phase_shed_control ? 0 : 1; - - table->ACPILevel.SclkFrequency = adev->clock.spll.reference_freq; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_SCLK, - table->ACPILevel.SclkFrequency, false, ÷rs); - if (ret) - return ret; - - table->ACPILevel.SclkDid = (u8)dividers.post_divider; - table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; - table->ACPILevel.DeepSleepDivId = 0; - - spll_func_cntl &= ~CG_SPLL_FUNC_CNTL__SPLL_PWRON_MASK; - spll_func_cntl |= CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK; - - spll_func_cntl_2 &= ~CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL_MASK; - spll_func_cntl_2 |= (4 << CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL__SHIFT); - - table->ACPILevel.CgSpllFuncCntl = spll_func_cntl; - table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2; - table->ACPILevel.CgSpllFuncCntl3 = pi->clock_registers.cg_spll_func_cntl_3; - table->ACPILevel.CgSpllFuncCntl4 = pi->clock_registers.cg_spll_func_cntl_4; - table->ACPILevel.SpllSpreadSpectrum = pi->clock_registers.cg_spll_spread_spectrum; - table->ACPILevel.SpllSpreadSpectrum2 = pi->clock_registers.cg_spll_spread_spectrum_2; - table->ACPILevel.CcPwrDynRm = 0; - table->ACPILevel.CcPwrDynRm1 = 0; - - table->ACPILevel.Flags = cpu_to_be32(table->ACPILevel.Flags); - table->ACPILevel.MinVddcPhases = cpu_to_be32(table->ACPILevel.MinVddcPhases); - table->ACPILevel.SclkFrequency = cpu_to_be32(table->ACPILevel.SclkFrequency); - table->ACPILevel.CgSpllFuncCntl = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl); - table->ACPILevel.CgSpllFuncCntl2 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl2); - table->ACPILevel.CgSpllFuncCntl3 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl3); - table->ACPILevel.CgSpllFuncCntl4 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl4); - table->ACPILevel.SpllSpreadSpectrum = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum); - table->ACPILevel.SpllSpreadSpectrum2 = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum2); - table->ACPILevel.CcPwrDynRm = cpu_to_be32(table->ACPILevel.CcPwrDynRm); - table->ACPILevel.CcPwrDynRm1 = cpu_to_be32(table->ACPILevel.CcPwrDynRm1); - - table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc; - table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases; - - if (pi->vddci_control != CISLANDS_VOLTAGE_CONTROL_NONE) { - if (pi->acpi_vddci) - table->MemoryACPILevel.MinVddci = - cpu_to_be32(pi->acpi_vddci * VOLTAGE_SCALE); - else - table->MemoryACPILevel.MinVddci = - cpu_to_be32(pi->min_vddci_in_pp_table * VOLTAGE_SCALE); - } - - if (ci_populate_mvdd_value(adev, 0, &voltage_level)) - table->MemoryACPILevel.MinMvdd = 0; - else - table->MemoryACPILevel.MinMvdd = - cpu_to_be32(voltage_level.Voltage * VOLTAGE_SCALE); - - mclk_pwrmgt_cntl |= MCLK_PWRMGT_CNTL__MRDCK0_RESET_MASK | - MCLK_PWRMGT_CNTL__MRDCK1_RESET_MASK; - mclk_pwrmgt_cntl &= ~(MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK | - MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK); - - dll_cntl &= ~(DLL_CNTL__MRDCK0_BYPASS_MASK | DLL_CNTL__MRDCK1_BYPASS_MASK); - - table->MemoryACPILevel.DllCntl = cpu_to_be32(dll_cntl); - table->MemoryACPILevel.MclkPwrmgtCntl = cpu_to_be32(mclk_pwrmgt_cntl); - table->MemoryACPILevel.MpllAdFuncCntl = - cpu_to_be32(pi->clock_registers.mpll_ad_func_cntl); - table->MemoryACPILevel.MpllDqFuncCntl = - cpu_to_be32(pi->clock_registers.mpll_dq_func_cntl); - table->MemoryACPILevel.MpllFuncCntl = - cpu_to_be32(pi->clock_registers.mpll_func_cntl); - table->MemoryACPILevel.MpllFuncCntl_1 = - cpu_to_be32(pi->clock_registers.mpll_func_cntl_1); - table->MemoryACPILevel.MpllFuncCntl_2 = - cpu_to_be32(pi->clock_registers.mpll_func_cntl_2); - table->MemoryACPILevel.MpllSs1 = cpu_to_be32(pi->clock_registers.mpll_ss1); - table->MemoryACPILevel.MpllSs2 = cpu_to_be32(pi->clock_registers.mpll_ss2); - - table->MemoryACPILevel.EnabledForThrottle = 0; - table->MemoryACPILevel.EnabledForActivity = 0; - table->MemoryACPILevel.UpH = 0; - table->MemoryACPILevel.DownH = 100; - table->MemoryACPILevel.VoltageDownH = 0; - table->MemoryACPILevel.ActivityLevel = - cpu_to_be16((u16)pi->mclk_activity_target); - - table->MemoryACPILevel.StutterEnable = false; - table->MemoryACPILevel.StrobeEnable = false; - table->MemoryACPILevel.EdcReadEnable = false; - table->MemoryACPILevel.EdcWriteEnable = false; - table->MemoryACPILevel.RttEnable = false; - - return 0; -} - - -static int ci_enable_ulv(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ulv_parm *ulv = &pi->ulv; - - if (ulv->supported) { - if (enable) - return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ? - 0 : -EINVAL; - else - return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ? - 0 : -EINVAL; - } - - return 0; -} - -static int ci_populate_ulv_level(struct amdgpu_device *adev, - SMU7_Discrete_Ulv *state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u16 ulv_voltage = adev->pm.dpm.backbias_response_time; - - state->CcPwrDynRm = 0; - state->CcPwrDynRm1 = 0; - - if (ulv_voltage == 0) { - pi->ulv.supported = false; - return 0; - } - - if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { - if (ulv_voltage > adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v) - state->VddcOffset = 0; - else - state->VddcOffset = - adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage; - } else { - if (ulv_voltage > adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v) - state->VddcOffsetVid = 0; - else - state->VddcOffsetVid = (u8) - ((adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage) * - VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1); - } - state->VddcPhase = pi->vddc_phase_shed_control ? 0 : 1; - - state->CcPwrDynRm = cpu_to_be32(state->CcPwrDynRm); - state->CcPwrDynRm1 = cpu_to_be32(state->CcPwrDynRm1); - state->VddcOffset = cpu_to_be16(state->VddcOffset); - - return 0; -} - -static int ci_calculate_sclk_params(struct amdgpu_device *adev, - u32 engine_clock, - SMU7_Discrete_GraphicsLevel *sclk) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct atom_clock_dividers dividers; - u32 spll_func_cntl_3 = pi->clock_registers.cg_spll_func_cntl_3; - u32 spll_func_cntl_4 = pi->clock_registers.cg_spll_func_cntl_4; - u32 cg_spll_spread_spectrum = pi->clock_registers.cg_spll_spread_spectrum; - u32 cg_spll_spread_spectrum_2 = pi->clock_registers.cg_spll_spread_spectrum_2; - u32 reference_clock = adev->clock.spll.reference_freq; - u32 reference_divider; - u32 fbdiv; - int ret; - - ret = amdgpu_atombios_get_clock_dividers(adev, - COMPUTE_GPUCLK_INPUT_FLAG_SCLK, - engine_clock, false, ÷rs); - if (ret) - return ret; - - reference_divider = 1 + dividers.ref_div; - fbdiv = dividers.fb_div & 0x3FFFFFF; - - spll_func_cntl_3 &= ~CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV_MASK; - spll_func_cntl_3 |= (fbdiv << CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV__SHIFT); - spll_func_cntl_3 |= CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN_MASK; - - if (pi->caps_sclk_ss_support) { - struct amdgpu_atom_ss ss; - u32 vco_freq = engine_clock * dividers.post_div; - - if (amdgpu_atombios_get_asic_ss_info(adev, &ss, - ASIC_INTERNAL_ENGINE_SS, vco_freq)) { - u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); - u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000); - - cg_spll_spread_spectrum &= ~(CG_SPLL_SPREAD_SPECTRUM__CLKS_MASK | CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK); - cg_spll_spread_spectrum |= (clk_s << CG_SPLL_SPREAD_SPECTRUM__CLKS__SHIFT); - cg_spll_spread_spectrum |= (1 << CG_SPLL_SPREAD_SPECTRUM__SSEN__SHIFT); - - cg_spll_spread_spectrum_2 &= ~CG_SPLL_SPREAD_SPECTRUM_2__CLKV_MASK; - cg_spll_spread_spectrum_2 |= (clk_v << CG_SPLL_SPREAD_SPECTRUM_2__CLKV__SHIFT); - } - } - - sclk->SclkFrequency = engine_clock; - sclk->CgSpllFuncCntl3 = spll_func_cntl_3; - sclk->CgSpllFuncCntl4 = spll_func_cntl_4; - sclk->SpllSpreadSpectrum = cg_spll_spread_spectrum; - sclk->SpllSpreadSpectrum2 = cg_spll_spread_spectrum_2; - sclk->SclkDid = (u8)dividers.post_divider; - - return 0; -} - -static int ci_populate_single_graphic_level(struct amdgpu_device *adev, - u32 engine_clock, - u16 sclk_activity_level_t, - SMU7_Discrete_GraphicsLevel *graphic_level) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - - ret = ci_calculate_sclk_params(adev, engine_clock, graphic_level); - if (ret) - return ret; - - ret = ci_get_dependency_volt_by_clk(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk, - engine_clock, &graphic_level->MinVddc); - if (ret) - return ret; - - graphic_level->SclkFrequency = engine_clock; - - graphic_level->Flags = 0; - graphic_level->MinVddcPhases = 1; - - if (pi->vddc_phase_shed_control) - ci_populate_phase_value_based_on_sclk(adev, - &adev->pm.dpm.dyn_state.phase_shedding_limits_table, - engine_clock, - &graphic_level->MinVddcPhases); - - graphic_level->ActivityLevel = sclk_activity_level_t; - - graphic_level->CcPwrDynRm = 0; - graphic_level->CcPwrDynRm1 = 0; - graphic_level->EnabledForThrottle = 1; - graphic_level->UpH = 0; - graphic_level->DownH = 0; - graphic_level->VoltageDownH = 0; - graphic_level->PowerThrottle = 0; - - if (pi->caps_sclk_ds) - graphic_level->DeepSleepDivId = ci_get_sleep_divider_id_from_clock(engine_clock, - CISLAND_MINIMUM_ENGINE_CLOCK); - - graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; - - graphic_level->Flags = cpu_to_be32(graphic_level->Flags); - graphic_level->MinVddc = cpu_to_be32(graphic_level->MinVddc * VOLTAGE_SCALE); - graphic_level->MinVddcPhases = cpu_to_be32(graphic_level->MinVddcPhases); - graphic_level->SclkFrequency = cpu_to_be32(graphic_level->SclkFrequency); - graphic_level->ActivityLevel = cpu_to_be16(graphic_level->ActivityLevel); - graphic_level->CgSpllFuncCntl3 = cpu_to_be32(graphic_level->CgSpllFuncCntl3); - graphic_level->CgSpllFuncCntl4 = cpu_to_be32(graphic_level->CgSpllFuncCntl4); - graphic_level->SpllSpreadSpectrum = cpu_to_be32(graphic_level->SpllSpreadSpectrum); - graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2); - graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm); - graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1); - - return 0; -} - -static int ci_populate_all_graphic_levels(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_dpm_table *dpm_table = &pi->dpm_table; - u32 level_array_address = pi->dpm_table_start + - offsetof(SMU7_Discrete_DpmTable, GraphicsLevel); - u32 level_array_size = sizeof(SMU7_Discrete_GraphicsLevel) * - SMU7_MAX_LEVELS_GRAPHICS; - SMU7_Discrete_GraphicsLevel *levels = pi->smc_state_table.GraphicsLevel; - u32 i, ret; - - memset(levels, 0, level_array_size); - - for (i = 0; i < dpm_table->sclk_table.count; i++) { - ret = ci_populate_single_graphic_level(adev, - dpm_table->sclk_table.dpm_levels[i].value, - (u16)pi->activity_target[i], - &pi->smc_state_table.GraphicsLevel[i]); - if (ret) - return ret; - if (i > 1) - pi->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0; - if (i == (dpm_table->sclk_table.count - 1)) - pi->smc_state_table.GraphicsLevel[i].DisplayWatermark = - PPSMC_DISPLAY_WATERMARK_HIGH; - } - pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1; - - pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count; - pi->dpm_level_enable_mask.sclk_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&dpm_table->sclk_table); - - ret = amdgpu_ci_copy_bytes_to_smc(adev, level_array_address, - (u8 *)levels, level_array_size, - pi->sram_end); - if (ret) - return ret; - - return 0; -} - -static int ci_populate_ulv_state(struct amdgpu_device *adev, - SMU7_Discrete_Ulv *ulv_level) -{ - return ci_populate_ulv_level(adev, ulv_level); -} - -static int ci_populate_all_memory_levels(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_dpm_table *dpm_table = &pi->dpm_table; - u32 level_array_address = pi->dpm_table_start + - offsetof(SMU7_Discrete_DpmTable, MemoryLevel); - u32 level_array_size = sizeof(SMU7_Discrete_MemoryLevel) * - SMU7_MAX_LEVELS_MEMORY; - SMU7_Discrete_MemoryLevel *levels = pi->smc_state_table.MemoryLevel; - u32 i, ret; - - memset(levels, 0, level_array_size); - - for (i = 0; i < dpm_table->mclk_table.count; i++) { - if (dpm_table->mclk_table.dpm_levels[i].value == 0) - return -EINVAL; - ret = ci_populate_single_memory_level(adev, - dpm_table->mclk_table.dpm_levels[i].value, - &pi->smc_state_table.MemoryLevel[i]); - if (ret) - return ret; - } - - if ((dpm_table->mclk_table.count >= 2) && - ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) { - pi->smc_state_table.MemoryLevel[1].MinVddc = - pi->smc_state_table.MemoryLevel[0].MinVddc; - pi->smc_state_table.MemoryLevel[1].MinVddcPhases = - pi->smc_state_table.MemoryLevel[0].MinVddcPhases; - } - - pi->smc_state_table.MemoryLevel[0].ActivityLevel = cpu_to_be16(0x1F); - - pi->smc_state_table.MemoryDpmLevelCount = (u8)dpm_table->mclk_table.count; - pi->dpm_level_enable_mask.mclk_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&dpm_table->mclk_table); - - pi->smc_state_table.MemoryLevel[dpm_table->mclk_table.count - 1].DisplayWatermark = - PPSMC_DISPLAY_WATERMARK_HIGH; - - ret = amdgpu_ci_copy_bytes_to_smc(adev, level_array_address, - (u8 *)levels, level_array_size, - pi->sram_end); - if (ret) - return ret; - - return 0; -} - -static void ci_reset_single_dpm_table(struct amdgpu_device *adev, - struct ci_single_dpm_table* dpm_table, - u32 count) -{ - u32 i; - - dpm_table->count = count; - for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) - dpm_table->dpm_levels[i].enabled = false; -} - -static void ci_setup_pcie_table_entry(struct ci_single_dpm_table* dpm_table, - u32 index, u32 pcie_gen, u32 pcie_lanes) -{ - dpm_table->dpm_levels[index].value = pcie_gen; - dpm_table->dpm_levels[index].param1 = pcie_lanes; - dpm_table->dpm_levels[index].enabled = true; -} - -static int ci_setup_default_pcie_tables(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (!pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) - return -EINVAL; - - if (pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) { - pi->pcie_gen_powersaving = pi->pcie_gen_performance; - pi->pcie_lane_powersaving = pi->pcie_lane_performance; - } else if (!pi->use_pcie_performance_levels && pi->use_pcie_powersaving_levels) { - pi->pcie_gen_performance = pi->pcie_gen_powersaving; - pi->pcie_lane_performance = pi->pcie_lane_powersaving; - } - - ci_reset_single_dpm_table(adev, - &pi->dpm_table.pcie_speed_table, - SMU7_MAX_LEVELS_LINK); - - if (adev->asic_type == CHIP_BONAIRE) - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0, - pi->pcie_gen_powersaving.min, - pi->pcie_lane_powersaving.max); - else - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0, - pi->pcie_gen_powersaving.min, - pi->pcie_lane_powersaving.min); - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 1, - pi->pcie_gen_performance.min, - pi->pcie_lane_performance.min); - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 2, - pi->pcie_gen_powersaving.min, - pi->pcie_lane_powersaving.max); - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 3, - pi->pcie_gen_performance.min, - pi->pcie_lane_performance.max); - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 4, - pi->pcie_gen_powersaving.max, - pi->pcie_lane_powersaving.max); - ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 5, - pi->pcie_gen_performance.max, - pi->pcie_lane_performance.max); - - pi->dpm_table.pcie_speed_table.count = 6; - - return 0; -} - -static int ci_setup_default_dpm_tables(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_clock_voltage_dependency_table *allowed_sclk_vddc_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk; - struct amdgpu_clock_voltage_dependency_table *allowed_mclk_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk; - struct amdgpu_cac_leakage_table *std_voltage_table = - &adev->pm.dpm.dyn_state.cac_leakage_table; - u32 i; - - if (allowed_sclk_vddc_table == NULL) - return -EINVAL; - if (allowed_sclk_vddc_table->count < 1) - return -EINVAL; - if (allowed_mclk_table == NULL) - return -EINVAL; - if (allowed_mclk_table->count < 1) - return -EINVAL; - - memset(&pi->dpm_table, 0, sizeof(struct ci_dpm_table)); - - ci_reset_single_dpm_table(adev, - &pi->dpm_table.sclk_table, - SMU7_MAX_LEVELS_GRAPHICS); - ci_reset_single_dpm_table(adev, - &pi->dpm_table.mclk_table, - SMU7_MAX_LEVELS_MEMORY); - ci_reset_single_dpm_table(adev, - &pi->dpm_table.vddc_table, - SMU7_MAX_LEVELS_VDDC); - ci_reset_single_dpm_table(adev, - &pi->dpm_table.vddci_table, - SMU7_MAX_LEVELS_VDDCI); - ci_reset_single_dpm_table(adev, - &pi->dpm_table.mvdd_table, - SMU7_MAX_LEVELS_MVDD); - - pi->dpm_table.sclk_table.count = 0; - for (i = 0; i < allowed_sclk_vddc_table->count; i++) { - if ((i == 0) || - (pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count-1].value != - allowed_sclk_vddc_table->entries[i].clk)) { - pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].value = - allowed_sclk_vddc_table->entries[i].clk; - pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].enabled = - (i == 0) ? true : false; - pi->dpm_table.sclk_table.count++; - } - } - - pi->dpm_table.mclk_table.count = 0; - for (i = 0; i < allowed_mclk_table->count; i++) { - if ((i == 0) || - (pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count-1].value != - allowed_mclk_table->entries[i].clk)) { - pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].value = - allowed_mclk_table->entries[i].clk; - pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].enabled = - (i == 0) ? true : false; - pi->dpm_table.mclk_table.count++; - } - } - - for (i = 0; i < allowed_sclk_vddc_table->count; i++) { - pi->dpm_table.vddc_table.dpm_levels[i].value = - allowed_sclk_vddc_table->entries[i].v; - pi->dpm_table.vddc_table.dpm_levels[i].param1 = - std_voltage_table->entries[i].leakage; - pi->dpm_table.vddc_table.dpm_levels[i].enabled = true; - } - pi->dpm_table.vddc_table.count = allowed_sclk_vddc_table->count; - - allowed_mclk_table = &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk; - if (allowed_mclk_table) { - for (i = 0; i < allowed_mclk_table->count; i++) { - pi->dpm_table.vddci_table.dpm_levels[i].value = - allowed_mclk_table->entries[i].v; - pi->dpm_table.vddci_table.dpm_levels[i].enabled = true; - } - pi->dpm_table.vddci_table.count = allowed_mclk_table->count; - } - - allowed_mclk_table = &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk; - if (allowed_mclk_table) { - for (i = 0; i < allowed_mclk_table->count; i++) { - pi->dpm_table.mvdd_table.dpm_levels[i].value = - allowed_mclk_table->entries[i].v; - pi->dpm_table.mvdd_table.dpm_levels[i].enabled = true; - } - pi->dpm_table.mvdd_table.count = allowed_mclk_table->count; - } - - ci_setup_default_pcie_tables(adev); - - /* save a copy of the default DPM table */ - memcpy(&(pi->golden_dpm_table), &(pi->dpm_table), - sizeof(struct ci_dpm_table)); - - return 0; -} - -static int ci_find_boot_level(struct ci_single_dpm_table *table, - u32 value, u32 *boot_level) -{ - u32 i; - int ret = -EINVAL; - - for(i = 0; i < table->count; i++) { - if (value == table->dpm_levels[i].value) { - *boot_level = i; - ret = 0; - } - } - - return ret; -} - -static int ci_init_smc_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ulv_parm *ulv = &pi->ulv; - struct amdgpu_ps *amdgpu_boot_state = adev->pm.dpm.boot_ps; - SMU7_Discrete_DpmTable *table = &pi->smc_state_table; - int ret; - - ret = ci_setup_default_dpm_tables(adev); - if (ret) - return ret; - - if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) - ci_populate_smc_voltage_tables(adev, table); - - ci_init_fps_limits(adev); - - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) - table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; - - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) - table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; - - if (adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) - table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5; - - if (ulv->supported) { - ret = ci_populate_ulv_state(adev, &pi->smc_state_table.Ulv); - if (ret) - return ret; - WREG32_SMC(ixCG_ULV_PARAMETER, ulv->cg_ulv_parameter); - } - - ret = ci_populate_all_graphic_levels(adev); - if (ret) - return ret; - - ret = ci_populate_all_memory_levels(adev); - if (ret) - return ret; - - ci_populate_smc_link_level(adev, table); - - ret = ci_populate_smc_acpi_level(adev, table); - if (ret) - return ret; - - ret = ci_populate_smc_vce_level(adev, table); - if (ret) - return ret; - - ret = ci_populate_smc_acp_level(adev, table); - if (ret) - return ret; - - ret = ci_populate_smc_samu_level(adev, table); - if (ret) - return ret; - - ret = ci_do_program_memory_timing_parameters(adev); - if (ret) - return ret; - - ret = ci_populate_smc_uvd_level(adev, table); - if (ret) - return ret; - - table->UvdBootLevel = 0; - table->VceBootLevel = 0; - table->AcpBootLevel = 0; - table->SamuBootLevel = 0; - table->GraphicsBootLevel = 0; - table->MemoryBootLevel = 0; - - ret = ci_find_boot_level(&pi->dpm_table.sclk_table, - pi->vbios_boot_state.sclk_bootup_value, - (u32 *)&pi->smc_state_table.GraphicsBootLevel); - - ret = ci_find_boot_level(&pi->dpm_table.mclk_table, - pi->vbios_boot_state.mclk_bootup_value, - (u32 *)&pi->smc_state_table.MemoryBootLevel); - - table->BootVddc = pi->vbios_boot_state.vddc_bootup_value; - table->BootVddci = pi->vbios_boot_state.vddci_bootup_value; - table->BootMVdd = pi->vbios_boot_state.mvdd_bootup_value; - - ci_populate_smc_initial_state(adev, amdgpu_boot_state); - - ret = ci_populate_bapm_parameters_in_dpm_table(adev); - if (ret) - return ret; - - table->UVDInterval = 1; - table->VCEInterval = 1; - table->ACPInterval = 1; - table->SAMUInterval = 1; - table->GraphicsVoltageChangeEnable = 1; - table->GraphicsThermThrottleEnable = 1; - table->GraphicsInterval = 1; - table->VoltageInterval = 1; - table->ThermalInterval = 1; - table->TemperatureLimitHigh = (u16)((pi->thermal_temp_setting.temperature_high * - CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000); - table->TemperatureLimitLow = (u16)((pi->thermal_temp_setting.temperature_low * - CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000); - table->MemoryVoltageChangeEnable = 1; - table->MemoryInterval = 1; - table->VoltageResponseTime = 0; - table->VddcVddciDelta = 4000; - table->PhaseResponseTime = 0; - table->MemoryThermThrottleEnable = 1; - table->PCIeBootLinkLevel = pi->dpm_table.pcie_speed_table.count - 1; - table->PCIeGenInterval = 1; - if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) - table->SVI2Enable = 1; - else - table->SVI2Enable = 0; - - table->ThermGpio = 17; - table->SclkStepSize = 0x4000; - - table->SystemFlags = cpu_to_be32(table->SystemFlags); - table->SmioMaskVddcVid = cpu_to_be32(table->SmioMaskVddcVid); - table->SmioMaskVddcPhase = cpu_to_be32(table->SmioMaskVddcPhase); - table->SmioMaskVddciVid = cpu_to_be32(table->SmioMaskVddciVid); - table->SmioMaskMvddVid = cpu_to_be32(table->SmioMaskMvddVid); - table->SclkStepSize = cpu_to_be32(table->SclkStepSize); - table->TemperatureLimitHigh = cpu_to_be16(table->TemperatureLimitHigh); - table->TemperatureLimitLow = cpu_to_be16(table->TemperatureLimitLow); - table->VddcVddciDelta = cpu_to_be16(table->VddcVddciDelta); - table->VoltageResponseTime = cpu_to_be16(table->VoltageResponseTime); - table->PhaseResponseTime = cpu_to_be16(table->PhaseResponseTime); - table->BootVddc = cpu_to_be16(table->BootVddc * VOLTAGE_SCALE); - table->BootVddci = cpu_to_be16(table->BootVddci * VOLTAGE_SCALE); - table->BootMVdd = cpu_to_be16(table->BootMVdd * VOLTAGE_SCALE); - - ret = amdgpu_ci_copy_bytes_to_smc(adev, - pi->dpm_table_start + - offsetof(SMU7_Discrete_DpmTable, SystemFlags), - (u8 *)&table->SystemFlags, - sizeof(SMU7_Discrete_DpmTable) - 3 * sizeof(SMU7_PIDController), - pi->sram_end); - if (ret) - return ret; - - return 0; -} - -static void ci_trim_single_dpm_states(struct amdgpu_device *adev, - struct ci_single_dpm_table *dpm_table, - u32 low_limit, u32 high_limit) -{ - u32 i; - - for (i = 0; i < dpm_table->count; i++) { - if ((dpm_table->dpm_levels[i].value < low_limit) || - (dpm_table->dpm_levels[i].value > high_limit)) - dpm_table->dpm_levels[i].enabled = false; - else - dpm_table->dpm_levels[i].enabled = true; - } -} - -static void ci_trim_pcie_dpm_states(struct amdgpu_device *adev, - u32 speed_low, u32 lanes_low, - u32 speed_high, u32 lanes_high) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table; - u32 i, j; - - for (i = 0; i < pcie_table->count; i++) { - if ((pcie_table->dpm_levels[i].value < speed_low) || - (pcie_table->dpm_levels[i].param1 < lanes_low) || - (pcie_table->dpm_levels[i].value > speed_high) || - (pcie_table->dpm_levels[i].param1 > lanes_high)) - pcie_table->dpm_levels[i].enabled = false; - else - pcie_table->dpm_levels[i].enabled = true; - } - - for (i = 0; i < pcie_table->count; i++) { - if (pcie_table->dpm_levels[i].enabled) { - for (j = i + 1; j < pcie_table->count; j++) { - if (pcie_table->dpm_levels[j].enabled) { - if ((pcie_table->dpm_levels[i].value == pcie_table->dpm_levels[j].value) && - (pcie_table->dpm_levels[i].param1 == pcie_table->dpm_levels[j].param1)) - pcie_table->dpm_levels[j].enabled = false; - } - } - } - } -} - -static int ci_trim_dpm_states(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_state) -{ - struct ci_ps *state = ci_get_ps(amdgpu_state); - struct ci_power_info *pi = ci_get_pi(adev); - u32 high_limit_count; - - if (state->performance_level_count < 1) - return -EINVAL; - - if (state->performance_level_count == 1) - high_limit_count = 0; - else - high_limit_count = 1; - - ci_trim_single_dpm_states(adev, - &pi->dpm_table.sclk_table, - state->performance_levels[0].sclk, - state->performance_levels[high_limit_count].sclk); - - ci_trim_single_dpm_states(adev, - &pi->dpm_table.mclk_table, - state->performance_levels[0].mclk, - state->performance_levels[high_limit_count].mclk); - - ci_trim_pcie_dpm_states(adev, - state->performance_levels[0].pcie_gen, - state->performance_levels[0].pcie_lane, - state->performance_levels[high_limit_count].pcie_gen, - state->performance_levels[high_limit_count].pcie_lane); - - return 0; -} - -static int ci_apply_disp_minimum_voltage_request(struct amdgpu_device *adev) -{ - struct amdgpu_clock_voltage_dependency_table *disp_voltage_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk; - struct amdgpu_clock_voltage_dependency_table *vddc_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk; - u32 requested_voltage = 0; - u32 i; - - if (disp_voltage_table == NULL) - return -EINVAL; - if (!disp_voltage_table->count) - return -EINVAL; - - for (i = 0; i < disp_voltage_table->count; i++) { - if (adev->clock.current_dispclk == disp_voltage_table->entries[i].clk) - requested_voltage = disp_voltage_table->entries[i].v; - } - - for (i = 0; i < vddc_table->count; i++) { - if (requested_voltage <= vddc_table->entries[i].v) { - requested_voltage = vddc_table->entries[i].v; - return (amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_VddC_Request, - requested_voltage * VOLTAGE_SCALE) == PPSMC_Result_OK) ? - 0 : -EINVAL; - } - } - - return -EINVAL; -} - -static int ci_upload_dpm_level_enable_mask(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - PPSMC_Result result; - - ci_apply_disp_minimum_voltage_request(adev); - - if (!pi->sclk_dpm_key_disabled) { - if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { - result = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_SCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.sclk_dpm_enable_mask); - if (result != PPSMC_Result_OK) - return -EINVAL; - } - } - - if (!pi->mclk_dpm_key_disabled) { - if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { - result = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_MCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.mclk_dpm_enable_mask); - if (result != PPSMC_Result_OK) - return -EINVAL; - } - } - -#if 0 - if (!pi->pcie_dpm_key_disabled) { - if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { - result = amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_PCIeDPM_SetEnabledMask, - pi->dpm_level_enable_mask.pcie_dpm_enable_mask); - if (result != PPSMC_Result_OK) - return -EINVAL; - } - } -#endif - - return 0; -} - -static void ci_find_dpm_states_clocks_in_dpm_table(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *state = ci_get_ps(amdgpu_state); - struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table; - u32 sclk = state->performance_levels[state->performance_level_count-1].sclk; - struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table; - u32 mclk = state->performance_levels[state->performance_level_count-1].mclk; - u32 i; - - pi->need_update_smu7_dpm_table = 0; - - for (i = 0; i < sclk_table->count; i++) { - if (sclk == sclk_table->dpm_levels[i].value) - break; - } - - if (i >= sclk_table->count) { - pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK; - } else { - /* XXX check display min clock requirements */ - if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK) - pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK; - } - - for (i = 0; i < mclk_table->count; i++) { - if (mclk == mclk_table->dpm_levels[i].value) - break; - } - - if (i >= mclk_table->count) - pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK; - - if (adev->pm.dpm.current_active_crtc_count != - adev->pm.dpm.new_active_crtc_count) - pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK; -} - -static int ci_populate_and_upload_sclk_mclk_dpm_levels(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *state = ci_get_ps(amdgpu_state); - u32 sclk = state->performance_levels[state->performance_level_count-1].sclk; - u32 mclk = state->performance_levels[state->performance_level_count-1].mclk; - struct ci_dpm_table *dpm_table = &pi->dpm_table; - int ret; - - if (!pi->need_update_smu7_dpm_table) - return 0; - - if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) - dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value = sclk; - - if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) - dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value = mclk; - - if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK)) { - ret = ci_populate_all_graphic_levels(adev); - if (ret) - return ret; - } - - if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK | DPMTABLE_UPDATE_MCLK)) { - ret = ci_populate_all_memory_levels(adev); - if (ret) - return ret; - } - - return 0; -} - -static int ci_enable_uvd_dpm(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct amdgpu_clock_and_voltage_limits *max_limits; - int i; - - if (adev->pm.ac_power) - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - - if (enable) { - pi->dpm_level_enable_mask.uvd_dpm_enable_mask = 0; - - for (i = adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; i >= 0; i--) { - if (adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { - pi->dpm_level_enable_mask.uvd_dpm_enable_mask |= 1 << i; - - if (!pi->caps_uvd_dpm) - break; - } - } - - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_UVDDPM_SetEnabledMask, - pi->dpm_level_enable_mask.uvd_dpm_enable_mask); - - if (pi->last_mclk_dpm_enable_mask & 0x1) { - pi->uvd_enabled = true; - pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE; - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_MCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.mclk_dpm_enable_mask); - } - } else { - if (pi->uvd_enabled) { - pi->uvd_enabled = false; - pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1; - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_MCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.mclk_dpm_enable_mask); - } - } - - return (amdgpu_ci_send_msg_to_smc(adev, enable ? - PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable) == PPSMC_Result_OK) ? - 0 : -EINVAL; -} - -static int ci_enable_vce_dpm(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct amdgpu_clock_and_voltage_limits *max_limits; - int i; - - if (adev->pm.ac_power) - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - - if (enable) { - pi->dpm_level_enable_mask.vce_dpm_enable_mask = 0; - for (i = adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count - 1; i >= 0; i--) { - if (adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { - pi->dpm_level_enable_mask.vce_dpm_enable_mask |= 1 << i; - - if (!pi->caps_vce_dpm) - break; - } - } - - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_VCEDPM_SetEnabledMask, - pi->dpm_level_enable_mask.vce_dpm_enable_mask); - } - - return (amdgpu_ci_send_msg_to_smc(adev, enable ? - PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable) == PPSMC_Result_OK) ? - 0 : -EINVAL; -} - -#if 0 -static int ci_enable_samu_dpm(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct amdgpu_clock_and_voltage_limits *max_limits; - int i; - - if (adev->pm.ac_power) - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - - if (enable) { - pi->dpm_level_enable_mask.samu_dpm_enable_mask = 0; - for (i = adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count - 1; i >= 0; i--) { - if (adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { - pi->dpm_level_enable_mask.samu_dpm_enable_mask |= 1 << i; - - if (!pi->caps_samu_dpm) - break; - } - } - - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_SAMUDPM_SetEnabledMask, - pi->dpm_level_enable_mask.samu_dpm_enable_mask); - } - return (amdgpu_ci_send_msg_to_smc(adev, enable ? - PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable) == PPSMC_Result_OK) ? - 0 : -EINVAL; -} - -static int ci_enable_acp_dpm(struct amdgpu_device *adev, bool enable) -{ - struct ci_power_info *pi = ci_get_pi(adev); - const struct amdgpu_clock_and_voltage_limits *max_limits; - int i; - - if (adev->pm.ac_power) - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else - max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - - if (enable) { - pi->dpm_level_enable_mask.acp_dpm_enable_mask = 0; - for (i = adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count - 1; i >= 0; i--) { - if (adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { - pi->dpm_level_enable_mask.acp_dpm_enable_mask |= 1 << i; - - if (!pi->caps_acp_dpm) - break; - } - } - - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_ACPDPM_SetEnabledMask, - pi->dpm_level_enable_mask.acp_dpm_enable_mask); - } - - return (amdgpu_ci_send_msg_to_smc(adev, enable ? - PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable) == PPSMC_Result_OK) ? - 0 : -EINVAL; -} -#endif - -static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - int ret = 0; - - if (!gate) { - /* turn the clocks on when decoding */ - if (pi->caps_uvd_dpm || - (adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0)) - pi->smc_state_table.UvdBootLevel = 0; - else - pi->smc_state_table.UvdBootLevel = - adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; - - tmp = RREG32_SMC(ixDPM_TABLE_475); - tmp &= ~DPM_TABLE_475__UvdBootLevel_MASK; - tmp |= (pi->smc_state_table.UvdBootLevel << DPM_TABLE_475__UvdBootLevel__SHIFT); - WREG32_SMC(ixDPM_TABLE_475, tmp); - ret = ci_enable_uvd_dpm(adev, true); - } else { - ret = ci_enable_uvd_dpm(adev, false); - if (ret) - return ret; - } - - return ret; -} - -static u8 ci_get_vce_boot_level(struct amdgpu_device *adev) -{ - u8 i; - u32 min_evclk = 30000; /* ??? */ - struct amdgpu_vce_clock_voltage_dependency_table *table = - &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; - - for (i = 0; i < table->count; i++) { - if (table->entries[i].evclk >= min_evclk) - return i; - } - - return table->count - 1; -} - -static int ci_update_vce_dpm(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_new_state, - struct amdgpu_ps *amdgpu_current_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret = 0; - u32 tmp; - - if (amdgpu_current_state->evclk != amdgpu_new_state->evclk) { - if (amdgpu_new_state->evclk) { - pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(adev); - tmp = RREG32_SMC(ixDPM_TABLE_475); - tmp &= ~DPM_TABLE_475__VceBootLevel_MASK; - tmp |= (pi->smc_state_table.VceBootLevel << DPM_TABLE_475__VceBootLevel__SHIFT); - WREG32_SMC(ixDPM_TABLE_475, tmp); - - ret = ci_enable_vce_dpm(adev, true); - } else { - ret = ci_enable_vce_dpm(adev, false); - if (ret) - return ret; - } - } - return ret; -} - -#if 0 -static int ci_update_samu_dpm(struct amdgpu_device *adev, bool gate) -{ - return ci_enable_samu_dpm(adev, gate); -} - -static int ci_update_acp_dpm(struct amdgpu_device *adev, bool gate) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp; - - if (!gate) { - pi->smc_state_table.AcpBootLevel = 0; - - tmp = RREG32_SMC(ixDPM_TABLE_475); - tmp &= ~AcpBootLevel_MASK; - tmp |= AcpBootLevel(pi->smc_state_table.AcpBootLevel); - WREG32_SMC(ixDPM_TABLE_475, tmp); - } - - return ci_enable_acp_dpm(adev, !gate); -} -#endif - -static int ci_generate_dpm_level_enable_mask(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - - ret = ci_trim_dpm_states(adev, amdgpu_state); - if (ret) - return ret; - - pi->dpm_level_enable_mask.sclk_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&pi->dpm_table.sclk_table); - pi->dpm_level_enable_mask.mclk_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&pi->dpm_table.mclk_table); - pi->last_mclk_dpm_enable_mask = - pi->dpm_level_enable_mask.mclk_dpm_enable_mask; - if (pi->uvd_enabled) { - if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask & 1) - pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE; - } - pi->dpm_level_enable_mask.pcie_dpm_enable_mask = - ci_get_dpm_level_enable_mask_value(&pi->dpm_table.pcie_speed_table); - - return 0; -} - -static u32 ci_get_lowest_enabled_level(struct amdgpu_device *adev, - u32 level_mask) -{ - u32 level = 0; - - while ((level_mask & (1 << level)) == 0) - level++; - - return level; -} - - -static int ci_dpm_force_performance_level(void *handle, - enum amd_dpm_forced_level level) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - u32 tmp, levels, i; - int ret; - - if (level == AMD_DPM_FORCED_LEVEL_HIGH) { - if ((!pi->pcie_dpm_key_disabled) && - pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { - levels = 0; - tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask; - while (tmp >>= 1) - levels++; - if (levels) { - ret = ci_dpm_force_state_pcie(adev, level); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) & - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } - if ((!pi->sclk_dpm_key_disabled) && - pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { - levels = 0; - tmp = pi->dpm_level_enable_mask.sclk_dpm_enable_mask; - while (tmp >>= 1) - levels++; - if (levels) { - ret = ci_dpm_force_state_sclk(adev, levels); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) & - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } - if ((!pi->mclk_dpm_key_disabled) && - pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { - levels = 0; - tmp = pi->dpm_level_enable_mask.mclk_dpm_enable_mask; - while (tmp >>= 1) - levels++; - if (levels) { - ret = ci_dpm_force_state_mclk(adev, levels); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) & - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } - } else if (level == AMD_DPM_FORCED_LEVEL_LOW) { - if ((!pi->sclk_dpm_key_disabled) && - pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { - levels = ci_get_lowest_enabled_level(adev, - pi->dpm_level_enable_mask.sclk_dpm_enable_mask); - ret = ci_dpm_force_state_sclk(adev, levels); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) & - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - if ((!pi->mclk_dpm_key_disabled) && - pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { - levels = ci_get_lowest_enabled_level(adev, - pi->dpm_level_enable_mask.mclk_dpm_enable_mask); - ret = ci_dpm_force_state_mclk(adev, levels); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) & - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - if ((!pi->pcie_dpm_key_disabled) && - pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { - levels = ci_get_lowest_enabled_level(adev, - pi->dpm_level_enable_mask.pcie_dpm_enable_mask); - ret = ci_dpm_force_state_pcie(adev, levels); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) & - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } else if (level == AMD_DPM_FORCED_LEVEL_AUTO) { - if (!pi->pcie_dpm_key_disabled) { - PPSMC_Result smc_result; - - smc_result = amdgpu_ci_send_msg_to_smc(adev, - PPSMC_MSG_PCIeDPM_UnForceLevel); - if (smc_result != PPSMC_Result_OK) - return -EINVAL; - } - ret = ci_upload_dpm_level_enable_mask(adev); - if (ret) - return ret; - } - - adev->pm.dpm.forced_level = level; - - return 0; -} - -static int ci_set_mc_special_registers(struct amdgpu_device *adev, - struct ci_mc_reg_table *table) -{ - u8 i, j, k; - u32 temp_reg; - - for (i = 0, j = table->last; i < table->last; i++) { - if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - switch(table->mc_reg_address[i].s1) { - case mmMC_SEQ_MISC1: - temp_reg = RREG32(mmMC_PMG_CMD_EMRS); - table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS; - table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP; - for (k = 0; k < table->num_entries; k++) { - table->mc_reg_table_entry[k].mc_data[j] = - ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); - } - j++; - - if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - temp_reg = RREG32(mmMC_PMG_CMD_MRS); - table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS; - table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP; - for (k = 0; k < table->num_entries; k++) { - table->mc_reg_table_entry[k].mc_data[j] = - (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); - if (adev->gmc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) - table->mc_reg_table_entry[k].mc_data[j] |= 0x100; - } - j++; - - if (adev->gmc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) { - if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD; - table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD; - for (k = 0; k < table->num_entries; k++) { - table->mc_reg_table_entry[k].mc_data[j] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16; - } - j++; - } - break; - case mmMC_SEQ_RESERVE_M: - temp_reg = RREG32(mmMC_PMG_CMD_MRS1); - table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1; - table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP; - for (k = 0; k < table->num_entries; k++) { - table->mc_reg_table_entry[k].mc_data[j] = - (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); - } - j++; - break; - default: - break; - } - - } - - table->last = j; - - return 0; -} - -static bool ci_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg) -{ - bool result = true; - - switch(in_reg) { - case mmMC_SEQ_RAS_TIMING: - *out_reg = mmMC_SEQ_RAS_TIMING_LP; - break; - case mmMC_SEQ_DLL_STBY: - *out_reg = mmMC_SEQ_DLL_STBY_LP; - break; - case mmMC_SEQ_G5PDX_CMD0: - *out_reg = mmMC_SEQ_G5PDX_CMD0_LP; - break; - case mmMC_SEQ_G5PDX_CMD1: - *out_reg = mmMC_SEQ_G5PDX_CMD1_LP; - break; - case mmMC_SEQ_G5PDX_CTRL: - *out_reg = mmMC_SEQ_G5PDX_CTRL_LP; - break; - case mmMC_SEQ_CAS_TIMING: - *out_reg = mmMC_SEQ_CAS_TIMING_LP; - break; - case mmMC_SEQ_MISC_TIMING: - *out_reg = mmMC_SEQ_MISC_TIMING_LP; - break; - case mmMC_SEQ_MISC_TIMING2: - *out_reg = mmMC_SEQ_MISC_TIMING2_LP; - break; - case mmMC_SEQ_PMG_DVS_CMD: - *out_reg = mmMC_SEQ_PMG_DVS_CMD_LP; - break; - case mmMC_SEQ_PMG_DVS_CTL: - *out_reg = mmMC_SEQ_PMG_DVS_CTL_LP; - break; - case mmMC_SEQ_RD_CTL_D0: - *out_reg = mmMC_SEQ_RD_CTL_D0_LP; - break; - case mmMC_SEQ_RD_CTL_D1: - *out_reg = mmMC_SEQ_RD_CTL_D1_LP; - break; - case mmMC_SEQ_WR_CTL_D0: - *out_reg = mmMC_SEQ_WR_CTL_D0_LP; - break; - case mmMC_SEQ_WR_CTL_D1: - *out_reg = mmMC_SEQ_WR_CTL_D1_LP; - break; - case mmMC_PMG_CMD_EMRS: - *out_reg = mmMC_SEQ_PMG_CMD_EMRS_LP; - break; - case mmMC_PMG_CMD_MRS: - *out_reg = mmMC_SEQ_PMG_CMD_MRS_LP; - break; - case mmMC_PMG_CMD_MRS1: - *out_reg = mmMC_SEQ_PMG_CMD_MRS1_LP; - break; - case mmMC_SEQ_PMG_TIMING: - *out_reg = mmMC_SEQ_PMG_TIMING_LP; - break; - case mmMC_PMG_CMD_MRS2: - *out_reg = mmMC_SEQ_PMG_CMD_MRS2_LP; - break; - case mmMC_SEQ_WR_CTL_2: - *out_reg = mmMC_SEQ_WR_CTL_2_LP; - break; - default: - result = false; - break; - } - - return result; -} - -static void ci_set_valid_flag(struct ci_mc_reg_table *table) -{ - u8 i, j; - - for (i = 0; i < table->last; i++) { - for (j = 1; j < table->num_entries; j++) { - if (table->mc_reg_table_entry[j-1].mc_data[i] != - table->mc_reg_table_entry[j].mc_data[i]) { - table->valid_flag |= 1 << i; - break; - } - } - } -} - -static void ci_set_s0_mc_reg_index(struct ci_mc_reg_table *table) -{ - u32 i; - u16 address; - - for (i = 0; i < table->last; i++) { - table->mc_reg_address[i].s0 = - ci_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ? - address : table->mc_reg_address[i].s1; - } -} - -static int ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table *table, - struct ci_mc_reg_table *ci_table) -{ - u8 i, j; - - if (table->last > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - if (table->num_entries > MAX_AC_TIMING_ENTRIES) - return -EINVAL; - - for (i = 0; i < table->last; i++) - ci_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1; - - ci_table->last = table->last; - - for (i = 0; i < table->num_entries; i++) { - ci_table->mc_reg_table_entry[i].mclk_max = - table->mc_reg_table_entry[i].mclk_max; - for (j = 0; j < table->last; j++) - ci_table->mc_reg_table_entry[i].mc_data[j] = - table->mc_reg_table_entry[i].mc_data[j]; - } - ci_table->num_entries = table->num_entries; - - return 0; -} - -static int ci_register_patching_mc_seq(struct amdgpu_device *adev, - struct ci_mc_reg_table *table) -{ - u8 i, k; - u32 tmp; - bool patch; - - tmp = RREG32(mmMC_SEQ_MISC0); - patch = ((tmp & 0x0000f00) == 0x300) ? true : false; - - if (patch && - ((adev->pdev->device == 0x67B0) || - (adev->pdev->device == 0x67B1))) { - for (i = 0; i < table->last; i++) { - if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - switch (table->mc_reg_address[i].s1) { - case mmMC_SEQ_MISC1: - for (k = 0; k < table->num_entries; k++) { - if ((table->mc_reg_table_entry[k].mclk_max == 125000) || - (table->mc_reg_table_entry[k].mclk_max == 137500)) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFF8) | - 0x00000007; - } - break; - case mmMC_SEQ_WR_CTL_D0: - for (k = 0; k < table->num_entries; k++) { - if ((table->mc_reg_table_entry[k].mclk_max == 125000) || - (table->mc_reg_table_entry[k].mclk_max == 137500)) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) | - 0x0000D0DD; - } - break; - case mmMC_SEQ_WR_CTL_D1: - for (k = 0; k < table->num_entries; k++) { - if ((table->mc_reg_table_entry[k].mclk_max == 125000) || - (table->mc_reg_table_entry[k].mclk_max == 137500)) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) | - 0x0000D0DD; - } - break; - case mmMC_SEQ_WR_CTL_2: - for (k = 0; k < table->num_entries; k++) { - if ((table->mc_reg_table_entry[k].mclk_max == 125000) || - (table->mc_reg_table_entry[k].mclk_max == 137500)) - table->mc_reg_table_entry[k].mc_data[i] = 0; - } - break; - case mmMC_SEQ_CAS_TIMING: - for (k = 0; k < table->num_entries; k++) { - if (table->mc_reg_table_entry[k].mclk_max == 125000) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) | - 0x000C0140; - else if (table->mc_reg_table_entry[k].mclk_max == 137500) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) | - 0x000C0150; - } - break; - case mmMC_SEQ_MISC_TIMING: - for (k = 0; k < table->num_entries; k++) { - if (table->mc_reg_table_entry[k].mclk_max == 125000) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) | - 0x00000030; - else if (table->mc_reg_table_entry[k].mclk_max == 137500) - table->mc_reg_table_entry[k].mc_data[i] = - (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) | - 0x00000035; - } - break; - default: - break; - } - } - - WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 3); - tmp = RREG32(mmMC_SEQ_IO_DEBUG_DATA); - tmp = (tmp & 0xFFF8FFFF) | (1 << 16); - WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 3); - WREG32(mmMC_SEQ_IO_DEBUG_DATA, tmp); - } - - return 0; -} - -static int ci_initialize_mc_reg_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct atom_mc_reg_table *table; - struct ci_mc_reg_table *ci_table = &pi->mc_reg_table; - u8 module_index = ci_get_memory_module_index(adev); - int ret; - - table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL); - if (!table) - return -ENOMEM; - - WREG32(mmMC_SEQ_RAS_TIMING_LP, RREG32(mmMC_SEQ_RAS_TIMING)); - WREG32(mmMC_SEQ_CAS_TIMING_LP, RREG32(mmMC_SEQ_CAS_TIMING)); - WREG32(mmMC_SEQ_DLL_STBY_LP, RREG32(mmMC_SEQ_DLL_STBY)); - WREG32(mmMC_SEQ_G5PDX_CMD0_LP, RREG32(mmMC_SEQ_G5PDX_CMD0)); - WREG32(mmMC_SEQ_G5PDX_CMD1_LP, RREG32(mmMC_SEQ_G5PDX_CMD1)); - WREG32(mmMC_SEQ_G5PDX_CTRL_LP, RREG32(mmMC_SEQ_G5PDX_CTRL)); - WREG32(mmMC_SEQ_PMG_DVS_CMD_LP, RREG32(mmMC_SEQ_PMG_DVS_CMD)); - WREG32(mmMC_SEQ_PMG_DVS_CTL_LP, RREG32(mmMC_SEQ_PMG_DVS_CTL)); - WREG32(mmMC_SEQ_MISC_TIMING_LP, RREG32(mmMC_SEQ_MISC_TIMING)); - WREG32(mmMC_SEQ_MISC_TIMING2_LP, RREG32(mmMC_SEQ_MISC_TIMING2)); - WREG32(mmMC_SEQ_PMG_CMD_EMRS_LP, RREG32(mmMC_PMG_CMD_EMRS)); - WREG32(mmMC_SEQ_PMG_CMD_MRS_LP, RREG32(mmMC_PMG_CMD_MRS)); - WREG32(mmMC_SEQ_PMG_CMD_MRS1_LP, RREG32(mmMC_PMG_CMD_MRS1)); - WREG32(mmMC_SEQ_WR_CTL_D0_LP, RREG32(mmMC_SEQ_WR_CTL_D0)); - WREG32(mmMC_SEQ_WR_CTL_D1_LP, RREG32(mmMC_SEQ_WR_CTL_D1)); - WREG32(mmMC_SEQ_RD_CTL_D0_LP, RREG32(mmMC_SEQ_RD_CTL_D0)); - WREG32(mmMC_SEQ_RD_CTL_D1_LP, RREG32(mmMC_SEQ_RD_CTL_D1)); - WREG32(mmMC_SEQ_PMG_TIMING_LP, RREG32(mmMC_SEQ_PMG_TIMING)); - WREG32(mmMC_SEQ_PMG_CMD_MRS2_LP, RREG32(mmMC_PMG_CMD_MRS2)); - WREG32(mmMC_SEQ_WR_CTL_2_LP, RREG32(mmMC_SEQ_WR_CTL_2)); - - ret = amdgpu_atombios_init_mc_reg_table(adev, module_index, table); - if (ret) - goto init_mc_done; - - ret = ci_copy_vbios_mc_reg_table(table, ci_table); - if (ret) - goto init_mc_done; - - ci_set_s0_mc_reg_index(ci_table); - - ret = ci_register_patching_mc_seq(adev, ci_table); - if (ret) - goto init_mc_done; - - ret = ci_set_mc_special_registers(adev, ci_table); - if (ret) - goto init_mc_done; - - ci_set_valid_flag(ci_table); - -init_mc_done: - kfree(table); - - return ret; -} - -static int ci_populate_mc_reg_addresses(struct amdgpu_device *adev, - SMU7_Discrete_MCRegisters *mc_reg_table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 i, j; - - for (i = 0, j = 0; j < pi->mc_reg_table.last; j++) { - if (pi->mc_reg_table.valid_flag & (1 << j)) { - if (i >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) - return -EINVAL; - mc_reg_table->address[i].s0 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s0); - mc_reg_table->address[i].s1 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s1); - i++; - } - } - - mc_reg_table->last = (u8)i; - - return 0; -} - -static void ci_convert_mc_registers(const struct ci_mc_reg_entry *entry, - SMU7_Discrete_MCRegisterSet *data, - u32 num_entries, u32 valid_flag) -{ - u32 i, j; - - for (i = 0, j = 0; j < num_entries; j++) { - if (valid_flag & (1 << j)) { - data->value[i] = cpu_to_be32(entry->mc_data[j]); - i++; - } - } -} - -static void ci_convert_mc_reg_table_entry_to_smc(struct amdgpu_device *adev, - const u32 memory_clock, - SMU7_Discrete_MCRegisterSet *mc_reg_table_data) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 i = 0; - - for(i = 0; i < pi->mc_reg_table.num_entries; i++) { - if (memory_clock <= pi->mc_reg_table.mc_reg_table_entry[i].mclk_max) - break; - } - - if ((i == pi->mc_reg_table.num_entries) && (i > 0)) - --i; - - ci_convert_mc_registers(&pi->mc_reg_table.mc_reg_table_entry[i], - mc_reg_table_data, pi->mc_reg_table.last, - pi->mc_reg_table.valid_flag); -} - -static void ci_convert_mc_reg_table_to_smc(struct amdgpu_device *adev, - SMU7_Discrete_MCRegisters *mc_reg_table) -{ - struct ci_power_info *pi = ci_get_pi(adev); - u32 i; - - for (i = 0; i < pi->dpm_table.mclk_table.count; i++) - ci_convert_mc_reg_table_entry_to_smc(adev, - pi->dpm_table.mclk_table.dpm_levels[i].value, - &mc_reg_table->data[i]); -} - -static int ci_populate_initial_mc_reg_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - int ret; - - memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters)); - - ret = ci_populate_mc_reg_addresses(adev, &pi->smc_mc_reg_table); - if (ret) - return ret; - ci_convert_mc_reg_table_to_smc(adev, &pi->smc_mc_reg_table); - - return amdgpu_ci_copy_bytes_to_smc(adev, - pi->mc_reg_table_start, - (u8 *)&pi->smc_mc_reg_table, - sizeof(SMU7_Discrete_MCRegisters), - pi->sram_end); -} - -static int ci_update_and_upload_mc_reg_table(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - - if (!(pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) - return 0; - - memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters)); - - ci_convert_mc_reg_table_to_smc(adev, &pi->smc_mc_reg_table); - - return amdgpu_ci_copy_bytes_to_smc(adev, - pi->mc_reg_table_start + - offsetof(SMU7_Discrete_MCRegisters, data[0]), - (u8 *)&pi->smc_mc_reg_table.data[0], - sizeof(SMU7_Discrete_MCRegisterSet) * - pi->dpm_table.mclk_table.count, - pi->sram_end); -} - -static void ci_enable_voltage_control(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT); - - tmp |= GENERAL_PWRMGT__VOLT_PWRMGT_EN_MASK; - WREG32_SMC(ixGENERAL_PWRMGT, tmp); -} - -static enum amdgpu_pcie_gen ci_get_maximum_link_speed(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_state) -{ - struct ci_ps *state = ci_get_ps(amdgpu_state); - int i; - u16 pcie_speed, max_speed = 0; - - for (i = 0; i < state->performance_level_count; i++) { - pcie_speed = state->performance_levels[i].pcie_gen; - if (max_speed < pcie_speed) - max_speed = pcie_speed; - } - - return max_speed; -} - -static u16 ci_get_current_pcie_speed(struct amdgpu_device *adev) -{ - u32 speed_cntl = 0; - - speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL) & - PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK; - speed_cntl >>= PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - - return (u16)speed_cntl; -} - -static int ci_get_current_pcie_lane_number(struct amdgpu_device *adev) -{ - u32 link_width = 0; - - link_width = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL) & - PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK; - link_width >>= PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; - - switch (link_width) { - case 1: - return 1; - case 2: - return 2; - case 3: - return 4; - case 4: - return 8; - case 0: - case 6: - default: - return 16; - } -} - -static void ci_request_link_speed_change_before_state_change(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_new_state, - struct amdgpu_ps *amdgpu_current_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - enum amdgpu_pcie_gen target_link_speed = - ci_get_maximum_link_speed(adev, amdgpu_new_state); - enum amdgpu_pcie_gen current_link_speed; - - if (pi->force_pcie_gen == AMDGPU_PCIE_GEN_INVALID) - current_link_speed = ci_get_maximum_link_speed(adev, amdgpu_current_state); - else - current_link_speed = pi->force_pcie_gen; - - pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID; - pi->pspp_notify_required = false; - if (target_link_speed > current_link_speed) { - switch (target_link_speed) { -#ifdef CONFIG_ACPI - case AMDGPU_PCIE_GEN3: - if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN3, false) == 0) - break; - pi->force_pcie_gen = AMDGPU_PCIE_GEN2; - if (current_link_speed == AMDGPU_PCIE_GEN2) - break; - case AMDGPU_PCIE_GEN2: - if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) - break; -#endif - default: - pi->force_pcie_gen = ci_get_current_pcie_speed(adev); - break; - } - } else { - if (target_link_speed < current_link_speed) - pi->pspp_notify_required = true; - } -} - -static void ci_notify_link_speed_change_after_state_change(struct amdgpu_device *adev, - struct amdgpu_ps *amdgpu_new_state, - struct amdgpu_ps *amdgpu_current_state) -{ - struct ci_power_info *pi = ci_get_pi(adev); - enum amdgpu_pcie_gen target_link_speed = - ci_get_maximum_link_speed(adev, amdgpu_new_state); - u8 request; - - if (pi->pspp_notify_required) { - if (target_link_speed == AMDGPU_PCIE_GEN3) - request = PCIE_PERF_REQ_PECI_GEN3; - else if (target_link_speed == AMDGPU_PCIE_GEN2) - request = PCIE_PERF_REQ_PECI_GEN2; - else - request = PCIE_PERF_REQ_PECI_GEN1; - - if ((request == PCIE_PERF_REQ_PECI_GEN1) && - (ci_get_current_pcie_speed(adev) > 0)) - return; - -#ifdef CONFIG_ACPI - amdgpu_acpi_pcie_performance_request(adev, request, false); -#endif - } -} - -static int ci_set_private_data_variables_based_on_pptable(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_clock_voltage_dependency_table *allowed_sclk_vddc_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk; - struct amdgpu_clock_voltage_dependency_table *allowed_mclk_vddc_table = - &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk; - struct amdgpu_clock_voltage_dependency_table *allowed_mclk_vddci_table = - &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk; - - if (allowed_sclk_vddc_table == NULL) - return -EINVAL; - if (allowed_sclk_vddc_table->count < 1) - return -EINVAL; - if (allowed_mclk_vddc_table == NULL) - return -EINVAL; - if (allowed_mclk_vddc_table->count < 1) - return -EINVAL; - if (allowed_mclk_vddci_table == NULL) - return -EINVAL; - if (allowed_mclk_vddci_table->count < 1) - return -EINVAL; - - pi->min_vddc_in_pp_table = allowed_sclk_vddc_table->entries[0].v; - pi->max_vddc_in_pp_table = - allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v; - - pi->min_vddci_in_pp_table = allowed_mclk_vddci_table->entries[0].v; - pi->max_vddci_in_pp_table = - allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v; - - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = - allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = - allowed_mclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = - allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = - allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v; - - return 0; -} - -static void ci_patch_with_vddc_leakage(struct amdgpu_device *adev, u16 *vddc) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_leakage_voltage *leakage_table = &pi->vddc_leakage; - u32 leakage_index; - - for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) { - if (leakage_table->leakage_id[leakage_index] == *vddc) { - *vddc = leakage_table->actual_voltage[leakage_index]; - break; - } - } -} - -static void ci_patch_with_vddci_leakage(struct amdgpu_device *adev, u16 *vddci) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_leakage_voltage *leakage_table = &pi->vddci_leakage; - u32 leakage_index; - - for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) { - if (leakage_table->leakage_id[leakage_index] == *vddci) { - *vddci = leakage_table->actual_voltage[leakage_index]; - break; - } - } -} - -static void ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_clock_voltage_dependency_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddc_leakage(adev, &table->entries[i].v); - } -} - -static void ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct amdgpu_device *adev, - struct amdgpu_clock_voltage_dependency_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddci_leakage(adev, &table->entries[i].v); - } -} - -static void ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_vce_clock_voltage_dependency_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddc_leakage(adev, &table->entries[i].v); - } -} - -static void ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_uvd_clock_voltage_dependency_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddc_leakage(adev, &table->entries[i].v); - } -} - -static void ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_phase_shedding_limits_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddc_leakage(adev, &table->entries[i].voltage); - } -} - -static void ci_patch_clock_voltage_limits_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_clock_and_voltage_limits *table) -{ - if (table) { - ci_patch_with_vddc_leakage(adev, (u16 *)&table->vddc); - ci_patch_with_vddci_leakage(adev, (u16 *)&table->vddci); - } -} - -static void ci_patch_cac_leakage_table_with_vddc_leakage(struct amdgpu_device *adev, - struct amdgpu_cac_leakage_table *table) -{ - u32 i; - - if (table) { - for (i = 0; i < table->count; i++) - ci_patch_with_vddc_leakage(adev, &table->entries[i].vddc); - } -} - -static void ci_patch_dependency_tables_with_leakage(struct amdgpu_device *adev) -{ - - ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk); - ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk); - ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk); - ci_patch_clock_voltage_dependency_table_with_vddci_leakage(adev, - &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk); - ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table); - ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table); - ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table); - ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table); - ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.phase_shedding_limits_table); - ci_patch_clock_voltage_limits_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac); - ci_patch_clock_voltage_limits_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc); - ci_patch_cac_leakage_table_with_vddc_leakage(adev, - &adev->pm.dpm.dyn_state.cac_leakage_table); - -} - -static void ci_update_current_ps(struct amdgpu_device *adev, - struct amdgpu_ps *rps) -{ - struct ci_ps *new_ps = ci_get_ps(rps); - struct ci_power_info *pi = ci_get_pi(adev); - - pi->current_rps = *rps; - pi->current_ps = *new_ps; - pi->current_rps.ps_priv = &pi->current_ps; - adev->pm.dpm.current_ps = &pi->current_rps; -} - -static void ci_update_requested_ps(struct amdgpu_device *adev, - struct amdgpu_ps *rps) -{ - struct ci_ps *new_ps = ci_get_ps(rps); - struct ci_power_info *pi = ci_get_pi(adev); - - pi->requested_rps = *rps; - pi->requested_ps = *new_ps; - pi->requested_rps.ps_priv = &pi->requested_ps; - adev->pm.dpm.requested_ps = &pi->requested_rps; -} - -static int ci_dpm_pre_set_power_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps; - struct amdgpu_ps *new_ps = &requested_ps; - - ci_update_requested_ps(adev, new_ps); - - ci_apply_state_adjust_rules(adev, &pi->requested_rps); - - return 0; -} - -static void ci_dpm_post_set_power_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps *new_ps = &pi->requested_rps; - - ci_update_current_ps(adev, new_ps); -} - - -static void ci_dpm_setup_asic(struct amdgpu_device *adev) -{ - ci_read_clock_registers(adev); - ci_enable_acpi_power_management(adev); - ci_init_sclk_t(adev); -} - -static int ci_dpm_enable(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps; - int ret; - - if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) { - ci_enable_voltage_control(adev); - ret = ci_construct_voltage_tables(adev); - if (ret) { - DRM_ERROR("ci_construct_voltage_tables failed\n"); - return ret; - } - } - if (pi->caps_dynamic_ac_timing) { - ret = ci_initialize_mc_reg_table(adev); - if (ret) - pi->caps_dynamic_ac_timing = false; - } - if (pi->dynamic_ss) - ci_enable_spread_spectrum(adev, true); - if (pi->thermal_protection) - ci_enable_thermal_protection(adev, true); - ci_program_sstp(adev); - ci_enable_display_gap(adev); - ci_program_vc(adev); - ret = ci_upload_firmware(adev); - if (ret) { - DRM_ERROR("ci_upload_firmware failed\n"); - return ret; - } - ret = ci_process_firmware_header(adev); - if (ret) { - DRM_ERROR("ci_process_firmware_header failed\n"); - return ret; - } - ret = ci_initial_switch_from_arb_f0_to_f1(adev); - if (ret) { - DRM_ERROR("ci_initial_switch_from_arb_f0_to_f1 failed\n"); - return ret; - } - ret = ci_init_smc_table(adev); - if (ret) { - DRM_ERROR("ci_init_smc_table failed\n"); - return ret; - } - ret = ci_init_arb_table_index(adev); - if (ret) { - DRM_ERROR("ci_init_arb_table_index failed\n"); - return ret; - } - if (pi->caps_dynamic_ac_timing) { - ret = ci_populate_initial_mc_reg_table(adev); - if (ret) { - DRM_ERROR("ci_populate_initial_mc_reg_table failed\n"); - return ret; - } - } - ret = ci_populate_pm_base(adev); - if (ret) { - DRM_ERROR("ci_populate_pm_base failed\n"); - return ret; - } - ci_dpm_start_smc(adev); - ci_enable_vr_hot_gpio_interrupt(adev); - ret = ci_notify_smc_display_change(adev, false); - if (ret) { - DRM_ERROR("ci_notify_smc_display_change failed\n"); - return ret; - } - ci_enable_sclk_control(adev, true); - ret = ci_enable_ulv(adev, true); - if (ret) { - DRM_ERROR("ci_enable_ulv failed\n"); - return ret; - } - ret = ci_enable_ds_master_switch(adev, true); - if (ret) { - DRM_ERROR("ci_enable_ds_master_switch failed\n"); - return ret; - } - ret = ci_start_dpm(adev); - if (ret) { - DRM_ERROR("ci_start_dpm failed\n"); - return ret; - } - ret = ci_enable_didt(adev, true); - if (ret) { - DRM_ERROR("ci_enable_didt failed\n"); - return ret; - } - ret = ci_enable_smc_cac(adev, true); - if (ret) { - DRM_ERROR("ci_enable_smc_cac failed\n"); - return ret; - } - ret = ci_enable_power_containment(adev, true); - if (ret) { - DRM_ERROR("ci_enable_power_containment failed\n"); - return ret; - } - - ret = ci_power_control_set_level(adev); - if (ret) { - DRM_ERROR("ci_power_control_set_level failed\n"); - return ret; - } - - ci_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, true); - - ret = ci_enable_thermal_based_sclk_dpm(adev, true); - if (ret) { - DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n"); - return ret; - } - - ci_thermal_start_thermal_controller(adev); - - ci_update_current_ps(adev, boot_ps); - - return 0; -} - -static void ci_dpm_disable(struct amdgpu_device *adev) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps; - - amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_LOW_TO_HIGH); - amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_HIGH_TO_LOW); - - ci_dpm_powergate_uvd(adev, true); - - if (!amdgpu_ci_is_smc_running(adev)) - return; - - ci_thermal_stop_thermal_controller(adev); - - if (pi->thermal_protection) - ci_enable_thermal_protection(adev, false); - ci_enable_power_containment(adev, false); - ci_enable_smc_cac(adev, false); - ci_enable_didt(adev, false); - ci_enable_spread_spectrum(adev, false); - ci_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, false); - ci_stop_dpm(adev); - ci_enable_ds_master_switch(adev, false); - ci_enable_ulv(adev, false); - ci_clear_vc(adev); - ci_reset_to_default(adev); - ci_dpm_stop_smc(adev); - ci_force_switch_to_arb_f0(adev); - ci_enable_thermal_based_sclk_dpm(adev, false); - - ci_update_current_ps(adev, boot_ps); -} - -static int ci_dpm_set_power_state(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps *new_ps = &pi->requested_rps; - struct amdgpu_ps *old_ps = &pi->current_rps; - int ret; - - ci_find_dpm_states_clocks_in_dpm_table(adev, new_ps); - if (pi->pcie_performance_request) - ci_request_link_speed_change_before_state_change(adev, new_ps, old_ps); - ret = ci_freeze_sclk_mclk_dpm(adev); - if (ret) { - DRM_ERROR("ci_freeze_sclk_mclk_dpm failed\n"); - return ret; - } - ret = ci_populate_and_upload_sclk_mclk_dpm_levels(adev, new_ps); - if (ret) { - DRM_ERROR("ci_populate_and_upload_sclk_mclk_dpm_levels failed\n"); - return ret; - } - ret = ci_generate_dpm_level_enable_mask(adev, new_ps); - if (ret) { - DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n"); - return ret; - } - - ret = ci_update_vce_dpm(adev, new_ps, old_ps); - if (ret) { - DRM_ERROR("ci_update_vce_dpm failed\n"); - return ret; - } - - ret = ci_update_sclk_t(adev); - if (ret) { - DRM_ERROR("ci_update_sclk_t failed\n"); - return ret; - } - if (pi->caps_dynamic_ac_timing) { - ret = ci_update_and_upload_mc_reg_table(adev); - if (ret) { - DRM_ERROR("ci_update_and_upload_mc_reg_table failed\n"); - return ret; - } - } - ret = ci_program_memory_timing_parameters(adev); - if (ret) { - DRM_ERROR("ci_program_memory_timing_parameters failed\n"); - return ret; - } - ret = ci_unfreeze_sclk_mclk_dpm(adev); - if (ret) { - DRM_ERROR("ci_unfreeze_sclk_mclk_dpm failed\n"); - return ret; - } - ret = ci_upload_dpm_level_enable_mask(adev); - if (ret) { - DRM_ERROR("ci_upload_dpm_level_enable_mask failed\n"); - return ret; - } - if (pi->pcie_performance_request) - ci_notify_link_speed_change_after_state_change(adev, new_ps, old_ps); - - return 0; -} - -#if 0 -static void ci_dpm_reset_asic(struct amdgpu_device *adev) -{ - ci_set_boot_state(adev); -} -#endif - -static void ci_dpm_display_configuration_changed(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - ci_program_display_gap(adev); -} - -union power_info { - struct _ATOM_POWERPLAY_INFO info; - struct _ATOM_POWERPLAY_INFO_V2 info_2; - struct _ATOM_POWERPLAY_INFO_V3 info_3; - struct _ATOM_PPLIB_POWERPLAYTABLE pplib; - struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; - struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; -}; - -union pplib_clock_info { - struct _ATOM_PPLIB_R600_CLOCK_INFO r600; - struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; - struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; - struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; - struct _ATOM_PPLIB_SI_CLOCK_INFO si; - struct _ATOM_PPLIB_CI_CLOCK_INFO ci; -}; - -union pplib_power_state { - struct _ATOM_PPLIB_STATE v1; - struct _ATOM_PPLIB_STATE_V2 v2; -}; - -static void ci_parse_pplib_non_clock_info(struct amdgpu_device *adev, - struct amdgpu_ps *rps, - struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, - u8 table_rev) -{ - rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); - rps->class = le16_to_cpu(non_clock_info->usClassification); - rps->class2 = le16_to_cpu(non_clock_info->usClassification2); - - if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { - rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); - rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); - } else { - rps->vclk = 0; - rps->dclk = 0; - } - - if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) - adev->pm.dpm.boot_ps = rps; - if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) - adev->pm.dpm.uvd_ps = rps; -} - -static void ci_parse_pplib_clock_info(struct amdgpu_device *adev, - struct amdgpu_ps *rps, int index, - union pplib_clock_info *clock_info) -{ - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *ps = ci_get_ps(rps); - struct ci_pl *pl = &ps->performance_levels[index]; - - ps->performance_level_count = index + 1; - - pl->sclk = le16_to_cpu(clock_info->ci.usEngineClockLow); - pl->sclk |= clock_info->ci.ucEngineClockHigh << 16; - pl->mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow); - pl->mclk |= clock_info->ci.ucMemoryClockHigh << 16; - - pl->pcie_gen = amdgpu_get_pcie_gen_support(adev, - pi->sys_pcie_mask, - pi->vbios_boot_state.pcie_gen_bootup_value, - clock_info->ci.ucPCIEGen); - pl->pcie_lane = amdgpu_get_pcie_lane_support(adev, - pi->vbios_boot_state.pcie_lane_bootup_value, - le16_to_cpu(clock_info->ci.usPCIELane)); - - if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) { - pi->acpi_pcie_gen = pl->pcie_gen; - } - - if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) { - pi->ulv.supported = true; - pi->ulv.pl = *pl; - pi->ulv.cg_ulv_parameter = CISLANDS_CGULVPARAMETER_DFLT; - } - - /* patch up boot state */ - if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { - pl->mclk = pi->vbios_boot_state.mclk_bootup_value; - pl->sclk = pi->vbios_boot_state.sclk_bootup_value; - pl->pcie_gen = pi->vbios_boot_state.pcie_gen_bootup_value; - pl->pcie_lane = pi->vbios_boot_state.pcie_lane_bootup_value; - } - - switch (rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) { - case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY: - pi->use_pcie_powersaving_levels = true; - if (pi->pcie_gen_powersaving.max < pl->pcie_gen) - pi->pcie_gen_powersaving.max = pl->pcie_gen; - if (pi->pcie_gen_powersaving.min > pl->pcie_gen) - pi->pcie_gen_powersaving.min = pl->pcie_gen; - if (pi->pcie_lane_powersaving.max < pl->pcie_lane) - pi->pcie_lane_powersaving.max = pl->pcie_lane; - if (pi->pcie_lane_powersaving.min > pl->pcie_lane) - pi->pcie_lane_powersaving.min = pl->pcie_lane; - break; - case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE: - pi->use_pcie_performance_levels = true; - if (pi->pcie_gen_performance.max < pl->pcie_gen) - pi->pcie_gen_performance.max = pl->pcie_gen; - if (pi->pcie_gen_performance.min > pl->pcie_gen) - pi->pcie_gen_performance.min = pl->pcie_gen; - if (pi->pcie_lane_performance.max < pl->pcie_lane) - pi->pcie_lane_performance.max = pl->pcie_lane; - if (pi->pcie_lane_performance.min > pl->pcie_lane) - pi->pcie_lane_performance.min = pl->pcie_lane; - break; - default: - break; - } -} - -static int ci_parse_power_table(struct amdgpu_device *adev) -{ - struct amdgpu_mode_info *mode_info = &adev->mode_info; - struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; - union pplib_power_state *power_state; - int i, j, k, non_clock_array_index, clock_array_index; - union pplib_clock_info *clock_info; - struct _StateArray *state_array; - struct _ClockInfoArray *clock_info_array; - struct _NonClockInfoArray *non_clock_info_array; - union power_info *power_info; - int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); - u16 data_offset; - u8 frev, crev; - u8 *power_state_offset; - struct ci_ps *ps; - - if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, - &frev, &crev, &data_offset)) - return -EINVAL; - power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); - - amdgpu_add_thermal_controller(adev); - - state_array = (struct _StateArray *) - (mode_info->atom_context->bios + data_offset + - le16_to_cpu(power_info->pplib.usStateArrayOffset)); - clock_info_array = (struct _ClockInfoArray *) - (mode_info->atom_context->bios + data_offset + - le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); - non_clock_info_array = (struct _NonClockInfoArray *) - (mode_info->atom_context->bios + data_offset + - le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); - - adev->pm.dpm.ps = kcalloc(state_array->ucNumEntries, - sizeof(struct amdgpu_ps), - GFP_KERNEL); - if (!adev->pm.dpm.ps) - return -ENOMEM; - power_state_offset = (u8 *)state_array->states; - for (i = 0; i < state_array->ucNumEntries; i++) { - u8 *idx; - power_state = (union pplib_power_state *)power_state_offset; - non_clock_array_index = power_state->v2.nonClockInfoIndex; - non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) - &non_clock_info_array->nonClockInfo[non_clock_array_index]; - ps = kzalloc(sizeof(struct ci_ps), GFP_KERNEL); - if (ps == NULL) { - kfree(adev->pm.dpm.ps); - return -ENOMEM; - } - adev->pm.dpm.ps[i].ps_priv = ps; - ci_parse_pplib_non_clock_info(adev, &adev->pm.dpm.ps[i], - non_clock_info, - non_clock_info_array->ucEntrySize); - k = 0; - idx = (u8 *)&power_state->v2.clockInfoIndex[0]; - for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { - clock_array_index = idx[j]; - if (clock_array_index >= clock_info_array->ucNumEntries) - continue; - if (k >= CISLANDS_MAX_HARDWARE_POWERLEVELS) - break; - clock_info = (union pplib_clock_info *) - ((u8 *)&clock_info_array->clockInfo[0] + - (clock_array_index * clock_info_array->ucEntrySize)); - ci_parse_pplib_clock_info(adev, - &adev->pm.dpm.ps[i], k, - clock_info); - k++; - } - power_state_offset += 2 + power_state->v2.ucNumDPMLevels; - } - adev->pm.dpm.num_ps = state_array->ucNumEntries; - - /* fill in the vce power states */ - for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) { - u32 sclk, mclk; - clock_array_index = adev->pm.dpm.vce_states[i].clk_idx; - clock_info = (union pplib_clock_info *) - &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize]; - sclk = le16_to_cpu(clock_info->ci.usEngineClockLow); - sclk |= clock_info->ci.ucEngineClockHigh << 16; - mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow); - mclk |= clock_info->ci.ucMemoryClockHigh << 16; - adev->pm.dpm.vce_states[i].sclk = sclk; - adev->pm.dpm.vce_states[i].mclk = mclk; - } - - return 0; -} - -static int ci_get_vbios_boot_values(struct amdgpu_device *adev, - struct ci_vbios_boot_state *boot_state) -{ - struct amdgpu_mode_info *mode_info = &adev->mode_info; - int index = GetIndexIntoMasterTable(DATA, FirmwareInfo); - ATOM_FIRMWARE_INFO_V2_2 *firmware_info; - u8 frev, crev; - u16 data_offset; - - if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, - &frev, &crev, &data_offset)) { - firmware_info = - (ATOM_FIRMWARE_INFO_V2_2 *)(mode_info->atom_context->bios + - data_offset); - boot_state->mvdd_bootup_value = le16_to_cpu(firmware_info->usBootUpMVDDCVoltage); - boot_state->vddc_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCVoltage); - boot_state->vddci_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCIVoltage); - boot_state->pcie_gen_bootup_value = ci_get_current_pcie_speed(adev); - boot_state->pcie_lane_bootup_value = ci_get_current_pcie_lane_number(adev); - boot_state->sclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultEngineClock); - boot_state->mclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultMemoryClock); - - return 0; - } - return -EINVAL; -} - -static void ci_dpm_fini(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->pm.dpm.num_ps; i++) { - kfree(adev->pm.dpm.ps[i].ps_priv); - } - kfree(adev->pm.dpm.ps); - kfree(adev->pm.dpm.priv); - kfree(adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries); - amdgpu_free_extended_power_table(adev); -} - -/** - * ci_dpm_init_microcode - load ucode images from disk - * - * @adev: amdgpu_device pointer - * - * Use the firmware interface to load the ucode images into - * the driver (not loaded into hw). - * Returns 0 on success, error on failure. - */ -static int ci_dpm_init_microcode(struct amdgpu_device *adev) -{ - const char *chip_name; - char fw_name[30]; - int err; - - DRM_DEBUG("\n"); - - switch (adev->asic_type) { - case CHIP_BONAIRE: - if ((adev->pdev->revision == 0x80) || - (adev->pdev->revision == 0x81) || - (adev->pdev->device == 0x665f)) - chip_name = "bonaire_k"; - else - chip_name = "bonaire"; - break; - case CHIP_HAWAII: - if (adev->pdev->revision == 0x80) - chip_name = "hawaii_k"; - else - chip_name = "hawaii"; - break; - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - default: BUG(); - } - - snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_smc.bin", chip_name); - err = request_firmware(&adev->pm.fw, fw_name, adev->dev); - if (err) - goto out; - err = amdgpu_ucode_validate(adev->pm.fw); - -out: - if (err) { - pr_err("cik_smc: Failed to load firmware \"%s\"\n", fw_name); - release_firmware(adev->pm.fw); - adev->pm.fw = NULL; - } - return err; -} - -static int ci_dpm_init(struct amdgpu_device *adev) -{ - int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info); - SMU7_Discrete_DpmTable *dpm_table; - struct amdgpu_gpio_rec gpio; - u16 data_offset, size; - u8 frev, crev; - struct ci_power_info *pi; - int ret; - - pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL); - if (pi == NULL) - return -ENOMEM; - adev->pm.dpm.priv = pi; - - pi->sys_pcie_mask = - adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK; - - pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID; - - pi->pcie_gen_performance.max = AMDGPU_PCIE_GEN1; - pi->pcie_gen_performance.min = AMDGPU_PCIE_GEN3; - pi->pcie_gen_powersaving.max = AMDGPU_PCIE_GEN1; - pi->pcie_gen_powersaving.min = AMDGPU_PCIE_GEN3; - - pi->pcie_lane_performance.max = 0; - pi->pcie_lane_performance.min = 16; - pi->pcie_lane_powersaving.max = 0; - pi->pcie_lane_powersaving.min = 16; - - ret = ci_get_vbios_boot_values(adev, &pi->vbios_boot_state); - if (ret) { - ci_dpm_fini(adev); - return ret; - } - - ret = amdgpu_get_platform_caps(adev); - if (ret) { - ci_dpm_fini(adev); - return ret; - } - - ret = amdgpu_parse_extended_power_table(adev); - if (ret) { - ci_dpm_fini(adev); - return ret; - } - - ret = ci_parse_power_table(adev); - if (ret) { - ci_dpm_fini(adev); - return ret; - } - - pi->dll_default_on = false; - pi->sram_end = SMC_RAM_END; - - pi->activity_target[0] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[1] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[2] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[3] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[4] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[5] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[6] = CISLAND_TARGETACTIVITY_DFLT; - pi->activity_target[7] = CISLAND_TARGETACTIVITY_DFLT; - - pi->mclk_activity_target = CISLAND_MCLK_TARGETACTIVITY_DFLT; - - pi->sclk_dpm_key_disabled = 0; - pi->mclk_dpm_key_disabled = 0; - pi->pcie_dpm_key_disabled = 0; - pi->thermal_sclk_dpm_enabled = 0; - - if (adev->powerplay.pp_feature & PP_SCLK_DEEP_SLEEP_MASK) - pi->caps_sclk_ds = true; - else - pi->caps_sclk_ds = false; - - pi->mclk_strobe_mode_threshold = 40000; - pi->mclk_stutter_mode_threshold = 40000; - pi->mclk_edc_enable_threshold = 40000; - pi->mclk_edc_wr_enable_threshold = 40000; - - ci_initialize_powertune_defaults(adev); - - pi->caps_fps = false; - - pi->caps_sclk_throttle_low_notification = false; - - pi->caps_uvd_dpm = true; - pi->caps_vce_dpm = true; - - ci_get_leakage_voltages(adev); - ci_patch_dependency_tables_with_leakage(adev); - ci_set_private_data_variables_based_on_pptable(adev); - - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries = - kcalloc(4, - sizeof(struct amdgpu_clock_voltage_dependency_entry), - GFP_KERNEL); - if (!adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { - ci_dpm_fini(adev); - return -ENOMEM; - } - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000; - adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900; - - adev->pm.dpm.dyn_state.mclk_sclk_ratio = 4; - adev->pm.dpm.dyn_state.sclk_mclk_delta = 15000; - adev->pm.dpm.dyn_state.vddc_vddci_delta = 200; - - adev->pm.dpm.dyn_state.valid_sclk_values.count = 0; - adev->pm.dpm.dyn_state.valid_sclk_values.values = NULL; - adev->pm.dpm.dyn_state.valid_mclk_values.count = 0; - adev->pm.dpm.dyn_state.valid_mclk_values.values = NULL; - - if (adev->asic_type == CHIP_HAWAII) { - pi->thermal_temp_setting.temperature_low = 94500; - pi->thermal_temp_setting.temperature_high = 95000; - pi->thermal_temp_setting.temperature_shutdown = 104000; - } else { - pi->thermal_temp_setting.temperature_low = 99500; - pi->thermal_temp_setting.temperature_high = 100000; - pi->thermal_temp_setting.temperature_shutdown = 104000; - } - - pi->uvd_enabled = false; - - dpm_table = &pi->smc_state_table; - - gpio = amdgpu_atombios_lookup_gpio(adev, VDDC_VRHOT_GPIO_PINID); - if (gpio.valid) { - dpm_table->VRHotGpio = gpio.shift; - adev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_REGULATOR_HOT; - } else { - dpm_table->VRHotGpio = CISLANDS_UNUSED_GPIO_PIN; - adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_REGULATOR_HOT; - } - - gpio = amdgpu_atombios_lookup_gpio(adev, PP_AC_DC_SWITCH_GPIO_PINID); - if (gpio.valid) { - dpm_table->AcDcGpio = gpio.shift; - adev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_HARDWAREDC; - } else { - dpm_table->AcDcGpio = CISLANDS_UNUSED_GPIO_PIN; - adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_HARDWAREDC; - } - - gpio = amdgpu_atombios_lookup_gpio(adev, VDDC_PCC_GPIO_PINID); - if (gpio.valid) { - u32 tmp = RREG32_SMC(ixCNB_PWRMGT_CNTL); - - switch (gpio.shift) { - case 0: - tmp &= ~CNB_PWRMGT_CNTL__GNB_SLOW_MODE_MASK; - tmp |= 1 << CNB_PWRMGT_CNTL__GNB_SLOW_MODE__SHIFT; - break; - case 1: - tmp &= ~CNB_PWRMGT_CNTL__GNB_SLOW_MODE_MASK; - tmp |= 2 << CNB_PWRMGT_CNTL__GNB_SLOW_MODE__SHIFT; - break; - case 2: - tmp |= CNB_PWRMGT_CNTL__GNB_SLOW_MASK; - break; - case 3: - tmp |= CNB_PWRMGT_CNTL__FORCE_NB_PS1_MASK; - break; - case 4: - tmp |= CNB_PWRMGT_CNTL__DPM_ENABLED_MASK; - break; - default: - DRM_INFO("Invalid PCC GPIO: %u!\n", gpio.shift); - break; - } - WREG32_SMC(ixCNB_PWRMGT_CNTL, tmp); - } - - pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_NONE; - pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_NONE; - pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_NONE; - if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT)) - pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; - else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) - pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; - - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL) { - if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT)) - pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; - else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2)) - pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; - else - adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL; - } - - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL) { - if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT)) - pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; - else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2)) - pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; - else - adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_MVDDCONTROL; - } - - pi->vddc_phase_shed_control = true; - -#if defined(CONFIG_ACPI) - pi->pcie_performance_request = - amdgpu_acpi_is_pcie_performance_request_supported(adev); -#else - pi->pcie_performance_request = false; -#endif - - if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, - &frev, &crev, &data_offset)) { - pi->caps_sclk_ss_support = true; - pi->caps_mclk_ss_support = true; - pi->dynamic_ss = true; - } else { - pi->caps_sclk_ss_support = false; - pi->caps_mclk_ss_support = false; - pi->dynamic_ss = true; - } - - if (adev->pm.int_thermal_type != THERMAL_TYPE_NONE) - pi->thermal_protection = true; - else - pi->thermal_protection = false; - - pi->caps_dynamic_ac_timing = true; - - pi->uvd_power_gated = true; - - /* make sure dc limits are valid */ - if ((adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) || - (adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0)) - adev->pm.dpm.dyn_state.max_clock_voltage_on_dc = - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - - pi->fan_ctrl_is_in_default_mode = true; - - return 0; -} - -static void -ci_dpm_debugfs_print_current_performance_level(void *handle, - struct seq_file *m) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct amdgpu_ps *rps = &pi->current_rps; - u32 sclk = ci_get_average_sclk_freq(adev); - u32 mclk = ci_get_average_mclk_freq(adev); - u32 activity_percent = 50; - int ret; - - ret = ci_read_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, AverageGraphicsA), - &activity_percent); - - if (ret == 0) { - activity_percent += 0x80; - activity_percent >>= 8; - activity_percent = activity_percent > 100 ? 100 : activity_percent; - } - - seq_printf(m, "uvd %sabled\n", pi->uvd_power_gated ? "dis" : "en"); - seq_printf(m, "vce %sabled\n", rps->vce_active ? "en" : "dis"); - seq_printf(m, "power level avg sclk: %u mclk: %u\n", - sclk, mclk); - seq_printf(m, "GPU load: %u %%\n", activity_percent); -} - -static void ci_dpm_print_power_state(void *handle, void *current_ps) -{ - struct amdgpu_ps *rps = (struct amdgpu_ps *)current_ps; - struct ci_ps *ps = ci_get_ps(rps); - struct ci_pl *pl; - int i; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_dpm_print_class_info(rps->class, rps->class2); - amdgpu_dpm_print_cap_info(rps->caps); - printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); - for (i = 0; i < ps->performance_level_count; i++) { - pl = &ps->performance_levels[i]; - printk("\t\tpower level %d sclk: %u mclk: %u pcie gen: %u pcie lanes: %u\n", - i, pl->sclk, pl->mclk, pl->pcie_gen + 1, pl->pcie_lane); - } - amdgpu_dpm_print_ps_status(adev, rps); -} - -static inline bool ci_are_power_levels_equal(const struct ci_pl *ci_cpl1, - const struct ci_pl *ci_cpl2) -{ - return ((ci_cpl1->mclk == ci_cpl2->mclk) && - (ci_cpl1->sclk == ci_cpl2->sclk) && - (ci_cpl1->pcie_gen == ci_cpl2->pcie_gen) && - (ci_cpl1->pcie_lane == ci_cpl2->pcie_lane)); -} - -static int ci_check_state_equal(void *handle, - void *current_ps, - void *request_ps, - bool *equal) -{ - struct ci_ps *ci_cps; - struct ci_ps *ci_rps; - int i; - struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps; - struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev == NULL || cps == NULL || rps == NULL || equal == NULL) - return -EINVAL; - - ci_cps = ci_get_ps((struct amdgpu_ps *)cps); - ci_rps = ci_get_ps((struct amdgpu_ps *)rps); - - if (ci_cps == NULL) { - *equal = false; - return 0; - } - - if (ci_cps->performance_level_count != ci_rps->performance_level_count) { - - *equal = false; - return 0; - } - - for (i = 0; i < ci_cps->performance_level_count; i++) { - if (!ci_are_power_levels_equal(&(ci_cps->performance_levels[i]), - &(ci_rps->performance_levels[i]))) { - *equal = false; - return 0; - } - } - - /* If all performance levels are the same try to use the UVD clocks to break the tie.*/ - *equal = ((cps->vclk == rps->vclk) && (cps->dclk == rps->dclk)); - *equal &= ((cps->evclk == rps->evclk) && (cps->ecclk == rps->ecclk)); - - return 0; -} - -static u32 ci_dpm_get_sclk(void *handle, bool low) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps); - - if (low) - return requested_state->performance_levels[0].sclk; - else - return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk; -} - -static u32 ci_dpm_get_mclk(void *handle, bool low) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps); - - if (low) - return requested_state->performance_levels[0].mclk; - else - return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk; -} - -/* get temperature in millidegrees */ -static int ci_dpm_get_temp(void *handle) -{ - u32 temp; - int actual_temp = 0; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - temp = (RREG32_SMC(ixCG_MULT_THERMAL_STATUS) & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >> - CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT; - - if (temp & 0x200) - actual_temp = 255; - else - actual_temp = temp & 0x1ff; - - actual_temp = actual_temp * 1000; - - return actual_temp; -} - -static int ci_set_temperature_range(struct amdgpu_device *adev) -{ - int ret; - - ret = ci_thermal_enable_alert(adev, false); - if (ret) - return ret; - ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN, - CISLANDS_TEMP_RANGE_MAX); - if (ret) - return ret; - ret = ci_thermal_enable_alert(adev, true); - if (ret) - return ret; - return ret; -} - -static int ci_dpm_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - adev->powerplay.pp_funcs = &ci_dpm_funcs; - adev->powerplay.pp_handle = adev; - ci_dpm_set_irq_funcs(adev); - - return 0; -} - -static int ci_dpm_late_init(void *handle) -{ - int ret; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (!adev->pm.dpm_enabled) - return 0; - - /* init the sysfs and debugfs files late */ - ret = amdgpu_pm_sysfs_init(adev); - if (ret) - return ret; - - ret = ci_set_temperature_range(adev); - if (ret) - return ret; - - return 0; -} - -static int ci_dpm_sw_init(void *handle) -{ - int ret; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - ret = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 230, - &adev->pm.dpm.thermal.irq); - if (ret) - return ret; - - ret = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 231, - &adev->pm.dpm.thermal.irq); - if (ret) - return ret; - - /* default to balanced state */ - adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED; - adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED; - adev->pm.dpm.forced_level = AMD_DPM_FORCED_LEVEL_AUTO; - adev->pm.default_sclk = adev->clock.default_sclk; - adev->pm.default_mclk = adev->clock.default_mclk; - adev->pm.current_sclk = adev->clock.default_sclk; - adev->pm.current_mclk = adev->clock.default_mclk; - adev->pm.int_thermal_type = THERMAL_TYPE_NONE; - - ret = ci_dpm_init_microcode(adev); - if (ret) - return ret; - - if (amdgpu_dpm == 0) - return 0; - - INIT_WORK(&adev->pm.dpm.thermal.work, amdgpu_dpm_thermal_work_handler); - mutex_lock(&adev->pm.mutex); - ret = ci_dpm_init(adev); - if (ret) - goto dpm_failed; - adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps; - if (amdgpu_dpm == 1) - amdgpu_pm_print_power_states(adev); - mutex_unlock(&adev->pm.mutex); - DRM_INFO("amdgpu: dpm initialized\n"); - - return 0; - -dpm_failed: - ci_dpm_fini(adev); - mutex_unlock(&adev->pm.mutex); - DRM_ERROR("amdgpu: dpm initialization failed\n"); - return ret; -} - -static int ci_dpm_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - flush_work(&adev->pm.dpm.thermal.work); - - mutex_lock(&adev->pm.mutex); - ci_dpm_fini(adev); - mutex_unlock(&adev->pm.mutex); - - release_firmware(adev->pm.fw); - adev->pm.fw = NULL; - - return 0; -} - -static int ci_dpm_hw_init(void *handle) -{ - int ret; - - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (!amdgpu_dpm) { - ret = ci_upload_firmware(adev); - if (ret) { - DRM_ERROR("ci_upload_firmware failed\n"); - return ret; - } - ci_dpm_start_smc(adev); - return 0; - } - - mutex_lock(&adev->pm.mutex); - ci_dpm_setup_asic(adev); - ret = ci_dpm_enable(adev); - if (ret) - adev->pm.dpm_enabled = false; - else - adev->pm.dpm_enabled = true; - mutex_unlock(&adev->pm.mutex); - - return ret; -} - -static int ci_dpm_hw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev->pm.dpm_enabled) { - mutex_lock(&adev->pm.mutex); - ci_dpm_disable(adev); - mutex_unlock(&adev->pm.mutex); - } else { - ci_dpm_stop_smc(adev); - } - - return 0; -} - -static int ci_dpm_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev->pm.dpm_enabled) { - mutex_lock(&adev->pm.mutex); - amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_LOW_TO_HIGH); - amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_HIGH_TO_LOW); - adev->pm.dpm.last_user_state = adev->pm.dpm.user_state; - adev->pm.dpm.last_state = adev->pm.dpm.state; - adev->pm.dpm.user_state = POWER_STATE_TYPE_INTERNAL_BOOT; - adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_BOOT; - mutex_unlock(&adev->pm.mutex); - amdgpu_pm_compute_clocks(adev); - - } - - return 0; -} - -static int ci_dpm_resume(void *handle) -{ - int ret; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - if (adev->pm.dpm_enabled) { - /* asic init will reset to the boot state */ - mutex_lock(&adev->pm.mutex); - ci_dpm_setup_asic(adev); - ret = ci_dpm_enable(adev); - if (ret) - adev->pm.dpm_enabled = false; - else - adev->pm.dpm_enabled = true; - adev->pm.dpm.user_state = adev->pm.dpm.last_user_state; - adev->pm.dpm.state = adev->pm.dpm.last_state; - mutex_unlock(&adev->pm.mutex); - if (adev->pm.dpm_enabled) - amdgpu_pm_compute_clocks(adev); - } - return 0; -} - -static bool ci_dpm_is_idle(void *handle) -{ - /* XXX */ - return true; -} - -static int ci_dpm_wait_for_idle(void *handle) -{ - /* XXX */ - return 0; -} - -static int ci_dpm_soft_reset(void *handle) -{ - return 0; -} - -static int ci_dpm_set_interrupt_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned type, - enum amdgpu_interrupt_state state) -{ - u32 cg_thermal_int; - - switch (type) { - case AMDGPU_THERMAL_IRQ_LOW_TO_HIGH: - switch (state) { - case AMDGPU_IRQ_STATE_DISABLE: - cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT); - cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK; - WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int); - break; - case AMDGPU_IRQ_STATE_ENABLE: - cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT); - cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK; - WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int); - break; - default: - break; - } - break; - - case AMDGPU_THERMAL_IRQ_HIGH_TO_LOW: - switch (state) { - case AMDGPU_IRQ_STATE_DISABLE: - cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT); - cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK; - WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int); - break; - case AMDGPU_IRQ_STATE_ENABLE: - cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT); - cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK; - WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int); - break; - default: - break; - } - break; - - default: - break; - } - return 0; -} - -static int ci_dpm_process_interrupt(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - struct amdgpu_iv_entry *entry) -{ - bool queue_thermal = false; - - if (entry == NULL) - return -EINVAL; - - switch (entry->src_id) { - case 230: /* thermal low to high */ - DRM_DEBUG("IH: thermal low to high\n"); - adev->pm.dpm.thermal.high_to_low = false; - queue_thermal = true; - break; - case 231: /* thermal high to low */ - DRM_DEBUG("IH: thermal high to low\n"); - adev->pm.dpm.thermal.high_to_low = true; - queue_thermal = true; - break; - default: - break; - } - - if (queue_thermal) - schedule_work(&adev->pm.dpm.thermal.work); - - return 0; -} - -static int ci_dpm_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - return 0; -} - -static int ci_dpm_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -static int ci_dpm_print_clock_levels(void *handle, - enum pp_clock_type type, char *buf) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table; - struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table; - struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table; - - int i, now, size = 0; - uint32_t clock, pcie_speed; - - switch (type) { - case PP_SCLK: - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetSclkFrequency); - clock = RREG32(mmSMC_MSG_ARG_0); - - for (i = 0; i < sclk_table->count; i++) { - if (clock > sclk_table->dpm_levels[i].value) - continue; - break; - } - now = i; - - for (i = 0; i < sclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", - i, sclk_table->dpm_levels[i].value / 100, - (i == now) ? "*" : ""); - break; - case PP_MCLK: - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetMclkFrequency); - clock = RREG32(mmSMC_MSG_ARG_0); - - for (i = 0; i < mclk_table->count; i++) { - if (clock > mclk_table->dpm_levels[i].value) - continue; - break; - } - now = i; - - for (i = 0; i < mclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", - i, mclk_table->dpm_levels[i].value / 100, - (i == now) ? "*" : ""); - break; - case PP_PCIE: - pcie_speed = ci_get_current_pcie_speed(adev); - for (i = 0; i < pcie_table->count; i++) { - if (pcie_speed != pcie_table->dpm_levels[i].value) - continue; - break; - } - now = i; - - for (i = 0; i < pcie_table->count; i++) - size += sprintf(buf + size, "%d: %s %s\n", i, - (pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x1" : - (pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" : - (pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "", - (i == now) ? "*" : ""); - break; - default: - break; - } - - return size; -} - -static int ci_dpm_force_clock_level(void *handle, - enum pp_clock_type type, uint32_t mask) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - - if (adev->pm.dpm.forced_level != AMD_DPM_FORCED_LEVEL_MANUAL) - return -EINVAL; - - if (mask == 0) - return -EINVAL; - - switch (type) { - case PP_SCLK: - if (!pi->sclk_dpm_key_disabled) - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_SCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.sclk_dpm_enable_mask & mask); - break; - - case PP_MCLK: - if (!pi->mclk_dpm_key_disabled) - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_MCLKDPM_SetEnabledMask, - pi->dpm_level_enable_mask.mclk_dpm_enable_mask & mask); - break; - - case PP_PCIE: - { - uint32_t tmp = mask & pi->dpm_level_enable_mask.pcie_dpm_enable_mask; - - if (!pi->pcie_dpm_key_disabled) { - if (fls(tmp) != ffs(tmp)) - amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PCIeDPM_UnForceLevel); - else - amdgpu_ci_send_msg_to_smc_with_parameter(adev, - PPSMC_MSG_PCIeDPM_ForceLevel, - fls(tmp) - 1); - } - break; - } - default: - break; - } - - return 0; -} - -static int ci_dpm_get_sclk_od(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_single_dpm_table *sclk_table = &(pi->dpm_table.sclk_table); - struct ci_single_dpm_table *golden_sclk_table = - &(pi->golden_dpm_table.sclk_table); - int value; - - value = (sclk_table->dpm_levels[sclk_table->count - 1].value - - golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) * - 100 / - golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; - - return value; -} - -static int ci_dpm_set_sclk_od(void *handle, uint32_t value) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps); - struct ci_single_dpm_table *golden_sclk_table = - &(pi->golden_dpm_table.sclk_table); - - if (value > 20) - value = 20; - - ps->performance_levels[ps->performance_level_count - 1].sclk = - golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value * - value / 100 + - golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; - - return 0; -} - -static int ci_dpm_get_mclk_od(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_single_dpm_table *mclk_table = &(pi->dpm_table.mclk_table); - struct ci_single_dpm_table *golden_mclk_table = - &(pi->golden_dpm_table.mclk_table); - int value; - - value = (mclk_table->dpm_levels[mclk_table->count - 1].value - - golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) * - 100 / - golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; - - return value; -} - -static int ci_dpm_set_mclk_od(void *handle, uint32_t value) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - struct ci_power_info *pi = ci_get_pi(adev); - struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps); - struct ci_single_dpm_table *golden_mclk_table = - &(pi->golden_dpm_table.mclk_table); - - if (value > 20) - value = 20; - - ps->performance_levels[ps->performance_level_count - 1].mclk = - golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value * - value / 100 + - golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; - - return 0; -} - -static int ci_dpm_read_sensor(void *handle, int idx, - void *value, int *size) -{ - u32 activity_percent = 50; - int ret; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* size must be at least 4 bytes for all sensors */ - if (*size < 4) - return -EINVAL; - - switch (idx) { - case AMDGPU_PP_SENSOR_GFX_SCLK: - *((uint32_t *)value) = ci_get_average_sclk_freq(adev); - *size = 4; - return 0; - case AMDGPU_PP_SENSOR_GFX_MCLK: - *((uint32_t *)value) = ci_get_average_mclk_freq(adev); - *size = 4; - return 0; - case AMDGPU_PP_SENSOR_GPU_TEMP: - *((uint32_t *)value) = ci_dpm_get_temp(adev); - *size = 4; - return 0; - case AMDGPU_PP_SENSOR_GPU_LOAD: - ret = ci_read_smc_soft_register(adev, - offsetof(SMU7_SoftRegisters, - AverageGraphicsA), - &activity_percent); - if (ret == 0) { - activity_percent += 0x80; - activity_percent >>= 8; - activity_percent = - activity_percent > 100 ? 100 : activity_percent; - } - *((uint32_t *)value) = activity_percent; - *size = 4; - return 0; - default: - return -EINVAL; - } -} - -static int ci_set_powergating_by_smu(void *handle, - uint32_t block_type, bool gate) -{ - switch (block_type) { - case AMD_IP_BLOCK_TYPE_UVD: - ci_dpm_powergate_uvd(handle, gate); - break; - default: - break; - } - return 0; -} - -static const struct amd_ip_funcs ci_dpm_ip_funcs = { - .name = "ci_dpm", - .early_init = ci_dpm_early_init, - .late_init = ci_dpm_late_init, - .sw_init = ci_dpm_sw_init, - .sw_fini = ci_dpm_sw_fini, - .hw_init = ci_dpm_hw_init, - .hw_fini = ci_dpm_hw_fini, - .suspend = ci_dpm_suspend, - .resume = ci_dpm_resume, - .is_idle = ci_dpm_is_idle, - .wait_for_idle = ci_dpm_wait_for_idle, - .soft_reset = ci_dpm_soft_reset, - .set_clockgating_state = ci_dpm_set_clockgating_state, - .set_powergating_state = ci_dpm_set_powergating_state, -}; - -const struct amdgpu_ip_block_version ci_smu_ip_block = -{ - .type = AMD_IP_BLOCK_TYPE_SMC, - .major = 7, - .minor = 0, - .rev = 0, - .funcs = &ci_dpm_ip_funcs, -}; - -static const struct amd_pm_funcs ci_dpm_funcs = { - .pre_set_power_state = &ci_dpm_pre_set_power_state, - .set_power_state = &ci_dpm_set_power_state, - .post_set_power_state = &ci_dpm_post_set_power_state, - .display_configuration_changed = &ci_dpm_display_configuration_changed, - .get_sclk = &ci_dpm_get_sclk, - .get_mclk = &ci_dpm_get_mclk, - .print_power_state = &ci_dpm_print_power_state, - .debugfs_print_current_performance_level = &ci_dpm_debugfs_print_current_performance_level, - .force_performance_level = &ci_dpm_force_performance_level, - .vblank_too_short = &ci_dpm_vblank_too_short, - .set_powergating_by_smu = &ci_set_powergating_by_smu, - .set_fan_control_mode = &ci_dpm_set_fan_control_mode, - .get_fan_control_mode = &ci_dpm_get_fan_control_mode, - .set_fan_speed_percent = &ci_dpm_set_fan_speed_percent, - .get_fan_speed_percent = &ci_dpm_get_fan_speed_percent, - .print_clock_levels = ci_dpm_print_clock_levels, - .force_clock_level = ci_dpm_force_clock_level, - .get_sclk_od = ci_dpm_get_sclk_od, - .set_sclk_od = ci_dpm_set_sclk_od, - .get_mclk_od = ci_dpm_get_mclk_od, - .set_mclk_od = ci_dpm_set_mclk_od, - .check_state_equal = ci_check_state_equal, - .get_vce_clock_state = amdgpu_get_vce_clock_state, - .read_sensor = ci_dpm_read_sensor, -}; - -static const struct amdgpu_irq_src_funcs ci_dpm_irq_funcs = { - .set = ci_dpm_set_interrupt_state, - .process = ci_dpm_process_interrupt, -}; - -static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev) -{ - adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST; - adev->pm.dpm.thermal.irq.funcs = &ci_dpm_irq_funcs; -} diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h b/drivers/gpu/drm/amd/amdgpu/ci_dpm.h deleted file mode 100644 index 91be2996ae7c..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright 2013 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ -#ifndef __CI_DPM_H__ -#define __CI_DPM_H__ - -#include "amdgpu_atombios.h" -#include "ppsmc.h" - -#define SMU__NUM_SCLK_DPM_STATE 8 -#define SMU__NUM_MCLK_DPM_LEVELS 6 -#define SMU__NUM_LCLK_DPM_LEVELS 8 -#define SMU__NUM_PCIE_DPM_LEVELS 8 -#include "smu7_discrete.h" - -#define CISLANDS_MAX_HARDWARE_POWERLEVELS 2 - -#define CISLANDS_UNUSED_GPIO_PIN 0x7F - -struct ci_pl { - u32 mclk; - u32 sclk; - enum amdgpu_pcie_gen pcie_gen; - u16 pcie_lane; -}; - -struct ci_ps { - u16 performance_level_count; - bool dc_compatible; - u32 sclk_t; - struct ci_pl performance_levels[CISLANDS_MAX_HARDWARE_POWERLEVELS]; -}; - -struct ci_dpm_level { - bool enabled; - u32 value; - u32 param1; -}; - -#define CISLAND_MAX_DEEPSLEEP_DIVIDER_ID 5 -#define MAX_REGULAR_DPM_NUMBER 8 -#define CISLAND_MINIMUM_ENGINE_CLOCK 800 - -struct ci_single_dpm_table { - u32 count; - struct ci_dpm_level dpm_levels[MAX_REGULAR_DPM_NUMBER]; -}; - -struct ci_dpm_table { - struct ci_single_dpm_table sclk_table; - struct ci_single_dpm_table mclk_table; - struct ci_single_dpm_table pcie_speed_table; - struct ci_single_dpm_table vddc_table; - struct ci_single_dpm_table vddci_table; - struct ci_single_dpm_table mvdd_table; -}; - -struct ci_mc_reg_entry { - u32 mclk_max; - u32 mc_data[SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE]; -}; - -struct ci_mc_reg_table { - u8 last; - u8 num_entries; - u16 valid_flag; - struct ci_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES]; - SMU7_Discrete_MCRegisterAddress mc_reg_address[SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE]; -}; - -struct ci_ulv_parm -{ - bool supported; - u32 cg_ulv_parameter; - u32 volt_change_delay; - struct ci_pl pl; -}; - -#define CISLANDS_MAX_LEAKAGE_COUNT 8 - -struct ci_leakage_voltage { - u16 count; - u16 leakage_id[CISLANDS_MAX_LEAKAGE_COUNT]; - u16 actual_voltage[CISLANDS_MAX_LEAKAGE_COUNT]; -}; - -struct ci_dpm_level_enable_mask { - u32 uvd_dpm_enable_mask; - u32 vce_dpm_enable_mask; - u32 acp_dpm_enable_mask; - u32 samu_dpm_enable_mask; - u32 sclk_dpm_enable_mask; - u32 mclk_dpm_enable_mask; - u32 pcie_dpm_enable_mask; -}; - -struct ci_vbios_boot_state -{ - u16 mvdd_bootup_value; - u16 vddc_bootup_value; - u16 vddci_bootup_value; - u32 sclk_bootup_value; - u32 mclk_bootup_value; - u16 pcie_gen_bootup_value; - u16 pcie_lane_bootup_value; -}; - -struct ci_clock_registers { - u32 cg_spll_func_cntl; - u32 cg_spll_func_cntl_2; - u32 cg_spll_func_cntl_3; - u32 cg_spll_func_cntl_4; - u32 cg_spll_spread_spectrum; - u32 cg_spll_spread_spectrum_2; - u32 dll_cntl; - u32 mclk_pwrmgt_cntl; - u32 mpll_ad_func_cntl; - u32 mpll_dq_func_cntl; - u32 mpll_func_cntl; - u32 mpll_func_cntl_1; - u32 mpll_func_cntl_2; - u32 mpll_ss1; - u32 mpll_ss2; -}; - -struct ci_thermal_temperature_setting { - s32 temperature_low; - s32 temperature_high; - s32 temperature_shutdown; -}; - -struct ci_pcie_perf_range { - u16 max; - u16 min; -}; - -enum ci_pt_config_reg_type { - CISLANDS_CONFIGREG_MMR = 0, - CISLANDS_CONFIGREG_SMC_IND, - CISLANDS_CONFIGREG_DIDT_IND, - CISLANDS_CONFIGREG_CACHE, - CISLANDS_CONFIGREG_MAX -}; - -#define POWERCONTAINMENT_FEATURE_BAPM 0x00000001 -#define POWERCONTAINMENT_FEATURE_TDCLimit 0x00000002 -#define POWERCONTAINMENT_FEATURE_PkgPwrLimit 0x00000004 - -struct ci_pt_config_reg { - u32 offset; - u32 mask; - u32 shift; - u32 value; - enum ci_pt_config_reg_type type; -}; - -struct ci_pt_defaults { - u8 svi_load_line_en; - u8 svi_load_line_vddc; - u8 tdc_vddc_throttle_release_limit_perc; - u8 tdc_mawt; - u8 tdc_waterfall_ctl; - u8 dte_ambient_temp_base; - u32 display_cac; - u32 bapm_temp_gradient; - u16 bapmti_r[SMU7_DTE_ITERATIONS * SMU7_DTE_SOURCES * SMU7_DTE_SINKS]; - u16 bapmti_rc[SMU7_DTE_ITERATIONS * SMU7_DTE_SOURCES * SMU7_DTE_SINKS]; -}; - -#define DPMTABLE_OD_UPDATE_SCLK 0x00000001 -#define DPMTABLE_OD_UPDATE_MCLK 0x00000002 -#define DPMTABLE_UPDATE_SCLK 0x00000004 -#define DPMTABLE_UPDATE_MCLK 0x00000008 - -struct ci_power_info { - struct ci_dpm_table dpm_table; - struct ci_dpm_table golden_dpm_table; - u32 voltage_control; - u32 mvdd_control; - u32 vddci_control; - u32 active_auto_throttle_sources; - struct ci_clock_registers clock_registers; - u16 acpi_vddc; - u16 acpi_vddci; - enum amdgpu_pcie_gen force_pcie_gen; - enum amdgpu_pcie_gen acpi_pcie_gen; - struct ci_leakage_voltage vddc_leakage; - struct ci_leakage_voltage vddci_leakage; - u16 max_vddc_in_pp_table; - u16 min_vddc_in_pp_table; - u16 max_vddci_in_pp_table; - u16 min_vddci_in_pp_table; - u32 mclk_strobe_mode_threshold; - u32 mclk_stutter_mode_threshold; - u32 mclk_edc_enable_threshold; - u32 mclk_edc_wr_enable_threshold; - struct ci_vbios_boot_state vbios_boot_state; - /* smc offsets */ - u32 sram_end; - u32 dpm_table_start; - u32 soft_regs_start; - u32 mc_reg_table_start; - u32 fan_table_start; - u32 arb_table_start; - /* smc tables */ - SMU7_Discrete_DpmTable smc_state_table; - SMU7_Discrete_MCRegisters smc_mc_reg_table; - SMU7_Discrete_PmFuses smc_powertune_table; - /* other stuff */ - struct ci_mc_reg_table mc_reg_table; - struct atom_voltage_table vddc_voltage_table; - struct atom_voltage_table vddci_voltage_table; - struct atom_voltage_table mvdd_voltage_table; - struct ci_ulv_parm ulv; - u32 power_containment_features; - const struct ci_pt_defaults *powertune_defaults; - u32 dte_tj_offset; - bool vddc_phase_shed_control; - struct ci_thermal_temperature_setting thermal_temp_setting; - struct ci_dpm_level_enable_mask dpm_level_enable_mask; - u32 need_update_smu7_dpm_table; - u32 sclk_dpm_key_disabled; - u32 mclk_dpm_key_disabled; - u32 pcie_dpm_key_disabled; - u32 thermal_sclk_dpm_enabled; - struct ci_pcie_perf_range pcie_gen_performance; - struct ci_pcie_perf_range pcie_lane_performance; - struct ci_pcie_perf_range pcie_gen_powersaving; - struct ci_pcie_perf_range pcie_lane_powersaving; - u32 activity_target[SMU7_MAX_LEVELS_GRAPHICS]; - u32 mclk_activity_target; - u32 low_sclk_interrupt_t; - u32 last_mclk_dpm_enable_mask; - u32 sys_pcie_mask; - /* caps */ - bool caps_power_containment; - bool caps_cac; - bool caps_sq_ramping; - bool caps_db_ramping; - bool caps_td_ramping; - bool caps_tcp_ramping; - bool caps_fps; - bool caps_sclk_ds; - bool caps_sclk_ss_support; - bool caps_mclk_ss_support; - bool caps_uvd_dpm; - bool caps_vce_dpm; - bool caps_samu_dpm; - bool caps_acp_dpm; - bool caps_automatic_dc_transition; - bool caps_sclk_throttle_low_notification; - bool caps_dynamic_ac_timing; - bool caps_od_fuzzy_fan_control_support; - /* flags */ - bool thermal_protection; - bool pcie_performance_request; - bool dynamic_ss; - bool dll_default_on; - bool cac_enabled; - bool uvd_enabled; - bool battery_state; - bool pspp_notify_required; - bool enable_bapm_feature; - bool enable_tdc_limit_feature; - bool enable_pkg_pwr_tracking_feature; - bool use_pcie_performance_levels; - bool use_pcie_powersaving_levels; - bool uvd_power_gated; - /* driver states */ - struct amdgpu_ps current_rps; - struct ci_ps current_ps; - struct amdgpu_ps requested_rps; - struct ci_ps requested_ps; - /* fan control */ - bool fan_ctrl_is_in_default_mode; - bool fan_is_controlled_by_smc; - u32 t_min; - u32 fan_ctrl_default_mode; -}; - -#define CISLANDS_VOLTAGE_CONTROL_NONE 0x0 -#define CISLANDS_VOLTAGE_CONTROL_BY_GPIO 0x1 -#define CISLANDS_VOLTAGE_CONTROL_BY_SVID2 0x2 - -#define CISLANDS_Q88_FORMAT_CONVERSION_UNIT 256 - -#define CISLANDS_VRC_DFLT0 0x3FFFC000 -#define CISLANDS_VRC_DFLT1 0x000400 -#define CISLANDS_VRC_DFLT2 0xC00080 -#define CISLANDS_VRC_DFLT3 0xC00200 -#define CISLANDS_VRC_DFLT4 0xC01680 -#define CISLANDS_VRC_DFLT5 0xC00033 -#define CISLANDS_VRC_DFLT6 0xC00033 -#define CISLANDS_VRC_DFLT7 0x3FFFC000 - -#define CISLANDS_CGULVPARAMETER_DFLT 0x00040035 -#define CISLAND_TARGETACTIVITY_DFLT 30 -#define CISLAND_MCLK_TARGETACTIVITY_DFLT 10 - -#define PCIE_PERF_REQ_REMOVE_REGISTRY 0 -#define PCIE_PERF_REQ_FORCE_LOWPOWER 1 -#define PCIE_PERF_REQ_PECI_GEN1 2 -#define PCIE_PERF_REQ_PECI_GEN2 3 -#define PCIE_PERF_REQ_PECI_GEN3 4 - -#define CISLANDS_SSTU_DFLT 0 -#define CISLANDS_SST_DFLT 0x00C8 - -/* XXX are these ok? */ -#define CISLANDS_TEMP_RANGE_MIN (90 * 1000) -#define CISLANDS_TEMP_RANGE_MAX (120 * 1000) - -int amdgpu_ci_copy_bytes_to_smc(struct amdgpu_device *adev, - u32 smc_start_address, - const u8 *src, u32 byte_count, u32 limit); -void amdgpu_ci_start_smc(struct amdgpu_device *adev); -void amdgpu_ci_reset_smc(struct amdgpu_device *adev); -int amdgpu_ci_program_jump_on_start(struct amdgpu_device *adev); -void amdgpu_ci_stop_smc_clock(struct amdgpu_device *adev); -void amdgpu_ci_start_smc_clock(struct amdgpu_device *adev); -bool amdgpu_ci_is_smc_running(struct amdgpu_device *adev); -PPSMC_Result amdgpu_ci_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg); -PPSMC_Result amdgpu_ci_wait_for_smc_inactive(struct amdgpu_device *adev); -int amdgpu_ci_load_smc_ucode(struct amdgpu_device *adev, u32 limit); -int amdgpu_ci_read_smc_sram_dword(struct amdgpu_device *adev, - u32 smc_address, u32 *value, u32 limit); -int amdgpu_ci_write_smc_sram_dword(struct amdgpu_device *adev, - u32 smc_address, u32 value, u32 limit); - -#endif diff --git a/drivers/gpu/drm/amd/amdgpu/ci_smc.c b/drivers/gpu/drm/amd/amdgpu/ci_smc.c deleted file mode 100644 index b8ba51e045b5..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/ci_smc.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright 2011 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Alex Deucher - */ - -#include <linux/firmware.h> -#include <drm/drmP.h> -#include "amdgpu.h" -#include "cikd.h" -#include "ppsmc.h" -#include "amdgpu_ucode.h" -#include "ci_dpm.h" - -#include "smu/smu_7_0_1_d.h" -#include "smu/smu_7_0_1_sh_mask.h" - -static int ci_set_smc_sram_address(struct amdgpu_device *adev, - u32 smc_address, u32 limit) -{ - if (smc_address & 3) - return -EINVAL; - if ((smc_address + 3) > limit) - return -EINVAL; - - WREG32(mmSMC_IND_INDEX_0, smc_address); - WREG32_P(mmSMC_IND_ACCESS_CNTL, 0, ~SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK); - - return 0; -} - -int amdgpu_ci_copy_bytes_to_smc(struct amdgpu_device *adev, - u32 smc_start_address, - const u8 *src, u32 byte_count, u32 limit) -{ - unsigned long flags; - u32 data, original_data; - u32 addr; - u32 extra_shift; - int ret = 0; - - if (smc_start_address & 3) - return -EINVAL; - if ((smc_start_address + byte_count) > limit) - return -EINVAL; - - addr = smc_start_address; - - spin_lock_irqsave(&adev->smc_idx_lock, flags); - while (byte_count >= 4) { - /* SMC address space is BE */ - data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; - - ret = ci_set_smc_sram_address(adev, addr, limit); - if (ret) - goto done; - - WREG32(mmSMC_IND_DATA_0, data); - - src += 4; - byte_count -= 4; - addr += 4; - } - - /* RMW for the final bytes */ - if (byte_count > 0) { - data = 0; - - ret = ci_set_smc_sram_address(adev, addr, limit); - if (ret) - goto done; - - original_data = RREG32(mmSMC_IND_DATA_0); - - extra_shift = 8 * (4 - byte_count); - - while (byte_count > 0) { - data = (data << 8) + *src++; - byte_count--; - } - - data <<= extra_shift; - - data |= (original_data & ~((~0UL) << extra_shift)); - - ret = ci_set_smc_sram_address(adev, addr, limit); - if (ret) - goto done; - - WREG32(mmSMC_IND_DATA_0, data); - } - -done: - spin_unlock_irqrestore(&adev->smc_idx_lock, flags); - - return ret; -} - -void amdgpu_ci_start_smc(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); - - tmp &= ~SMC_SYSCON_RESET_CNTL__rst_reg_MASK; - WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, tmp); -} - -void amdgpu_ci_reset_smc(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); - - tmp |= SMC_SYSCON_RESET_CNTL__rst_reg_MASK; - WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, tmp); -} - -int amdgpu_ci_program_jump_on_start(struct amdgpu_device *adev) -{ - static u8 data[] = { 0xE0, 0x00, 0x80, 0x40 }; - - return amdgpu_ci_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1); -} - -void amdgpu_ci_stop_smc_clock(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); - - tmp |= SMC_SYSCON_CLOCK_CNTL_0__ck_disable_MASK; - - WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, tmp); -} - -void amdgpu_ci_start_smc_clock(struct amdgpu_device *adev) -{ - u32 tmp = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); - - tmp &= ~SMC_SYSCON_CLOCK_CNTL_0__ck_disable_MASK; - - WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, tmp); -} - -bool amdgpu_ci_is_smc_running(struct amdgpu_device *adev) -{ - u32 clk = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); - u32 pc_c = RREG32_SMC(ixSMC_PC_C); - - if (!(clk & SMC_SYSCON_CLOCK_CNTL_0__ck_disable_MASK) && (0x20100 <= pc_c)) - return true; - - return false; -} - -PPSMC_Result amdgpu_ci_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg) -{ - u32 tmp; - int i; - - if (!amdgpu_ci_is_smc_running(adev)) - return PPSMC_Result_Failed; - - WREG32(mmSMC_MESSAGE_0, msg); - - for (i = 0; i < adev->usec_timeout; i++) { - tmp = RREG32(mmSMC_RESP_0); - if (tmp != 0) - break; - udelay(1); - } - tmp = RREG32(mmSMC_RESP_0); - - return (PPSMC_Result)tmp; -} - -PPSMC_Result amdgpu_ci_wait_for_smc_inactive(struct amdgpu_device *adev) -{ - u32 tmp; - int i; - - if (!amdgpu_ci_is_smc_running(adev)) - return PPSMC_Result_OK; - - for (i = 0; i < adev->usec_timeout; i++) { - tmp = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); - if ((tmp & SMC_SYSCON_CLOCK_CNTL_0__cken_MASK) == 0) - break; - udelay(1); - } - - return PPSMC_Result_OK; -} - -int amdgpu_ci_load_smc_ucode(struct amdgpu_device *adev, u32 limit) -{ - const struct smc_firmware_header_v1_0 *hdr; - unsigned long flags; - u32 ucode_start_address; - u32 ucode_size; - const u8 *src; - u32 data; - - if (!adev->pm.fw) - return -EINVAL; - - hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data; - amdgpu_ucode_print_smc_hdr(&hdr->header); - - adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version); - ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); - ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); - src = (const u8 *) - (adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); - - if (ucode_size & 3) - return -EINVAL; - - spin_lock_irqsave(&adev->smc_idx_lock, flags); - WREG32(mmSMC_IND_INDEX_0, ucode_start_address); - WREG32_P(mmSMC_IND_ACCESS_CNTL, SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK, - ~SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK); - while (ucode_size >= 4) { - /* SMC address space is BE */ - data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; - - WREG32(mmSMC_IND_DATA_0, data); - - src += 4; - ucode_size -= 4; - } - WREG32_P(mmSMC_IND_ACCESS_CNTL, 0, ~SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK); - spin_unlock_irqrestore(&adev->smc_idx_lock, flags); - - return 0; -} - -int amdgpu_ci_read_smc_sram_dword(struct amdgpu_device *adev, - u32 smc_address, u32 *value, u32 limit) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&adev->smc_idx_lock, flags); - ret = ci_set_smc_sram_address(adev, smc_address, limit); - if (ret == 0) - *value = RREG32(mmSMC_IND_DATA_0); - spin_unlock_irqrestore(&adev->smc_idx_lock, flags); - - return ret; -} - -int amdgpu_ci_write_smc_sram_dword(struct amdgpu_device *adev, - u32 smc_address, u32 value, u32 limit) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&adev->smc_idx_lock, flags); - ret = ci_set_smc_sram_address(adev, smc_address, limit); - if (ret == 0) - WREG32(mmSMC_IND_DATA_0, value); - spin_unlock_irqrestore(&adev->smc_idx_lock, flags); - - return ret; -} diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 71c50d8900e3..07c1f239e9c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -1741,6 +1741,69 @@ static bool cik_need_full_reset(struct amdgpu_device *adev) return true; } +static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1) +{ + uint32_t perfctr = 0; + uint64_t cnt0_of, cnt1_of; + int tmp; + + /* This reports 0 on APUs, so return to avoid writing/reading registers + * that may or may not be different from their GPU counterparts + */ + if (adev->flags & AMD_IS_APU) + return; + + /* Set the 2 events that we wish to watch, defined above */ + /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */ + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); + + /* Write to enable desired perf counters */ + WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr); + /* Zero out and enable the perf counters + * Write 0x5: + * Bit 0 = Start all counters(1) + * Bit 2 = Global counter reset enable(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005); + + msleep(1000); + + /* Load the shadow and disable the perf counters + * Write 0x2: + * Bit 0 = Stop counters(0) + * Bit 1 = Load the shadow counters(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002); + + /* Read register values to get any >32bit overflow */ + tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK); + cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); + cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); + + /* Get the values and add the overflow */ + *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); + *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); +} + +static bool cik_need_reset_on_init(struct amdgpu_device *adev) +{ + u32 clock_cntl, pc; + + if (adev->flags & AMD_IS_APU) + return false; + + /* check if the SMC is already running */ + clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + pc = RREG32_SMC(ixSMC_PC_C); + if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) && + (0x20100 <= pc)) + return true; + + return false; +} + static const struct amdgpu_asic_funcs cik_asic_funcs = { .read_disabled_bios = &cik_read_disabled_bios, @@ -1756,6 +1819,8 @@ static const struct amdgpu_asic_funcs cik_asic_funcs = .invalidate_hdp = &cik_invalidate_hdp, .need_full_reset = &cik_need_full_reset, .init_doorbell_index = &legacy_doorbell_index_init, + .get_pcie_usage = &cik_get_pcie_usage, + .need_reset_on_init = &cik_need_reset_on_init, }; static int cik_common_early_init(void *handle) @@ -2005,10 +2070,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block); amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); - if (amdgpu_dpm == -1) - amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); - else - amdgpu_device_ip_block_add(adev, &ci_smu_ip_block); + amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); #if defined(CONFIG_DRM_AMD_DC) @@ -2026,10 +2088,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block); amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); - if (amdgpu_dpm == -1) - amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); - else - amdgpu_device_ip_block_add(adev, &ci_smu_ip_block); + amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); #if defined(CONFIG_DRM_AMD_DC) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h index 2a086610f74d..2fcc4b60153c 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h +++ b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h @@ -24,7 +24,6 @@ #ifndef __CIK_DPM_H__ #define __CIK_DPM_H__ -extern const struct amdgpu_ip_block_version ci_smu_ip_block; extern const struct amdgpu_ip_block_version kv_smu_ip_block; #endif diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c index 8a8b4967a101..721c757156e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c @@ -103,9 +103,9 @@ static void cik_ih_disable_interrupts(struct amdgpu_device *adev) */ static int cik_ih_irq_init(struct amdgpu_device *adev) { + struct amdgpu_ih_ring *ih = &adev->irq.ih; int rb_bufsz; u32 interrupt_cntl, ih_cntl, ih_rb_cntl; - u64 wptr_off; /* disable irqs */ cik_ih_disable_interrupts(adev); @@ -131,9 +131,8 @@ static int cik_ih_irq_init(struct amdgpu_device *adev) ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK; /* set the writeback address whether it's enabled or not */ - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); WREG32(mmIH_RB_CNTL, ih_rb_cntl); @@ -183,11 +182,12 @@ static void cik_ih_irq_disable(struct amdgpu_device *adev) * Used by cik_irq_process(). * Returns the value of the wptr. */ -static u32 cik_ih_get_wptr(struct amdgpu_device *adev) +static u32 cik_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { u32 wptr, tmp; - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) { wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK; @@ -196,13 +196,13 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev) * this should allow us to catchup. */ dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; tmp = RREG32(mmIH_RB_CNTL); tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK; WREG32(mmIH_RB_CNTL, tmp); } - return (wptr & adev->irq.ih.ptr_mask); + return (wptr & ih->ptr_mask); } /* CIK IV Ring @@ -237,16 +237,17 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev) * position and also advance the position. */ static void cik_ih_decode_iv(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, struct amdgpu_iv_entry *entry) { /* wptr/rptr are in bytes! */ - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[4]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; entry->src_id = dw[0] & 0xff; @@ -256,7 +257,7 @@ static void cik_ih_decode_iv(struct amdgpu_device *adev, entry->pasid = (dw[2] >> 16) & 0xffff; /* wptr/rptr are in bytes! */ - adev->irq.ih.rptr += 16; + ih->rptr += 16; } /** @@ -266,9 +267,10 @@ static void cik_ih_decode_iv(struct amdgpu_device *adev, * * Set the IH ring buffer rptr. */ -static void cik_ih_set_rptr(struct amdgpu_device *adev) +static void cik_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); + WREG32(mmIH_RB_RPTR, ih->rptr); } static int cik_ih_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 45795191de1f..189599b694e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -220,7 +220,7 @@ static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 extra_bits = vmid & 0xf; diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c index 9d3ea298e116..61024b9c7a4b 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c @@ -103,9 +103,9 @@ static void cz_ih_disable_interrupts(struct amdgpu_device *adev) */ static int cz_ih_irq_init(struct amdgpu_device *adev) { - int rb_bufsz; + struct amdgpu_ih_ring *ih = &adev->irq.ih; u32 interrupt_cntl, ih_cntl, ih_rb_cntl; - u64 wptr_off; + int rb_bufsz; /* disable irqs */ cz_ih_disable_interrupts(adev); @@ -133,9 +133,8 @@ static int cz_ih_irq_init(struct amdgpu_device *adev) ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1); /* set the writeback address whether it's enabled or not */ - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); WREG32(mmIH_RB_CNTL, ih_rb_cntl); @@ -185,11 +184,12 @@ static void cz_ih_irq_disable(struct amdgpu_device *adev) * Used by cz_irq_process(VI). * Returns the value of the wptr. */ -static u32 cz_ih_get_wptr(struct amdgpu_device *adev) +static u32 cz_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { u32 wptr, tmp; - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); @@ -198,13 +198,13 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev) * this should allow us to catchup. */ dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; tmp = RREG32(mmIH_RB_CNTL); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); WREG32(mmIH_RB_CNTL, tmp); } - return (wptr & adev->irq.ih.ptr_mask); + return (wptr & ih->ptr_mask); } /** @@ -216,16 +216,17 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev) * position and also advance the position. */ static void cz_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) { /* wptr/rptr are in bytes! */ - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[4]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; entry->src_id = dw[0] & 0xff; @@ -235,7 +236,7 @@ static void cz_ih_decode_iv(struct amdgpu_device *adev, entry->pasid = (dw[2] >> 16) & 0xffff; /* wptr/rptr are in bytes! */ - adev->irq.ih.rptr += 16; + ih->rptr += 16; } /** @@ -245,9 +246,10 @@ static void cz_ih_decode_iv(struct amdgpu_device *adev, * * Set the IH ring buffer rptr. */ -static void cz_ih_set_rptr(struct amdgpu_device *adev) +static void cz_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); + WREG32(mmIH_RB_RPTR, ih->rptr); } static int cz_ih_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 4cfecdce29a3..1f0426d2fc2a 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -1682,7 +1682,7 @@ static void dce_v10_0_afmt_setmode(struct drm_encoder *encoder, dce_v10_0_audio_write_sad_regs(encoder); dce_v10_0_audio_write_latency_fields(encoder, mode); - err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false); + err = drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); if (err < 0) { DRM_ERROR("failed to setup AVI infoframe: %zd\n", err); return; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 7c868916d90f..2280b971d758 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -1724,7 +1724,7 @@ static void dce_v11_0_afmt_setmode(struct drm_encoder *encoder, dce_v11_0_audio_write_sad_regs(encoder); dce_v11_0_audio_write_latency_fields(encoder, mode); - err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false); + err = drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); if (err < 0) { DRM_ERROR("failed to setup AVI infoframe: %zd\n", err); return; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index 17eaaba36017..bea32f076b91 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -1423,6 +1423,7 @@ static void dce_v6_0_audio_set_avi_infoframe(struct drm_encoder *encoder, struct amdgpu_device *adev = dev->dev_private; struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); struct hdmi_avi_infoframe frame; u8 buffer[HDMI_INFOFRAME_HEADER_SIZE + HDMI_AVI_INFOFRAME_SIZE]; uint8_t *payload = buffer + 3; @@ -1430,7 +1431,7 @@ static void dce_v6_0_audio_set_avi_infoframe(struct drm_encoder *encoder, ssize_t err; u32 tmp; - err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false); + err = drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); if (err < 0) { DRM_ERROR("failed to setup AVI infoframe: %zd\n", err); return; @@ -2979,7 +2980,7 @@ static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { - unsigned long flags; + unsigned long flags; unsigned crtc_id; struct amdgpu_crtc *amdgpu_crtc; struct amdgpu_flip_work *works; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 8c0576978d36..13da915991dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -1616,7 +1616,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder, dce_v8_0_audio_write_sad_regs(encoder); dce_v8_0_audio_write_latency_fields(encoder, mode); - err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false); + err = drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); if (err < 0) { DRM_ERROR("failed to setup AVI infoframe: %zd\n", err); return; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index 1dc3013ea1d5..305276c7e4bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -1842,13 +1842,13 @@ static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; /* insert SWITCH_BUFFER packet before first IB in the ring frame */ - if (ctx_switch) { + if (flags & AMDGPU_HAVE_CTX_SWITCH) { amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); amdgpu_ring_write(ring, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 3a9fb6018c16..a59e0fdf5a97 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2228,13 +2228,13 @@ static void gfx_v7_0_ring_emit_fence_compute(struct amdgpu_ring *ring, static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; /* insert SWITCH_BUFFER packet before first IB in the ring frame */ - if (ctx_switch) { + if (flags & AMDGPU_HAVE_CTX_SWITCH) { amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); amdgpu_ring_write(ring, 0); } @@ -2259,11 +2259,27 @@ static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); + /* Currently, there is a high possibility to get wave ID mismatch + * between ME and GDS, leading to a hw deadlock, because ME generates + * different wave IDs than the GDS expects. This situation happens + * randomly when at least 5 compute pipes use GDS ordered append. + * The wave IDs generated by ME are also wrong after suspend/resume. + * Those are probably bugs somewhere else in the kernel driver. + * + * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and + * GDS to 0 for this ring (me/pipe). + */ + if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { + amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); + amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); + } + amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN @@ -5000,7 +5016,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = { 7 + /* gfx_v7_0_ring_emit_pipeline_sync */ CIK_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v7_0_ring_emit_vm_flush */ 7 + 7 + 7, /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */ - .emit_ib_size = 4, /* gfx_v7_0_ring_emit_ib_compute */ + .emit_ib_size = 7, /* gfx_v7_0_ring_emit_ib_compute */ .emit_ib = gfx_v7_0_ring_emit_ib_compute, .emit_fence = gfx_v7_0_ring_emit_fence_compute, .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync, @@ -5057,6 +5073,7 @@ static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev) adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE); adev->gds.gws.total_size = 64; adev->gds.oa.total_size = 16; + adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); if (adev->gds.mem.total_size == 64 * 1024) { adev->gds.mem.gfx_partition_size = 4096; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 57cb3a51bda7..b8e50a34bdb3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -6047,7 +6047,7 @@ static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; @@ -6079,11 +6079,27 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); + /* Currently, there is a high possibility to get wave ID mismatch + * between ME and GDS, leading to a hw deadlock, because ME generates + * different wave IDs than the GDS expects. This situation happens + * randomly when at least 5 compute pipes use GDS ordered append. + * The wave IDs generated by ME are also wrong after suspend/resume. + * Those are probably bugs somewhere else in the kernel driver. + * + * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and + * GDS to 0 for this ring (me/pipe). + */ + if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { + amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); + amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); + } + amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN @@ -6890,7 +6906,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ - .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */ + .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ .emit_ib = gfx_v8_0_ring_emit_ib_compute, .emit_fence = gfx_v8_0_ring_emit_fence_compute, .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, @@ -6920,7 +6936,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 17 + /* gfx_v8_0_ring_emit_vm_flush */ 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */ - .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */ + .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ .emit_fence = gfx_v8_0_ring_emit_fence_kiq, .test_ring = gfx_v8_0_ring_test_ring, .insert_nop = amdgpu_ring_insert_nop, @@ -6996,6 +7012,7 @@ static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE); adev->gds.gws.total_size = 64; adev->gds.oa.total_size = 16; + adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); if (adev->gds.mem.total_size == 64 * 1024) { adev->gds.mem.gfx_partition_size = 4096; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index fbca0494f871..5533f6e4f4a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -3972,7 +3972,7 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; @@ -4005,11 +4005,27 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); + /* Currently, there is a high possibility to get wave ID mismatch + * between ME and GDS, leading to a hw deadlock, because ME generates + * different wave IDs than the GDS expects. This situation happens + * randomly when at least 5 compute pipes use GDS ordered append. + * The wave IDs generated by ME are also wrong after suspend/resume. + * Those are probably bugs somewhere else in the kernel driver. + * + * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and + * GDS to 0 for this ring (me/pipe). + */ + if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { + amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); + amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); + } + amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, @@ -4729,7 +4745,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 2 + /* gfx_v9_0_ring_emit_vm_flush */ 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ - .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */ + .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ .emit_ib = gfx_v9_0_ring_emit_ib_compute, .emit_fence = gfx_v9_0_ring_emit_fence, .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, @@ -4764,7 +4780,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 2 + /* gfx_v9_0_ring_emit_vm_flush */ 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ - .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */ + .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ .emit_fence = gfx_v9_0_ring_emit_fence_kiq, .test_ring = gfx_v9_0_ring_test_ring, .insert_nop = amdgpu_ring_insert_nop, @@ -4846,6 +4862,26 @@ static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) break; } + switch (adev->asic_type) { + case CHIP_VEGA10: + case CHIP_VEGA20: + adev->gds.gds_compute_max_wave_id = 0x7ff; + break; + case CHIP_VEGA12: + adev->gds.gds_compute_max_wave_id = 0x27f; + break; + case CHIP_RAVEN: + if (adev->rev_id >= 0x8) + adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ + else + adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ + break; + default: + /* this really depends on the chip */ + adev->gds.gds_compute_max_wave_id = 0x7ff; + break; + } + adev->gds.gws.total_size = 64; adev->gds.oa.total_size = 16; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 1ad7e6b8ed1d..34440672f938 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1471,8 +1471,9 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, gmc_v8_0_set_fault_enable_default(adev, false); if (printk_ratelimit()) { - struct amdgpu_task_info task_info = { 0 }; + struct amdgpu_task_info task_info; + memset(&task_info, 0, sizeof(struct amdgpu_task_info)); amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); dev_err(adev->dev, "GPU fault detected: %d 0x%08x for process %s pid %d thread %s pid %d\n", diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index bacdaef77b6c..600259b4e291 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -305,6 +305,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; + bool retry_fault = !!(entry->src_data[1] & 0x80); uint32_t status = 0; u64 addr; @@ -320,13 +321,16 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, } if (printk_ratelimit()) { - struct amdgpu_task_info task_info = { 0 }; + struct amdgpu_task_info task_info; + memset(&task_info, 0, sizeof(struct amdgpu_task_info)); amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); dev_err(adev->dev, - "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n", + "[%s] %s page fault (src_id:%u ring:%u vmid:%u " + "pasid:%u, for process %s pid %d thread %s pid %d)\n", entry->vmid_src ? "mmhub" : "gfxhub", + retry_fault ? "retry" : "no-retry", entry->src_id, entry->ring_id, entry->vmid, entry->pasid, task_info.process_name, task_info.tgid, task_info.task_name, task_info.pid); @@ -961,7 +965,11 @@ static int gmc_v9_0_sw_init(void *handle) * vm size is 256TB (48bit), maximum size of Vega10, * block size 512 (9bit) */ - amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); + /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ + if (amdgpu_sriov_vf(adev)) + amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); + else + amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c index a3984d10b604..b1626e1d2f5d 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c @@ -103,9 +103,9 @@ static void iceland_ih_disable_interrupts(struct amdgpu_device *adev) */ static int iceland_ih_irq_init(struct amdgpu_device *adev) { + struct amdgpu_ih_ring *ih = &adev->irq.ih; int rb_bufsz; u32 interrupt_cntl, ih_cntl, ih_rb_cntl; - u64 wptr_off; /* disable irqs */ iceland_ih_disable_interrupts(adev); @@ -133,9 +133,8 @@ static int iceland_ih_irq_init(struct amdgpu_device *adev) ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1); /* set the writeback address whether it's enabled or not */ - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); WREG32(mmIH_RB_CNTL, ih_rb_cntl); @@ -185,11 +184,12 @@ static void iceland_ih_irq_disable(struct amdgpu_device *adev) * Used by cz_irq_process(VI). * Returns the value of the wptr. */ -static u32 iceland_ih_get_wptr(struct amdgpu_device *adev) +static u32 iceland_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { u32 wptr, tmp; - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); @@ -198,13 +198,13 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev) * this should allow us to catchup. */ dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; tmp = RREG32(mmIH_RB_CNTL); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); WREG32(mmIH_RB_CNTL, tmp); } - return (wptr & adev->irq.ih.ptr_mask); + return (wptr & ih->ptr_mask); } /** @@ -216,16 +216,17 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev) * position and also advance the position. */ static void iceland_ih_decode_iv(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, struct amdgpu_iv_entry *entry) { /* wptr/rptr are in bytes! */ - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[4]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; entry->src_id = dw[0] & 0xff; @@ -235,7 +236,7 @@ static void iceland_ih_decode_iv(struct amdgpu_device *adev, entry->pasid = (dw[2] >> 16) & 0xffff; /* wptr/rptr are in bytes! */ - adev->irq.ih.rptr += 16; + ih->rptr += 16; } /** @@ -245,9 +246,10 @@ static void iceland_ih_decode_iv(struct amdgpu_device *adev, * * Set the IH ring buffer rptr. */ -static void iceland_ih_set_rptr(struct amdgpu_device *adev) +static void iceland_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); + WREG32(mmIH_RB_RPTR, ih->rptr); } static int iceland_ih_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index b11a1c17a7f2..73851ebb3833 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -266,7 +266,8 @@ flr_done: } /* Trigger recovery for world switch failure if no TDR */ - if (amdgpu_device_should_recover_gpu(adev)) + if (amdgpu_device_should_recover_gpu(adev) + && amdgpu_lockup_timeout == MAX_SCHEDULE_TIMEOUT) amdgpu_device_gpu_recover(adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c index accdedd63c98..cc967dbfd631 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c @@ -27,13 +27,9 @@ #include "nbio/nbio_6_1_default.h" #include "nbio/nbio_6_1_offset.h" #include "nbio/nbio_6_1_sh_mask.h" +#include "nbio/nbio_6_1_smn.h" #include "vega10_enum.h" -#define smnCPM_CONTROL 0x11180460 -#define smnPCIE_CNTL2 0x11180070 -#define smnPCIE_CONFIG_CNTL 0x11180044 -#define smnPCIE_CI_CNTL 0x11180080 - static u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev) { u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); @@ -72,7 +68,7 @@ static u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev) } static void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instance, - bool use_doorbell, int doorbell_index) + bool use_doorbell, int doorbell_index, int doorbell_size) { u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) : SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE); @@ -81,7 +77,7 @@ static void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instan if (use_doorbell) { doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index); - doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 2); + doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, doorbell_size); } else doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c index df34dc79d444..1cdb98ad2db3 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c @@ -27,13 +27,11 @@ #include "nbio/nbio_7_0_default.h" #include "nbio/nbio_7_0_offset.h" #include "nbio/nbio_7_0_sh_mask.h" +#include "nbio/nbio_7_0_smn.h" #include "vega10_enum.h" #define smnNBIF_MGCG_CTRL_LCLK 0x1013a05c -#define smnCPM_CONTROL 0x11180460 -#define smnPCIE_CNTL2 0x11180070 - static u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev) { u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); @@ -69,7 +67,7 @@ static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev) } static void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instance, - bool use_doorbell, int doorbell_index) + bool use_doorbell, int doorbell_index, int doorbell_size) { u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) : SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE); @@ -78,7 +76,7 @@ static void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instan if (use_doorbell) { doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index); - doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 2); + doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, doorbell_size); } else doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index 186db182f924..c69d51598cfe 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -26,16 +26,13 @@ #include "nbio/nbio_7_4_offset.h" #include "nbio/nbio_7_4_sh_mask.h" +#include "nbio/nbio_7_4_0_smn.h" #define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c -#define smnCPM_CONTROL 0x11180460 -#define smnPCIE_CNTL2 0x11180070 -#define smnPCIE_CI_CNTL 0x11180080 - static u32 nbio_v7_4_get_rev_id(struct amdgpu_device *adev) { - u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); + u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); tmp &= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK; tmp >>= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT; @@ -68,7 +65,7 @@ static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev) } static void nbio_v7_4_sdma_doorbell_range(struct amdgpu_device *adev, int instance, - bool use_doorbell, int doorbell_index) + bool use_doorbell, int doorbell_index, int doorbell_size) { u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) : SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE); @@ -77,7 +74,7 @@ static void nbio_v7_4_sdma_doorbell_range(struct amdgpu_device *adev, int instan if (use_doorbell) { doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index); - doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 2); + doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, doorbell_size); } else doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h index 0de00fbe9233..f3a7d207af07 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h +++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h @@ -191,7 +191,7 @@ enum psp_gfx_fw_type GFX_FW_TYPE_MMSCH = 19, GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM = 20, GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM = 21, - GFX_FW_TYPE_RLC_RESTORE_LIST_CNTL = 22, + GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL = 22, GFX_FW_TYPE_UVD1 = 23, GFX_FW_TYPE_MAX = 24 }; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c index d78b4306a36f..77c2bc344dfc 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c @@ -38,75 +38,6 @@ MODULE_FIRMWARE("amdgpu/raven_asd.bin"); MODULE_FIRMWARE("amdgpu/picasso_asd.bin"); MODULE_FIRMWARE("amdgpu/raven2_asd.bin"); -static int -psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type) -{ - switch(ucode->ucode_id) { - case AMDGPU_UCODE_ID_SDMA0: - *type = GFX_FW_TYPE_SDMA0; - break; - case AMDGPU_UCODE_ID_SDMA1: - *type = GFX_FW_TYPE_SDMA1; - break; - case AMDGPU_UCODE_ID_CP_CE: - *type = GFX_FW_TYPE_CP_CE; - break; - case AMDGPU_UCODE_ID_CP_PFP: - *type = GFX_FW_TYPE_CP_PFP; - break; - case AMDGPU_UCODE_ID_CP_ME: - *type = GFX_FW_TYPE_CP_ME; - break; - case AMDGPU_UCODE_ID_CP_MEC1: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC1_JT: - *type = GFX_FW_TYPE_CP_MEC_ME1; - break; - case AMDGPU_UCODE_ID_CP_MEC2: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC2_JT: - *type = GFX_FW_TYPE_CP_MEC_ME2; - break; - case AMDGPU_UCODE_ID_RLC_G: - *type = GFX_FW_TYPE_RLC_G; - break; - case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL: - *type = GFX_FW_TYPE_RLC_RESTORE_LIST_CNTL; - break; - case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM: - *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM; - break; - case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM: - *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM; - break; - case AMDGPU_UCODE_ID_SMC: - *type = GFX_FW_TYPE_SMU; - break; - case AMDGPU_UCODE_ID_UVD: - *type = GFX_FW_TYPE_UVD; - break; - case AMDGPU_UCODE_ID_VCE: - *type = GFX_FW_TYPE_VCE; - break; - case AMDGPU_UCODE_ID_VCN: - *type = GFX_FW_TYPE_VCN; - break; - case AMDGPU_UCODE_ID_DMCU_ERAM: - *type = GFX_FW_TYPE_DMCU_ERAM; - break; - case AMDGPU_UCODE_ID_DMCU_INTV: - *type = GFX_FW_TYPE_DMCU_ISR; - break; - case AMDGPU_UCODE_ID_MAXIMUM: - default: - return -EINVAL; - } - - return 0; -} - static int psp_v10_0_init_microcode(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -158,26 +89,6 @@ out: return err; } -static int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, - struct psp_gfx_cmd_resp *cmd) -{ - int ret; - uint64_t fw_mem_mc_addr = ucode->mc_addr; - - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); - - cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; - - ret = psp_v10_0_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); - if (ret) - DRM_ERROR("Unknown firmware type\n"); - - return ret; -} - static int psp_v10_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -454,7 +365,6 @@ static int psp_v10_0_mode1_reset(struct psp_context *psp) static const struct psp_funcs psp_v10_0_funcs = { .init_microcode = psp_v10_0_init_microcode, - .prep_cmd_buf = psp_v10_0_prep_cmd_buf, .ring_init = psp_v10_0_ring_init, .ring_create = psp_v10_0_ring_create, .ring_stop = psp_v10_0_ring_stop, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index 189fcb004579..860b70d80d3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -40,60 +40,6 @@ MODULE_FIRMWARE("amdgpu/vega20_ta.bin"); /* address block */ #define smnMP1_FIRMWARE_FLAGS 0x3010024 -static int -psp_v11_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type) -{ - switch (ucode->ucode_id) { - case AMDGPU_UCODE_ID_SDMA0: - *type = GFX_FW_TYPE_SDMA0; - break; - case AMDGPU_UCODE_ID_SDMA1: - *type = GFX_FW_TYPE_SDMA1; - break; - case AMDGPU_UCODE_ID_CP_CE: - *type = GFX_FW_TYPE_CP_CE; - break; - case AMDGPU_UCODE_ID_CP_PFP: - *type = GFX_FW_TYPE_CP_PFP; - break; - case AMDGPU_UCODE_ID_CP_ME: - *type = GFX_FW_TYPE_CP_ME; - break; - case AMDGPU_UCODE_ID_CP_MEC1: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC1_JT: - *type = GFX_FW_TYPE_CP_MEC_ME1; - break; - case AMDGPU_UCODE_ID_CP_MEC2: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC2_JT: - *type = GFX_FW_TYPE_CP_MEC_ME2; - break; - case AMDGPU_UCODE_ID_RLC_G: - *type = GFX_FW_TYPE_RLC_G; - break; - case AMDGPU_UCODE_ID_SMC: - *type = GFX_FW_TYPE_SMU; - break; - case AMDGPU_UCODE_ID_UVD: - *type = GFX_FW_TYPE_UVD; - break; - case AMDGPU_UCODE_ID_VCE: - *type = GFX_FW_TYPE_VCE; - break; - case AMDGPU_UCODE_ID_UVD1: - *type = GFX_FW_TYPE_UVD1; - break; - case AMDGPU_UCODE_ID_MAXIMUM: - default: - return -EINVAL; - } - - return 0; -} - static int psp_v11_0_init_microcode(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -271,26 +217,6 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; } -static int psp_v11_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, - struct psp_gfx_cmd_resp *cmd) -{ - int ret; - uint64_t fw_mem_mc_addr = ucode->mc_addr; - - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); - - cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; - - ret = psp_v11_0_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); - if (ret) - DRM_ERROR("Unknown firmware type\n"); - - return ret; -} - static int psp_v11_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -757,7 +683,6 @@ static const struct psp_funcs psp_v11_0_funcs = { .init_microcode = psp_v11_0_init_microcode, .bootloader_load_sysdrv = psp_v11_0_bootloader_load_sysdrv, .bootloader_load_sos = psp_v11_0_bootloader_load_sos, - .prep_cmd_buf = psp_v11_0_prep_cmd_buf, .ring_init = psp_v11_0_ring_init, .ring_create = psp_v11_0_ring_create, .ring_stop = psp_v11_0_ring_stop, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index 79694ff16969..c63de945c021 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -47,57 +47,6 @@ MODULE_FIRMWARE("amdgpu/vega12_asd.bin"); static uint32_t sos_old_versions[] = {1517616, 1510592, 1448594, 1446554}; -static int -psp_v3_1_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type) -{ - switch(ucode->ucode_id) { - case AMDGPU_UCODE_ID_SDMA0: - *type = GFX_FW_TYPE_SDMA0; - break; - case AMDGPU_UCODE_ID_SDMA1: - *type = GFX_FW_TYPE_SDMA1; - break; - case AMDGPU_UCODE_ID_CP_CE: - *type = GFX_FW_TYPE_CP_CE; - break; - case AMDGPU_UCODE_ID_CP_PFP: - *type = GFX_FW_TYPE_CP_PFP; - break; - case AMDGPU_UCODE_ID_CP_ME: - *type = GFX_FW_TYPE_CP_ME; - break; - case AMDGPU_UCODE_ID_CP_MEC1: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC1_JT: - *type = GFX_FW_TYPE_CP_MEC_ME1; - break; - case AMDGPU_UCODE_ID_CP_MEC2: - *type = GFX_FW_TYPE_CP_MEC; - break; - case AMDGPU_UCODE_ID_CP_MEC2_JT: - *type = GFX_FW_TYPE_CP_MEC_ME2; - break; - case AMDGPU_UCODE_ID_RLC_G: - *type = GFX_FW_TYPE_RLC_G; - break; - case AMDGPU_UCODE_ID_SMC: - *type = GFX_FW_TYPE_SMU; - break; - case AMDGPU_UCODE_ID_UVD: - *type = GFX_FW_TYPE_UVD; - break; - case AMDGPU_UCODE_ID_VCE: - *type = GFX_FW_TYPE_VCE; - break; - case AMDGPU_UCODE_ID_MAXIMUM: - default: - return -EINVAL; - } - - return 0; -} - static int psp_v3_1_init_microcode(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -277,26 +226,6 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp) return ret; } -static int psp_v3_1_prep_cmd_buf(struct amdgpu_firmware_info *ucode, - struct psp_gfx_cmd_resp *cmd) -{ - int ret; - uint64_t fw_mem_mc_addr = ucode->mc_addr; - - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); - - cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); - cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; - - ret = psp_v3_1_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); - if (ret) - DRM_ERROR("Unknown firmware type\n"); - - return ret; -} - static int psp_v3_1_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -615,7 +544,6 @@ static const struct psp_funcs psp_v3_1_funcs = { .init_microcode = psp_v3_1_init_microcode, .bootloader_load_sysdrv = psp_v3_1_bootloader_load_sysdrv, .bootloader_load_sos = psp_v3_1_bootloader_load_sos, - .prep_cmd_buf = psp_v3_1_prep_cmd_buf, .ring_init = psp_v3_1_ring_init, .ring_create = psp_v3_1_ring_create, .ring_stop = psp_v3_1_ring_stop, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 9f3cb2aec7c2..cca3552b36ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -247,7 +247,7 @@ static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 1bccc5fe2d9d..0ce8331baeb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -421,7 +421,7 @@ static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -1145,8 +1145,7 @@ static int sdma_v3_0_sw_init(void *handle) ring->ring_obj = NULL; if (!amdgpu_sriov_vf(adev)) { ring->use_doorbell = true; - ring->doorbell_index = (i == 0) ? - adev->doorbell_index.sdma_engine0 : adev->doorbell_index.sdma_engine1; + ring->doorbell_index = adev->doorbell_index.sdma_engine[i]; } else { ring->use_pollmem = true; } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index aa2f71cc1eba..c816e55d43a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -500,7 +500,7 @@ static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -834,8 +834,6 @@ static void sdma_v4_0_gfx_resume(struct amdgpu_device *adev, unsigned int i) OFFSET, ring->doorbell_index); WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL, doorbell); WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET, doorbell_offset); - adev->nbio_funcs->sdma_doorbell_range(adev, i, ring->use_doorbell, - ring->doorbell_index); sdma_v4_0_ring_set_wptr(ring); @@ -1522,9 +1520,7 @@ static int sdma_v4_0_sw_init(void *handle) ring->use_doorbell?"true":"false"); /* doorbell size is 2 dwords, get DWORD offset */ - ring->doorbell_index = (i == 0) ? - (adev->doorbell_index.sdma_engine0 << 1) - : (adev->doorbell_index.sdma_engine1 << 1); + ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1; sprintf(ring->name, "sdma%d", i); r = amdgpu_ring_init(adev, ring, 1024, @@ -1543,9 +1539,7 @@ static int sdma_v4_0_sw_init(void *handle) /* paging queue use same doorbell index/routing as gfx queue * with 0x400 (4096 dwords) offset on second doorbell page */ - ring->doorbell_index = (i == 0) ? - (adev->doorbell_index.sdma_engine0 << 1) - : (adev->doorbell_index.sdma_engine1 << 1); + ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1; ring->doorbell_index += 0x400; sprintf(ring->name, "page%d", i); diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index f8408f88cd37..9d8df68893b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -47,6 +47,7 @@ #include "dce/dce_6_0_d.h" #include "uvd/uvd_4_0_d.h" #include "bif/bif_3_0_d.h" +#include "bif/bif_3_0_sh_mask.h" static const u32 tahiti_golden_registers[] = { @@ -1258,6 +1259,11 @@ static bool si_need_full_reset(struct amdgpu_device *adev) return true; } +static bool si_need_reset_on_init(struct amdgpu_device *adev) +{ + return false; +} + static int si_get_pcie_lanes(struct amdgpu_device *adev) { u32 link_width_cntl; @@ -1323,6 +1329,52 @@ static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes) WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } +static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1) +{ + uint32_t perfctr = 0; + uint64_t cnt0_of, cnt1_of; + int tmp; + + /* This reports 0 on APUs, so return to avoid writing/reading registers + * that may or may not be different from their GPU counterparts + */ + if (adev->flags & AMD_IS_APU) + return; + + /* Set the 2 events that we wish to watch, defined above */ + /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */ + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); + + /* Write to enable desired perf counters */ + WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr); + /* Zero out and enable the perf counters + * Write 0x5: + * Bit 0 = Start all counters(1) + * Bit 2 = Global counter reset enable(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005); + + msleep(1000); + + /* Load the shadow and disable the perf counters + * Write 0x2: + * Bit 0 = Stop counters(0) + * Bit 1 = Load the shadow counters(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002); + + /* Read register values to get any >32bit overflow */ + tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK); + cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); + cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); + + /* Get the values and add the overflow */ + *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); + *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); +} + static const struct amdgpu_asic_funcs si_asic_funcs = { .read_disabled_bios = &si_read_disabled_bios, @@ -1339,6 +1391,8 @@ static const struct amdgpu_asic_funcs si_asic_funcs = .flush_hdp = &si_flush_hdp, .invalidate_hdp = &si_invalidate_hdp, .need_full_reset = &si_need_full_reset, + .get_pcie_usage = &si_get_pcie_usage, + .need_reset_on_init = &si_need_reset_on_init, }; static uint32_t si_get_rev_id(struct amdgpu_device *adev) @@ -1382,7 +1436,7 @@ static int si_common_early_init(void *handle) AMD_CG_SUPPORT_UVD_MGCG | AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG; - adev->pg_flags = 0; + adev->pg_flags = 0; adev->external_rev_id = (adev->rev_id == 0) ? 1 : (adev->rev_id == 1) ? 5 : 6; break; diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index b6e473134e19..f15f196684ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -63,7 +63,7 @@ static void si_dma_ring_set_wptr(struct amdgpu_ring *ring) static void si_dma_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c index da58040fdbdc..41e01a7f57a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c @@ -6216,10 +6216,12 @@ static void si_request_link_speed_change_before_state_change(struct amdgpu_devic si_pi->force_pcie_gen = AMDGPU_PCIE_GEN2; if (current_link_speed == AMDGPU_PCIE_GEN2) break; + /* fall through */ case AMDGPU_PCIE_GEN2: if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; #endif + /* fall through */ default: si_pi->force_pcie_gen = si_get_current_pcie_speed(adev); break; diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c index 2938fb9f17cc..8c50c9cab455 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c @@ -57,9 +57,9 @@ static void si_ih_disable_interrupts(struct amdgpu_device *adev) static int si_ih_irq_init(struct amdgpu_device *adev) { + struct amdgpu_ih_ring *ih = &adev->irq.ih; int rb_bufsz; u32 interrupt_cntl, ih_cntl, ih_rb_cntl; - u64 wptr_off; si_ih_disable_interrupts(adev); WREG32(INTERRUPT_CNTL2, adev->irq.ih.gpu_addr >> 8); @@ -76,9 +76,8 @@ static int si_ih_irq_init(struct amdgpu_device *adev) (rb_bufsz << 1) | IH_WPTR_WRITEBACK_ENABLE; - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32(IH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); + WREG32(IH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); WREG32(IH_RB_CNTL, ih_rb_cntl); WREG32(IH_RB_RPTR, 0); WREG32(IH_RB_WPTR, 0); @@ -100,34 +99,36 @@ static void si_ih_irq_disable(struct amdgpu_device *adev) mdelay(1); } -static u32 si_ih_get_wptr(struct amdgpu_device *adev) +static u32 si_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { u32 wptr, tmp; - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) { wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK; dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; tmp = RREG32(IH_RB_CNTL); tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK; WREG32(IH_RB_CNTL, tmp); } - return (wptr & adev->irq.ih.ptr_mask); + return (wptr & ih->ptr_mask); } static void si_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) { - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[4]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; entry->src_id = dw[0] & 0xff; @@ -135,12 +136,13 @@ static void si_ih_decode_iv(struct amdgpu_device *adev, entry->ring_id = dw[2] & 0xff; entry->vmid = (dw[2] >> 8) & 0xff; - adev->irq.ih.rptr += 16; + ih->rptr += 16; } -static void si_ih_set_rptr(struct amdgpu_device *adev) +static void si_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - WREG32(IH_RB_RPTR, adev->irq.ih.rptr); + WREG32(IH_RB_RPTR, ih->rptr); } static int si_ih_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 9b639974c70c..99ebcf29dcb0 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -43,6 +43,10 @@ #include "hdp/hdp_4_0_sh_mask.h" #include "smuio/smuio_9_0_offset.h" #include "smuio/smuio_9_0_sh_mask.h" +#include "nbio/nbio_7_0_default.h" +#include "nbio/nbio_7_0_sh_mask.h" +#include "nbio/nbio_7_0_smn.h" +#include "mp/mp_9_0_offset.h" #include "soc15.h" #include "soc15_common.h" @@ -385,14 +389,13 @@ void soc15_program_register_sequence(struct amdgpu_device *adev, } - -static int soc15_asic_reset(struct amdgpu_device *adev) +static int soc15_asic_mode1_reset(struct amdgpu_device *adev) { u32 i; amdgpu_atombios_scratch_regs_engine_hung(adev, true); - dev_info(adev->dev, "GPU reset\n"); + dev_info(adev->dev, "GPU mode1 reset\n"); /* disable BM */ pci_clear_master(adev->pdev); @@ -417,6 +420,63 @@ static int soc15_asic_reset(struct amdgpu_device *adev) return 0; } +static int soc15_asic_get_baco_capability(struct amdgpu_device *adev, bool *cap) +{ + void *pp_handle = adev->powerplay.pp_handle; + const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; + + if (!pp_funcs || !pp_funcs->get_asic_baco_capability) { + *cap = false; + return -ENOENT; + } + + return pp_funcs->get_asic_baco_capability(pp_handle, cap); +} + +static int soc15_asic_baco_reset(struct amdgpu_device *adev) +{ + void *pp_handle = adev->powerplay.pp_handle; + const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; + + if (!pp_funcs ||!pp_funcs->get_asic_baco_state ||!pp_funcs->set_asic_baco_state) + return -ENOENT; + + /* enter BACO state */ + if (pp_funcs->set_asic_baco_state(pp_handle, 1)) + return -EIO; + + /* exit BACO state */ + if (pp_funcs->set_asic_baco_state(pp_handle, 0)) + return -EIO; + + dev_info(adev->dev, "GPU BACO reset\n"); + + return 0; +} + +static int soc15_asic_reset(struct amdgpu_device *adev) +{ + int ret; + bool baco_reset; + + switch (adev->asic_type) { + case CHIP_VEGA10: + case CHIP_VEGA20: + soc15_asic_get_baco_capability(adev, &baco_reset); + break; + default: + baco_reset = false; + break; + } + + if (baco_reset) + ret = soc15_asic_baco_reset(adev); + else + ret = soc15_asic_mode1_reset(adev); + + return ret; +} + /*static int soc15_set_uvd_clock(struct amdgpu_device *adev, u32 clock, u32 cntl_reg, u32 status_reg) { @@ -535,10 +595,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &vega10_common_ip_block); amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); - if (adev->asic_type == CHIP_VEGA20) - amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); - else - amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); + if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { + if (adev->asic_type == CHIP_VEGA20) + amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); + else + amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); + } amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); if (!amdgpu_sriov_vf(adev)) @@ -560,7 +622,8 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &vega10_common_ip_block); amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); - amdgpu_device_ip_block_add(adev, &psp_v10_0_ip_block); + if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) + amdgpu_device_ip_block_add(adev, &psp_v10_0_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); @@ -601,6 +664,68 @@ static bool soc15_need_full_reset(struct amdgpu_device *adev) /* change this when we implement soft reset */ return true; } +static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1) +{ + uint32_t perfctr = 0; + uint64_t cnt0_of, cnt1_of; + int tmp; + + /* This reports 0 on APUs, so return to avoid writing/reading registers + * that may or may not be different from their GPU counterparts + */ + if (adev->flags & AMD_IS_APU) + return; + + /* Set the 2 events that we wish to watch, defined above */ + /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */ + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); + + /* Write to enable desired perf counters */ + WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK, perfctr); + /* Zero out and enable the perf counters + * Write 0x5: + * Bit 0 = Start all counters(1) + * Bit 2 = Global counter reset enable(1) + */ + WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000005); + + msleep(1000); + + /* Load the shadow and disable the perf counters + * Write 0x2: + * Bit 0 = Stop counters(0) + * Bit 1 = Load the shadow counters(1) + */ + WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000002); + + /* Read register values to get any >32bit overflow */ + tmp = RREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK); + cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); + cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); + + /* Get the values and add the overflow */ + *count0 = RREG32_PCIE(smnPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); + *count1 = RREG32_PCIE(smnPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); +} + +static bool soc15_need_reset_on_init(struct amdgpu_device *adev) +{ + u32 sol_reg; + + if (adev->flags & AMD_IS_APU) + return false; + + /* Check sOS sign of life register to confirm sys driver and sOS + * are already been loaded. + */ + sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81); + if (sol_reg) + return true; + + return false; +} static const struct amdgpu_asic_funcs soc15_asic_funcs = { @@ -617,6 +742,8 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs = .invalidate_hdp = &soc15_invalidate_hdp, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega10_doorbell_index_init, + .get_pcie_usage = &soc15_get_pcie_usage, + .need_reset_on_init = &soc15_need_reset_on_init, }; static const struct amdgpu_asic_funcs vega20_asic_funcs = @@ -634,6 +761,8 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = .invalidate_hdp = &soc15_invalidate_hdp, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega20_doorbell_index_init, + .get_pcie_usage = &soc15_get_pcie_usage, + .need_reset_on_init = &soc15_need_reset_on_init, }; static int soc15_common_early_init(void *handle) @@ -842,6 +971,22 @@ static int soc15_common_sw_fini(void *handle) return 0; } +static void soc15_doorbell_range_init(struct amdgpu_device *adev) +{ + int i; + struct amdgpu_ring *ring; + + for (i = 0; i < adev->sdma.num_instances; i++) { + ring = &adev->sdma.instance[i].ring; + adev->nbio_funcs->sdma_doorbell_range(adev, i, + ring->use_doorbell, ring->doorbell_index, + adev->doorbell_index.sdma_doorbell_range); + } + + adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, + adev->irq.ih.doorbell_index); +} + static int soc15_common_hw_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -854,6 +999,12 @@ static int soc15_common_hw_init(void *handle) adev->nbio_funcs->init_registers(adev); /* enable the doorbell aperture */ soc15_enable_doorbell_aperture(adev, true); + /* HW doorbell routing policy: doorbell writing not + * in SDMA/IH/MM/ACV range will be routed to CP. So + * we need to init SDMA/IH/MM/ACV doorbell range prior + * to CP ip block init and ring test. + */ + soc15_doorbell_range_init(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c index 15da06ddeb75..a20b711a6756 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c @@ -99,9 +99,9 @@ static void tonga_ih_disable_interrupts(struct amdgpu_device *adev) */ static int tonga_ih_irq_init(struct amdgpu_device *adev) { - int rb_bufsz; u32 interrupt_cntl, ih_rb_cntl, ih_doorbell_rtpr; - u64 wptr_off; + struct amdgpu_ih_ring *ih = &adev->irq.ih; + int rb_bufsz; /* disable irqs */ tonga_ih_disable_interrupts(adev); @@ -118,10 +118,7 @@ static int tonga_ih_irq_init(struct amdgpu_device *adev) WREG32(mmINTERRUPT_CNTL, interrupt_cntl); /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - if (adev->irq.ih.use_bus_addr) - WREG32(mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8); - else - WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); + WREG32(mmIH_RB_BASE, ih->gpu_addr >> 8); rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); @@ -136,12 +133,8 @@ static int tonga_ih_irq_init(struct amdgpu_device *adev) WREG32(mmIH_RB_CNTL, ih_rb_cntl); /* set the writeback address whether it's enabled or not */ - if (adev->irq.ih.use_bus_addr) - wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4); - else - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); /* set rptr, wptr to 0 */ WREG32(mmIH_RB_RPTR, 0); @@ -193,14 +186,12 @@ static void tonga_ih_irq_disable(struct amdgpu_device *adev) * Used by cz_irq_process(VI). * Returns the value of the wptr. */ -static u32 tonga_ih_get_wptr(struct amdgpu_device *adev) +static u32 tonga_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { u32 wptr, tmp; - if (adev->irq.ih.use_bus_addr) - wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]); - else - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); @@ -209,13 +200,13 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev) * this should allow us to catchup. */ dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; tmp = RREG32(mmIH_RB_CNTL); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); WREG32(mmIH_RB_CNTL, tmp); } - return (wptr & adev->irq.ih.ptr_mask); + return (wptr & ih->ptr_mask); } /** @@ -227,16 +218,17 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev) * position and also advance the position. */ static void tonga_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) { /* wptr/rptr are in bytes! */ - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[4]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; entry->src_id = dw[0] & 0xff; @@ -246,7 +238,7 @@ static void tonga_ih_decode_iv(struct amdgpu_device *adev, entry->pasid = (dw[2] >> 16) & 0xffff; /* wptr/rptr are in bytes! */ - adev->irq.ih.rptr += 16; + ih->rptr += 16; } /** @@ -256,17 +248,15 @@ static void tonga_ih_decode_iv(struct amdgpu_device *adev, * * Set the IH ring buffer rptr. */ -static void tonga_ih_set_rptr(struct amdgpu_device *adev) +static void tonga_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - if (adev->irq.ih.use_doorbell) { + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ - if (adev->irq.ih.use_bus_addr) - adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; - else - adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; - WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr); + *ih->rptr_cpu = ih->rptr; + WDOORBELL32(ih->doorbell_index, ih->rptr); } else { - WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); + WREG32(mmIH_RB_RPTR, ih->rptr); } } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index d69c8f6daaf8..c4fb58667fd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -511,7 +511,7 @@ static int uvd_v4_2_ring_test_ring(struct amdgpu_ring *ring) static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { amdgpu_ring_write(ring, PACKET0(mmUVD_RBC_IB_BASE, 0)); amdgpu_ring_write(ring, ib->gpu_addr); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index ee8cd06ddc38..52bd8a654734 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -526,7 +526,7 @@ static int uvd_v5_0_ring_test_ring(struct amdgpu_ring *ring) static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0)); amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index d4f4a66f8324..c9edddf9f88a 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -977,7 +977,7 @@ static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring) static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -1003,7 +1003,7 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring, static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index aef924026a28..dc461df48da0 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c @@ -1272,7 +1272,7 @@ static int uvd_v7_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { struct amdgpu_device *adev = ring->adev; unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -1303,7 +1303,7 @@ static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring, static void uvd_v7_0_enc_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 2668effadd27..6ec65cf11112 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -834,7 +834,7 @@ out: static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c index 9fb34b7d8e03..aadc3e66ebd7 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c @@ -947,7 +947,7 @@ static int vce_v4_0_set_powergating_state(void *handle, #endif static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, - struct amdgpu_ib *ib, bool ctx_switch) + struct amdgpu_ib *ib, uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index 89bb2fef90eb..3dbc51f9d3b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -1371,7 +1371,7 @@ static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { struct amdgpu_device *adev = ring->adev; unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -1531,7 +1531,7 @@ static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring) static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); @@ -1736,7 +1736,7 @@ static void vcn_v1_0_jpeg_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u6 static void vcn_v1_0_jpeg_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, struct amdgpu_ib *ib, - bool ctx_switch) + uint32_t flags) { struct amdgpu_device *adev = ring->adev; unsigned vmid = AMDGPU_JOB_GET_VMID(job); diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index 2c250b01a903..6d1f804277f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -50,6 +50,22 @@ static void vega10_ih_enable_interrupts(struct amdgpu_device *adev) ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); adev->irq.ih.enabled = true; + + if (adev->irq.ih1.ring_size) { + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, + RB_ENABLE, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + adev->irq.ih1.enabled = true; + } + + if (adev->irq.ih2.ring_size) { + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, + RB_ENABLE, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); + adev->irq.ih2.enabled = true; + } } /** @@ -71,6 +87,53 @@ static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); adev->irq.ih.enabled = false; adev->irq.ih.rptr = 0; + + if (adev->irq.ih1.ring_size) { + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, + RB_ENABLE, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + /* set rptr, wptr to 0 */ + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); + adev->irq.ih1.enabled = false; + adev->irq.ih1.rptr = 0; + } + + if (adev->irq.ih2.ring_size) { + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, + RB_ENABLE, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); + /* set rptr, wptr to 0 */ + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); + adev->irq.ih2.enabled = false; + adev->irq.ih2.rptr = 0; + } +} + +static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) +{ + int rb_bufsz = order_base_2(ih->ring_size / 4); + + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + MC_SPACE, ih->use_bus_addr ? 1 : 4); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_CLEAR, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); + /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register + * value is written to memory + */ + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_WRITEBACK_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); + + return ih_rb_cntl; } /** @@ -86,50 +149,32 @@ static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) */ static int vega10_ih_irq_init(struct amdgpu_device *adev) { + struct amdgpu_ih_ring *ih; int ret = 0; - int rb_bufsz; u32 ih_rb_cntl, ih_doorbell_rtpr; u32 tmp; - u64 wptr_off; /* disable irqs */ vega10_ih_disable_interrupts(adev); adev->nbio_funcs->ih_control(adev); - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + ih = &adev->irq.ih; /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - if (adev->irq.ih.use_bus_addr) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1); - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4); - } - rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); - /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */ - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); - - if (adev->irq.msi_enabled) - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, + !!adev->irq.msi_enabled); WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); /* set the writeback address whether it's enabled or not */ - if (adev->irq.ih.use_bus_addr) - wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4); - else - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, + lower_32_bits(ih->wptr_addr)); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, + upper_32_bits(ih->wptr_addr) & 0xFFFF); /* set rptr, wptr to 0 */ WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); @@ -137,17 +182,48 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR); if (adev->irq.ih.use_doorbell) { - ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR, - OFFSET, adev->irq.ih.doorbell_index); - ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR, + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, OFFSET, + adev->irq.ih.doorbell_index); + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, ENABLE, 1); } else { - ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR, + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, ENABLE, 0); } WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr); - adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, - adev->irq.ih.doorbell_index); + + ih = &adev->irq.ih1; + if (ih->ring_size) { + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, + (ih->gpu_addr >> 40) & 0xff); + + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + + /* set rptr, wptr to 0 */ + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); + } + + ih = &adev->irq.ih2; + if (ih->ring_size) { + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, + (ih->gpu_addr >> 40) & 0xff); + + ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); + + /* set rptr, wptr to 0 */ + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); + WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); + } tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, @@ -191,32 +267,58 @@ static void vega10_ih_irq_disable(struct amdgpu_device *adev) * ring buffer overflow and deal with it. * Returns the value of the wptr. */ -static u32 vega10_ih_get_wptr(struct amdgpu_device *adev) +static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - u32 wptr, tmp; + u32 wptr, reg, tmp; - if (adev->irq.ih.use_bus_addr) - wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]); + wptr = le32_to_cpu(*ih->wptr_cpu); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + + if (ih == &adev->irq.ih) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + else if (ih == &adev->irq.ih1) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + else if (ih == &adev->irq.ih2) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); else - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); - - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 32). Hopefully - * this should allow us to catchup. - */ - tmp = (wptr + 32) & adev->irq.ih.ptr_mask; - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, adev->irq.ih.rptr, tmp); - adev->irq.ih.rptr = tmp; - - tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL)); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp); - } - return (wptr & adev->irq.ih.ptr_mask); + BUG(); + + wptr = RREG32_NO_KIQ(reg); + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 32). Hopefully + * this should allow us to catchup. + */ + tmp = (wptr + 32) & ih->ptr_mask; + dev_warn(adev->dev, "IH ring buffer overflow " + "(0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, tmp); + ih->rptr = tmp; + + if (ih == &adev->irq.ih) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + else if (ih == &adev->irq.ih1) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + else if (ih == &adev->irq.ih2) + reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + else + BUG(); + + tmp = RREG32_NO_KIQ(reg); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32_NO_KIQ(reg, tmp); + +out: + return (wptr & ih->ptr_mask); } /** @@ -228,20 +330,21 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev) * position and also advance the position. */ static void vega10_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) { /* wptr/rptr are in bytes! */ - u32 ring_index = adev->irq.ih.rptr >> 2; + u32 ring_index = ih->rptr >> 2; uint32_t dw[8]; - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); - dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]); - dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]); - dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]); - dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]); + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); + dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); + dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); + dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); + dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); entry->client_id = dw[0] & 0xff; entry->src_id = (dw[0] >> 8) & 0xff; @@ -257,9 +360,8 @@ static void vega10_ih_decode_iv(struct amdgpu_device *adev, entry->src_data[2] = dw[6]; entry->src_data[3] = dw[7]; - /* wptr/rptr are in bytes! */ - adev->irq.ih.rptr += 32; + ih->rptr += 32; } /** @@ -269,37 +371,95 @@ static void vega10_ih_decode_iv(struct amdgpu_device *adev, * * Set the IH ring buffer rptr. */ -static void vega10_ih_set_rptr(struct amdgpu_device *adev) +static void vega10_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { - if (adev->irq.ih.use_doorbell) { + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ - if (adev->irq.ih.use_bus_addr) - adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; - else - adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; - WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr); - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr); + *ih->rptr_cpu = ih->rptr; + WDOORBELL32(ih->doorbell_index, ih->rptr); + } else if (ih == &adev->irq.ih) { + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); + } else if (ih == &adev->irq.ih1) { + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); + } else if (ih == &adev->irq.ih2) { + WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); } } +/** + * vega10_ih_self_irq - dispatch work for ring 1 and 2 + * + * @adev: amdgpu_device pointer + * @source: irq source + * @entry: IV with WPTR update + * + * Update the WPTR from the IV and schedule work to handle the entries. + */ +static int vega10_ih_self_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + uint32_t wptr = cpu_to_le32(entry->src_data[0]); + + switch (entry->ring_id) { + case 1: + *adev->irq.ih1.wptr_cpu = wptr; + schedule_work(&adev->irq.ih1_work); + break; + case 2: + *adev->irq.ih2.wptr_cpu = wptr; + schedule_work(&adev->irq.ih2_work); + break; + default: break; + } + return 0; +} + +static const struct amdgpu_irq_src_funcs vega10_ih_self_irq_funcs = { + .process = vega10_ih_self_irq, +}; + +static void vega10_ih_set_self_irq_funcs(struct amdgpu_device *adev) +{ + adev->irq.self_irq.num_types = 0; + adev->irq.self_irq.funcs = &vega10_ih_self_irq_funcs; +} + static int vega10_ih_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; vega10_ih_set_interrupt_funcs(adev); + vega10_ih_set_self_irq_funcs(adev); return 0; } static int vega10_ih_sw_init(void *handle) { - int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r; + + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, + &adev->irq.self_irq); + if (r) + return r; r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true); if (r) return r; + if (adev->asic_type == CHIP_VEGA10) { + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; + } + + /* TODO add doorbell for IH1 & IH2 as well */ adev->irq.ih.use_doorbell = true; adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; @@ -313,6 +473,8 @@ static int vega10_ih_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih2); + amdgpu_ih_ring_fini(adev, &adev->irq.ih1); amdgpu_ih_ring_fini(adev, &adev->irq.ih); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c index 422674bb3cdf..a8e92638a2e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c @@ -70,8 +70,8 @@ void vega10_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.userqueue_start = AMDGPU_DOORBELL64_USERQUEUE_START; adev->doorbell_index.userqueue_end = AMDGPU_DOORBELL64_USERQUEUE_END; adev->doorbell_index.gfx_ring0 = AMDGPU_DOORBELL64_GFX_RING0; - adev->doorbell_index.sdma_engine0 = AMDGPU_DOORBELL64_sDMA_ENGINE0; - adev->doorbell_index.sdma_engine1 = AMDGPU_DOORBELL64_sDMA_ENGINE1; + adev->doorbell_index.sdma_engine[0] = AMDGPU_DOORBELL64_sDMA_ENGINE0; + adev->doorbell_index.sdma_engine[1] = AMDGPU_DOORBELL64_sDMA_ENGINE1; adev->doorbell_index.ih = AMDGPU_DOORBELL64_IH; adev->doorbell_index.uvd_vce.uvd_ring0_1 = AMDGPU_DOORBELL64_UVD_RING0_1; adev->doorbell_index.uvd_vce.uvd_ring2_3 = AMDGPU_DOORBELL64_UVD_RING2_3; @@ -81,7 +81,12 @@ void vega10_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.uvd_vce.vce_ring2_3 = AMDGPU_DOORBELL64_VCE_RING2_3; adev->doorbell_index.uvd_vce.vce_ring4_5 = AMDGPU_DOORBELL64_VCE_RING4_5; adev->doorbell_index.uvd_vce.vce_ring6_7 = AMDGPU_DOORBELL64_VCE_RING6_7; + + adev->doorbell_index.first_non_cp = AMDGPU_DOORBELL64_FIRST_NON_CP; + adev->doorbell_index.last_non_cp = AMDGPU_DOORBELL64_LAST_NON_CP; + /* In unit of dword doorbell */ adev->doorbell_index.max_assignment = AMDGPU_DOORBELL64_MAX_ASSIGNMENT << 1; + adev->doorbell_index.sdma_doorbell_range = 4; } diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c index edce413fda9a..0db84386252a 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c @@ -68,14 +68,14 @@ void vega20_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.userqueue_start = AMDGPU_VEGA20_DOORBELL_USERQUEUE_START; adev->doorbell_index.userqueue_end = AMDGPU_VEGA20_DOORBELL_USERQUEUE_END; adev->doorbell_index.gfx_ring0 = AMDGPU_VEGA20_DOORBELL_GFX_RING0; - adev->doorbell_index.sdma_engine0 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0; - adev->doorbell_index.sdma_engine1 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1; - adev->doorbell_index.sdma_engine2 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2; - adev->doorbell_index.sdma_engine3 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3; - adev->doorbell_index.sdma_engine4 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4; - adev->doorbell_index.sdma_engine5 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5; - adev->doorbell_index.sdma_engine6 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6; - adev->doorbell_index.sdma_engine7 = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7; + adev->doorbell_index.sdma_engine[0] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0; + adev->doorbell_index.sdma_engine[1] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1; + adev->doorbell_index.sdma_engine[2] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2; + adev->doorbell_index.sdma_engine[3] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3; + adev->doorbell_index.sdma_engine[4] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4; + adev->doorbell_index.sdma_engine[5] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5; + adev->doorbell_index.sdma_engine[6] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6; + adev->doorbell_index.sdma_engine[7] = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7; adev->doorbell_index.ih = AMDGPU_VEGA20_DOORBELL_IH; adev->doorbell_index.uvd_vce.uvd_ring0_1 = AMDGPU_VEGA20_DOORBELL64_UVD_RING0_1; adev->doorbell_index.uvd_vce.uvd_ring2_3 = AMDGPU_VEGA20_DOORBELL64_UVD_RING2_3; @@ -85,6 +85,11 @@ void vega20_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.uvd_vce.vce_ring2_3 = AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3; adev->doorbell_index.uvd_vce.vce_ring4_5 = AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5; adev->doorbell_index.uvd_vce.vce_ring6_7 = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7; + + adev->doorbell_index.first_non_cp = AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP; + adev->doorbell_index.last_non_cp = AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP; + adev->doorbell_index.max_assignment = AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT << 1; + adev->doorbell_index.sdma_doorbell_range = 20; } diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 77e367459101..5e5b42a0744a 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -941,6 +941,69 @@ static bool vi_need_full_reset(struct amdgpu_device *adev) } } +static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1) +{ + uint32_t perfctr = 0; + uint64_t cnt0_of, cnt1_of; + int tmp; + + /* This reports 0 on APUs, so return to avoid writing/reading registers + * that may or may not be different from their GPU counterparts + */ + if (adev->flags & AMD_IS_APU) + return; + + /* Set the 2 events that we wish to watch, defined above */ + /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */ + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); + + /* Write to enable desired perf counters */ + WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr); + /* Zero out and enable the perf counters + * Write 0x5: + * Bit 0 = Start all counters(1) + * Bit 2 = Global counter reset enable(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005); + + msleep(1000); + + /* Load the shadow and disable the perf counters + * Write 0x2: + * Bit 0 = Stop counters(0) + * Bit 1 = Load the shadow counters(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002); + + /* Read register values to get any >32bit overflow */ + tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK); + cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); + cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); + + /* Get the values and add the overflow */ + *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); + *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); +} + +static bool vi_need_reset_on_init(struct amdgpu_device *adev) +{ + u32 clock_cntl, pc; + + if (adev->flags & AMD_IS_APU) + return false; + + /* check if the SMC is already running */ + clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + pc = RREG32_SMC(ixSMC_PC_C); + if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) && + (0x20100 <= pc)) + return true; + + return false; +} + static const struct amdgpu_asic_funcs vi_asic_funcs = { .read_disabled_bios = &vi_read_disabled_bios, @@ -956,6 +1019,8 @@ static const struct amdgpu_asic_funcs vi_asic_funcs = .invalidate_hdp = &vi_invalidate_hdp, .need_full_reset = &vi_need_full_reset, .init_doorbell_index = &legacy_doorbell_index_init, + .get_pcie_usage = &vi_get_pcie_usage, + .need_reset_on_init = &vi_need_reset_on_init, }; #define CZ_REV_BRISTOL(rev) \ @@ -1726,8 +1791,8 @@ void legacy_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.mec_ring6 = AMDGPU_DOORBELL_MEC_RING6; adev->doorbell_index.mec_ring7 = AMDGPU_DOORBELL_MEC_RING7; adev->doorbell_index.gfx_ring0 = AMDGPU_DOORBELL_GFX_RING0; - adev->doorbell_index.sdma_engine0 = AMDGPU_DOORBELL_sDMA_ENGINE0; - adev->doorbell_index.sdma_engine1 = AMDGPU_DOORBELL_sDMA_ENGINE1; + adev->doorbell_index.sdma_engine[0] = AMDGPU_DOORBELL_sDMA_ENGINE0; + adev->doorbell_index.sdma_engine[1] = AMDGPU_DOORBELL_sDMA_ENGINE1; adev->doorbell_index.ih = AMDGPU_DOORBELL_IH; adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_MAX_ASSIGNMENT; } |