diff options
Diffstat (limited to 'drivers/gpu/drm')
312 files changed, 6217 insertions, 3134 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 9efa681d0878..8d0d7f3dd5fb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -411,6 +411,8 @@ struct amdgpu_fpriv { struct amdgpu_ctx_mgr ctx_mgr; }; +int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); + int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned size, struct amdgpu_ib *ib); void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index e957e42c539a..fe1d7368c1e6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -131,7 +131,7 @@ static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) { - int i, n; + int i; int last_valid_bit; if (adev->kfd.dev) { @@ -142,7 +142,9 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) .gpuvm_size = min(adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GMC_HOLE_START), - .drm_render_minor = adev->ddev->render->index + .drm_render_minor = adev->ddev->render->index, + .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, + }; /* this is going to have a few of the MSBs set that we need to @@ -172,35 +174,20 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) &gpu_resources.doorbell_aperture_size, &gpu_resources.doorbell_start_offset); - if (adev->asic_type < CHIP_VEGA10) { - kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); - return; - } - - n = (adev->asic_type < CHIP_VEGA20) ? 2 : 8; - - for (i = 0; i < n; i += 2) { - /* On SOC15 the BIF is involved in routing - * doorbells using the low 12 bits of the - * address. Communicate the assignments to - * KFD. KFD uses two doorbell pages per - * process in case of 64-bit doorbells so we - * can use each doorbell assignment twice. - */ - gpu_resources.sdma_doorbell[0][i] = - adev->doorbell_index.sdma_engine[0] + (i >> 1); - gpu_resources.sdma_doorbell[0][i+1] = - adev->doorbell_index.sdma_engine[0] + 0x200 + (i >> 1); - gpu_resources.sdma_doorbell[1][i] = - adev->doorbell_index.sdma_engine[1] + (i >> 1); - gpu_resources.sdma_doorbell[1][i+1] = - adev->doorbell_index.sdma_engine[1] + 0x200 + (i >> 1); - } - /* Doorbells 0x0e0-0ff and 0x2e0-2ff are reserved for - * SDMA, IH and VCN. So don't use them for the CP. + /* Since SOC15, BIF starts to statically use the + * lower 12 bits of doorbell addresses for routing + * based on settings in registers like + * SDMA0_DOORBELL_RANGE etc.. + * In order to route a doorbell to CP engine, the lower + * 12 bits of its address has to be outside the range + * set for SDMA, VCN, and IH blocks. */ - gpu_resources.reserved_doorbell_mask = 0x1e0; - gpu_resources.reserved_doorbell_val = 0x0e0; + if (adev->asic_type >= CHIP_VEGA10) { + gpu_resources.non_cp_doorbells_start = + adev->doorbell_index.first_non_cp; + gpu_resources.non_cp_doorbells_end = + adev->doorbell_index.last_non_cp; + } kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index d7b10d79f1de..1921dec3df7a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -204,38 +204,25 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) } -/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence(s) from BO's +/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO - * @ef: [IN] If ef is specified, then this eviction fence is removed if it + * @ef: [IN] This eviction fence is removed if it * is present in the shared list. - * @ef_list: [OUT] Returns list of eviction fences. These fences are removed - * from BO's reservation object shared list. - * @ef_count: [OUT] Number of fences in ef_list. * - * NOTE: If called with ef_list, then amdgpu_amdkfd_add_eviction_fence must be - * called to restore the eviction fences and to avoid memory leak. This is - * useful for shared BOs. * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, - struct amdgpu_amdkfd_fence *ef, - struct amdgpu_amdkfd_fence ***ef_list, - unsigned int *ef_count) + struct amdgpu_amdkfd_fence *ef) { struct reservation_object *resv = bo->tbo.resv; struct reservation_object_list *old, *new; unsigned int i, j, k; - if (!ef && !ef_list) + if (!ef) return -EINVAL; - if (ef_list) { - *ef_list = NULL; - *ef_count = 0; - } - old = reservation_object_get_list(resv); if (!old) return 0; @@ -254,8 +241,7 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, f = rcu_dereference_protected(old->shared[i], reservation_object_held(resv)); - if ((ef && f->context == ef->base.context) || - (!ef && to_amdgpu_amdkfd_fence(f))) + if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); @@ -263,21 +249,6 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, new->shared_max = old->shared_max; new->shared_count = k; - if (!ef) { - unsigned int count = old->shared_count - j; - - /* Alloc memory for count number of eviction fence pointers. - * Fill the ef_list array and ef_count - */ - *ef_list = kcalloc(count, sizeof(**ef_list), GFP_KERNEL); - *ef_count = count; - - if (!*ef_list) { - kfree(new); - return -ENOMEM; - } - } - /* Install the new fence list, seqcount provides the barriers */ preempt_disable(); write_seqcount_begin(&resv->seq); @@ -291,46 +262,13 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, f = rcu_dereference_protected(new->shared[i], reservation_object_held(resv)); - if (!ef) - (*ef_list)[k++] = to_amdgpu_amdkfd_fence(f); - else - dma_fence_put(f); + dma_fence_put(f); } kfree_rcu(old, rcu); return 0; } -/* amdgpu_amdkfd_add_eviction_fence - Adds eviction fence(s) back into BO's - * reservation object. - * - * @bo: [IN] Add eviction fences to this BO - * @ef_list: [IN] List of eviction fences to be added - * @ef_count: [IN] Number of fences in ef_list. - * - * NOTE: Must call amdgpu_amdkfd_remove_eviction_fence before calling this - * function. - */ -static void amdgpu_amdkfd_add_eviction_fence(struct amdgpu_bo *bo, - struct amdgpu_amdkfd_fence **ef_list, - unsigned int ef_count) -{ - int i; - - if (!ef_list || !ef_count) - return; - - for (i = 0; i < ef_count; i++) { - amdgpu_bo_fence(bo, &ef_list[i]->base, true); - /* Re-adding the fence takes an additional reference. Drop that - * reference. - */ - dma_fence_put(&ef_list[i]->base); - } - - kfree(ef_list); -} - static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { @@ -346,18 +284,8 @@ static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; - if (wait) { - struct amdgpu_amdkfd_fence **ef_list; - unsigned int ef_count; - - ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL, &ef_list, - &ef_count); - if (ret) - goto validate_fail; - - ttm_bo_wait(&bo->tbo, false, false); - amdgpu_amdkfd_add_eviction_fence(bo, ef_list, ef_count); - } + if (wait) + amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; @@ -444,7 +372,6 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, { int ret; struct kfd_bo_va_list *bo_va_entry; - struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; @@ -484,14 +411,8 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, *p_bo_va_entry = bo_va_entry; /* Allocate new page tables if needed and validate - * them. Clearing of new page tables and validate need to wait - * on move fences. We don't want that to trigger the eviction - * fence, so remove it temporarily. + * them. */ - amdgpu_amdkfd_remove_eviction_fence(pd, - vm->process_info->eviction_fence, - NULL, NULL); - ret = amdgpu_vm_alloc_pts(adev, vm, va, amdgpu_bo_size(bo)); if (ret) { pr_err("Failed to allocate pts, err=%d\n", ret); @@ -504,13 +425,9 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, goto err_alloc_pts; } - /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); - return 0; err_alloc_pts: - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); list_del(&bo_va_entry->bo_list); err_vmadd: @@ -809,24 +726,11 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev, { struct amdgpu_bo_va *bo_va = entry->bo_va; struct amdgpu_vm *vm = bo_va->base.vm; - struct amdgpu_bo *pd = vm->root.base.bo; - /* Remove eviction fence from PD (and thereby from PTs too as - * they share the resv. object). Otherwise during PT update - * job (see amdgpu_vm_bo_update_mapping), eviction fence would - * get added to job->sync object and job execution would - * trigger the eviction fence. - */ - amdgpu_amdkfd_remove_eviction_fence(pd, - vm->process_info->eviction_fence, - NULL, NULL); amdgpu_vm_bo_unmap(adev, bo_va, entry->va); amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); - /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); - amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false); return 0; @@ -1002,7 +906,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, pr_err("validate_pt_pd_bos() failed\n"); goto validate_pd_fail; } - ret = ttm_bo_wait(&vm->root.base.bo->tbo, false, false); + amdgpu_bo_sync_wait(vm->root.base.bo, AMDGPU_FENCE_OWNER_KFD, false); if (ret) goto wait_pd_fail; amdgpu_bo_fence(vm->root.base.bo, @@ -1389,8 +1293,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( * attached */ amdgpu_amdkfd_remove_eviction_fence(mem->bo, - process_info->eviction_fence, - NULL, NULL); + process_info->eviction_fence); pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, mem->va + bo_size * (1 + mem->aql_queue)); @@ -1617,8 +1520,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( if (mem->mapped_to_gpu_memory == 0 && !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count) amdgpu_amdkfd_remove_eviction_fence(mem->bo, - process_info->eviction_fence, - NULL, NULL); + process_info->eviction_fence); unreserve_out: unreserve_bo_and_vms(&ctx, false, false); @@ -1679,7 +1581,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, } amdgpu_amdkfd_remove_eviction_fence( - bo, mem->process_info->eviction_fence, NULL, NULL); + bo, mem->process_info->eviction_fence); list_del_init(&mem->validate_list.head); if (size) @@ -1945,16 +1847,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) amdgpu_sync_create(&sync); - /* Avoid triggering eviction fences when unmapping invalid - * userptr BOs (waits for all fences, doesn't use - * FENCE_OWNER_VM) - */ - list_for_each_entry(peer_vm, &process_info->vm_list_head, - vm_list_node) - amdgpu_amdkfd_remove_eviction_fence(peer_vm->root.base.bo, - process_info->eviction_fence, - NULL, NULL); - ret = process_validate_vms(process_info); if (ret) goto unreserve_out; @@ -2015,10 +1907,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) ret = process_update_pds(process_info, &sync); unreserve_out: - list_for_each_entry(peer_vm, &process_info->vm_list_head, - vm_list_node) - amdgpu_bo_fence(peer_vm->root.base.bo, - &process_info->eviction_fence->base, true); ttm_eu_backoff_reservation(&ticket, &resv_list); amdgpu_sync_wait(&sync, false); amdgpu_sync_free(&sync); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index d85184b5b35c..7b526593eb77 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -124,6 +124,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; struct drm_sched_rq *rqs[AMDGPU_MAX_RINGS]; unsigned num_rings; + unsigned num_rqs = 0; switch (i) { case AMDGPU_HW_IP_GFX: @@ -166,12 +167,16 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, break; } - for (j = 0; j < num_rings; ++j) - rqs[j] = &rings[j]->sched.sched_rq[priority]; + for (j = 0; j < num_rings; ++j) { + if (!rings[j]->adev) + continue; + + rqs[num_rqs++] = &rings[j]->sched.sched_rq[priority]; + } for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) r = drm_sched_entity_init(&ctx->entities[i][j].entity, - rqs, num_rings, &ctx->guilty); + rqs, num_rqs, &ctx->guilty); if (r) goto error_cleanup_entities; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index dd9a4fb9ce39..4ae3ff9a1d4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -158,9 +158,6 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, while (size) { uint32_t value; - if (*pos > adev->rmmio_size) - goto end; - if (read) { value = RREG32(*pos >> 2); r = put_user(value, (uint32_t *)buf); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h index 1cfec06f81d4..68959b923f89 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h @@ -71,6 +71,8 @@ struct amdgpu_doorbell_index { uint32_t vce_ring6_7; } uvd_vce; }; + uint32_t first_non_cp; + uint32_t last_non_cp; uint32_t max_assignment; /* Per engine SDMA doorbell size in dword */ uint32_t sdma_doorbell_range; @@ -143,6 +145,10 @@ typedef enum _AMDGPU_VEGA20_DOORBELL_ASSIGNMENT AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3 = 0x18D, AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5 = 0x18E, AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7 = 0x18F, + + AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0, + AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7, + AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT = 0x18F, AMDGPU_VEGA20_DOORBELL_INVALID = 0xFFFF } AMDGPU_VEGA20_DOORBELL_ASSIGNMENT; @@ -222,6 +228,9 @@ typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT AMDGPU_DOORBELL64_VCE_RING4_5 = 0xFE, AMDGPU_DOORBELL64_VCE_RING6_7 = 0xFF, + AMDGPU_DOORBELL64_FIRST_NON_CP = AMDGPU_DOORBELL64_sDMA_ENGINE0, + AMDGPU_DOORBELL64_LAST_NON_CP = AMDGPU_DOORBELL64_VCE_RING6_7, + AMDGPU_DOORBELL64_MAX_ASSIGNMENT = 0xFF, AMDGPU_DOORBELL64_INVALID = 0xFFFF } AMDGPU_DOORBELL64_ASSIGNMENT; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c index 1c4595562f8f..344967df3137 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c @@ -184,61 +184,6 @@ u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev) return vrefresh; } -void amdgpu_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, - u32 *p, u32 *u) -{ - u32 b_c = 0; - u32 i_c; - u32 tmp; - - i_c = (i * r_c) / 100; - tmp = i_c >> p_b; - - while (tmp) { - b_c++; - tmp >>= 1; - } - - *u = (b_c + 1) / 2; - *p = i_c / (1 << (2 * (*u))); -} - -int amdgpu_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th) -{ - u32 k, a, ah, al; - u32 t1; - - if ((fl == 0) || (fh == 0) || (fl > fh)) - return -EINVAL; - - k = (100 * fh) / fl; - t1 = (t * (k - 100)); - a = (1000 * (100 * h + t1)) / (10000 + (t1 / 100)); - a = (a + 5) / 10; - ah = ((a * t) + 5000) / 10000; - al = a - ah; - - *th = t - ah; - *tl = t + al; - - return 0; -} - -bool amdgpu_is_uvd_state(u32 class, u32 class2) -{ - if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) - return true; - if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) - return true; - if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) - return true; - return false; -} - bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor) { switch (sensor) { @@ -949,39 +894,6 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, return AMDGPU_PCIE_GEN1; } -u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev, - u16 asic_lanes, - u16 default_lanes) -{ - switch (asic_lanes) { - case 0: - default: - return default_lanes; - case 1: - return 1; - case 2: - return 2; - case 4: - return 4; - case 8: - return 8; - case 12: - return 12; - case 16: - return 16; - } -} - -u8 amdgpu_encode_pci_lane_width(u32 lanes) -{ - u8 encoded_lanes[] = { 0, 1, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6 }; - - if (lanes > 16) - return 0; - - return encoded_lanes[lanes]; -} - struct amd_vce_state* amdgpu_get_vce_clock_state(void *handle, u32 idx) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h index 2f61e9edb1c1..e871e022c129 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h @@ -486,10 +486,6 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev); u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev); void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev); -bool amdgpu_is_uvd_state(u32 class, u32 class2); -void amdgpu_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, - u32 *p, u32 *u); -int amdgpu_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th); bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor); @@ -505,11 +501,6 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, enum amdgpu_pcie_gen asic_gen, enum amdgpu_pcie_gen default_gen); -u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev, - u16 asic_lanes, - u16 default_lanes); -u8 amdgpu_encode_pci_lane_width(u32 lanes); - struct amd_vce_state* amdgpu_get_vce_clock_state(void *handle, u32 idx); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7f3aa7b7e1d8..7419ea8a388b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -73,9 +73,10 @@ * - 3.27.0 - Add new chunk to to AMDGPU_CS to enable BO_LIST creation. * - 3.28.0 - Add AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES * - 3.29.0 - Add AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID + * - 3.30.0 - Add AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE. */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 29 +#define KMS_DRIVER_MINOR 30 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit = 0; @@ -1178,6 +1179,22 @@ static const struct file_operations amdgpu_driver_kms_fops = { #endif }; +int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv) +{ + struct drm_file *file; + + if (!filp) + return -EINVAL; + + if (filp->f_op != &amdgpu_driver_kms_fops) { + return -EINVAL; + } + + file = filp->private_data; + *fpriv = file->driver_priv; + return 0; +} + static bool amdgpu_get_crtc_scanout_position(struct drm_device *dev, unsigned int pipe, bool in_vblank_irq, int *vpos, int *hpos, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index d0a5db777b6d..1c50be3ab8a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -140,9 +140,7 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) * Interrupt hander (VI), walk the IH ring. * Returns irq process return code. */ -int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, - void (*callback)(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih)) +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { u32 wptr; @@ -162,7 +160,7 @@ restart_ih: rmb(); while (ih->rptr != wptr) { - callback(adev, ih); + amdgpu_irq_dispatch(adev, ih); ih->rptr &= ih->ptr_mask; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index 1ccb1831382a..113a1ba13d4a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -69,8 +69,6 @@ struct amdgpu_ih_funcs { int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, unsigned ring_size, bool use_bus_addr); void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); -int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, - void (*callback)(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih)); +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 8bfb3dab46f7..af4c3b1af322 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -131,29 +131,6 @@ void amdgpu_irq_disable_all(struct amdgpu_device *adev) } /** - * amdgpu_irq_callback - callback from the IH ring - * - * @adev: amdgpu device pointer - * @ih: amdgpu ih ring - * - * Callback from IH ring processing to handle the entry at the current position - * and advance the read pointer. - */ -static void amdgpu_irq_callback(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih) -{ - u32 ring_index = ih->rptr >> 2; - struct amdgpu_iv_entry entry; - - entry.iv_entry = (const uint32_t *)&ih->ring[ring_index]; - amdgpu_ih_decode_iv(adev, &entry); - - trace_amdgpu_iv(ih - &adev->irq.ih, &entry); - - amdgpu_irq_dispatch(adev, &entry); -} - -/** * amdgpu_irq_handler - IRQ handler * * @irq: IRQ number (unused) @@ -170,7 +147,7 @@ irqreturn_t amdgpu_irq_handler(int irq, void *arg) struct amdgpu_device *adev = dev->dev_private; irqreturn_t ret; - ret = amdgpu_ih_process(adev, &adev->irq.ih, amdgpu_irq_callback); + ret = amdgpu_ih_process(adev, &adev->irq.ih); if (ret == IRQ_HANDLED) pm_runtime_mark_last_busy(dev->dev); return ret; @@ -188,7 +165,7 @@ static void amdgpu_irq_handle_ih1(struct work_struct *work) struct amdgpu_device *adev = container_of(work, struct amdgpu_device, irq.ih1_work); - amdgpu_ih_process(adev, &adev->irq.ih1, amdgpu_irq_callback); + amdgpu_ih_process(adev, &adev->irq.ih1); } /** @@ -203,7 +180,7 @@ static void amdgpu_irq_handle_ih2(struct work_struct *work) struct amdgpu_device *adev = container_of(work, struct amdgpu_device, irq.ih2_work); - amdgpu_ih_process(adev, &adev->irq.ih2, amdgpu_irq_callback); + amdgpu_ih_process(adev, &adev->irq.ih2); } /** @@ -394,14 +371,23 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev, * Dispatches IRQ to IP blocks. */ void amdgpu_irq_dispatch(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry) + struct amdgpu_ih_ring *ih) { - unsigned client_id = entry->client_id; - unsigned src_id = entry->src_id; + u32 ring_index = ih->rptr >> 2; + struct amdgpu_iv_entry entry; + unsigned client_id, src_id; struct amdgpu_irq_src *src; bool handled = false; int r; + entry.iv_entry = (const uint32_t *)&ih->ring[ring_index]; + amdgpu_ih_decode_iv(adev, &entry); + + trace_amdgpu_iv(ih - &adev->irq.ih, &entry); + + client_id = entry.client_id; + src_id = entry.src_id; + if (client_id >= AMDGPU_IRQ_CLIENTID_MAX) { DRM_DEBUG("Invalid client_id in IV: %d\n", client_id); @@ -416,7 +402,7 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, client_id, src_id); } else if ((src = adev->irq.client[client_id].sources[src_id])) { - r = src->funcs->process(adev, src, entry); + r = src->funcs->process(adev, src, &entry); if (r < 0) DRM_ERROR("error processing interrupt (%d)\n", r); else if (r) @@ -428,7 +414,7 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, /* Send it to amdkfd as well if it isn't already handled */ if (!handled) - amdgpu_amdkfd_interrupt(adev, entry->iv_entry); + amdgpu_amdkfd_interrupt(adev, entry.iv_entry); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index c27decfda494..c718e94a55c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -108,7 +108,7 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev, unsigned client_id, unsigned src_id, struct amdgpu_irq_src *source); void amdgpu_irq_dispatch(struct amdgpu_device *adev, - struct amdgpu_iv_entry *entry); + struct amdgpu_ih_ring *ih); int amdgpu_irq_update(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type); int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index bc62bf41b7e9..e860412043bb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -207,11 +207,12 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) if (!r) { acpi_status = amdgpu_acpi_init(adev); if (acpi_status) - dev_dbg(&dev->pdev->dev, + dev_dbg(&dev->pdev->dev, "Error during ACPI methods call\n"); } if (amdgpu_device_is_px(dev)) { + dev_pm_set_driver_flags(dev->dev, DPM_FLAG_NEVER_SKIP); pm_runtime_use_autosuspend(dev->dev); pm_runtime_set_autosuspend_delay(dev->dev, 5000); pm_runtime_set_active(dev->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index 698fd8a2f775..889e443eeee7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -406,6 +406,7 @@ struct amdgpu_crtc { struct amdgpu_flip_work *pflip_works; enum amdgpu_flip_status pflip_status; int deferred_flip_completion; + u64 last_flip_vblank; /* pll sharing */ struct amdgpu_atom_ss ss; bool ss_enabled; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index fd9c4beeaaa4..ec9e45004bff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1285,6 +1285,30 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, } /** + * amdgpu_sync_wait_resv - Wait for BO reservation fences + * + * @bo: buffer object + * @owner: fence owner + * @intr: Whether the wait is interruptible + * + * Returns: + * 0 on success, errno otherwise. + */ +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_sync sync; + int r; + + amdgpu_sync_create(&sync); + amdgpu_sync_resv(adev, &sync, bo->tbo.resv, owner, false); + r = amdgpu_sync_wait(&sync, intr); + amdgpu_sync_free(&sync); + + return r; +} + +/** * amdgpu_bo_gpu_offset - return GPU offset of bo * @bo: amdgpu object for which we query the offset * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 9291c2f837e9..220a6a7b1bc1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -266,6 +266,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, bool shared); +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr); u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); int amdgpu_bo_validate(struct amdgpu_bo *bo); int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c index 71913a18d142..a38e0fb4a6fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c @@ -38,6 +38,7 @@ #include "amdgpu_gem.h" #include <drm/amdgpu_drm.h> #include <linux/dma-buf.h> +#include <linux/dma-fence-array.h> /** * amdgpu_gem_prime_get_sg_table - &drm_driver.gem_prime_get_sg_table @@ -187,6 +188,48 @@ error: return ERR_PTR(ret); } +static int +__reservation_object_make_exclusive(struct reservation_object *obj) +{ + struct dma_fence **fences; + unsigned int count; + int r; + + if (!reservation_object_get_list(obj)) /* no shared fences to convert */ + return 0; + + r = reservation_object_get_fences_rcu(obj, NULL, &count, &fences); + if (r) + return r; + + if (count == 0) { + /* Now that was unexpected. */ + } else if (count == 1) { + reservation_object_add_excl_fence(obj, fences[0]); + dma_fence_put(fences[0]); + kfree(fences); + } else { + struct dma_fence_array *array; + + array = dma_fence_array_create(count, fences, + dma_fence_context_alloc(1), 0, + false); + if (!array) + goto err_fences_put; + + reservation_object_add_excl_fence(obj, &array->base); + dma_fence_put(&array->base); + } + + return 0; + +err_fences_put: + while (count--) + dma_fence_put(fences[count]); + kfree(fences); + return -ENOMEM; +} + /** * amdgpu_gem_map_attach - &dma_buf_ops.attach implementation * @dma_buf: Shared DMA buffer @@ -218,16 +261,16 @@ static int amdgpu_gem_map_attach(struct dma_buf *dma_buf, if (attach->dev->driver != adev->dev->driver) { /* - * Wait for all shared fences to complete before we switch to future - * use of exclusive fence on this prime shared bo. + * We only create shared fences for internal use, but importers + * of the dmabuf rely on exclusive fences for implicitly + * tracking write hazards. As any of the current fences may + * correspond to a write, we need to convert all existing + * fences on the reservation object into a single exclusive + * fence. */ - r = reservation_object_wait_timeout_rcu(bo->tbo.resv, - true, false, - MAX_SCHEDULE_TIMEOUT); - if (unlikely(r < 0)) { - DRM_DEBUG_PRIME("Fence wait failed: %li\n", r); + r = __reservation_object_make_exclusive(bo->tbo.resv); + if (r) goto error_unreserve; - } } /* pin buffer into GTT */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index d87f165e3a23..3091488cd8cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -84,8 +84,10 @@ static int psp_sw_fini(void *handle) adev->psp.sos_fw = NULL; release_firmware(adev->psp.asd_fw); adev->psp.asd_fw = NULL; - release_firmware(adev->psp.ta_fw); - adev->psp.ta_fw = NULL; + if (adev->psp.ta_fw) { + release_firmware(adev->psp.ta_fw); + adev->psp.ta_fw = NULL; + } return 0; } @@ -440,6 +442,9 @@ static int psp_xgmi_initialize(struct psp_context *psp) struct ta_xgmi_shared_memory *xgmi_cmd; int ret; + if (!psp->adev->psp.ta_fw) + return -ENOENT; + if (!psp->xgmi_context.initialized) { ret = psp_xgmi_init_shared_buf(psp); if (ret) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 1cafe8d83a4d..0767a93e4d91 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -54,16 +54,20 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, enum drm_sched_priority priority) { struct file *filp = fget(fd); - struct drm_file *file; struct amdgpu_fpriv *fpriv; struct amdgpu_ctx *ctx; uint32_t id; + int r; if (!filp) return -EINVAL; - file = filp->private_data; - fpriv = file->driver_priv; + r = amdgpu_file_to_fpriv(filp, &fpriv); + if (r) { + fput(filp); + return r; + } + idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id) amdgpu_ctx_priority_override(ctx, priority); @@ -72,6 +76,39 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, return 0; } +static int amdgpu_sched_context_priority_override(struct amdgpu_device *adev, + int fd, + unsigned ctx_id, + enum drm_sched_priority priority) +{ + struct file *filp = fget(fd); + struct amdgpu_fpriv *fpriv; + struct amdgpu_ctx *ctx; + int r; + + if (!filp) + return -EINVAL; + + r = amdgpu_file_to_fpriv(filp, &fpriv); + if (r) { + fput(filp); + return r; + } + + ctx = amdgpu_ctx_get(fpriv, ctx_id); + + if (!ctx) { + fput(filp); + return -EINVAL; + } + + amdgpu_ctx_priority_override(ctx, priority); + amdgpu_ctx_put(ctx); + fput(filp); + + return 0; +} + int amdgpu_sched_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { @@ -81,7 +118,7 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, int r; priority = amdgpu_to_sched_priority(args->in.priority); - if (args->in.flags || priority == DRM_SCHED_PRIORITY_INVALID) + if (priority == DRM_SCHED_PRIORITY_INVALID) return -EINVAL; switch (args->in.op) { @@ -90,6 +127,12 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, args->in.fd, priority); break; + case AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE: + r = amdgpu_sched_context_priority_override(adev, + args->in.fd, + args->in.ctx_id, + priority); + break; default: DRM_ERROR("Invalid sched op specified: %d\n", args->in.op); r = -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 75481cf3348f..bfa9062ce6b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -652,12 +652,14 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, struct ttm_bo_global *glob = adev->mman.bdev.glob; struct amdgpu_vm_bo_base *bo_base; +#if 0 if (vm->bulk_moveable) { spin_lock(&glob->lru_lock); ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); spin_unlock(&glob->lru_lock); return; } +#endif memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); @@ -698,8 +700,6 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_bo_base *bo_base, *tmp; int r = 0; - vm->bulk_moveable &= list_empty(&vm->evicted); - list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { struct amdgpu_bo *bo = bo_base->bo; @@ -828,7 +828,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, WARN_ON(job->ibs[0].length_dw > 64); r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv, - AMDGPU_FENCE_OWNER_UNDEFINED, false); + AMDGPU_FENCE_OWNER_KFD, false); if (r) goto error_free; @@ -947,10 +947,6 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, if (r) return r; - r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats); - if (r) - goto error_free_pt; - if (vm->use_cpu_for_update) { r = amdgpu_bo_kmap(pt, NULL); if (r) @@ -963,6 +959,10 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, pt->parent = amdgpu_bo_ref(cursor.parent->base.bo); amdgpu_vm_bo_base_init(&entry->base, vm, pt); + + r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats); + if (r) + goto error_free_pt; } return 0; @@ -1332,31 +1332,6 @@ static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params, } } - -/** - * amdgpu_vm_wait_pd - Wait for PT BOs to be free. - * - * @adev: amdgpu_device pointer - * @vm: related vm - * @owner: fence owner - * - * Returns: - * 0 on success, errno otherwise. - */ -static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm, - void *owner) -{ - struct amdgpu_sync sync; - int r; - - amdgpu_sync_create(&sync); - amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false); - r = amdgpu_sync_wait(&sync, true); - amdgpu_sync_free(&sync); - - return r; -} - /** * amdgpu_vm_update_func - helper to call update function * @@ -1451,7 +1426,8 @@ restart: params.adev = adev; if (vm->use_cpu_for_update) { - r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM); + r = amdgpu_bo_sync_wait(vm->root.base.bo, + AMDGPU_FENCE_OWNER_VM, true); if (unlikely(r)) return r; @@ -1772,9 +1748,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, params.adev = adev; params.vm = vm; - /* sync to everything on unmapping */ + /* sync to everything except eviction fences on unmapping */ if (!(flags & AMDGPU_PTE_VALID)) - owner = AMDGPU_FENCE_OWNER_UNDEFINED; + owner = AMDGPU_FENCE_OWNER_KFD; if (vm->use_cpu_for_update) { /* params.src is used as flag to indicate system Memory */ @@ -1784,7 +1760,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, /* Wait for PT BOs to be idle. PTs share the same resv. object * as the root PD BO */ - r = amdgpu_vm_wait_pd(adev, vm, owner); + r = amdgpu_bo_sync_wait(vm->root.base.bo, owner, true); if (unlikely(r)) return r; @@ -3057,13 +3033,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (r) goto error_unreserve; + amdgpu_vm_bo_base_init(&vm->root.base, vm, root); + r = amdgpu_vm_clear_bo(adev, vm, root, adev->vm_manager.root_level, vm->pte_support_ats); if (r) goto error_unreserve; - amdgpu_vm_bo_base_init(&vm->root.base, vm, root); amdgpu_bo_unreserve(vm->root.base.bo); if (pasid) { @@ -3398,14 +3375,15 @@ void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, struct amdgpu_task_info *task_info) { struct amdgpu_vm *vm; + unsigned long flags; - spin_lock(&adev->vm_manager.pasid_lock); + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); vm = idr_find(&adev->vm_manager.pasid_idr, pasid); if (vm) *task_info = vm->task_info; - spin_unlock(&adev->vm_manager.pasid_lock); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index db443ec53d3a..bea32f076b91 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -2980,7 +2980,7 @@ static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { - unsigned long flags; + unsigned long flags; unsigned crtc_id; struct amdgpu_crtc *amdgpu_crtc; struct amdgpu_flip_work *works; diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index b11a1c17a7f2..73851ebb3833 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -266,7 +266,8 @@ flr_done: } /* Trigger recovery for world switch failure if no TDR */ - if (amdgpu_device_should_recover_gpu(adev)) + if (amdgpu_device_should_recover_gpu(adev) + && amdgpu_lockup_timeout == MAX_SCHEDULE_TIMEOUT) amdgpu_device_gpu_recover(adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index e347b407bd03..c69d51598cfe 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -32,7 +32,7 @@ static u32 nbio_v7_4_get_rev_id(struct amdgpu_device *adev) { - u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); + u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); tmp &= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK; tmp >>= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT; @@ -90,7 +90,20 @@ static void nbio_v7_4_enable_doorbell_aperture(struct amdgpu_device *adev, static void nbio_v7_4_enable_doorbell_selfring_aperture(struct amdgpu_device *adev, bool enable) { + u32 tmp = 0; + if (enable) { + tmp = REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_EN, 1) | + REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_MODE, 1) | + REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_SIZE, 0); + + WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_BASE_LOW, + lower_32_bits(adev->doorbell.base)); + WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_BASE_HIGH, + upper_32_bits(adev->doorbell.base)); + } + + WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_CNTL, tmp); } static void nbio_v7_4_ih_doorbell_range(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index f71384be1f97..860b70d80d3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -98,18 +98,22 @@ static int psp_v11_0_init_microcode(struct psp_context *psp) snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name); err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev); - if (err) - goto out2; - - err = amdgpu_ucode_validate(adev->psp.ta_fw); - if (err) - goto out2; - - ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data; - adev->psp.ta_xgmi_ucode_version = le32_to_cpu(ta_hdr->ta_xgmi_ucode_version); - adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes); - adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr + - le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); + if (err) { + release_firmware(adev->psp.ta_fw); + adev->psp.ta_fw = NULL; + dev_info(adev->dev, + "psp v11.0: Failed to load firmware \"%s\"\n", fw_name); + } else { + err = amdgpu_ucode_validate(adev->psp.ta_fw); + if (err) + goto out2; + + ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data; + adev->psp.ta_xgmi_ucode_version = le32_to_cpu(ta_hdr->ta_xgmi_ucode_version); + adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes); + adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr + + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); + } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index c63de945c021..0487e3a4e9e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -500,9 +500,7 @@ static bool psp_v3_1_smu_reload_quirk(struct psp_context *psp) struct amdgpu_device *adev = psp->adev; uint32_t reg; - reg = smnMP1_FIRMWARE_FLAGS | 0x03b00000; - WREG32_SOC15(NBIO, 0, mmPCIE_INDEX2, reg); - reg = RREG32_SOC15(NBIO, 0, mmPCIE_DATA2); + reg = RREG32_PCIE(smnMP1_FIRMWARE_FLAGS | 0x03b00000); return (reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) ? true : false; } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index 127b85983e8f..c816e55d43a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -128,7 +128,7 @@ static const struct soc15_reg_golden golden_settings_sdma0_4_2_init[] = { static const struct soc15_reg_golden golden_settings_sdma0_4_2[] = { - SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07), + SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07), SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100), SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002), SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002), @@ -158,7 +158,7 @@ static const struct soc15_reg_golden golden_settings_sdma0_4_2[] = }; static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = { - SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07), + SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07), SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100), SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002), SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002), diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 79c1a9bbcc21..9d8df68893b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -1436,7 +1436,7 @@ static int si_common_early_init(void *handle) AMD_CG_SUPPORT_UVD_MGCG | AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG; - adev->pg_flags = 0; + adev->pg_flags = 0; adev->external_rev_id = (adev->rev_id == 0) ? 1 : (adev->rev_id == 1) ? 5 : 6; break; diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c index da58040fdbdc..41e01a7f57a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c @@ -6216,10 +6216,12 @@ static void si_request_link_speed_change_before_state_change(struct amdgpu_devic si_pi->force_pcie_gen = AMDGPU_PCIE_GEN2; if (current_link_speed == AMDGPU_PCIE_GEN2) break; + /* fall through */ case AMDGPU_PCIE_GEN2: if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; #endif + /* fall through */ default: si_pi->force_pcie_gen = si_get_current_pcie_speed(adev); break; diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 62d272b4be19..ed89a101f73f 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -461,7 +461,6 @@ static int soc15_asic_reset(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_VEGA10: - case CHIP_VEGA20: soc15_asic_get_baco_capability(adev, &baco_reset); break; default: @@ -858,11 +857,13 @@ static int soc15_common_early_init(void *handle) case CHIP_RAVEN: adev->asic_funcs = &soc15_asic_funcs; if (adev->rev_id >= 0x8) - adev->external_rev_id = adev->rev_id + 0x81; + adev->external_rev_id = adev->rev_id + 0x79; else if (adev->pdev->device == 0x15d8) adev->external_rev_id = adev->rev_id + 0x41; + else if (adev->rev_id == 1) + adev->external_rev_id = adev->rev_id + 0x20; else - adev->external_rev_id = 0x1; + adev->external_rev_id = adev->rev_id + 0x01; if (adev->rev_id >= 0x8) { adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c index 4b5d60ea3e78..a8e92638a2e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c @@ -81,6 +81,10 @@ void vega10_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.uvd_vce.vce_ring2_3 = AMDGPU_DOORBELL64_VCE_RING2_3; adev->doorbell_index.uvd_vce.vce_ring4_5 = AMDGPU_DOORBELL64_VCE_RING4_5; adev->doorbell_index.uvd_vce.vce_ring6_7 = AMDGPU_DOORBELL64_VCE_RING6_7; + + adev->doorbell_index.first_non_cp = AMDGPU_DOORBELL64_FIRST_NON_CP; + adev->doorbell_index.last_non_cp = AMDGPU_DOORBELL64_LAST_NON_CP; + /* In unit of dword doorbell */ adev->doorbell_index.max_assignment = AMDGPU_DOORBELL64_MAX_ASSIGNMENT << 1; adev->doorbell_index.sdma_doorbell_range = 4; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c index 53716c593b2b..0db84386252a 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c @@ -85,6 +85,10 @@ void vega20_doorbell_index_init(struct amdgpu_device *adev) adev->doorbell_index.uvd_vce.vce_ring2_3 = AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3; adev->doorbell_index.uvd_vce.vce_ring4_5 = AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5; adev->doorbell_index.uvd_vce.vce_ring6_7 = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7; + + adev->doorbell_index.first_non_cp = AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP; + adev->doorbell_index.last_non_cp = AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP; + adev->doorbell_index.max_assignment = AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT << 1; adev->doorbell_index.sdma_doorbell_range = 20; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 5d85ff341385..2e7c44955f43 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -863,7 +863,7 @@ static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size, return 0; } -#if CONFIG_X86_64 +#ifdef CONFIG_X86_64 static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size, uint32_t *num_entries, struct crat_subtype_iolink *sub_type_hdr) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 8372556b52eb..c6c9530e704e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -134,12 +134,18 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) */ q->doorbell_id = q->properties.queue_id; } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { - /* For SDMA queues on SOC15, use static doorbell - * assignments based on the engine and queue. + /* For SDMA queues on SOC15 with 8-byte doorbell, use static + * doorbell assignments based on the engine and queue id. + * The doobell index distance between RLC (2*i) and (2*i+1) + * for a SDMA engine is 512. */ - q->doorbell_id = dev->shared_resources.sdma_doorbell - [q->properties.sdma_engine_id] - [q->properties.sdma_queue_id]; + uint32_t *idx_offset = + dev->shared_resources.sdma_doorbell_idx; + + q->doorbell_id = idx_offset[q->properties.sdma_engine_id] + + (q->properties.sdma_queue_id & 1) + * KFD_QUEUE_DOORBELL_MIRROR_OFFSET + + (q->properties.sdma_queue_id >> 1); } else { /* For CP queues on SOC15 reserve a free doorbell ID */ unsigned int found; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c index 47243165a082..ae90a99909ef 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c @@ -323,57 +323,7 @@ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, struct queue_properties *q) { - uint64_t addr; - struct cik_mqd *m; - int retval; - - retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), - mqd_mem_obj); - - if (retval != 0) - return -ENOMEM; - - m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; - addr = (*mqd_mem_obj)->gpu_addr; - - memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); - - m->header = 0xC0310800; - m->compute_pipelinestat_enable = 1; - m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; - - m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | - PRELOAD_REQ; - m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | - QUANTUM_DURATION(10); - - m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; - m->cp_mqd_base_addr_lo = lower_32_bits(addr); - m->cp_mqd_base_addr_hi = upper_32_bits(addr); - - m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; - - /* - * Pipe Priority - * Identifies the pipe relative priority when this queue is connected - * to the pipeline. The pipe priority is against the GFX pipe and HP3D. - * In KFD we are using a fixed pipe priority set to CS_MEDIUM. - * 0 = CS_LOW (typically below GFX) - * 1 = CS_MEDIUM (typically between HP3D and GFX - * 2 = CS_HIGH (typically above HP3D) - */ - m->cp_hqd_pipe_priority = 1; - m->cp_hqd_queue_priority = 15; - - *mqd = m; - if (gart_addr) - *gart_addr = addr; - retval = mm->update_mqd(mm, m, q); - - return retval; + return init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); } static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 12b66330fc6d..0eeee3c6d6dc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -97,17 +97,29 @@ #define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) #define KFD_CWSR_TMA_OFFSET PAGE_SIZE +#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ + (KFD_MAX_NUM_OF_PROCESSES * \ + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) + +#define KFD_KERNEL_QUEUE_SIZE 2048 + +/* + * 512 = 0x200 + * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the + * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA. + * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC + * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in + * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE. + */ +#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512 + + /* * Kernel module parameter to specify maximum number of supported queues per * device */ extern int max_num_of_queues_per_device; -#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ - (KFD_MAX_NUM_OF_PROCESSES * \ - KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) - -#define KFD_KERNEL_QUEUE_SIZE 2048 /* Kernel module parameter to specify the scheduling policy */ extern int sched_policy; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 80b36e860a0a..4bdae78bab8e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -607,13 +607,17 @@ static int init_doorbell_bitmap(struct qcm_process_device *qpd, if (!qpd->doorbell_bitmap) return -ENOMEM; - /* Mask out any reserved doorbells */ - for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS; i++) - if ((dev->shared_resources.reserved_doorbell_mask & i) == - dev->shared_resources.reserved_doorbell_val) { + /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */ + for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) { + if (i >= dev->shared_resources.non_cp_doorbells_start + && i <= dev->shared_resources.non_cp_doorbells_end) { set_bit(i, qpd->doorbell_bitmap); - pr_debug("reserved doorbell 0x%03x\n", i); + set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, + qpd->doorbell_bitmap); + pr_debug("reserved doorbell 0x%03x and 0x%03x\n", i, + i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET); } + } return 0; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index ad31d7b9912f..fb27783d7a54 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -303,12 +303,11 @@ static void dm_pflip_high_irq(void *interrupt_params) return; } + /* Update to correct count(s) if racing with vblank irq */ + amdgpu_crtc->last_flip_vblank = drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); /* wake up userspace */ if (amdgpu_crtc->event) { - /* Update to correct count(s) if racing with vblank irq */ - drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); - drm_crtc_send_vblank_event(&amdgpu_crtc->base, amdgpu_crtc->event); /* page flip completed. clean up */ @@ -699,22 +698,36 @@ static void s3_handle_mst(struct drm_device *dev, bool suspend) { struct amdgpu_dm_connector *aconnector; struct drm_connector *connector; + struct drm_dp_mst_topology_mgr *mgr; + int ret; + bool need_hotplug = false; drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - aconnector = to_amdgpu_dm_connector(connector); - if (aconnector->dc_link->type == dc_connection_mst_branch && - !aconnector->mst_port) { + list_for_each_entry(connector, &dev->mode_config.connector_list, + head) { + aconnector = to_amdgpu_dm_connector(connector); + if (aconnector->dc_link->type != dc_connection_mst_branch || + aconnector->mst_port) + continue; - if (suspend) - drm_dp_mst_topology_mgr_suspend(&aconnector->mst_mgr); - else - drm_dp_mst_topology_mgr_resume(&aconnector->mst_mgr); - } + mgr = &aconnector->mst_mgr; + + if (suspend) { + drm_dp_mst_topology_mgr_suspend(mgr); + } else { + ret = drm_dp_mst_topology_mgr_resume(mgr); + if (ret < 0) { + drm_dp_mst_topology_mgr_set_mst(mgr, false); + need_hotplug = true; + } + } } drm_modeset_unlock(&dev->mode_config.connection_mutex); + + if (need_hotplug) + drm_kms_helper_hotplug_event(dev); } /** @@ -772,12 +785,13 @@ static int dm_suspend(void *handle) struct amdgpu_display_manager *dm = &adev->dm; int ret = 0; + WARN_ON(adev->dm.cached_state); + adev->dm.cached_state = drm_atomic_helper_suspend(adev->ddev); + s3_handle_mst(adev->ddev, true); amdgpu_dm_irq_suspend(adev); - WARN_ON(adev->dm.cached_state); - adev->dm.cached_state = drm_atomic_helper_suspend(adev->ddev); dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D3); @@ -872,6 +886,7 @@ static void emulated_link_detect(struct dc_link *link) return; } + /* dc_sink_create returns a new reference */ link->local_sink = sink; edid_status = dm_helpers_read_local_edid( @@ -898,7 +913,6 @@ static int dm_resume(void *handle) struct drm_plane_state *new_plane_state; struct dm_plane_state *dm_new_plane_state; enum dc_connection_type new_connection_type = dc_connection_none; - int ret; int i; /* power on hardware */ @@ -939,6 +953,8 @@ static int dm_resume(void *handle) if (aconnector->fake_enable && aconnector->dc_link->local_sink) aconnector->fake_enable = false; + if (aconnector->dc_sink) + dc_sink_release(aconnector->dc_sink); aconnector->dc_sink = NULL; amdgpu_dm_update_connector_after_detect(aconnector); mutex_unlock(&aconnector->hpd_lock); @@ -971,13 +987,13 @@ static int dm_resume(void *handle) } } - ret = drm_atomic_helper_resume(ddev, dm->cached_state); + drm_atomic_helper_resume(ddev, dm->cached_state); dm->cached_state = NULL; amdgpu_dm_irq_resume_late(adev); - return ret; + return 0; } /** @@ -1048,6 +1064,8 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) sink = aconnector->dc_link->local_sink; + if (sink) + dc_sink_retain(sink); /* * Edid mgmt connector gets first update only in mode_valid hook and then @@ -1072,21 +1090,24 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) * to it anymore after disconnect, so on next crtc to connector * reshuffle by UMD we will get into unwanted dc_sink release */ - if (aconnector->dc_sink != aconnector->dc_em_sink) - dc_sink_release(aconnector->dc_sink); + dc_sink_release(aconnector->dc_sink); } aconnector->dc_sink = sink; + dc_sink_retain(aconnector->dc_sink); amdgpu_dm_update_freesync_caps(connector, aconnector->edid); } else { amdgpu_dm_update_freesync_caps(connector, NULL); - if (!aconnector->dc_sink) + if (!aconnector->dc_sink) { aconnector->dc_sink = aconnector->dc_em_sink; - else if (aconnector->dc_sink != aconnector->dc_em_sink) dc_sink_retain(aconnector->dc_sink); + } } mutex_unlock(&dev->mode_config.mutex); + + if (sink) + dc_sink_release(sink); return; } @@ -1094,8 +1115,10 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) * TODO: temporary guard to look for proper fix * if this sink is MST sink, we should not do anything */ - if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + dc_sink_release(sink); return; + } if (aconnector->dc_sink == sink) { /* @@ -1104,6 +1127,8 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) */ DRM_DEBUG_DRIVER("DCHPD: connector_id=%d: dc_sink didn't change.\n", aconnector->connector_id); + if (sink) + dc_sink_release(sink); return; } @@ -1125,6 +1150,7 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) amdgpu_dm_update_freesync_caps(connector, NULL); aconnector->dc_sink = sink; + dc_sink_retain(aconnector->dc_sink); if (sink->dc_edid.length == 0) { aconnector->edid = NULL; drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); @@ -1145,11 +1171,15 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) amdgpu_dm_update_freesync_caps(connector, NULL); drm_connector_update_edid_property(connector, NULL); aconnector->num_modes = 0; + dc_sink_release(aconnector->dc_sink); aconnector->dc_sink = NULL; aconnector->edid = NULL; } mutex_unlock(&dev->mode_config.mutex); + + if (sink) + dc_sink_release(sink); } static void handle_hpd_irq(void *param) @@ -2964,6 +2994,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, return stream; } else { sink = aconnector->dc_sink; + dc_sink_retain(sink); } stream = dc_create_stream_for_sink(sink); @@ -3029,8 +3060,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, update_stream_signal(stream, sink); finish: - if (sink && sink->sink_signal == SIGNAL_TYPE_VIRTUAL && aconnector->base.force != DRM_FORCE_ON) - dc_sink_release(sink); + dc_sink_release(sink); return stream; } @@ -3288,6 +3318,14 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) dm->backlight_dev = NULL; } #endif + + if (aconnector->dc_em_sink) + dc_sink_release(aconnector->dc_em_sink); + aconnector->dc_em_sink = NULL; + if (aconnector->dc_sink) + dc_sink_release(aconnector->dc_sink); + aconnector->dc_sink = NULL; + drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux); drm_connector_unregister(connector); drm_connector_cleanup(connector); @@ -3385,10 +3423,12 @@ static void create_eml_sink(struct amdgpu_dm_connector *aconnector) (edid->extensions + 1) * EDID_LENGTH, &init_params); - if (aconnector->base.force == DRM_FORCE_ON) + if (aconnector->base.force == DRM_FORCE_ON) { aconnector->dc_sink = aconnector->dc_link->local_sink ? aconnector->dc_link->local_sink : aconnector->dc_em_sink; + dc_sink_retain(aconnector->dc_sink); + } } static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector) @@ -3777,7 +3817,6 @@ static const struct drm_plane_helper_funcs dm_plane_helper_funcs = { * check will succeed, and let DC implement proper check */ static const uint32_t rgb_formats[] = { - DRM_FORMAT_RGB888, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_RGBA8888, @@ -4148,7 +4187,8 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, } if (connector_type == DRM_MODE_CONNECTOR_HDMIA || - connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + connector_type == DRM_MODE_CONNECTOR_DisplayPort || + connector_type == DRM_MODE_CONNECTOR_eDP) { drm_connector_attach_vrr_capable_property( &aconnector->base); } @@ -4632,6 +4672,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct amdgpu_bo *abo; uint64_t tiling_flags, dcc_address; uint32_t target, target_vblank; + uint64_t last_flip_vblank; + bool vrr_active = acrtc_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; struct { struct dc_surface_update surface_updates[MAX_SURFACES]; @@ -4664,10 +4706,9 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct dc_plane_state *dc_plane; struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state); - if (plane->type == DRM_PLANE_TYPE_CURSOR) { - handle_cursor_update(plane, old_plane_state); + /* Cursor plane is handled after stream updates */ + if (plane->type == DRM_PLANE_TYPE_CURSOR) continue; - } if (!fb || !crtc || pcrtc != crtc) continue; @@ -4698,14 +4739,21 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, */ abo = gem_to_amdgpu_bo(fb->obj[0]); r = amdgpu_bo_reserve(abo, true); - if (unlikely(r != 0)) { + if (unlikely(r != 0)) DRM_ERROR("failed to reserve buffer before flip\n"); - WARN_ON(1); - } - /* Wait for all fences on this FB */ - WARN_ON(reservation_object_wait_timeout_rcu(abo->tbo.resv, true, false, - MAX_SCHEDULE_TIMEOUT) < 0); + /* + * Wait for all fences on this FB. Do limited wait to avoid + * deadlock during GPU reset when this fence will not signal + * but we hold reservation lock for the BO. + */ + r = reservation_object_wait_timeout_rcu(abo->tbo.resv, + true, false, + msecs_to_jiffies(5000)); + if (unlikely(r == 0)) + DRM_ERROR("Waiting for fences timed out."); + + amdgpu_bo_get_tiling_flags(abo, &tiling_flags); @@ -4785,7 +4833,31 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, * hopefully eliminating dc_*_update structs in their entirety. */ if (flip_count) { - target = (uint32_t)drm_crtc_vblank_count(pcrtc) + *wait_for_vblank; + if (!vrr_active) { + /* Use old throttling in non-vrr fixed refresh rate mode + * to keep flip scheduling based on target vblank counts + * working in a backwards compatible way, e.g., for + * clients using the GLX_OML_sync_control extension or + * DRI3/Present extension with defined target_msc. + */ + last_flip_vblank = drm_crtc_vblank_count(pcrtc); + } + else { + /* For variable refresh rate mode only: + * Get vblank of last completed flip to avoid > 1 vrr + * flips per video frame by use of throttling, but allow + * flip programming anywhere in the possibly large + * variable vrr vblank interval for fine-grained flip + * timing control and more opportunity to avoid stutter + * on late submission of flips. + */ + spin_lock_irqsave(&pcrtc->dev->event_lock, flags); + last_flip_vblank = acrtc_attach->last_flip_vblank; + spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); + } + + target = (uint32_t)last_flip_vblank + *wait_for_vblank; + /* Prepare wait for target vblank early - before the fence-waits */ target_vblank = target - (uint32_t)drm_crtc_vblank_count(pcrtc) + amdgpu_get_vblank_counter_kms(pcrtc->dev, acrtc_attach->crtc_id); @@ -4860,6 +4932,10 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, mutex_unlock(&dm->dc_lock); } + for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) + if (plane->type == DRM_PLANE_TYPE_CURSOR) + handle_cursor_update(plane, old_plane_state); + cleanup: kfree(flip); kfree(full); @@ -5785,14 +5861,13 @@ dm_determine_update_type_for_commit(struct dc *dc, old_dm_crtc_state = to_dm_crtc_state(old_crtc_state); num_plane = 0; - if (!new_dm_crtc_state->stream) { - if (!new_dm_crtc_state->stream && old_dm_crtc_state->stream) { - update_type = UPDATE_TYPE_FULL; - goto cleanup; - } + if (new_dm_crtc_state->stream != old_dm_crtc_state->stream) { + update_type = UPDATE_TYPE_FULL; + goto cleanup; + } + if (!new_dm_crtc_state->stream) continue; - } for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, j) { new_plane_crtc = new_plane_state->crtc; @@ -5803,6 +5878,11 @@ dm_determine_update_type_for_commit(struct dc *dc, if (plane->type == DRM_PLANE_TYPE_CURSOR) continue; + if (new_dm_plane_state->dc_state != old_dm_plane_state->dc_state) { + update_type = UPDATE_TYPE_FULL; + goto cleanup; + } + if (!state->allow_modeset) continue; @@ -5941,6 +6021,42 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, goto fail; } + /* + * Add all primary and overlay planes on the CRTC to the state + * whenever a plane is enabled to maintain correct z-ordering + * and to enable fast surface updates. + */ + drm_for_each_crtc(crtc, dev) { + bool modified = false; + + for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { + if (plane->type == DRM_PLANE_TYPE_CURSOR) + continue; + + if (new_plane_state->crtc == crtc || + old_plane_state->crtc == crtc) { + modified = true; + break; + } + } + + if (!modified) + continue; + + drm_for_each_plane_mask(plane, state->dev, crtc->state->plane_mask) { + if (plane->type == DRM_PLANE_TYPE_CURSOR) + continue; + + new_plane_state = + drm_atomic_get_plane_state(state, plane); + + if (IS_ERR(new_plane_state)) { + ret = PTR_ERR(new_plane_state); + goto fail; + } + } + } + /* Remove exiting planes if they are modified */ for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { ret = dm_update_plane_state(dc, state, plane, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index cca3e16cda4f..4a55cde027cf 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -671,6 +671,25 @@ static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __us return bytes_from_user; } +/* + * Returns the min and max vrr vfreq through the connector's debugfs file. + * Example usage: cat /sys/kernel/debug/dri/0/DP-1/vrr_range + */ +static int vrr_range_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + + if (connector->status != connector_status_connected) + return -ENODEV; + + seq_printf(m, "Min: %u\n", (unsigned int)aconnector->min_vfreq); + seq_printf(m, "Max: %u\n", (unsigned int)aconnector->max_vfreq); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(vrr_range); + static const struct file_operations dp_link_settings_debugfs_fops = { .owner = THIS_MODULE, .read = dp_link_settings_read, @@ -697,7 +716,8 @@ static const struct { } dp_debugfs_entries[] = { {"link_settings", &dp_link_settings_debugfs_fops}, {"phy_settings", &dp_phy_settings_debugfs_fop}, - {"test_pattern", &dp_phy_test_pattern_fops} + {"test_pattern", &dp_phy_test_pattern_fops}, + {"vrr_range", &vrr_range_fops} }; int connector_debugfs_init(struct amdgpu_dm_connector *connector) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index f51d52eb52e6..c4ea3a91f17a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -191,6 +191,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector) &init_params); dc_sink->priv = aconnector; + /* dc_link_add_remote_sink returns a new reference */ aconnector->dc_sink = dc_sink; if (aconnector->dc_sink) diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index a1c56f29cfeb..fd5266a58297 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -265,6 +265,7 @@ static struct atom_display_object_path_v2 *get_bios_object( && id.enum_id == obj_id.enum_id) return &bp->object_info_tbl.v1_4->display_path[i]; } + /* fall through */ case OBJECT_TYPE_CONNECTOR: case OBJECT_TYPE_GENERIC: /* Both Generic and Connector Object ID @@ -277,6 +278,7 @@ static struct atom_display_object_path_v2 *get_bios_object( && id.enum_id == obj_id.enum_id) return &bp->object_info_tbl.v1_4->display_path[i]; } + /* fall through */ default: return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c index 12d1842079ae..eb62d10bb65c 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c @@ -1348,12 +1348,12 @@ void dcn_bw_update_from_pplib(struct dc *dc) struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0}; bool res; - kernel_fpu_begin(); - /* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */ res = dm_pp_get_clock_levels_by_type_with_voltage( ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks); + kernel_fpu_begin(); + if (res) res = verify_clock_values(&fclks); @@ -1372,9 +1372,13 @@ void dcn_bw_update_from_pplib(struct dc *dc) } else BREAK_TO_DEBUGGER(); + kernel_fpu_end(); + res = dm_pp_get_clock_levels_by_type_with_voltage( ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks); + kernel_fpu_begin(); + if (res) res = verify_clock_values(&dcfclks); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 52f838442e21..c68fbd55db3c 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1138,6 +1138,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c /* pplib is notified if disp_num changed */ dc->hwss.optimize_bandwidth(dc, context); + for (i = 0; i < context->stream_count; i++) + context->streams[i]->mode_changed = false; + dc_release_state(dc->current_state); dc->current_state = context; @@ -1623,13 +1626,13 @@ static void commit_planes_do_stream_update(struct dc *dc, stream_update->adjust->v_total_min, stream_update->adjust->v_total_max); - if (stream_update->periodic_vsync_config && pipe_ctx->stream_res.tg->funcs->program_vline_interrupt) - pipe_ctx->stream_res.tg->funcs->program_vline_interrupt( - pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing, VLINE0, &stream->periodic_vsync_config); + if (stream_update->periodic_interrupt0 && + dc->hwss.setup_periodic_interrupt) + dc->hwss.setup_periodic_interrupt(pipe_ctx, VLINE0); - if (stream_update->enhanced_sync_config && pipe_ctx->stream_res.tg->funcs->program_vline_interrupt) - pipe_ctx->stream_res.tg->funcs->program_vline_interrupt( - pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing, VLINE1, &stream->enhanced_sync_config); + if (stream_update->periodic_interrupt1 && + dc->hwss.setup_periodic_interrupt) + dc->hwss.setup_periodic_interrupt(pipe_ctx, VLINE1); if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) || stream_update->vrr_infopacket || diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 7f5a947ad31d..4eba3c4800b6 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -794,6 +794,7 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason) sink->link->dongle_max_pix_clk = sink_caps.max_hdmi_pixel_clock; sink->converter_disable_audio = converter_disable_audio; + /* dc_sink_create returns a new reference */ link->local_sink = sink; edid_status = dm_helpers_read_local_edid( @@ -2037,6 +2038,9 @@ static enum dc_status enable_link( break; } + if (status == DC_OK) + pipe_ctx->stream->link->link_status.link_active = true; + return status; } @@ -2060,6 +2064,14 @@ static void disable_link(struct dc_link *link, enum signal_type signal) dp_disable_link_phy_mst(link, signal); } else link->link_enc->funcs->disable_output(link->link_enc, signal); + + if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + /* MST disable link only when no stream use the link */ + if (link->mst_stream_alloc_table.stream_count <= 0) + link->link_status.link_active = false; + } else { + link->link_status.link_active = false; + } } static bool dp_active_dongle_validate_timing( @@ -2623,8 +2635,6 @@ void core_link_enable_stream( } } - stream->link->link_status.link_active = true; - core_dc->hwss.enable_audio_stream(pipe_ctx); /* turn off otg test pattern if enable */ @@ -2659,8 +2669,6 @@ void core_link_disable_stream(struct pipe_ctx *pipe_ctx, int option) core_dc->hwss.disable_stream(pipe_ctx, option); disable_link(pipe_ctx->stream->link, pipe_ctx->stream->signal); - - pipe_ctx->stream->link->link_status.link_active = false; } void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index a798694992b9..5657cb3a2ad3 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -51,9 +51,19 @@ struct freesync_context { bool dummy; }; -union vline_config { - unsigned int line_number; - unsigned long long delta_in_ns; +enum vertical_interrupt_ref_point { + START_V_UPDATE = 0, + START_V_SYNC, + INVALID_POINT + + //For now, only v_update interrupt is used. + //START_V_BLANK, + //START_V_ACTIVE +}; + +struct periodic_interrupt_config { + enum vertical_interrupt_ref_point ref_point; + int lines_offset; }; @@ -106,8 +116,8 @@ struct dc_stream_state { /* DMCU info */ unsigned int abm_level; - union vline_config periodic_vsync_config; - union vline_config enhanced_sync_config; + struct periodic_interrupt_config periodic_interrupt0; + struct periodic_interrupt_config periodic_interrupt1; /* from core_stream struct */ struct dc_context *ctx; @@ -158,8 +168,8 @@ struct dc_stream_update { struct dc_info_packet *hdr_static_metadata; unsigned int *abm_level; - union vline_config *periodic_vsync_config; - union vline_config *enhanced_sync_config; + struct periodic_interrupt_config *periodic_interrupt0; + struct periodic_interrupt_config *periodic_interrupt1; struct dc_crtc_timing_adjust *adjust; struct dc_info_packet *vrr_infopacket; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c index 01e56f1a9f34..da96229db53a 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c @@ -53,6 +53,27 @@ #define MCP_DISABLE_ABM_IMMEDIATELY 255 +static bool dce_abm_set_pipe(struct abm *abm, uint32_t controller_id) +{ + struct dce_abm *abm_dce = TO_DCE_ABM(abm); + uint32_t rampingBoundary = 0xFFFF; + + REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, + 1, 80000); + + /* set ramping boundary */ + REG_WRITE(MASTER_COMM_DATA_REG1, rampingBoundary); + + /* setDMCUParam_Pipe */ + REG_UPDATE_2(MASTER_COMM_CMD_REG, + MASTER_COMM_CMD_REG_BYTE0, MCP_ABM_PIPE_SET, + MASTER_COMM_CMD_REG_BYTE1, controller_id); + + /* notifyDMCUMsg */ + REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1); + + return true; +} static unsigned int calculate_16_bit_backlight_from_pwm(struct dce_abm *abm_dce) { @@ -175,7 +196,6 @@ static void dmcu_set_backlight_level( uint32_t controller_id) { unsigned int backlight_8_bit = 0; - uint32_t rampingBoundary = 0xFFFF; uint32_t s2; if (backlight_pwm_u16_16 & 0x10000) @@ -185,16 +205,7 @@ static void dmcu_set_backlight_level( // Take MSB of fractional part since backlight is not max backlight_8_bit = (backlight_pwm_u16_16 >> 8) & 0xFF; - /* set ramping boundary */ - REG_WRITE(MASTER_COMM_DATA_REG1, rampingBoundary); - - /* setDMCUParam_Pipe */ - REG_UPDATE_2(MASTER_COMM_CMD_REG, - MASTER_COMM_CMD_REG_BYTE0, MCP_ABM_PIPE_SET, - MASTER_COMM_CMD_REG_BYTE1, controller_id); - - /* notifyDMCUMsg */ - REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1); + dce_abm_set_pipe(&abm_dce->base, controller_id); /* waitDMCUReadyForCmd */ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, @@ -309,16 +320,7 @@ static bool dce_abm_immediate_disable(struct abm *abm) { struct dce_abm *abm_dce = TO_DCE_ABM(abm); - REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, - 1, 80000); - - /* setDMCUParam_ABMLevel */ - REG_UPDATE_2(MASTER_COMM_CMD_REG, - MASTER_COMM_CMD_REG_BYTE0, MCP_ABM_PIPE_SET, - MASTER_COMM_CMD_REG_BYTE1, MCP_DISABLE_ABM_IMMEDIATELY); - - /* notifyDMCUMsg */ - REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1); + dce_abm_set_pipe(abm, MCP_DISABLE_ABM_IMMEDIATELY); abm->stored_backlight_registers.BL_PWM_CNTL = REG_READ(BL_PWM_CNTL); @@ -419,6 +421,7 @@ static const struct abm_funcs dce_funcs = { .abm_init = dce_abm_init, .set_abm_level = dce_abm_set_level, .init_backlight = dce_abm_init_backlight, + .set_pipe = dce_abm_set_pipe, .set_backlight_level_pwm = dce_abm_set_backlight_level_pwm, .get_current_backlight = dce_abm_get_current_backlight, .get_target_backlight = dce_abm_get_target_backlight, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c index 3c52a4fc921d..6e142c2db986 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c @@ -627,7 +627,15 @@ static void dce11_pplib_apply_display_requirements( dc, context->bw.dce.sclk_khz); - pp_display_cfg->min_dcfclock_khz = pp_display_cfg->min_engine_clock_khz; + /* + * As workaround for >4x4K lightup set dcfclock to min_engine_clock value. + * This is not required for less than 5 displays, + * thus don't request decfclk in dc to avoid impact + * on power saving. + * + */ + pp_display_cfg->min_dcfclock_khz = (context->stream_count > 4)? + pp_display_cfg->min_engine_clock_khz : 0; pp_display_cfg->min_engine_clock_deep_sleep_khz = context->bw.dce.sclk_deep_sleep_khz; @@ -688,6 +696,11 @@ static void dce11_update_clocks(struct clk_mgr *clk_mgr, { struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr); struct dm_pp_power_level_change_request level_change_req; + int patched_disp_clk = context->bw.dce.dispclk_khz; + + /*TODO: W/A for dal3 linux, investigate why this works */ + if (!clk_mgr_dce->dfs_bypass_active) + patched_disp_clk = patched_disp_clk * 115 / 100; level_change_req.power_level = dce_get_required_clocks_state(clk_mgr, context); /* get max clock state from PPLIB */ @@ -697,9 +710,9 @@ static void dce11_update_clocks(struct clk_mgr *clk_mgr, clk_mgr_dce->cur_min_clks_state = level_change_req.power_level; } - if (should_set_clock(safe_to_lower, context->bw.dce.dispclk_khz, clk_mgr->clks.dispclk_khz)) { - context->bw.dce.dispclk_khz = dce_set_clock(clk_mgr, context->bw.dce.dispclk_khz); - clk_mgr->clks.dispclk_khz = context->bw.dce.dispclk_khz; + if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) { + context->bw.dce.dispclk_khz = dce_set_clock(clk_mgr, patched_disp_clk); + clk_mgr->clks.dispclk_khz = patched_disp_clk; } dce11_pplib_apply_display_requirements(clk_mgr->ctx->dc, context); } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c index 85686d917636..a24a2bda8656 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c @@ -479,7 +479,7 @@ static void program_grph_pixel_format( case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F: sign = 1; floating = 1; - /* no break */ + /* fall through */ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F: /* shouldn't this get float too? */ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616: grph_depth = 3; diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.h index acd418515346..a6b80fdaa666 100644 --- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.h @@ -37,6 +37,10 @@ void dce100_prepare_bandwidth( struct dc *dc, struct dc_state *context); +void dce100_optimize_bandwidth( + struct dc *dc, + struct dc_state *context); + bool dce100_enable_display_power_gating(struct dc *dc, uint8_t controller_id, struct dc_bios *dcb, enum pipe_gating_control power_gating); diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index e1b285ea01ac..5e4db3712eef 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -1300,6 +1300,10 @@ static enum dc_status apply_single_controller_ctx_to_hw( struct drr_params params = {0}; unsigned int event_triggers = 0; + if (dc->hwss.disable_stream_gating) { + dc->hwss.disable_stream_gating(dc, pipe_ctx); + } + if (pipe_ctx->stream_res.audio != NULL) { struct audio_output audio_output; @@ -1329,10 +1333,8 @@ static enum dc_status apply_single_controller_ctx_to_hw( if (!pipe_ctx->stream->apply_seamless_boot_optimization) dc->hwss.enable_stream_timing(pipe_ctx, context, dc); - if (pipe_ctx->stream_res.tg->funcs->program_vupdate_interrupt) - pipe_ctx->stream_res.tg->funcs->program_vupdate_interrupt( - pipe_ctx->stream_res.tg, - &stream->timing); + if (dc->hwss.setup_vupdate_interrupt) + dc->hwss.setup_vupdate_interrupt(pipe_ctx); params.vertical_total_min = stream->adjust.v_total_min; params.vertical_total_max = stream->adjust.v_total_max; @@ -1521,6 +1523,14 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context) struct dc_link *edp_link = get_link_for_edp(dc); bool can_edp_fast_boot_optimize = false; bool apply_edp_fast_boot_optimization = false; + bool can_apply_seamless_boot = false; + + for (i = 0; i < context->stream_count; i++) { + if (context->streams[i]->apply_seamless_boot_optimization) { + can_apply_seamless_boot = true; + break; + } + } if (edp_link) { /* this seems to cause blank screens on DCE8 */ @@ -1549,7 +1559,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context) } } - if (!apply_edp_fast_boot_optimization) { + if (!apply_edp_fast_boot_optimization && !can_apply_seamless_boot) { if (edp_link_to_turnoff) { /*turn off backlight before DP_blank and encoder powered down*/ dc->hwss.edp_backlight_control(edp_link_to_turnoff, false); @@ -2676,6 +2686,8 @@ static const struct hw_sequencer_funcs dce110_funcs = { .set_static_screen_control = set_static_screen_control, .reset_hw_ctx_wrap = dce110_reset_hw_ctx_wrap, .enable_stream_timing = dce110_enable_stream_timing, + .disable_stream_gating = NULL, + .enable_stream_gating = NULL, .setup_stereo = NULL, .set_avmute = dce110_set_avmute, .wait_for_mpcc_disconnect = dce110_wait_for_mpcc_disconnect, diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_hw_sequencer.c index a60a90e68d91..c4543178ba20 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_hw_sequencer.c @@ -77,6 +77,6 @@ void dce80_hw_sequencer_construct(struct dc *dc) dc->hwss.enable_display_power_gating = dce100_enable_display_power_gating; dc->hwss.pipe_control_lock = dce_pipe_control_lock; dc->hwss.prepare_bandwidth = dce100_prepare_bandwidth; - dc->hwss.optimize_bandwidth = dce100_prepare_bandwidth; + dc->hwss.optimize_bandwidth = dce100_optimize_bandwidth; } diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c index 2eca81b5cf2f..c109ace96be9 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c @@ -792,9 +792,22 @@ bool dce80_validate_bandwidth( struct dc *dc, struct dc_state *context) { - /* TODO implement when needed but for now hardcode max value*/ - context->bw.dce.dispclk_khz = 681000; - context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ; + int i; + bool at_least_one_pipe = false; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + if (context->res_ctx.pipe_ctx[i].stream) + at_least_one_pipe = true; + } + + if (at_least_one_pipe) { + /* TODO implement when needed but for now hardcode max value*/ + context->bw.dce.dispclk_khz = 681000; + context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ; + } else { + context->bw.dce.dispclk_khz = 0; + context->bw.dce.yclk_khz = 0; + } return true; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index 117d9d8227f7..d1a8f1c302a9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -959,9 +959,25 @@ static void dcn10_disable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx) static void dcn10_init_pipes(struct dc *dc, struct dc_state *context) { int i; + bool can_apply_seamless_boot = false; + + for (i = 0; i < context->stream_count; i++) { + if (context->streams[i]->apply_seamless_boot_optimization) { + can_apply_seamless_boot = true; + break; + } + } for (i = 0; i < dc->res_pool->pipe_count; i++) { struct timing_generator *tg = dc->res_pool->timing_generators[i]; + struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; + + /* There is assumption that pipe_ctx is not mapping irregularly + * to non-preferred front end. If pipe_ctx->stream is not NULL, + * we will use the pipe, so don't disable + */ + if (pipe_ctx->stream != NULL) + continue; if (tg->funcs->is_tg_enabled(tg)) tg->funcs->lock(tg); @@ -975,7 +991,9 @@ static void dcn10_init_pipes(struct dc *dc, struct dc_state *context) } } - dc->res_pool->mpc->funcs->mpc_init(dc->res_pool->mpc); + /* Cannot reset the MPC mux if seamless boot */ + if (!can_apply_seamless_boot) + dc->res_pool->mpc->funcs->mpc_init(dc->res_pool->mpc); for (i = 0; i < dc->res_pool->pipe_count; i++) { struct timing_generator *tg = dc->res_pool->timing_generators[i]; @@ -983,6 +1001,16 @@ static void dcn10_init_pipes(struct dc *dc, struct dc_state *context) struct dpp *dpp = dc->res_pool->dpps[i]; struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; + // W/A for issue with dc_post_update_surfaces_to_stream + hubp->power_gated = true; + + /* There is assumption that pipe_ctx is not mapping irregularly + * to non-preferred front end. If pipe_ctx->stream is not NULL, + * we will use the pipe, so don't disable + */ + if (pipe_ctx->stream != NULL) + continue; + dpp->funcs->dpp_reset(dpp); pipe_ctx->stream_res.tg = tg; @@ -1137,11 +1165,13 @@ static void reset_hw_ctx_wrap( struct clock_source *old_clk = pipe_ctx_old->clock_source; reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state); + if (dc->hwss.enable_stream_gating) { + dc->hwss.enable_stream_gating(dc, pipe_ctx); + } if (old_clk) old_clk->funcs->cs_power_down(old_clk); } } - } static bool patch_address_for_sbs_tb_stereo( @@ -2162,8 +2192,10 @@ static void dcn10_blank_pixel_data( if (!blank) { if (stream_res->tg->funcs->set_blank) stream_res->tg->funcs->set_blank(stream_res->tg, blank); - if (stream_res->abm) + if (stream_res->abm) { + stream_res->abm->funcs->set_pipe(stream_res->abm, stream_res->tg->inst + 1); stream_res->abm->funcs->set_abm_level(stream_res->abm, stream->abm_level); + } } else if (blank) { if (stream_res->abm) stream_res->abm->funcs->set_abm_immediate_disable(stream_res->abm); @@ -2661,8 +2693,8 @@ static void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) .mirror = pipe_ctx->plane_state->horizontal_mirror }; - pos_cpy.x -= pipe_ctx->plane_state->dst_rect.x; - pos_cpy.y -= pipe_ctx->plane_state->dst_rect.y; + pos_cpy.x_hotspot += pipe_ctx->plane_state->dst_rect.x; + pos_cpy.y_hotspot += pipe_ctx->plane_state->dst_rect.y; if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE) @@ -2709,6 +2741,147 @@ static void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx) pipe_ctx->plane_res.dpp, &opt_attr); } +/** +* apply_front_porch_workaround TODO FPGA still need? +* +* This is a workaround for a bug that has existed since R5xx and has not been +* fixed keep Front porch at minimum 2 for Interlaced mode or 1 for progressive. +*/ +static void apply_front_porch_workaround( + struct dc_crtc_timing *timing) +{ + if (timing->flags.INTERLACE == 1) { + if (timing->v_front_porch < 2) + timing->v_front_porch = 2; + } else { + if (timing->v_front_porch < 1) + timing->v_front_porch = 1; + } +} + +int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx) +{ + struct timing_generator *optc = pipe_ctx->stream_res.tg; + const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing; + struct dc_crtc_timing patched_crtc_timing; + int vesa_sync_start; + int asic_blank_end; + int interlace_factor; + int vertical_line_start; + + patched_crtc_timing = *dc_crtc_timing; + apply_front_porch_workaround(&patched_crtc_timing); + + interlace_factor = patched_crtc_timing.flags.INTERLACE ? 2 : 1; + + vesa_sync_start = patched_crtc_timing.v_addressable + + patched_crtc_timing.v_border_bottom + + patched_crtc_timing.v_front_porch; + + asic_blank_end = (patched_crtc_timing.v_total - + vesa_sync_start - + patched_crtc_timing.v_border_top) + * interlace_factor; + + vertical_line_start = asic_blank_end - + optc->dlg_otg_param.vstartup_start + 1; + + return vertical_line_start; +} + +static void calc_vupdate_position( + struct pipe_ctx *pipe_ctx, + uint32_t *start_line, + uint32_t *end_line) +{ + const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing; + int vline_int_offset_from_vupdate = + pipe_ctx->stream->periodic_interrupt0.lines_offset; + int vupdate_offset_from_vsync = get_vupdate_offset_from_vsync(pipe_ctx); + int start_position; + + if (vline_int_offset_from_vupdate > 0) + vline_int_offset_from_vupdate--; + else if (vline_int_offset_from_vupdate < 0) + vline_int_offset_from_vupdate++; + + start_position = vline_int_offset_from_vupdate + vupdate_offset_from_vsync; + + if (start_position >= 0) + *start_line = start_position; + else + *start_line = dc_crtc_timing->v_total + start_position - 1; + + *end_line = *start_line + 2; + + if (*end_line >= dc_crtc_timing->v_total) + *end_line = 2; +} + +static void cal_vline_position( + struct pipe_ctx *pipe_ctx, + enum vline_select vline, + uint32_t *start_line, + uint32_t *end_line) +{ + enum vertical_interrupt_ref_point ref_point = INVALID_POINT; + + if (vline == VLINE0) + ref_point = pipe_ctx->stream->periodic_interrupt0.ref_point; + else if (vline == VLINE1) + ref_point = pipe_ctx->stream->periodic_interrupt1.ref_point; + + switch (ref_point) { + case START_V_UPDATE: + calc_vupdate_position( + pipe_ctx, + start_line, + end_line); + break; + case START_V_SYNC: + // Suppose to do nothing because vsync is 0; + break; + default: + ASSERT(0); + break; + } +} + +static void dcn10_setup_periodic_interrupt( + struct pipe_ctx *pipe_ctx, + enum vline_select vline) +{ + struct timing_generator *tg = pipe_ctx->stream_res.tg; + + if (vline == VLINE0) { + uint32_t start_line = 0; + uint32_t end_line = 0; + + cal_vline_position(pipe_ctx, vline, &start_line, &end_line); + + tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line); + + } else if (vline == VLINE1) { + pipe_ctx->stream_res.tg->funcs->setup_vertical_interrupt1( + tg, + pipe_ctx->stream->periodic_interrupt1.lines_offset); + } +} + +static void dcn10_setup_vupdate_interrupt(struct pipe_ctx *pipe_ctx) +{ + struct timing_generator *tg = pipe_ctx->stream_res.tg; + int start_line = get_vupdate_offset_from_vsync(pipe_ctx); + + if (start_line < 0) { + ASSERT(0); + start_line = 0; + } + + if (tg->funcs->setup_vertical_interrupt2) + tg->funcs->setup_vertical_interrupt2(tg, start_line); +} + static const struct hw_sequencer_funcs dcn10_funcs = { .program_gamut_remap = program_gamut_remap, .init_hw = dcn10_init_hw, @@ -2756,7 +2929,11 @@ static const struct hw_sequencer_funcs dcn10_funcs = { .edp_wait_for_hpd_ready = hwss_edp_wait_for_hpd_ready, .set_cursor_position = dcn10_set_cursor_position, .set_cursor_attribute = dcn10_set_cursor_attribute, - .set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level + .set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level, + .disable_stream_gating = NULL, + .enable_stream_gating = NULL, + .setup_periodic_interrupt = dcn10_setup_periodic_interrupt, + .setup_vupdate_interrupt = dcn10_setup_vupdate_interrupt }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h index f8eea10e4c64..6d66084df55f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h @@ -81,4 +81,6 @@ struct pipe_ctx *find_top_pipe_for_stream( struct dc_state *context, const struct dc_stream_state *stream); +int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx); + #endif /* __DC_HWSS_DCN10_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index 2f78a84f0dcb..0345d51e9d6f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -92,134 +92,36 @@ static void optc1_disable_stereo(struct timing_generator *optc) OTG_3D_STRUCTURE_STEREO_SEL_OVR, 0); } -static uint32_t get_start_vline(struct timing_generator *optc, const struct dc_crtc_timing *dc_crtc_timing) -{ - struct dc_crtc_timing patched_crtc_timing; - int vesa_sync_start; - int asic_blank_end; - int interlace_factor; - int vertical_line_start; - - patched_crtc_timing = *dc_crtc_timing; - optc1_apply_front_porch_workaround(optc, &patched_crtc_timing); - - vesa_sync_start = patched_crtc_timing.h_addressable + - patched_crtc_timing.h_border_right + - patched_crtc_timing.h_front_porch; - - asic_blank_end = patched_crtc_timing.h_total - - vesa_sync_start - - patched_crtc_timing.h_border_left; - - interlace_factor = patched_crtc_timing.flags.INTERLACE ? 2 : 1; - - vesa_sync_start = patched_crtc_timing.v_addressable + - patched_crtc_timing.v_border_bottom + - patched_crtc_timing.v_front_porch; - - asic_blank_end = (patched_crtc_timing.v_total - - vesa_sync_start - - patched_crtc_timing.v_border_top) - * interlace_factor; - - vertical_line_start = asic_blank_end - optc->dlg_otg_param.vstartup_start + 1; - if (vertical_line_start < 0) { - ASSERT(0); - vertical_line_start = 0; - } - - return vertical_line_start; -} - -static void calc_vline_position( +void optc1_setup_vertical_interrupt0( struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing, - unsigned long long vsync_delta, - uint32_t *start_line, - uint32_t *end_line) + uint32_t start_line, + uint32_t end_line) { - unsigned long long req_delta_tens_of_usec = div64_u64((vsync_delta + 9999), 10000); - unsigned long long pix_clk_hundreds_khz = div64_u64((dc_crtc_timing->pix_clk_100hz + 999), 1000); - uint32_t req_delta_lines = (uint32_t) div64_u64( - (req_delta_tens_of_usec * pix_clk_hundreds_khz + dc_crtc_timing->h_total - 1), - dc_crtc_timing->h_total); - - uint32_t vsync_line = get_start_vline(optc, dc_crtc_timing); - - if (req_delta_lines != 0) - req_delta_lines--; - - if (req_delta_lines > vsync_line) - *start_line = dc_crtc_timing->v_total - (req_delta_lines - vsync_line) - 1; - else - *start_line = vsync_line - req_delta_lines; - - *end_line = *start_line + 2; + struct optc *optc1 = DCN10TG_FROM_TG(optc); - if (*end_line >= dc_crtc_timing->v_total) - *end_line = 2; + REG_SET_2(OTG_VERTICAL_INTERRUPT0_POSITION, 0, + OTG_VERTICAL_INTERRUPT0_LINE_START, start_line, + OTG_VERTICAL_INTERRUPT0_LINE_END, end_line); } -void optc1_program_vline_interrupt( +void optc1_setup_vertical_interrupt1( struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing, - enum vline_select vline, - const union vline_config *vline_config) + uint32_t start_line) { struct optc *optc1 = DCN10TG_FROM_TG(optc); - uint32_t start_line = 0; - uint32_t end_line = 0; - - switch (vline) { - case VLINE0: - calc_vline_position(optc, dc_crtc_timing, vline_config->delta_in_ns, &start_line, &end_line); - REG_SET_2(OTG_VERTICAL_INTERRUPT0_POSITION, 0, - OTG_VERTICAL_INTERRUPT0_LINE_START, start_line, - OTG_VERTICAL_INTERRUPT0_LINE_END, end_line); - break; - case VLINE1: - REG_SET(OTG_VERTICAL_INTERRUPT1_POSITION, 0, - OTG_VERTICAL_INTERRUPT1_LINE_START, vline_config->line_number); - break; - default: - break; - } + + REG_SET(OTG_VERTICAL_INTERRUPT1_POSITION, 0, + OTG_VERTICAL_INTERRUPT1_LINE_START, start_line); } -void optc1_program_vupdate_interrupt( +void optc1_setup_vertical_interrupt2( struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing) + uint32_t start_line) { struct optc *optc1 = DCN10TG_FROM_TG(optc); - int32_t vertical_line_start; - uint32_t asic_blank_end; - uint32_t vesa_sync_start; - struct dc_crtc_timing patched_crtc_timing; - - patched_crtc_timing = *dc_crtc_timing; - optc1_apply_front_porch_workaround(optc, &patched_crtc_timing); - - /* asic_h_blank_end = HsyncWidth + HbackPorch = - * vesa. usHorizontalTotal - vesa. usHorizontalSyncStart - - * vesa.h_left_border - */ - vesa_sync_start = patched_crtc_timing.h_addressable + - patched_crtc_timing.h_border_right + - patched_crtc_timing.h_front_porch; - - asic_blank_end = patched_crtc_timing.h_total - - vesa_sync_start - - patched_crtc_timing.h_border_left; - - /* Use OTG_VERTICAL_INTERRUPT2 replace VUPDATE interrupt, - * program the reg for interrupt postition. - */ - vertical_line_start = asic_blank_end - optc->dlg_otg_param.vstartup_start + 1; - if (vertical_line_start < 0) - vertical_line_start = 0; REG_SET(OTG_VERTICAL_INTERRUPT2_POSITION, 0, - OTG_VERTICAL_INTERRUPT2_LINE_START, vertical_line_start); + OTG_VERTICAL_INTERRUPT2_LINE_START, start_line); } /** @@ -1480,8 +1382,9 @@ bool optc1_get_crc(struct timing_generator *optc, static const struct timing_generator_funcs dcn10_tg_funcs = { .validate_timing = optc1_validate_timing, .program_timing = optc1_program_timing, - .program_vline_interrupt = optc1_program_vline_interrupt, - .program_vupdate_interrupt = optc1_program_vupdate_interrupt, + .setup_vertical_interrupt0 = optc1_setup_vertical_interrupt0, + .setup_vertical_interrupt1 = optc1_setup_vertical_interrupt1, + .setup_vertical_interrupt2 = optc1_setup_vertical_interrupt2, .program_global_sync = optc1_program_global_sync, .enable_crtc = optc1_enable_crtc, .disable_crtc = optc1_disable_crtc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h index 24452f11c598..4eb9a898c237 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h @@ -483,11 +483,16 @@ void optc1_program_timing( const struct dc_crtc_timing *dc_crtc_timing, bool use_vbios); -void optc1_program_vline_interrupt( +void optc1_setup_vertical_interrupt0( struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing, - enum vline_select vline, - const union vline_config *vline_config); + uint32_t start_line, + uint32_t end_line); +void optc1_setup_vertical_interrupt1( + struct timing_generator *optc, + uint32_t start_line); +void optc1_setup_vertical_interrupt2( + struct timing_generator *optc, + uint32_t start_line); void optc1_program_global_sync( struct timing_generator *optc); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h index abc961c0906e..86dc39a02408 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h @@ -46,6 +46,7 @@ struct abm_funcs { void (*abm_init)(struct abm *abm); bool (*set_abm_level)(struct abm *abm, unsigned int abm_level); bool (*set_abm_immediate_disable)(struct abm *abm); + bool (*set_pipe)(struct abm *abm, unsigned int controller_id); bool (*init_backlight)(struct abm *abm); /* backlight_pwm_u16_16 is unsigned 32 bit, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index 03ae941895f3..c25f7df7b5e3 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -134,14 +134,6 @@ struct dc_crtc_timing; struct drr_params; -union vline_config; - - -enum vline_select { - VLINE0, - VLINE1, - VLINE2 -}; struct timing_generator_funcs { bool (*validate_timing)(struct timing_generator *tg, @@ -149,14 +141,17 @@ struct timing_generator_funcs { void (*program_timing)(struct timing_generator *tg, const struct dc_crtc_timing *timing, bool use_vbios); - void (*program_vline_interrupt)( + void (*setup_vertical_interrupt0)( + struct timing_generator *optc, + uint32_t start_line, + uint32_t end_line); + void (*setup_vertical_interrupt1)( + struct timing_generator *optc, + uint32_t start_line); + void (*setup_vertical_interrupt2)( struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing, - enum vline_select vline, - const union vline_config *vline_config); + uint32_t start_line); - void (*program_vupdate_interrupt)(struct timing_generator *optc, - const struct dc_crtc_timing *dc_crtc_timing); bool (*enable_crtc)(struct timing_generator *tg); bool (*disable_crtc)(struct timing_generator *tg); bool (*is_counter_moving)(struct timing_generator *tg); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 341b4810288c..7676f25216b1 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -38,6 +38,11 @@ enum pipe_gating_control { PIPE_GATING_CONTROL_INIT }; +enum vline_select { + VLINE0, + VLINE1 +}; + struct dce_hwseq_wa { bool blnd_crtc_trigger; bool DEGVIDCN10_253; @@ -68,6 +73,10 @@ struct stream_resource; struct hw_sequencer_funcs { + void (*disable_stream_gating)(struct dc *dc, struct pipe_ctx *pipe_ctx); + + void (*enable_stream_gating)(struct dc *dc, struct pipe_ctx *pipe_ctx); + void (*init_hw)(struct dc *dc); void (*init_pipes)(struct dc *dc, struct dc_state *context); @@ -220,6 +229,9 @@ struct hw_sequencer_funcs { void (*set_cursor_attribute)(struct pipe_ctx *pipe); void (*set_cursor_sdr_white_level)(struct pipe_ctx *pipe); + void (*setup_periodic_interrupt)(struct pipe_ctx *pipe_ctx, enum vline_select vline); + void (*setup_vupdate_interrupt)(struct pipe_ctx *pipe_ctx); + }; void color_space_to_black_color( diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h index 4f501ddcfb8d..34d6fdcb32e2 100644 --- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h +++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h @@ -131,6 +131,7 @@ #define INTERNAL_REV_RAVEN_A0 0x00 /* First spin of Raven */ #define RAVEN_A0 0x01 #define RAVEN_B0 0x21 +#define PICASSO_A0 0x41 #if defined(CONFIG_DRM_AMD_DC_DCN1_01) /* DCN1_01 */ #define RAVEN2_A0 0x81 @@ -165,4 +166,6 @@ #define FAMILY_UNKNOWN 0xFF + + #endif /* __DAL_ASIC_ID_H__ */ diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 94a84bc57c7a..bfd27f10879e 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -724,7 +724,7 @@ static void build_vrr_infopacket_v1(enum signal_type signal, static void build_vrr_infopacket_v2(enum signal_type signal, const struct mod_vrr_params *vrr, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket) { unsigned int payload_size = 0; @@ -732,8 +732,7 @@ static void build_vrr_infopacket_v2(enum signal_type signal, build_vrr_infopacket_header_v2(signal, infopacket, &payload_size); build_vrr_infopacket_data(vrr, infopacket); - if (app_tf != NULL) - build_vrr_infopacket_fs2_data(*app_tf, infopacket); + build_vrr_infopacket_fs2_data(app_tf, infopacket); build_vrr_infopacket_checksum(&payload_size, infopacket); @@ -757,7 +756,7 @@ void mod_freesync_build_vrr_infopacket(struct mod_freesync *mod_freesync, const struct dc_stream_state *stream, const struct mod_vrr_params *vrr, enum vrr_packet_type packet_type, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket) { /* SPD info packet for FreeSync diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h b/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h index 4222e403b151..dcef85994c45 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h @@ -145,7 +145,7 @@ void mod_freesync_build_vrr_infopacket(struct mod_freesync *mod_freesync, const struct dc_stream_state *stream, const struct mod_vrr_params *vrr, enum vrr_packet_type packet_type, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket); void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c index 3ba87b076287..038b88221c5f 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c @@ -165,18 +165,11 @@ struct iram_table_v_2_2 { }; #pragma pack(pop) -static uint16_t backlight_8_to_16(unsigned int backlight_8bit) -{ - return (uint16_t)(backlight_8bit * 0x101); -} - static void fill_backlight_transform_table(struct dmcu_iram_parameters params, struct iram_table_v_2 *table) { unsigned int i; unsigned int num_entries = NUM_BL_CURVE_SEGS; - unsigned int query_input_8bit; - unsigned int query_output_8bit; unsigned int lut_index; table->backlight_thresholds[0] = 0; @@ -194,16 +187,13 @@ static void fill_backlight_transform_table(struct dmcu_iram_parameters params, * format U4.10. */ for (i = 1; i+1 < num_entries; i++) { - query_input_8bit = DIV_ROUNDUP((i * 256), num_entries); - lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1); ASSERT(lut_index < params.backlight_lut_array_size); - query_output_8bit = params.backlight_lut_array[lut_index] >> 8; table->backlight_thresholds[i] = - backlight_8_to_16(query_input_8bit); + cpu_to_be16(DIV_ROUNDUP((i * 65536), num_entries)); table->backlight_offsets[i] = - backlight_8_to_16(query_output_8bit); + cpu_to_be16(params.backlight_lut_array[lut_index]); } } @@ -212,8 +202,6 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par { unsigned int i; unsigned int num_entries = NUM_BL_CURVE_SEGS; - unsigned int query_input_8bit; - unsigned int query_output_8bit; unsigned int lut_index; table->backlight_thresholds[0] = 0; @@ -231,16 +219,13 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par * format U4.10. */ for (i = 1; i+1 < num_entries; i++) { - query_input_8bit = DIV_ROUNDUP((i * 256), num_entries); - lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1); ASSERT(lut_index < params.backlight_lut_array_size); - query_output_8bit = params.backlight_lut_array[lut_index] >> 8; table->backlight_thresholds[i] = - backlight_8_to_16(query_input_8bit); + cpu_to_be16(DIV_ROUNDUP((i * 65536), num_entries)); table->backlight_offsets[i] = - backlight_8_to_16(query_output_8bit); + cpu_to_be16(params.backlight_lut_array[lut_index]); } } diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 83d960110d23..5f3c10ebff08 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -137,20 +137,17 @@ struct kgd2kfd_shared_resources { /* Bit n == 1 means Queue n is available for KFD */ DECLARE_BITMAP(queue_bitmap, KGD_MAX_QUEUES); - /* Doorbell assignments (SOC15 and later chips only). Only + /* SDMA doorbell assignments (SOC15 and later chips only). Only * specific doorbells are routed to each SDMA engine. Others * are routed to IH and VCN. They are not usable by the CP. - * - * Any doorbell number D that satisfies the following condition - * is reserved: (D & reserved_doorbell_mask) == reserved_doorbell_val - * - * KFD currently uses 1024 (= 0x3ff) doorbells per process. If - * doorbells 0x0e0-0x0ff and 0x2e0-0x2ff are reserved, that means - * mask would be set to 0x1e0 and val set to 0x0e0. */ - unsigned int sdma_doorbell[2][8]; - unsigned int reserved_doorbell_mask; - unsigned int reserved_doorbell_val; + uint32_t *sdma_doorbell_idx; + + /* From SOC15 onward, the doorbell index range not usable for CP + * queues. + */ + uint32_t non_cp_doorbells_start; + uint32_t non_cp_doorbells_end; /* Base address of doorbell aperture. */ phys_addr_t doorbell_physical_address; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c index ce177d7f04cb..6bf48934fdc4 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c @@ -277,8 +277,7 @@ int psm_adjust_power_state_dynamic(struct pp_hwmgr *hwmgr, bool skip_display_set if (!skip_display_settings) phm_notify_smc_display_config_after_ps_adjustment(hwmgr); - if ((hwmgr->request_dpm_level != hwmgr->dpm_level) && - !phm_force_dpm_levels(hwmgr, hwmgr->request_dpm_level)) + if (!phm_force_dpm_levels(hwmgr, hwmgr->request_dpm_level)) hwmgr->dpm_level = hwmgr->request_dpm_level; if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) { diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c index 4588bddf8b33..615cf2c09e54 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c @@ -489,15 +489,16 @@ int pp_atomfwctrl_get_gpio_information(struct pp_hwmgr *hwmgr, } int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, - uint8_t id, uint32_t *frequency) + uint8_t clk_id, uint8_t syspll_id, + uint32_t *frequency) { struct amdgpu_device *adev = hwmgr->adev; struct atom_get_smu_clock_info_parameters_v3_1 parameters; struct atom_get_smu_clock_info_output_parameters_v3_1 *output; uint32_t ix; - parameters.clk_id = id; - parameters.syspll_id = 0; + parameters.clk_id = clk_id; + parameters.syspll_id = syspll_id; parameters.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ; parameters.dfsdid = 0; @@ -530,20 +531,23 @@ static void pp_atomfwctrl_copy_vbios_bootup_values_3_2(struct pp_hwmgr *hwmgr, boot_values->ulSocClk = 0; boot_values->ulDCEFClk = 0; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_SOCCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_SOCCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulSocClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCEFCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCEFCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulDCEFClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_ECLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_ECLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulEClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_VCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_VCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulVClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulDClk = frequency; + + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL1_0_FCLK_ID, SMU11_SYSPLL1_2_ID, &frequency)) + boot_values->ulFClk = frequency; } static void pp_atomfwctrl_copy_vbios_bootup_values_3_1(struct pp_hwmgr *hwmgr, @@ -563,19 +567,19 @@ static void pp_atomfwctrl_copy_vbios_bootup_values_3_1(struct pp_hwmgr *hwmgr, boot_values->ulSocClk = 0; boot_values->ulDCEFClk = 0; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, 0, &frequency)) boot_values->ulSocClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, 0, &frequency)) boot_values->ulDCEFClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_ECLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_ECLK_ID, 0, &frequency)) boot_values->ulEClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_VCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_VCLK_ID, 0, &frequency)) boot_values->ulVClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCLK_ID, 0, &frequency)) boot_values->ulDClk = frequency; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h index fe9e8ceef50e..b7e2651b570b 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h @@ -139,6 +139,7 @@ struct pp_atomfwctrl_bios_boot_up_values { uint32_t ulEClk; uint32_t ulVClk; uint32_t ulDClk; + uint32_t ulFClk; uint16_t usVddc; uint16_t usVddci; uint16_t usMvddc; @@ -236,7 +237,8 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr, int pp_atomfwctrl_get_smc_dpm_information(struct pp_hwmgr *hwmgr, struct pp_atomfwctrl_smc_dpm_parameters *param); int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, - uint8_t id, uint32_t *frequency); + uint8_t clk_id, uint8_t syspll_id, + uint32_t *frequency); #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c index f95c5f50eb0f..0ad8fe4a6277 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c @@ -139,12 +139,10 @@ static int smu10_construct_max_power_limits_table(struct pp_hwmgr *hwmgr, static int smu10_init_dynamic_state_adjustment_rule_settings( struct pp_hwmgr *hwmgr) { - uint32_t table_size = - sizeof(struct phm_clock_voltage_dependency_table) + - (7 * sizeof(struct phm_clock_voltage_dependency_record)); + struct phm_clock_voltage_dependency_table *table_clk_vlt; - struct phm_clock_voltage_dependency_table *table_clk_vlt = - kzalloc(table_size, GFP_KERNEL); + table_clk_vlt = kzalloc(struct_size(table_clk_vlt, entries, 7), + GFP_KERNEL); if (NULL == table_clk_vlt) { pr_err("Can not allocate memory!\n"); @@ -1033,6 +1031,7 @@ static int smu10_get_clock_by_type_with_latency(struct pp_hwmgr *hwmgr, break; case amd_pp_dpp_clock: pclk_vol_table = pinfo->vdd_dep_on_dppclk; + break; default: return -EINVAL; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c index c8f5c00dd1e7..48187acac59e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c @@ -3681,10 +3681,12 @@ static int smu7_request_link_speed_change_before_state_change( data->force_pcie_gen = PP_PCIEGen2; if (current_link_speed == PP_PCIEGen2) break; + /* fall through */ case PP_PCIEGen2: if (0 == amdgpu_acpi_pcie_performance_request(hwmgr->adev, PCIE_PERF_REQ_GEN2, false)) break; #endif + /* fall through */ default: data->force_pcie_gen = smu7_get_current_pcie_speed(hwmgr); break; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c index d138ddae563d..58f5589aaf12 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c @@ -1211,7 +1211,7 @@ int smu7_power_control_set_level(struct pp_hwmgr *hwmgr) hwmgr->platform_descriptor.TDPAdjustment : (-1 * hwmgr->platform_descriptor.TDPAdjustment); - if (hwmgr->chip_id > CHIP_TONGA) + if (hwmgr->chip_id > CHIP_TONGA) target_tdp = ((100 + adjust_percent) * (int)(cac_table->usTDP * 256)) / 100; else target_tdp = ((100 + adjust_percent) * (int)(cac_table->usConfigurableTDP * 256)) / 100; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c index 553a203ac47c..019d6a206492 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c @@ -272,12 +272,10 @@ static int smu8_init_dynamic_state_adjustment_rule_settings( struct pp_hwmgr *hwmgr, ATOM_CLK_VOLT_CAPABILITY *disp_voltage_table) { - uint32_t table_size = - sizeof(struct phm_clock_voltage_dependency_table) + - (7 * sizeof(struct phm_clock_voltage_dependency_record)); + struct phm_clock_voltage_dependency_table *table_clk_vlt; - struct phm_clock_voltage_dependency_table *table_clk_vlt = - kzalloc(table_size, GFP_KERNEL); + table_clk_vlt = kzalloc(struct_size(table_clk_vlt, entries, 7), + GFP_KERNEL); if (NULL == table_clk_vlt) { pr_err("Can not allocate memory!\n"); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c index f94dab27f486..7337be5602e4 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c @@ -1,3 +1,25 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ #include "amdgpu.h" #include "soc15.h" #include "soc15_hw_ip.h" @@ -114,7 +136,7 @@ int vega10_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state) if (soc15_baco_program_registers(hwmgr, pre_baco_tbl, ARRAY_SIZE(pre_baco_tbl))) { if (smum_send_msg_to_smc(hwmgr, PPSMC_MSG_EnterBaco)) - return -1; + return -EINVAL; if (soc15_baco_program_registers(hwmgr, enter_baco_tbl, ARRAY_SIZE(enter_baco_tbl))) @@ -132,5 +154,5 @@ int vega10_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state) } } - return -1; + return -EINVAL; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h index a93b1e6d1c66..f7a3ffa744b3 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h @@ -20,8 +20,8 @@ * OTHER DEALINGS IN THE SOFTWARE. * */ -#ifndef __VEGA10_BOCO_H__ -#define __VEGA10_BOCO_H__ +#ifndef __VEGA10_BACO_H__ +#define __VEGA10_BACO_H__ #include "hwmgr.h" #include "common_baco.h" diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c index 5479125ff4f6..5c4f701939ea 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c @@ -2575,10 +2575,10 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr) data->vbios_boot_state.gfx_clock = boot_up_values.ulGfxClk; data->vbios_boot_state.mem_clock = boot_up_values.ulUClk; pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, - SMU9_SYSPLL0_SOCCLK_ID, &boot_up_values.ulSocClk); + SMU9_SYSPLL0_SOCCLK_ID, 0, &boot_up_values.ulSocClk); pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, - SMU9_SYSPLL0_DCEFCLK_ID, &boot_up_values.ulDCEFClk); + SMU9_SYSPLL0_DCEFCLK_ID, 0, &boot_up_values.ulDCEFClk); data->vbios_boot_state.soc_clock = boot_up_values.ulSocClk; data->vbios_boot_state.dcef_clock = boot_up_values.ulDCEFClk; @@ -4407,9 +4407,9 @@ static int vega10_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c index 6c8e78611c03..bdb48e94eff6 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c @@ -2009,9 +2009,9 @@ static int vega12_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c index 0d883b358df2..5e8602a79b1c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c @@ -1,3 +1,25 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ #include "amdgpu.h" #include "soc15.h" #include "soc15_hw_ip.h" @@ -67,14 +89,14 @@ int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state) if(smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnterBaco, 0)) - return -1; + return -EINVAL; } else if (state == BACO_STATE_OUT) { if (smum_send_msg_to_smc(hwmgr, PPSMC_MSG_ExitBaco)) - return -1; + return -EINVAL; if (!soc15_baco_program_registers(hwmgr, clean_baco_tbl, ARRAY_SIZE(clean_baco_tbl))) - return -1; + return -EINVAL; } return 0; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h index c51988a9ed77..51c7f8392925 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h @@ -20,8 +20,8 @@ * OTHER DEALINGS IN THE SOFTWARE. * */ -#ifndef __VEGA20_BOCO_H__ -#define __VEGA20_BOCO_H__ +#ifndef __VEGA20_BACO_H__ +#define __VEGA20_BACO_H__ #include "hwmgr.h" #include "common_baco.h" diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c index 0769b1ec562b..9aa7bec1b5fe 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c @@ -463,9 +463,9 @@ static int vega20_setup_asic_task(struct pp_hwmgr *hwmgr) static void vega20_init_dpm_state(struct vega20_dpm_state *dpm_state) { dpm_state->soft_min_level = 0x0; - dpm_state->soft_max_level = 0xffff; + dpm_state->soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_state->hard_min_level = 0x0; - dpm_state->hard_max_level = 0xffff; + dpm_state->hard_max_level = VG20_CLOCK_MAX_DEFAULT; } static int vega20_get_number_of_dpm_level(struct pp_hwmgr *hwmgr, @@ -711,8 +711,10 @@ static int vega20_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) PP_ASSERT_WITH_CODE(!ret, "[SetupDefaultDpmTable] failed to get fclk dpm levels!", return ret); - } else - dpm_table->count = 0; + } else { + dpm_table->count = 1; + dpm_table->dpm_levels[0].value = data->vbios_boot_state.fclock / 100; + } vega20_init_dpm_state(&(dpm_table->dpm_state)); /* save a copy of the default DPM table */ @@ -754,6 +756,7 @@ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) data->vbios_boot_state.eclock = boot_up_values.ulEClk; data->vbios_boot_state.vclock = boot_up_values.ulVClk; data->vbios_boot_state.dclock = boot_up_values.ulDClk; + data->vbios_boot_state.fclock = boot_up_values.ulFClk; data->vbios_boot_state.uc_cooling_id = boot_up_values.ucCoolingID; smum_send_msg_to_smc_with_parameter(hwmgr, @@ -780,6 +783,8 @@ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) static int vega20_override_pcie_parameters(struct pp_hwmgr *hwmgr) { struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev); + struct vega20_hwmgr *data = + (struct vega20_hwmgr *)(hwmgr->backend); uint32_t pcie_gen = 0, pcie_width = 0, smu_pcie_arg; int ret; @@ -816,6 +821,10 @@ static int vega20_override_pcie_parameters(struct pp_hwmgr *hwmgr) "[OverridePcieParameters] Attempt to override pcie params failed!", return ret); + data->pcie_parameters_override = 1; + data->pcie_gen_level1 = pcie_gen; + data->pcie_width_level1 = pcie_width; + return 0; } @@ -979,6 +988,8 @@ static int vega20_od8_set_feature_capabilities( } if (data->smu_features[GNLD_DPM_UCLK].enabled) { + pptable_information->od_settings_min[OD8_SETTING_UCLK_FMAX] = + data->dpm_table.mem_table.dpm_levels[data->dpm_table.mem_table.count - 2].value; if (pptable_information->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_UCLK_MAX] && pptable_information->od_settings_min[OD8_SETTING_UCLK_FMAX] > 0 && pptable_information->od_settings_max[OD8_SETTING_UCLK_FMAX] > 0 && @@ -2314,32 +2325,8 @@ static int vega20_force_dpm_lowest(struct pp_hwmgr *hwmgr) static int vega20_unforce_dpm_levels(struct pp_hwmgr *hwmgr) { - struct vega20_hwmgr *data = - (struct vega20_hwmgr *)(hwmgr->backend); - uint32_t soft_min_level, soft_max_level; int ret = 0; - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.gfx_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.gfx_table)); - data->dpm_table.gfx_table.dpm_state.soft_min_level = - data->dpm_table.gfx_table.dpm_levels[soft_min_level].value; - data->dpm_table.gfx_table.dpm_state.soft_max_level = - data->dpm_table.gfx_table.dpm_levels[soft_max_level].value; - - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.mem_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.mem_table)); - data->dpm_table.mem_table.dpm_state.soft_min_level = - data->dpm_table.mem_table.dpm_levels[soft_min_level].value; - data->dpm_table.mem_table.dpm_state.soft_max_level = - data->dpm_table.mem_table.dpm_levels[soft_max_level].value; - - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.soc_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.soc_table)); - data->dpm_table.soc_table.dpm_state.soft_min_level = - data->dpm_table.soc_table.dpm_levels[soft_min_level].value; - data->dpm_table.soc_table.dpm_state.soft_max_level = - data->dpm_table.soc_table.dpm_levels[soft_max_level].value; - ret = vega20_upload_dpm_min_level(hwmgr, 0xFFFFFFFF); PP_ASSERT_WITH_CODE(!ret, "Failed to upload DPM Bootup Levels!", @@ -2641,9 +2628,8 @@ static int vega20_get_sclks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.gfx_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_GFXCLK].enabled, - "[GetSclks]: gfxclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_GFXCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2670,9 +2656,8 @@ static int vega20_get_memclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.mem_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_UCLK].enabled, - "[GetMclks]: uclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_UCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = data->mclk_latency_table.count = count; @@ -2696,9 +2681,8 @@ static int vega20_get_dcefclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.dcef_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_DCEFCLK].enabled, - "[GetDcfclocks]: dcefclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_DCEFCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2719,9 +2703,8 @@ static int vega20_get_socclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.soc_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_SOCCLK].enabled, - "[GetSocclks]: socclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_SOCCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2799,7 +2782,6 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, data->od8_settings.od8_settings_array; OverDriveTable_t *od_table = &(data->smc_state_table.overdrive_table); - struct pp_clock_levels_with_latency clocks; int32_t input_index, input_clk, input_vol, i; int od8_id; int ret; @@ -2858,11 +2840,6 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, return -EOPNOTSUPP; } - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get memory clk levels failed!", - return ret); - for (i = 0; i < size; i += 2) { if (i + 2 > size) { pr_info("invalid number of input parameters %d\n", @@ -2879,11 +2856,11 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, return -EINVAL; } - if (input_clk < clocks.data[0].clocks_in_khz / 1000 || + if (input_clk < od8_settings[OD8_SETTING_UCLK_FMAX].min_value || input_clk > od8_settings[OD8_SETTING_UCLK_FMAX].max_value) { pr_info("clock freq %d is not within allowed range [%d - %d]\n", input_clk, - clocks.data[0].clocks_in_khz / 1000, + od8_settings[OD8_SETTING_UCLK_FMAX].min_value, od8_settings[OD8_SETTING_UCLK_FMAX].max_value); return -EINVAL; } @@ -3088,9 +3065,9 @@ static int vega20_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); @@ -3128,7 +3105,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, &(data->dpm_table.fclk_table); int i, now, size = 0; int ret = 0; - uint32_t gen_speed, lane_width; + uint32_t gen_speed, lane_width, current_gen_speed, current_lane_width; switch (type) { case PP_SCLK: @@ -3137,10 +3114,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current gfx clk Failed!", return ret); - ret = vega20_get_sclks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get gfx clk levels Failed!", - return ret); + if (vega20_get_sclks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3154,10 +3132,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current mclk freq Failed!", return ret); - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get memory clk levels Failed!", - return ret); + if (vega20_get_memclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3171,10 +3150,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current socclk freq Failed!", return ret); - ret = vega20_get_socclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get soc clk levels Failed!", - return ret); + if (vega20_get_socclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3200,10 +3180,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current dcefclk freq Failed!", return ret); - ret = vega20_get_dcefclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get dcefclk levels Failed!", - return ret); + if (vega20_get_dcefclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3212,28 +3193,36 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, break; case PP_PCIE: - gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & + current_gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & + current_lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; - for (i = 0; i < NUM_LINK_LEVELS; i++) + for (i = 0; i < NUM_LINK_LEVELS; i++) { + if (i == 1 && data->pcie_parameters_override) { + gen_speed = data->pcie_gen_level1; + lane_width = data->pcie_width_level1; + } else { + gen_speed = pptable->PcieGenSpeed[i]; + lane_width = pptable->PcieLaneCount[i]; + } size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, - (pptable->PcieGenSpeed[i] == 0) ? "2.5GT/s," : - (pptable->PcieGenSpeed[i] == 1) ? "5.0GT/s," : - (pptable->PcieGenSpeed[i] == 2) ? "8.0GT/s," : - (pptable->PcieGenSpeed[i] == 3) ? "16.0GT/s," : "", - (pptable->PcieLaneCount[i] == 1) ? "x1" : - (pptable->PcieLaneCount[i] == 2) ? "x2" : - (pptable->PcieLaneCount[i] == 3) ? "x4" : - (pptable->PcieLaneCount[i] == 4) ? "x8" : - (pptable->PcieLaneCount[i] == 5) ? "x12" : - (pptable->PcieLaneCount[i] == 6) ? "x16" : "", + (gen_speed == 0) ? "2.5GT/s," : + (gen_speed == 1) ? "5.0GT/s," : + (gen_speed == 2) ? "8.0GT/s," : + (gen_speed == 3) ? "16.0GT/s," : "", + (lane_width == 1) ? "x1" : + (lane_width == 2) ? "x2" : + (lane_width == 3) ? "x4" : + (lane_width == 4) ? "x8" : + (lane_width == 5) ? "x12" : + (lane_width == 6) ? "x16" : "", pptable->LclkFreq[i], - (gen_speed == pptable->PcieGenSpeed[i]) && - (lane_width == pptable->PcieLaneCount[i]) ? + (current_gen_speed == gen_speed) && + (current_lane_width == lane_width) ? "*" : ""); + } break; case OD_SCLK: @@ -3288,13 +3277,8 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, } if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) { - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Fail to get memory clk levels!", - return ret); - size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", - clocks.data[0].clocks_in_khz / 1000, + od8_settings[OD8_SETTING_UCLK_FMAX].min_value, od8_settings[OD8_SETTING_UCLK_FMAX].max_value); } @@ -3356,6 +3340,31 @@ static int vega20_set_uclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr, return ret; } +static int vega20_set_fclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr) +{ + struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); + struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.fclk_table); + int ret = 0; + + if (data->smu_features[GNLD_DPM_FCLK].enabled) { + PP_ASSERT_WITH_CODE(dpm_table->count > 0, + "[SetFclkToHightestDpmLevel] Dpm table has no entry!", + return -EINVAL); + PP_ASSERT_WITH_CODE(dpm_table->count <= NUM_FCLK_DPM_LEVELS, + "[SetFclkToHightestDpmLevel] Dpm table has too many entries!", + return -EINVAL); + + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(hwmgr, + PPSMC_MSG_SetSoftMinByFreq, + (PPCLK_FCLK << 16 ) | dpm_table->dpm_state.soft_min_level)), + "[SetFclkToHightestDpmLevel] Set soft min fclk failed!", + return ret); + } + + return ret; +} + static int vega20_pre_display_configuration_changed_task(struct pp_hwmgr *hwmgr) { struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); @@ -3366,8 +3375,10 @@ static int vega20_pre_display_configuration_changed_task(struct pp_hwmgr *hwmgr) ret = vega20_set_uclk_to_highest_dpm_level(hwmgr, &data->dpm_table.mem_table); + if (ret) + return ret; - return ret; + return vega20_set_fclk_to_highest_dpm_level(hwmgr); } static int vega20_display_configuration_changed_task(struct pp_hwmgr *hwmgr) @@ -3456,14 +3467,14 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) disable_mclk_switching = ((1 < hwmgr->display_config->num_display) && !hwmgr->display_config->multi_monitor_in_sync) || vblank_too_short; - latency = hwmgr->display_config->dce_tolerable_mclk_in_active_latency; + latency = hwmgr->display_config->dce_tolerable_mclk_in_active_latency; /* gfxclk */ dpm_table = &(data->dpm_table.gfx_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_GFXCLK_LEVEL < dpm_table->count) { @@ -3485,9 +3496,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* memclk */ dpm_table = &(data->dpm_table.mem_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_MCLK_LEVEL < dpm_table->count) { @@ -3526,12 +3537,21 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) if (hwmgr->display_config->nb_pstate_switch_disable) dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + /* fclk */ + dpm_table = &(data->dpm_table.fclk_table); + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; + dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; + if (hwmgr->display_config->nb_pstate_switch_disable) + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + /* vclk */ dpm_table = &(data->dpm_table.vclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) { @@ -3548,9 +3568,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* dclk */ dpm_table = &(data->dpm_table.dclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) { @@ -3567,9 +3587,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* socclk */ dpm_table = &(data->dpm_table.soc_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_SOCCLK_LEVEL < dpm_table->count) { @@ -3586,9 +3606,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* eclk */ dpm_table = &(data->dpm_table.eclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_VCEMCLK_LEVEL < dpm_table->count) { diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h index 37f5f5e657da..a5bc758ae097 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h @@ -42,6 +42,8 @@ #define AVFS_CURVE 0 #define OD8_HOTCURVE_TEMPERATURE 85 +#define VG20_CLOCK_MAX_DEFAULT 0xFFFF + typedef uint32_t PP_Clock; enum { @@ -219,6 +221,7 @@ struct vega20_vbios_boot_state { uint32_t eclock; uint32_t dclock; uint32_t vclock; + uint32_t fclock; }; #define DPMTABLE_OD_UPDATE_SCLK 0x00000001 @@ -523,6 +526,10 @@ struct vega20_hwmgr { unsigned long metrics_time; SmuMetrics_t metrics_table; + + bool pcie_parameters_override; + uint32_t pcie_gen_level1; + uint32_t pcie_width_level1; }; #define VEGA20_DPM2_NEAR_TDP_DEC 10 diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c index 52abca065764..2d4cfe14f72e 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c @@ -2330,6 +2330,7 @@ static uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member) case DRAM_LOG_BUFF_SIZE: return offsetof(SMU74_SoftRegisters, DRAM_LOG_BUFF_SIZE); } + break; case SMU_Discrete_DpmTable: switch (member) { case UvdBootLevel: @@ -2339,6 +2340,7 @@ static uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member) case LowSclkInterruptThreshold: return offsetof(SMU74_Discrete_DpmTable, LowSclkInterruptThreshold); } + break; } pr_warn("can't get the offset of type %x member %x\n", type, member); return 0; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c index 079fc8e8f709..742b3dc1f6cb 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c @@ -40,10 +40,8 @@ bool smu9_is_smc_ram_running(struct pp_hwmgr *hwmgr) struct amdgpu_device *adev = hwmgr->adev; uint32_t mp1_fw_flags; - WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2, - (MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff))); - - mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2); + mp1_fw_flags = RREG32_PCIE(MP1_Public | + (smnMP1_FIRMWARE_FLAGS & 0xffffffff)); if (mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) return true; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c index a6edd5df33b0..4240aeec9000 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c @@ -29,6 +29,10 @@ #include <drm/amdgpu_drm.h> #include "smumgr.h" +MODULE_FIRMWARE("amdgpu/bonaire_smc.bin"); +MODULE_FIRMWARE("amdgpu/bonaire_k_smc.bin"); +MODULE_FIRMWARE("amdgpu/hawaii_smc.bin"); +MODULE_FIRMWARE("amdgpu/hawaii_k_smc.bin"); MODULE_FIRMWARE("amdgpu/topaz_smc.bin"); MODULE_FIRMWARE("amdgpu/topaz_k_smc.bin"); MODULE_FIRMWARE("amdgpu/tonga_smc.bin"); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c index b7ff7d4d6f44..ba00744c3413 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c @@ -49,10 +49,8 @@ static bool vega20_is_smc_ram_running(struct pp_hwmgr *hwmgr) struct amdgpu_device *adev = hwmgr->adev; uint32_t mp1_fw_flags; - WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2, - (MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff))); - - mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2); + mp1_fw_flags = RREG32_PCIE(MP1_Public | + (smnMP1_FIRMWARE_FLAGS & 0xffffffff)); if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >> MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT) diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c index cb55bdc36f3f..6b6e037258c3 100644 --- a/drivers/gpu/drm/bochs/bochs_drv.c +++ b/drivers/gpu/drm/bochs/bochs_drv.c @@ -145,6 +145,10 @@ static int bochs_pci_probe(struct pci_dev *pdev, if (IS_ERR(dev)) return PTR_ERR(dev); + ret = pci_enable_device(pdev); + if (ret) + goto err_free_dev; + dev->pdev = pdev; pci_set_drvdata(pdev, dev); diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c index bbc17530c23d..888980d4bc74 100644 --- a/drivers/gpu/drm/bridge/tc358767.c +++ b/drivers/gpu/drm/bridge/tc358767.c @@ -98,6 +98,8 @@ #define DP0_STARTVAL 0x064c #define DP0_ACTIVEVAL 0x0650 #define DP0_SYNCVAL 0x0654 +#define SYNCVAL_HS_POL_ACTIVE_LOW (1 << 15) +#define SYNCVAL_VS_POL_ACTIVE_LOW (1 << 31) #define DP0_MISC 0x0658 #define TU_SIZE_RECOMMENDED (63) /* LSCLK cycles per TU */ #define BPC_6 (0 << 5) @@ -142,6 +144,8 @@ #define DP0_LTLOOPCTRL 0x06d8 #define DP0_SNKLTCTRL 0x06e4 +#define DP1_SRCCTRL 0x07a0 + /* PHY */ #define DP_PHY_CTRL 0x0800 #define DP_PHY_RST BIT(28) /* DP PHY Global Soft Reset */ @@ -150,6 +154,7 @@ #define PHY_M1_RST BIT(12) /* Reset PHY1 Main Channel */ #define PHY_RDY BIT(16) /* PHY Main Channels Ready */ #define PHY_M0_RST BIT(8) /* Reset PHY0 Main Channel */ +#define PHY_2LANE BIT(2) /* PHY Enable 2 lanes */ #define PHY_A0_EN BIT(1) /* PHY Aux Channel0 Enable */ #define PHY_M0_EN BIT(0) /* PHY Main Channel0 Enable */ @@ -540,6 +545,7 @@ static int tc_aux_link_setup(struct tc_data *tc) unsigned long rate; u32 value; int ret; + u32 dp_phy_ctrl; rate = clk_get_rate(tc->refclk); switch (rate) { @@ -564,7 +570,10 @@ static int tc_aux_link_setup(struct tc_data *tc) value |= SYSCLK_SEL_LSCLK | LSCLK_DIV_2; tc_write(SYS_PLLPARAM, value); - tc_write(DP_PHY_CTRL, BGREN | PWR_SW_EN | BIT(2) | PHY_A0_EN); + dp_phy_ctrl = BGREN | PWR_SW_EN | PHY_A0_EN; + if (tc->link.base.num_lanes == 2) + dp_phy_ctrl |= PHY_2LANE; + tc_write(DP_PHY_CTRL, dp_phy_ctrl); /* * Initially PLLs are in bypass. Force PLL parameter update, @@ -720,7 +729,9 @@ static int tc_set_video_mode(struct tc_data *tc, tc_write(DP0_ACTIVEVAL, (mode->vdisplay << 16) | (mode->hdisplay)); - tc_write(DP0_SYNCVAL, (vsync_len << 16) | (hsync_len << 0)); + tc_write(DP0_SYNCVAL, (vsync_len << 16) | (hsync_len << 0) | + ((mode->flags & DRM_MODE_FLAG_NHSYNC) ? SYNCVAL_HS_POL_ACTIVE_LOW : 0) | + ((mode->flags & DRM_MODE_FLAG_NVSYNC) ? SYNCVAL_VS_POL_ACTIVE_LOW : 0)); tc_write(DPIPXLFMT, VS_POL_ACTIVE_LOW | HS_POL_ACTIVE_LOW | DE_POL_ACTIVE_HIGH | SUB_CFG_TYPE_CONFIG1 | DPI_BPP_RGB888); @@ -830,12 +841,11 @@ static int tc_main_link_setup(struct tc_data *tc) if (!tc->mode) return -EINVAL; - /* from excel file - DP0_SrcCtrl */ - tc_write(DP0_SRCCTRL, DP0_SRCCTRL_SCRMBLDIS | DP0_SRCCTRL_EN810B | - DP0_SRCCTRL_LANESKEW | DP0_SRCCTRL_LANES_2 | - DP0_SRCCTRL_BW27 | DP0_SRCCTRL_AUTOCORRECT); - /* from excel file - DP1_SrcCtrl */ - tc_write(0x07a0, 0x00003083); + tc_write(DP0_SRCCTRL, tc_srcctrl(tc)); + /* SSCG and BW27 on DP1 must be set to the same as on DP0 */ + tc_write(DP1_SRCCTRL, + (tc->link.spread ? DP0_SRCCTRL_SSCG : 0) | + ((tc->link.base.rate != 162000) ? DP0_SRCCTRL_BW27 : 0)); rate = clk_get_rate(tc->refclk); switch (rate) { @@ -856,8 +866,11 @@ static int tc_main_link_setup(struct tc_data *tc) } value |= SYSCLK_SEL_LSCLK | LSCLK_DIV_2; tc_write(SYS_PLLPARAM, value); + /* Setup Main Link */ - dp_phy_ctrl = BGREN | PWR_SW_EN | BIT(2) | PHY_A0_EN | PHY_M0_EN; + dp_phy_ctrl = BGREN | PWR_SW_EN | PHY_A0_EN | PHY_M0_EN; + if (tc->link.base.num_lanes == 2) + dp_phy_ctrl |= PHY_2LANE; tc_write(DP_PHY_CTRL, dp_phy_ctrl); msleep(100); @@ -1106,10 +1119,20 @@ static bool tc_bridge_mode_fixup(struct drm_bridge *bridge, static enum drm_mode_status tc_connector_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { + struct tc_data *tc = connector_to_tc(connector); + u32 req, avail; + u32 bits_per_pixel = 24; + /* DPI interface clock limitation: upto 154 MHz */ if (mode->clock > 154000) return MODE_CLOCK_HIGH; + req = mode->clock * bits_per_pixel / 8; + avail = tc->link.base.num_lanes * tc->link.base.rate; + + if (req > avail) + return MODE_BAD; + return MODE_OK; } @@ -1187,7 +1210,8 @@ static int tc_bridge_attach(struct drm_bridge *bridge) /* Create eDP connector */ drm_connector_helper_add(&tc->connector, &tc_connector_helper_funcs); ret = drm_connector_init(drm, &tc->connector, &tc_connector_funcs, - DRM_MODE_CONNECTOR_eDP); + tc->panel ? DRM_MODE_CONNECTOR_eDP : + DRM_MODE_CONNECTOR_DisplayPort); if (ret) return ret; @@ -1196,6 +1220,10 @@ static int tc_bridge_attach(struct drm_bridge *bridge) drm_display_info_set_bus_formats(&tc->connector.display_info, &bus_format, 1); + tc->connector.display_info.bus_flags = + DRM_BUS_FLAG_DE_HIGH | + DRM_BUS_FLAG_PIXDATA_NEGEDGE | + DRM_BUS_FLAG_SYNC_NEGEDGE; drm_connector_attach_encoder(&tc->connector, tc->bridge.encoder); return 0; diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 7a7231d00779..40ac19848034 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -1608,6 +1608,15 @@ int drm_atomic_helper_async_check(struct drm_device *dev, old_plane_state->crtc != new_plane_state->crtc) return -EINVAL; + /* + * FIXME: Since prepare_fb and cleanup_fb are always called on + * the new_plane_state for async updates we need to block framebuffer + * changes. This prevents use of a fb that's been cleaned up and + * double cleanups from occuring. + */ + if (old_plane_state->fb != new_plane_state->fb) + return -EINVAL; + funcs = plane->helper_private; if (!funcs->atomic_async_update) return -EINVAL; diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index defc03a9cd1c..0aabd401d3ca 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1296,12 +1296,11 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, (arg->flags & DRM_MODE_PAGE_FLIP_EVENT)) return -EINVAL; - drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); - state = drm_atomic_state_alloc(dev); if (!state) return -ENOMEM; + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); state->acquire_ctx = &ctx; state->allow_modeset = !!(arg->flags & DRM_MODE_ATOMIC_ALLOW_MODESET); diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 523b1067f1ed..0e9349ff2d16 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1621,6 +1621,64 @@ static bool drm_fb_pixel_format_equal(const struct fb_var_screeninfo *var_1, var_1->transp.msb_right == var_2->transp.msb_right; } +static void drm_fb_helper_fill_pixel_fmt(struct fb_var_screeninfo *var, + u8 depth) +{ + switch (depth) { + case 8: + var->red.offset = 0; + var->green.offset = 0; + var->blue.offset = 0; + var->red.length = 8; /* 8bit DAC */ + var->green.length = 8; + var->blue.length = 8; + var->transp.offset = 0; + var->transp.length = 0; + break; + case 15: + var->red.offset = 10; + var->green.offset = 5; + var->blue.offset = 0; + var->red.length = 5; + var->green.length = 5; + var->blue.length = 5; + var->transp.offset = 15; + var->transp.length = 1; + break; + case 16: + var->red.offset = 11; + var->green.offset = 5; + var->blue.offset = 0; + var->red.length = 5; + var->green.length = 6; + var->blue.length = 5; + var->transp.offset = 0; + break; + case 24: + var->red.offset = 16; + var->green.offset = 8; + var->blue.offset = 0; + var->red.length = 8; + var->green.length = 8; + var->blue.length = 8; + var->transp.offset = 0; + var->transp.length = 0; + break; + case 32: + var->red.offset = 16; + var->green.offset = 8; + var->blue.offset = 0; + var->red.length = 8; + var->green.length = 8; + var->blue.length = 8; + var->transp.offset = 24; + var->transp.length = 8; + break; + default: + break; + } +} + /** * drm_fb_helper_check_var - implementation for &fb_ops.fb_check_var * @var: screeninfo to check @@ -1632,9 +1690,14 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, struct drm_fb_helper *fb_helper = info->par; struct drm_framebuffer *fb = fb_helper->fb; - if (var->pixclock != 0 || in_dbg_master()) + if (in_dbg_master()) return -EINVAL; + if (var->pixclock != 0) { + DRM_DEBUG("fbdev emulation doesn't support changing the pixel clock, value of pixclock is ignored\n"); + var->pixclock = 0; + } + if ((drm_format_info_block_width(fb->format, 0) > 1) || (drm_format_info_block_height(fb->format, 0) > 1)) return -EINVAL; @@ -1655,6 +1718,20 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, } /* + * Workaround for SDL 1.2, which is known to be setting all pixel format + * fields values to zero in some cases. We treat this situation as a + * kind of "use some reasonable autodetected values". + */ + if (!var->red.offset && !var->green.offset && + !var->blue.offset && !var->transp.offset && + !var->red.length && !var->green.length && + !var->blue.length && !var->transp.length && + !var->red.msb_right && !var->green.msb_right && + !var->blue.msb_right && !var->transp.msb_right) { + drm_fb_helper_fill_pixel_fmt(var, fb->format->depth); + } + + /* * drm fbdev emulation doesn't support changing the pixel format at all, * so reject all pixel format changing requests. */ @@ -2020,59 +2097,7 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe info->var.yoffset = 0; info->var.activate = FB_ACTIVATE_NOW; - switch (fb->format->depth) { - case 8: - info->var.red.offset = 0; - info->var.green.offset = 0; - info->var.blue.offset = 0; - info->var.red.length = 8; /* 8bit DAC */ - info->var.green.length = 8; - info->var.blue.length = 8; - info->var.transp.offset = 0; - info->var.transp.length = 0; - break; - case 15: - info->var.red.offset = 10; - info->var.green.offset = 5; - info->var.blue.offset = 0; - info->var.red.length = 5; - info->var.green.length = 5; - info->var.blue.length = 5; - info->var.transp.offset = 15; - info->var.transp.length = 1; - break; - case 16: - info->var.red.offset = 11; - info->var.green.offset = 5; - info->var.blue.offset = 0; - info->var.red.length = 5; - info->var.green.length = 6; - info->var.blue.length = 5; - info->var.transp.offset = 0; - break; - case 24: - info->var.red.offset = 16; - info->var.green.offset = 8; - info->var.blue.offset = 0; - info->var.red.length = 8; - info->var.green.length = 8; - info->var.blue.length = 8; - info->var.transp.offset = 0; - info->var.transp.length = 0; - break; - case 32: - info->var.red.offset = 16; - info->var.green.offset = 8; - info->var.blue.offset = 0; - info->var.red.length = 8; - info->var.green.length = 8; - info->var.blue.length = 8; - info->var.transp.offset = 24; - info->var.transp.length = 8; - break; - default: - break; - } + drm_fb_helper_fill_pixel_fmt(&info->var, fb->format->depth); info->var.xres = fb_width; info->var.yres = fb_height; diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index b1838a41ad43..83a5bbca6e7e 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -262,6 +262,18 @@ void drm_file_free(struct drm_file *file) kfree(file); } +static void drm_close_helper(struct file *filp) +{ + struct drm_file *file_priv = filp->private_data; + struct drm_device *dev = file_priv->minor->dev; + + mutex_lock(&dev->filelist_mutex); + list_del(&file_priv->lhead); + mutex_unlock(&dev->filelist_mutex); + + drm_file_free(file_priv); +} + static int drm_setup(struct drm_device * dev) { int ret; @@ -318,8 +330,10 @@ int drm_open(struct inode *inode, struct file *filp) goto err_undo; if (need_setup) { retcode = drm_setup(dev); - if (retcode) + if (retcode) { + drm_close_helper(filp); goto err_undo; + } } return 0; @@ -473,11 +487,7 @@ int drm_release(struct inode *inode, struct file *filp) DRM_DEBUG("open_count = %d\n", dev->open_count); - mutex_lock(&dev->filelist_mutex); - list_del(&file_priv->lhead); - mutex_unlock(&dev->filelist_mutex); - - drm_file_free(file_priv); + drm_close_helper(filp); if (!--dev->open_count) { drm_lastclose(dev); diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index 7e6746b2d704..687943df58e1 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -508,6 +508,13 @@ int drm_version(struct drm_device *dev, void *data, return err; } +static inline bool +drm_render_driver_and_ioctl(const struct drm_device *dev, u32 flags) +{ + return drm_core_check_feature(dev, DRIVER_RENDER) && + (flags & DRM_RENDER_ALLOW); +} + /** * drm_ioctl_permit - Check ioctl permissions against caller * @@ -522,14 +529,19 @@ int drm_version(struct drm_device *dev, void *data, */ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) { + const struct drm_device *dev = file_priv->minor->dev; + /* ROOT_ONLY is only for CAP_SYS_ADMIN */ if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN))) return -EACCES; - /* AUTH is only for authenticated or render client */ - if (unlikely((flags & DRM_AUTH) && !drm_is_render_client(file_priv) && - !file_priv->authenticated)) - return -EACCES; + /* AUTH is only for master ... */ + if (unlikely((flags & DRM_AUTH) && drm_is_primary_client(file_priv))) { + /* authenticated ones, or render capable on DRM_RENDER_ALLOW. */ + if (!file_priv->authenticated && + !drm_render_driver_and_ioctl(dev, flags)) + return -EACCES; + } /* MASTER is only for master or control clients */ if (unlikely((flags & DRM_MASTER) && @@ -570,7 +582,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = { DRM_IOCTL_DEF(DRM_IOCTL_SET_UNIQUE, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_IOCTL_BLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_IOCTL_UNBLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_AUTH|DRM_UNLOCKED|DRM_MASTER), + DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_UNLOCKED|DRM_MASTER), DRM_IOCTL_DEF(DRM_IOCTL_ADD_MAP, drm_legacy_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF(DRM_IOCTL_RM_MAP, drm_legacy_rmmap_ioctl, DRM_AUTH), diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c index b735704653cb..603b0bd9c5ce 100644 --- a/drivers/gpu/drm/drm_lease.c +++ b/drivers/gpu/drm/drm_lease.c @@ -528,7 +528,8 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, object_count = cl->object_count; - object_ids = memdup_user(u64_to_user_ptr(cl->object_ids), object_count * sizeof(__u32)); + object_ids = memdup_user(u64_to_user_ptr(cl->object_ids), + array_size(object_count, sizeof(__u32))); if (IS_ERR(object_ids)) return PTR_ERR(object_ids); diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index a28e7a91c8be..869ac6f4671e 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -751,7 +751,7 @@ int drm_mode_hsync(const struct drm_display_mode *mode) if (mode->hsync) return mode->hsync; - if (mode->htotal < 0) + if (mode->htotal <= 0) return 0; calc_val = (mode->clock * 1000) / mode->htotal; /* hsync in Hz */ diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c index a9d9df6c85ad..693748ad8b88 100644 --- a/drivers/gpu/drm/drm_pci.c +++ b/drivers/gpu/drm/drm_pci.c @@ -61,8 +61,9 @@ drm_dma_handle_t *drm_pci_alloc(struct drm_device * dev, size_t size, size_t ali return NULL; dmah->size = size; - dmah->vaddr = dma_zalloc_coherent(&dev->pdev->dev, size, &dmah->busaddr, - GFP_KERNEL | __GFP_COMP); + dmah->vaddr = dma_alloc_coherent(&dev->pdev->dev, size, + &dmah->busaddr, + GFP_KERNEL | __GFP_COMP); if (dmah->vaddr == NULL) { kfree(dmah); diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 041a77e400d4..21df44b78df3 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -2,7 +2,6 @@ config DRM_ETNAVIV tristate "ETNAVIV (DRM support for Vivante GPU IP cores)" depends on DRM - depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST) depends on MMU select SHMEM select SYNC_FILE diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h index acb68c698363..4d5d1a77eb2a 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h @@ -15,8 +15,6 @@ struct etnaviv_perfmon_request; struct etnaviv_cmdbuf { /* suballocator this cmdbuf is allocated from */ struct etnaviv_cmdbuf_suballoc *suballoc; - /* user context key, must be unique between all active users */ - struct etnaviv_file_private *ctx; /* cmdbuf properties */ int suballoc_offset; void *vaddr; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index 3fbb4855396c..33854c94cb85 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -215,7 +215,7 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) mutex_lock(&obj->lock); pages = etnaviv_gem_get_pages(obj); mutex_unlock(&obj->lock); - if (pages) { + if (!IS_ERR(pages)) { int j; iter.hdr->data[0] = bomap - bomap_start; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index 76079c2291f8..f0abb744ef95 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -95,6 +95,7 @@ struct etnaviv_gem_submit_bo { struct etnaviv_gem_submit { struct drm_sched_job sched_job; struct kref refcount; + struct etnaviv_file_private *ctx; struct etnaviv_gpu *gpu; struct dma_fence *out_fence, *in_fence; int out_fence_id; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index 0566171f8df2..f21529e635e3 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -15,7 +15,7 @@ struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj) int npages = obj->size >> PAGE_SHIFT; if (WARN_ON(!etnaviv_obj->pages)) /* should have already pinned! */ - return NULL; + return ERR_PTR(-EINVAL); return drm_prime_pages_to_sg(etnaviv_obj->pages, npages); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index 30875f8f2933..b2fe3446bfbc 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -506,7 +506,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, if (ret) goto err_submit_objects; - submit->cmdbuf.ctx = file->driver_priv; + submit->ctx = file->driver_priv; submit->exec_state = args->exec_state; submit->flags = args->flags; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c index f1c88d8ad5ba..f794e04be9e6 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c @@ -320,8 +320,8 @@ etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) domain = &etnaviv_domain->base; domain->dev = gpu->dev; - domain->base = 0; - domain->size = (u64)SZ_1G * 4; + domain->base = SZ_4K; + domain->size = (u64)SZ_1G * 4 - SZ_4K; domain->ops = &etnaviv_iommuv2_ops; ret = etnaviv_iommuv2_init(etnaviv_domain); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c index 9980d81a26e3..4227a4006c34 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c @@ -113,7 +113,7 @@ static const struct etnaviv_pm_domain doms_3d[] = { .name = "PE", .profile_read = VIVS_MC_PROFILE_PE_READ, .profile_config = VIVS_MC_PROFILE_CONFIG0, - .nr_signals = 5, + .nr_signals = 4, .signal = (const struct etnaviv_pm_signal[]) { { "PIXEL_COUNT_KILLED_BY_COLOR_PIPE", @@ -435,7 +435,7 @@ int etnaviv_pm_query_sig(struct etnaviv_gpu *gpu, dom = meta->domains + signal->domain; - if (signal->iter > dom->nr_signals) + if (signal->iter >= dom->nr_signals) return -EINVAL; sig = &dom->signal[signal->iter]; @@ -461,7 +461,7 @@ int etnaviv_pm_req_validate(const struct drm_etnaviv_gem_submit_pmr *r, dom = meta->domains + r->domain; - if (r->signal > dom->nr_signals) + if (r->signal >= dom->nr_signals) return -EINVAL; return 0; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 67ae26602024..6d24fea1766b 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -153,7 +153,7 @@ int etnaviv_sched_push_job(struct drm_sched_entity *sched_entity, mutex_lock(&submit->gpu->fence_lock); ret = drm_sched_job_init(&submit->sched_job, sched_entity, - submit->cmdbuf.ctx); + submit->ctx); if (ret) goto out_unlock; diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 9c106e47e640..bc64b810e0d5 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -2799,6 +2799,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x21f0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); return 0; } diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index 5e01cc8d9b16..4862fb12778e 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -47,7 +47,7 @@ struct intel_gvt_mpt { int (*host_init)(struct device *dev, void *gvt, const void *ops); void (*host_exit)(struct device *dev); int (*attach_vgpu)(void *vgpu, unsigned long *handle); - void (*detach_vgpu)(unsigned long handle); + void (*detach_vgpu)(void *vgpu); int (*inject_msi)(unsigned long handle, u32 addr, u16 data); unsigned long (*from_virt_to_mfn)(void *p); int (*enable_page_track)(unsigned long handle, u64 gfn); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 63eef86a2a85..d5fcc447d22f 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -1147,7 +1147,7 @@ static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) { unsigned int index; u64 virtaddr; - unsigned long req_size, pgoff = 0; + unsigned long req_size, pgoff, req_start; pgprot_t pg_prot; struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); @@ -1165,7 +1165,17 @@ static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) pg_prot = vma->vm_page_prot; virtaddr = vma->vm_start; req_size = vma->vm_end - vma->vm_start; - pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; + pgoff = vma->vm_pgoff & + ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); + req_start = pgoff << PAGE_SHIFT; + + if (!intel_vgpu_in_aperture(vgpu, req_start)) + return -EINVAL; + if (req_start + req_size > + vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu)) + return -EINVAL; + + pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff; return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); } @@ -1813,9 +1823,21 @@ static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) return 0; } -static void kvmgt_detach_vgpu(unsigned long handle) +static void kvmgt_detach_vgpu(void *p_vgpu) { - /* nothing to do here */ + int i; + struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; + + if (!vgpu->vdev.region) + return; + + for (i = 0; i < vgpu->vdev.num_regions; i++) + if (vgpu->vdev.region[i].ops->release) + vgpu->vdev.region[i].ops->release(vgpu, + &vgpu->vdev.region[i]); + vgpu->vdev.num_regions = 0; + kfree(vgpu->vdev.region); + vgpu->vdev.region = NULL; } static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 5d8b8f228d8f..0f9440128123 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -99,7 +99,7 @@ static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu) if (!intel_gvt_host.mpt->detach_vgpu) return; - intel_gvt_host.mpt->detach_vgpu(vgpu->handle); + intel_gvt_host.mpt->detach_vgpu(vgpu); } #define MSI_CAP_CONTROL(offset) (offset + 2) diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index b7957eefb976..1bb8f936fdaa 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -333,6 +333,9 @@ static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); i915_gem_object_put(wa_ctx->indirect_ctx.obj); + + wa_ctx->indirect_ctx.obj = NULL; + wa_ctx->indirect_ctx.shadow_va = NULL; } static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, @@ -357,6 +360,33 @@ static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, return 0; } +static int +intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) +{ + struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; + struct i915_request *rq; + int ret = 0; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + if (workload->req) + goto out; + + rq = i915_request_alloc(engine, shadow_ctx); + if (IS_ERR(rq)) { + gvt_vgpu_err("fail to allocate gem request\n"); + ret = PTR_ERR(rq); + goto out; + } + workload->req = i915_request_get(rq); +out: + return ret; +} + /** * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and * shadow it as well, include ringbuffer,wa_ctx and ctx. @@ -373,12 +403,11 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; struct intel_context *ce; - struct i915_request *rq; int ret; lockdep_assert_held(&dev_priv->drm.struct_mutex); - if (workload->req) + if (workload->shadow) return 0; ret = set_context_ppgtt_from_shadow(workload, shadow_ctx); @@ -418,22 +447,8 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) goto err_shadow; } - rq = i915_request_alloc(engine, shadow_ctx); - if (IS_ERR(rq)) { - gvt_vgpu_err("fail to allocate gem request\n"); - ret = PTR_ERR(rq); - goto err_shadow; - } - workload->req = i915_request_get(rq); - - ret = populate_shadow_context(workload); - if (ret) - goto err_req; - + workload->shadow = true; return 0; -err_req: - rq = fetch_and_zero(&workload->req); - i915_request_put(rq); err_shadow: release_shadow_wa_ctx(&workload->wa_ctx); err_unpin: @@ -672,23 +687,31 @@ static int dispatch_workload(struct intel_vgpu_workload *workload) mutex_lock(&vgpu->vgpu_lock); mutex_lock(&dev_priv->drm.struct_mutex); + ret = intel_gvt_workload_req_alloc(workload); + if (ret) + goto err_req; + ret = intel_gvt_scan_and_shadow_workload(workload); if (ret) goto out; - ret = prepare_workload(workload); + ret = populate_shadow_context(workload); + if (ret) { + release_shadow_wa_ctx(&workload->wa_ctx); + goto out; + } + ret = prepare_workload(workload); out: - if (ret) - workload->status = ret; - if (!IS_ERR_OR_NULL(workload->req)) { gvt_dbg_sched("ring id %d submit workload to i915 %p\n", ring_id, workload->req); i915_request_add(workload->req); workload->dispatched = true; } - +err_req: + if (ret) + workload->status = ret; mutex_unlock(&dev_priv->drm.struct_mutex); mutex_unlock(&vgpu->vgpu_lock); return ret; @@ -892,11 +915,6 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) list_del_init(&workload->list); - if (!workload->status) { - release_shadow_batch_buffer(workload); - release_shadow_wa_ctx(&workload->wa_ctx); - } - if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) { /* if workload->status is not successful means HW GPU * has occurred GPU hang or something wrong with i915/GVT, @@ -1262,6 +1280,9 @@ void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) { struct intel_vgpu_submission *s = &workload->vgpu->submission; + release_shadow_batch_buffer(workload); + release_shadow_wa_ctx(&workload->wa_ctx); + if (workload->shadow_mm) intel_vgpu_mm_put(workload->shadow_mm); diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index 1e9eec6a32fe..0635b2c4bed7 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -83,6 +83,7 @@ struct intel_vgpu_workload { struct i915_request *req; /* if this workload has been dispatched to i915? */ bool dispatched; + bool shadow; /* if workload has done shadow of guest request */ int status; struct intel_vgpu_mm *shadow_mm; diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index 376ffe842e26..e8f694b57b8a 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -338,8 +338,8 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, bool *enabled, int width, int height) { struct drm_i915_private *dev_priv = to_i915(fb_helper->dev); - unsigned long conn_configured, conn_seq, mask; unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG); + unsigned long conn_configured, conn_seq; int i, j; bool *save_enabled; bool fallback = true, ret = true; @@ -357,10 +357,9 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, drm_modeset_backoff(&ctx); memcpy(save_enabled, enabled, count); - mask = GENMASK(count - 1, 0); + conn_seq = GENMASK(count - 1, 0); conn_configured = 0; retry: - conn_seq = conn_configured; for (i = 0; i < count; i++) { struct drm_fb_helper_connector *fb_conn; struct drm_connector *connector; @@ -373,7 +372,8 @@ retry: if (conn_configured & BIT(i)) continue; - if (conn_seq == 0 && !connector->has_tile) + /* First pass, only consider tiled connectors */ + if (conn_seq == GENMASK(count - 1, 0) && !connector->has_tile) continue; if (connector->status == connector_status_connected) @@ -477,8 +477,10 @@ retry: conn_configured |= BIT(i); } - if ((conn_configured & mask) != mask && conn_configured != conn_seq) + if (conn_configured != conn_seq) { /* repeat until no more are found */ + conn_seq = conn_configured; goto retry; + } /* * If the BIOS didn't enable everything it could, fall back to have the diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 30ae96c5c97c..5e00ee9270b5 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -54,7 +54,12 @@ struct opregion_header { u8 signature[16]; u32 size; - u32 opregion_ver; + struct { + u8 rsvd; + u8 revision; + u8 minor; + u8 major; + } __packed over; u8 bios_ver[32]; u8 vbios_ver[16]; u8 driver_ver[16]; @@ -118,7 +123,8 @@ struct opregion_asle { u64 fdss; u32 fdsp; u32 stat; - u64 rvda; /* Physical address of raw vbt data */ + u64 rvda; /* Physical (2.0) or relative from opregion (2.1+) + * address of raw VBT data. */ u32 rvds; /* Size of raw vbt data */ u8 rsvd[58]; } __packed; @@ -924,6 +930,11 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) opregion->header = base; opregion->lid_state = base + ACPI_CLID; + DRM_DEBUG_DRIVER("ACPI OpRegion version %u.%u.%u\n", + opregion->header->over.major, + opregion->header->over.minor, + opregion->header->over.revision); + mboxes = opregion->header->mboxes; if (mboxes & MBOX_ACPI) { DRM_DEBUG_DRIVER("Public ACPI methods supported\n"); @@ -952,11 +963,26 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) if (dmi_check_system(intel_no_opregion_vbt)) goto out; - if (opregion->header->opregion_ver >= 2 && opregion->asle && + if (opregion->header->over.major >= 2 && opregion->asle && opregion->asle->rvda && opregion->asle->rvds) { - opregion->rvda = memremap(opregion->asle->rvda, - opregion->asle->rvds, + resource_size_t rvda = opregion->asle->rvda; + + /* + * opregion 2.0: rvda is the physical VBT address. + * + * opregion 2.1+: rvda is unsigned, relative offset from + * opregion base, and should never point within opregion. + */ + if (opregion->header->over.major > 2 || + opregion->header->over.minor >= 1) { + WARN_ON(rvda < OPREGION_SIZE); + + rvda += asls; + } + + opregion->rvda = memremap(rvda, opregion->asle->rvds, MEMREMAP_WB); + vbt = opregion->rvda; vbt_size = opregion->asle->rvds; if (intel_bios_is_valid_vbt(vbt, vbt_size)) { @@ -966,6 +992,8 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) goto out; } else { DRM_DEBUG_KMS("Invalid VBT in ACPI OpRegion (RVDA)\n"); + memunmap(opregion->rvda); + opregion->rvda = NULL; } } diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig index c9e439c82241..c3c84a09e628 100644 --- a/drivers/gpu/drm/imx/Kconfig +++ b/drivers/gpu/drm/imx/Kconfig @@ -4,7 +4,7 @@ config DRM_IMX select VIDEOMODE_HELPERS select DRM_GEM_CMA_HELPER select DRM_KMS_CMA_HELPER - depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM) + depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM || COMPILE_TEST) depends on IMX_IPUV3_CORE help enable i.MX graphics support @@ -18,6 +18,7 @@ config DRM_IMX_PARALLEL_DISPLAY config DRM_IMX_TVE tristate "Support for TV and VGA displays" depends on DRM_IMX + depends on COMMON_CLK select REGMAP_MMIO help Choose this to enable the internal Television Encoder (TVe) diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c index 44da0f5d0ed9..c935cbe059a7 100644 --- a/drivers/gpu/drm/imx/imx-drm-core.c +++ b/drivers/gpu/drm/imx/imx-drm-core.c @@ -49,11 +49,7 @@ static int imx_drm_atomic_check(struct drm_device *dev, { int ret; - ret = drm_atomic_helper_check_modeset(dev, state); - if (ret) - return ret; - - ret = drm_atomic_helper_check_planes(dev, state); + ret = drm_atomic_helper_check(dev, state); if (ret) return ret; @@ -229,6 +225,7 @@ static int imx_drm_bind(struct device *dev) drm->mode_config.funcs = &imx_drm_mode_config_funcs; drm->mode_config.helper_private = &imx_drm_mode_config_helpers; drm->mode_config.allow_fb_modifiers = true; + drm->mode_config.normalize_zpos = true; drm_mode_config_init(drm); diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c index e2a129d4ae7b..383733302280 100644 --- a/drivers/gpu/drm/imx/imx-ldb.c +++ b/drivers/gpu/drm/imx/imx-ldb.c @@ -643,8 +643,10 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data) int bus_format; ret = of_property_read_u32(child, "reg", &i); - if (ret || i < 0 || i > 1) - return -EINVAL; + if (ret || i < 0 || i > 1) { + ret = -EINVAL; + goto free_child; + } if (!of_device_is_available(child)) continue; @@ -657,7 +659,6 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data) channel = &imx_ldb->channel[i]; channel->ldb = imx_ldb; channel->chno = i; - channel->child = child; /* * The output port is port@4 with an external 4-port mux or @@ -667,13 +668,13 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data) imx_ldb->lvds_mux ? 4 : 2, 0, &channel->panel, &channel->bridge); if (ret && ret != -ENODEV) - return ret; + goto free_child; /* panel ddc only if there is no bridge */ if (!channel->bridge) { ret = imx_ldb_panel_ddc(dev, channel, child); if (ret) - return ret; + goto free_child; } bus_format = of_get_bus_format(dev, child); @@ -689,18 +690,26 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data) if (bus_format < 0) { dev_err(dev, "could not determine data mapping: %d\n", bus_format); - return bus_format; + ret = bus_format; + goto free_child; } channel->bus_format = bus_format; + channel->child = child; ret = imx_ldb_register(drm, channel); - if (ret) - return ret; + if (ret) { + channel->child = NULL; + goto free_child; + } } dev_set_drvdata(dev, imx_ldb); return 0; + +free_child: + of_node_put(child); + return ret; } static void imx_ldb_unbind(struct device *dev, struct device *master, diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c index 3c62167a9251..ec3602ebbc1c 100644 --- a/drivers/gpu/drm/imx/ipuv3-crtc.c +++ b/drivers/gpu/drm/imx/ipuv3-crtc.c @@ -34,6 +34,7 @@ struct ipu_crtc { struct ipu_dc *dc; struct ipu_di *di; int irq; + struct drm_pending_vblank_event *event; }; static inline struct ipu_crtc *to_ipu_crtc(struct drm_crtc *crtc) @@ -173,8 +174,31 @@ static const struct drm_crtc_funcs ipu_crtc_funcs = { static irqreturn_t ipu_irq_handler(int irq, void *dev_id) { struct ipu_crtc *ipu_crtc = dev_id; + struct drm_crtc *crtc = &ipu_crtc->base; + unsigned long flags; + int i; + + drm_crtc_handle_vblank(crtc); + + if (ipu_crtc->event) { + for (i = 0; i < ARRAY_SIZE(ipu_crtc->plane); i++) { + struct ipu_plane *plane = ipu_crtc->plane[i]; - drm_crtc_handle_vblank(&ipu_crtc->base); + if (!plane) + continue; + + if (ipu_plane_atomic_update_pending(&plane->base)) + break; + } + + if (i == ARRAY_SIZE(ipu_crtc->plane)) { + spin_lock_irqsave(&crtc->dev->event_lock, flags); + drm_crtc_send_vblank_event(crtc, ipu_crtc->event); + ipu_crtc->event = NULL; + drm_crtc_vblank_put(crtc); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + } + } return IRQ_HANDLED; } @@ -223,8 +247,10 @@ static void ipu_crtc_atomic_flush(struct drm_crtc *crtc, { spin_lock_irq(&crtc->dev->event_lock); if (crtc->state->event) { + struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc); + WARN_ON(drm_crtc_vblank_get(crtc)); - drm_crtc_arm_vblank_event(crtc, crtc->state->event); + ipu_crtc->event = crtc->state->event; crtc->state->event = NULL; } spin_unlock_irq(&crtc->dev->event_lock); diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c index c390924de93d..d7a727a6e3d7 100644 --- a/drivers/gpu/drm/imx/ipuv3-plane.c +++ b/drivers/gpu/drm/imx/ipuv3-plane.c @@ -273,6 +273,7 @@ static void ipu_plane_destroy(struct drm_plane *plane) static void ipu_plane_state_reset(struct drm_plane *plane) { + unsigned int zpos = (plane->type == DRM_PLANE_TYPE_PRIMARY) ? 0 : 1; struct ipu_plane_state *ipu_state; if (plane->state) { @@ -284,8 +285,11 @@ static void ipu_plane_state_reset(struct drm_plane *plane) ipu_state = kzalloc(sizeof(*ipu_state), GFP_KERNEL); - if (ipu_state) + if (ipu_state) { __drm_atomic_helper_plane_reset(plane, &ipu_state->base); + ipu_state->base.zpos = zpos; + ipu_state->base.normalized_zpos = zpos; + } } static struct drm_plane_state * @@ -370,9 +374,9 @@ static int ipu_plane_atomic_check(struct drm_plane *plane, if (ret) return ret; - /* CRTC should be enabled */ + /* nothing to check when disabling or disabled */ if (!crtc_state->enable) - return -EINVAL; + return 0; switch (plane->type) { case DRM_PLANE_TYPE_PRIMARY: @@ -560,6 +564,25 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, if (ipu_plane->dp_flow == IPU_DP_FLOW_SYNC_FG) ipu_dp_set_window_pos(ipu_plane->dp, dst->x1, dst->y1); + switch (ipu_plane->dp_flow) { + case IPU_DP_FLOW_SYNC_BG: + if (state->normalized_zpos == 1) { + ipu_dp_set_global_alpha(ipu_plane->dp, + !fb->format->has_alpha, 0xff, + true); + } else { + ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true); + } + break; + case IPU_DP_FLOW_SYNC_FG: + if (state->normalized_zpos == 1) { + ipu_dp_set_global_alpha(ipu_plane->dp, + !fb->format->has_alpha, 0xff, + false); + } + break; + } + eba = drm_plane_state_to_eba(state, 0); /* @@ -582,6 +605,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, active = ipu_idmac_get_current_buffer(ipu_plane->ipu_ch); ipu_cpmem_set_buffer(ipu_plane->ipu_ch, !active, eba); ipu_idmac_select_buffer(ipu_plane->ipu_ch, !active); + ipu_plane->next_buf = !active; if (ipu_plane_separate_alpha(ipu_plane)) { active = ipu_idmac_get_current_buffer(ipu_plane->alpha_ch); ipu_cpmem_set_buffer(ipu_plane->alpha_ch, !active, @@ -595,34 +619,11 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, switch (ipu_plane->dp_flow) { case IPU_DP_FLOW_SYNC_BG: ipu_dp_setup_channel(ipu_plane->dp, ics, IPUV3_COLORSPACE_RGB); - ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true); break; case IPU_DP_FLOW_SYNC_FG: ipu_dp_setup_channel(ipu_plane->dp, ics, IPUV3_COLORSPACE_UNKNOWN); - /* Enable local alpha on partial plane */ - switch (fb->format->format) { - case DRM_FORMAT_ARGB1555: - case DRM_FORMAT_ABGR1555: - case DRM_FORMAT_RGBA5551: - case DRM_FORMAT_BGRA5551: - case DRM_FORMAT_ARGB4444: - case DRM_FORMAT_ARGB8888: - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_RGBA8888: - case DRM_FORMAT_BGRA8888: - case DRM_FORMAT_RGB565_A8: - case DRM_FORMAT_BGR565_A8: - case DRM_FORMAT_RGB888_A8: - case DRM_FORMAT_BGR888_A8: - case DRM_FORMAT_RGBX8888_A8: - case DRM_FORMAT_BGRX8888_A8: - ipu_dp_set_global_alpha(ipu_plane->dp, false, 0, false); - break; - default: - ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true); - break; - } + break; } ipu_dmfc_config_wait4eot(ipu_plane->dmfc, drm_rect_width(dst)); @@ -709,6 +710,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 1, eba); ipu_idmac_lock_enable(ipu_plane->ipu_ch, num_bursts); ipu_plane_enable(ipu_plane); + ipu_plane->next_buf = -1; } static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = { @@ -718,6 +720,24 @@ static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = { .atomic_update = ipu_plane_atomic_update, }; +bool ipu_plane_atomic_update_pending(struct drm_plane *plane) +{ + struct ipu_plane *ipu_plane = to_ipu_plane(plane); + struct drm_plane_state *state = plane->state; + struct ipu_plane_state *ipu_state = to_ipu_plane_state(state); + + /* disabled crtcs must not block the update */ + if (!state->crtc) + return false; + + if (ipu_state->use_pre) + return ipu_prg_channel_configure_pending(ipu_plane->ipu_ch); + else if (ipu_plane->next_buf >= 0) + return ipu_idmac_get_current_buffer(ipu_plane->ipu_ch) != + ipu_plane->next_buf; + + return false; +} int ipu_planes_assign_pre(struct drm_device *dev, struct drm_atomic_state *state) { @@ -806,6 +826,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu, { struct ipu_plane *ipu_plane; const uint64_t *modifiers = ipu_format_modifiers; + unsigned int zpos = (type == DRM_PLANE_TYPE_PRIMARY) ? 0 : 1; int ret; DRM_DEBUG_KMS("channel %d, dp flow %d, possible_crtcs=0x%x\n", @@ -836,5 +857,10 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu, drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs); + if (dp == IPU_DP_FLOW_SYNC_BG || dp == IPU_DP_FLOW_SYNC_FG) + drm_plane_create_zpos_property(&ipu_plane->base, zpos, 0, 1); + else + drm_plane_create_zpos_immutable_property(&ipu_plane->base, 0); + return ipu_plane; } diff --git a/drivers/gpu/drm/imx/ipuv3-plane.h b/drivers/gpu/drm/imx/ipuv3-plane.h index e563ea17a827..15e85e15d35c 100644 --- a/drivers/gpu/drm/imx/ipuv3-plane.h +++ b/drivers/gpu/drm/imx/ipuv3-plane.h @@ -27,6 +27,7 @@ struct ipu_plane { int dp_flow; bool disabling; + int next_buf; }; struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu, @@ -48,5 +49,6 @@ int ipu_plane_irq(struct ipu_plane *plane); void ipu_plane_disable(struct ipu_plane *ipu_plane, bool disable_dp_channel); void ipu_plane_disable_deferred(struct drm_plane *plane); +bool ipu_plane_atomic_update_pending(struct drm_plane *plane); #endif diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c index ec573c04206b..43e29984f8b1 100644 --- a/drivers/gpu/drm/meson/meson_crtc.c +++ b/drivers/gpu/drm/meson/meson_crtc.c @@ -46,7 +46,6 @@ struct meson_crtc { struct drm_crtc base; struct drm_pending_vblank_event *event; struct meson_drm *priv; - bool enabled; }; #define to_meson_crtc(x) container_of(x, struct meson_crtc, base) @@ -82,7 +81,8 @@ static const struct drm_crtc_funcs meson_crtc_funcs = { }; -static void meson_crtc_enable(struct drm_crtc *crtc) +static void meson_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_crtc_state *old_state) { struct meson_crtc *meson_crtc = to_meson_crtc(crtc); struct drm_crtc_state *crtc_state = crtc->state; @@ -108,20 +108,6 @@ static void meson_crtc_enable(struct drm_crtc *crtc) drm_crtc_vblank_on(crtc); - meson_crtc->enabled = true; -} - -static void meson_crtc_atomic_enable(struct drm_crtc *crtc, - struct drm_crtc_state *old_state) -{ - struct meson_crtc *meson_crtc = to_meson_crtc(crtc); - struct meson_drm *priv = meson_crtc->priv; - - DRM_DEBUG_DRIVER("\n"); - - if (!meson_crtc->enabled) - meson_crtc_enable(crtc); - priv->viu.osd1_enabled = true; } @@ -153,8 +139,6 @@ static void meson_crtc_atomic_disable(struct drm_crtc *crtc, crtc->state->event = NULL; } - - meson_crtc->enabled = false; } static void meson_crtc_atomic_begin(struct drm_crtc *crtc, @@ -163,9 +147,6 @@ static void meson_crtc_atomic_begin(struct drm_crtc *crtc, struct meson_crtc *meson_crtc = to_meson_crtc(crtc); unsigned long flags; - if (crtc->state->enable && !meson_crtc->enabled) - meson_crtc_enable(crtc); - if (crtc->state->event) { WARN_ON(drm_crtc_vblank_get(crtc) != 0); diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c index 27f38bdc8677..2281ed3eb774 100644 --- a/drivers/gpu/drm/meson/meson_drv.c +++ b/drivers/gpu/drm/meson/meson_drv.c @@ -75,6 +75,10 @@ static const struct drm_mode_config_funcs meson_mode_config_funcs = { .fb_create = drm_gem_fb_create, }; +static const struct drm_mode_config_helper_funcs meson_mode_config_helpers = { + .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, +}; + static irqreturn_t meson_irq(int irq, void *arg) { struct drm_device *dev = arg; @@ -286,6 +290,7 @@ static int meson_drv_bind_master(struct device *dev, bool has_components) drm->mode_config.max_width = 3840; drm->mode_config.max_height = 2160; drm->mode_config.funcs = &meson_mode_config_funcs; + drm->mode_config.helper_private = &meson_mode_config_helpers; /* Hardware Initialization */ @@ -408,8 +413,10 @@ static int meson_probe_remote(struct platform_device *pdev, remote_node = of_graph_get_remote_port_parent(ep); if (!remote_node || remote_node == parent || /* Ignore parent endpoint */ - !of_device_is_available(remote_node)) + !of_device_is_available(remote_node)) { + of_node_put(remote_node); continue; + } count += meson_probe_remote(pdev, match, remote, remote_node); @@ -428,10 +435,13 @@ static int meson_drv_probe(struct platform_device *pdev) for_each_endpoint_of_node(np, ep) { remote = of_graph_get_remote_port_parent(ep); - if (!remote || !of_device_is_available(remote)) + if (!remote || !of_device_is_available(remote)) { + of_node_put(remote); continue; + } count += meson_probe_remote(pdev, &match, np, remote); + of_node_put(remote); } if (count && !match) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 5beb83d1cf87..ce1b3cc4bf6d 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -944,7 +944,7 @@ static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq) np = dev_pm_opp_get_of_node(opp); if (np) { - of_property_read_u32(np, "qcom,level", &val); + of_property_read_u32(np, "opp-level", &val); of_node_put(np); } diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 2e4372ef17a3..2cfee1a4fe0b 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -765,7 +765,6 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, adreno_gpu->rev = config->rev; adreno_gpu_config.ioname = "kgsl_3d0_reg_memory"; - adreno_gpu_config.irqname = "kgsl_3d0_irq"; adreno_gpu_config.va_start = SZ_16M; adreno_gpu_config.va_end = 0xffffffff; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index d130825e2c75..b776fca571f3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -465,8 +465,6 @@ static void _dpu_crtc_setup_mixer_for_encoder( return; } - mixer->encoder = enc; - cstate->num_mixers++; DPU_DEBUG("setup mixer %d: lm %d\n", i, mixer->hw_lm->idx - LM_0); @@ -718,11 +716,8 @@ void dpu_crtc_commit_kickoff(struct drm_crtc *crtc, bool async) * may delay and flush at an irq event (e.g. ppdone) */ drm_for_each_encoder_mask(encoder, crtc->dev, - crtc->state->encoder_mask) { - struct dpu_encoder_kickoff_params params = { 0 }; - dpu_encoder_prepare_for_kickoff(encoder, ¶ms, async); - } - + crtc->state->encoder_mask) + dpu_encoder_prepare_for_kickoff(encoder, async); if (!async) { /* wait for frame_event_done completion */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h index dbfb38a1986c..e59d62be4980 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h @@ -84,14 +84,12 @@ struct dpu_crtc_smmu_state_data { * struct dpu_crtc_mixer: stores the map for each virtual pipeline in the CRTC * @hw_lm: LM HW Driver context * @lm_ctl: CTL Path HW driver context - * @encoder: Encoder attached to this lm & ctl * @mixer_op_mode: mixer blending operation mode * @flush_mask: mixer flush mask for ctl, mixer and pipe */ struct dpu_crtc_mixer { struct dpu_hw_mixer *hw_lm; struct dpu_hw_ctl *lm_ctl; - struct drm_encoder *encoder; u32 mixer_op_mode; u32 flush_mask; }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 36af231bb73f..5aa3307f3f0c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -205,7 +205,7 @@ struct dpu_encoder_virt { bool idle_pc_supported; struct mutex rc_lock; enum dpu_enc_rc_states rc_state; - struct kthread_delayed_work delayed_off_work; + struct delayed_work delayed_off_work; struct kthread_work vsync_event_work; struct msm_display_topology topology; bool mode_set_complete; @@ -742,7 +742,6 @@ static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, { struct dpu_encoder_virt *dpu_enc; struct msm_drm_private *priv; - struct msm_drm_thread *disp_thread; bool is_vid_mode = false; if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private || @@ -755,12 +754,6 @@ static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, is_vid_mode = dpu_enc->disp_info.capabilities & MSM_DISPLAY_CAP_VID_MODE; - if (drm_enc->crtc->index >= ARRAY_SIZE(priv->disp_thread)) { - DPU_ERROR("invalid crtc index\n"); - return -EINVAL; - } - disp_thread = &priv->disp_thread[drm_enc->crtc->index]; - /* * when idle_pc is not supported, process only KICKOFF, STOP and MODESET * events and return early for other events (ie wb display). @@ -777,8 +770,7 @@ static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, switch (sw_event) { case DPU_ENC_RC_EVENT_KICKOFF: /* cancel delayed off work, if any */ - if (kthread_cancel_delayed_work_sync( - &dpu_enc->delayed_off_work)) + if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work)) DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n", sw_event); @@ -837,10 +829,8 @@ static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, return 0; } - kthread_queue_delayed_work( - &disp_thread->worker, - &dpu_enc->delayed_off_work, - msecs_to_jiffies(dpu_enc->idle_timeout)); + queue_delayed_work(priv->wq, &dpu_enc->delayed_off_work, + msecs_to_jiffies(dpu_enc->idle_timeout)); trace_dpu_enc_rc(DRMID(drm_enc), sw_event, dpu_enc->idle_pc_supported, dpu_enc->rc_state, @@ -849,8 +839,7 @@ static int dpu_encoder_resource_control(struct drm_encoder *drm_enc, case DPU_ENC_RC_EVENT_PRE_STOP: /* cancel delayed off work, if any */ - if (kthread_cancel_delayed_work_sync( - &dpu_enc->delayed_off_work)) + if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work)) DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n", sw_event); @@ -1368,7 +1357,7 @@ static void dpu_encoder_frame_done_callback( } } -static void dpu_encoder_off_work(struct kthread_work *work) +static void dpu_encoder_off_work(struct work_struct *work) { struct dpu_encoder_virt *dpu_enc = container_of(work, struct dpu_encoder_virt, delayed_off_work.work); @@ -1756,15 +1745,14 @@ static void dpu_encoder_vsync_event_work_handler(struct kthread_work *work) nsecs_to_jiffies(ktime_to_ns(wakeup_time))); } -void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc, - struct dpu_encoder_kickoff_params *params, bool async) +void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc, bool async) { struct dpu_encoder_virt *dpu_enc; struct dpu_encoder_phys *phys; bool needs_hw_reset = false; unsigned int i; - if (!drm_enc || !params) { + if (!drm_enc) { DPU_ERROR("invalid args\n"); return; } @@ -1778,7 +1766,7 @@ void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc, phys = dpu_enc->phys_encs[i]; if (phys) { if (phys->ops.prepare_for_kickoff) - phys->ops.prepare_for_kickoff(phys, params); + phys->ops.prepare_for_kickoff(phys); if (phys->enable_state == DPU_ENC_ERR_NEEDS_HW_RESET) needs_hw_reset = true; } @@ -2193,7 +2181,7 @@ int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc, mutex_init(&dpu_enc->rc_lock); - kthread_init_delayed_work(&dpu_enc->delayed_off_work, + INIT_DELAYED_WORK(&dpu_enc->delayed_off_work, dpu_encoder_off_work); dpu_enc->idle_timeout = IDLE_TIMEOUT; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h index 3f5dafe00580..d77f74fb26d4 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h @@ -38,15 +38,6 @@ struct dpu_encoder_hw_resources { }; /** - * dpu_encoder_kickoff_params - info encoder requires at kickoff - * @affected_displays: bitmask, bit set means the ROI of the commit lies within - * the bounds of the physical display at the bit index - */ -struct dpu_encoder_kickoff_params { - unsigned long affected_displays; -}; - -/** * dpu_encoder_get_hw_resources - Populate table of required hardware resources * @encoder: encoder pointer * @hw_res: resource table to populate with encoder required resources @@ -88,11 +79,9 @@ void dpu_encoder_register_frame_event_callback(struct drm_encoder *encoder, * Immediately: if no previous commit is outstanding. * Delayed: Block until next trigger can be issued. * @encoder: encoder pointer - * @params: kickoff time parameters * @async: true if this is an asynchronous commit */ -void dpu_encoder_prepare_for_kickoff(struct drm_encoder *encoder, - struct dpu_encoder_kickoff_params *params, bool async); +void dpu_encoder_prepare_for_kickoff(struct drm_encoder *encoder, bool async); /** * dpu_encoder_trigger_kickoff_pending - Clear the flush bits from previous diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h index 44e6f8b68e70..db94f3d3bea3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h @@ -144,8 +144,7 @@ struct dpu_encoder_phys_ops { int (*wait_for_commit_done)(struct dpu_encoder_phys *phys_enc); int (*wait_for_tx_complete)(struct dpu_encoder_phys *phys_enc); int (*wait_for_vblank)(struct dpu_encoder_phys *phys_enc); - void (*prepare_for_kickoff)(struct dpu_encoder_phys *phys_enc, - struct dpu_encoder_kickoff_params *params); + void (*prepare_for_kickoff)(struct dpu_encoder_phys *phys_enc); void (*handle_post_kickoff)(struct dpu_encoder_phys *phys_enc); void (*trigger_start)(struct dpu_encoder_phys *phys_enc); bool (*needs_single_flush)(struct dpu_encoder_phys *phys_enc); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 99ab5ca9bed3..a399e1edd313 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -594,8 +594,7 @@ static void dpu_encoder_phys_cmd_get_hw_resources( } static void dpu_encoder_phys_cmd_prepare_for_kickoff( - struct dpu_encoder_phys *phys_enc, - struct dpu_encoder_kickoff_params *params) + struct dpu_encoder_phys *phys_enc) { struct dpu_encoder_phys_cmd *cmd_enc = to_dpu_encoder_phys_cmd(phys_enc); @@ -693,7 +692,7 @@ static int dpu_encoder_phys_cmd_wait_for_commit_done( /* required for both controllers */ if (!rc && cmd_enc->serialize_wait4pp) - dpu_encoder_phys_cmd_prepare_for_kickoff(phys_enc, NULL); + dpu_encoder_phys_cmd_prepare_for_kickoff(phys_enc); return rc; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c index acdab5b0db18..3c4eb470a82c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c @@ -587,14 +587,13 @@ static int dpu_encoder_phys_vid_wait_for_vblank( } static void dpu_encoder_phys_vid_prepare_for_kickoff( - struct dpu_encoder_phys *phys_enc, - struct dpu_encoder_kickoff_params *params) + struct dpu_encoder_phys *phys_enc) { struct dpu_encoder_phys_vid *vid_enc; struct dpu_hw_ctl *ctl; int rc; - if (!phys_enc || !params) { + if (!phys_enc) { DPU_ERROR("invalid encoder/parameters\n"); return; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c index 0874f0a53bf9..f59fe1a9f4b9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c @@ -263,13 +263,13 @@ static const struct dpu_format dpu_format_map[] = { INTERLEAVED_RGB_FMT(RGB565, 0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT, - C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3, + C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3, false, 2, 0, DPU_FETCH_LINEAR, 1), INTERLEAVED_RGB_FMT(BGR565, 0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT, - C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3, + C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3, false, 2, 0, DPU_FETCH_LINEAR, 1), @@ -1137,36 +1137,3 @@ const struct msm_format *dpu_get_msm_format( return &fmt->base; return NULL; } - -uint32_t dpu_populate_formats( - const struct dpu_format_extended *format_list, - uint32_t *pixel_formats, - uint64_t *pixel_modifiers, - uint32_t pixel_formats_max) -{ - uint32_t i, fourcc_format; - - if (!format_list || !pixel_formats) - return 0; - - for (i = 0, fourcc_format = 0; - format_list->fourcc_format && i < pixel_formats_max; - ++format_list) { - /* verify if listed format is in dpu_format_map? */ - - /* optionally return modified formats */ - if (pixel_modifiers) { - /* assume same modifier for all fb planes */ - pixel_formats[i] = format_list->fourcc_format; - pixel_modifiers[i++] = format_list->modifier; - } else { - /* assume base formats grouped together */ - if (fourcc_format != format_list->fourcc_format) { - fourcc_format = format_list->fourcc_format; - pixel_formats[i++] = fourcc_format; - } - } - } - - return i; -} diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h index a54451d8d011..c02c81e7a667 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h @@ -41,20 +41,6 @@ const struct msm_format *dpu_get_msm_format( const uint64_t modifiers); /** - * dpu_populate_formats - populate the given array with fourcc codes supported - * @format_list: pointer to list of possible formats - * @pixel_formats: array to populate with fourcc codes - * @pixel_modifiers: array to populate with drm modifiers, can be NULL - * @pixel_formats_max: length of pixel formats array - * Return: number of elements populated - */ -uint32_t dpu_populate_formats( - const struct dpu_format_extended *format_list, - uint32_t *pixel_formats, - uint64_t *pixel_modifiers, - uint32_t pixel_formats_max); - -/** * dpu_format_check_modified_format - validate format and buffers for * dpu non-standard, i.e. modified format * @kms: kms driver diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 512ac0834d2b..df6852cc98b9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -151,7 +151,9 @@ static const struct dpu_sspp_blks_common sdm845_sspp_common = { .id = DPU_SSPP_CSC_10BIT, \ .base = 0x1a00, .len = 0x100,}, \ .format_list = plane_formats_yuv, \ + .num_formats = ARRAY_SIZE(plane_formats_yuv), \ .virt_format_list = plane_formats, \ + .virt_num_formats = ARRAY_SIZE(plane_formats), \ } #define _DMA_SBLK(num, sdma_pri) \ @@ -163,7 +165,9 @@ static const struct dpu_sspp_blks_common sdm845_sspp_common = { .src_blk = {.name = STRCAT("sspp_src_", num), \ .id = DPU_SSPP_SRC, .base = 0x00, .len = 0x150,}, \ .format_list = plane_formats, \ + .num_formats = ARRAY_SIZE(plane_formats), \ .virt_format_list = plane_formats, \ + .virt_num_formats = ARRAY_SIZE(plane_formats), \ } static const struct dpu_sspp_sub_blks sdm845_vig_sblk_0 = _VIG_SBLK("0", 5); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h index 144358a3d0fb..a55653b2e466 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h @@ -252,17 +252,6 @@ struct dpu_pp_blk { }; /** - * struct dpu_format_extended - define dpu specific pixel format+modifier - * @fourcc_format: Base FOURCC pixel format code - * @modifier: 64-bit drm format modifier, same modifier must be applied to all - * framebuffer planes - */ -struct dpu_format_extended { - uint32_t fourcc_format; - uint64_t modifier; -}; - -/** * enum dpu_qos_lut_usage - define QoS LUT use cases */ enum dpu_qos_lut_usage { @@ -348,7 +337,9 @@ struct dpu_sspp_blks_common { * @pcc_blk: * @igc_blk: * @format_list: Pointer to list of supported formats + * @num_formats: Number of supported formats * @virt_format_list: Pointer to list of supported formats for virtual planes + * @virt_num_formats: Number of supported formats for virtual planes */ struct dpu_sspp_sub_blks { const struct dpu_sspp_blks_common *common; @@ -366,8 +357,10 @@ struct dpu_sspp_sub_blks { struct dpu_pp_blk pcc_blk; struct dpu_pp_blk igc_blk; - const struct dpu_format_extended *format_list; - const struct dpu_format_extended *virt_format_list; + const u32 *format_list; + u32 num_formats; + const u32 *virt_format_list; + u32 virt_num_formats; }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h index 3c9f028628ef..d09730985951 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h @@ -12,157 +12,81 @@ #include "dpu_hw_mdss.h" -static const struct dpu_format_extended plane_formats[] = { - {DRM_FORMAT_ARGB8888, 0}, - {DRM_FORMAT_ABGR8888, 0}, - {DRM_FORMAT_RGBA8888, 0}, - {DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_BGRA8888, 0}, - {DRM_FORMAT_XRGB8888, 0}, - {DRM_FORMAT_RGBX8888, 0}, - {DRM_FORMAT_BGRX8888, 0}, - {DRM_FORMAT_XBGR8888, 0}, - {DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_RGB888, 0}, - {DRM_FORMAT_BGR888, 0}, - {DRM_FORMAT_RGB565, 0}, - {DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_BGR565, 0}, - {DRM_FORMAT_ARGB1555, 0}, - {DRM_FORMAT_ABGR1555, 0}, - {DRM_FORMAT_RGBA5551, 0}, - {DRM_FORMAT_BGRA5551, 0}, - {DRM_FORMAT_XRGB1555, 0}, - {DRM_FORMAT_XBGR1555, 0}, - {DRM_FORMAT_RGBX5551, 0}, - {DRM_FORMAT_BGRX5551, 0}, - {DRM_FORMAT_ARGB4444, 0}, - {DRM_FORMAT_ABGR4444, 0}, - {DRM_FORMAT_RGBA4444, 0}, - {DRM_FORMAT_BGRA4444, 0}, - {DRM_FORMAT_XRGB4444, 0}, - {DRM_FORMAT_XBGR4444, 0}, - {DRM_FORMAT_RGBX4444, 0}, - {DRM_FORMAT_BGRX4444, 0}, - {0, 0}, +static const uint32_t qcom_compressed_supported_formats[] = { + DRM_FORMAT_ABGR8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_BGR565, }; -static const struct dpu_format_extended plane_formats_yuv[] = { - {DRM_FORMAT_ARGB8888, 0}, - {DRM_FORMAT_ABGR8888, 0}, - {DRM_FORMAT_RGBA8888, 0}, - {DRM_FORMAT_BGRX8888, 0}, - {DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_BGRA8888, 0}, - {DRM_FORMAT_XRGB8888, 0}, - {DRM_FORMAT_XBGR8888, 0}, - {DRM_FORMAT_RGBX8888, 0}, - {DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_RGB888, 0}, - {DRM_FORMAT_BGR888, 0}, - {DRM_FORMAT_RGB565, 0}, - {DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_BGR565, 0}, - {DRM_FORMAT_ARGB1555, 0}, - {DRM_FORMAT_ABGR1555, 0}, - {DRM_FORMAT_RGBA5551, 0}, - {DRM_FORMAT_BGRA5551, 0}, - {DRM_FORMAT_XRGB1555, 0}, - {DRM_FORMAT_XBGR1555, 0}, - {DRM_FORMAT_RGBX5551, 0}, - {DRM_FORMAT_BGRX5551, 0}, - {DRM_FORMAT_ARGB4444, 0}, - {DRM_FORMAT_ABGR4444, 0}, - {DRM_FORMAT_RGBA4444, 0}, - {DRM_FORMAT_BGRA4444, 0}, - {DRM_FORMAT_XRGB4444, 0}, - {DRM_FORMAT_XBGR4444, 0}, - {DRM_FORMAT_RGBX4444, 0}, - {DRM_FORMAT_BGRX4444, 0}, - - {DRM_FORMAT_NV12, 0}, - {DRM_FORMAT_NV12, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_NV21, 0}, - {DRM_FORMAT_NV16, 0}, - {DRM_FORMAT_NV61, 0}, - {DRM_FORMAT_VYUY, 0}, - {DRM_FORMAT_UYVY, 0}, - {DRM_FORMAT_YUYV, 0}, - {DRM_FORMAT_YVYU, 0}, - {DRM_FORMAT_YUV420, 0}, - {DRM_FORMAT_YVU420, 0}, - {0, 0}, -}; - -static const struct dpu_format_extended cursor_formats[] = { - {DRM_FORMAT_ARGB8888, 0}, - {DRM_FORMAT_ABGR8888, 0}, - {DRM_FORMAT_RGBA8888, 0}, - {DRM_FORMAT_BGRA8888, 0}, - {DRM_FORMAT_XRGB8888, 0}, - {DRM_FORMAT_ARGB1555, 0}, - {DRM_FORMAT_ABGR1555, 0}, - {DRM_FORMAT_RGBA5551, 0}, - {DRM_FORMAT_BGRA5551, 0}, - {DRM_FORMAT_ARGB4444, 0}, - {DRM_FORMAT_ABGR4444, 0}, - {DRM_FORMAT_RGBA4444, 0}, - {DRM_FORMAT_BGRA4444, 0}, - {0, 0}, +static const uint32_t plane_formats[] = { + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_RGBA8888, + DRM_FORMAT_BGRA8888, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_RGBX8888, + DRM_FORMAT_BGRX8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_RGB888, + DRM_FORMAT_BGR888, + DRM_FORMAT_RGB565, + DRM_FORMAT_BGR565, + DRM_FORMAT_ARGB1555, + DRM_FORMAT_ABGR1555, + DRM_FORMAT_RGBA5551, + DRM_FORMAT_BGRA5551, + DRM_FORMAT_XRGB1555, + DRM_FORMAT_XBGR1555, + DRM_FORMAT_RGBX5551, + DRM_FORMAT_BGRX5551, + DRM_FORMAT_ARGB4444, + DRM_FORMAT_ABGR4444, + DRM_FORMAT_RGBA4444, + DRM_FORMAT_BGRA4444, + DRM_FORMAT_XRGB4444, + DRM_FORMAT_XBGR4444, + DRM_FORMAT_RGBX4444, + DRM_FORMAT_BGRX4444, }; -static const struct dpu_format_extended wb2_formats[] = { - {DRM_FORMAT_RGB565, 0}, - {DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_RGB888, 0}, - {DRM_FORMAT_ARGB8888, 0}, - {DRM_FORMAT_RGBA8888, 0}, - {DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_XRGB8888, 0}, - {DRM_FORMAT_RGBX8888, 0}, - {DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_ARGB1555, 0}, - {DRM_FORMAT_RGBA5551, 0}, - {DRM_FORMAT_XRGB1555, 0}, - {DRM_FORMAT_RGBX5551, 0}, - {DRM_FORMAT_ARGB4444, 0}, - {DRM_FORMAT_RGBA4444, 0}, - {DRM_FORMAT_RGBX4444, 0}, - {DRM_FORMAT_XRGB4444, 0}, - - {DRM_FORMAT_BGR565, 0}, - {DRM_FORMAT_BGR888, 0}, - {DRM_FORMAT_ABGR8888, 0}, - {DRM_FORMAT_BGRA8888, 0}, - {DRM_FORMAT_BGRX8888, 0}, - {DRM_FORMAT_XBGR8888, 0}, - {DRM_FORMAT_ABGR1555, 0}, - {DRM_FORMAT_BGRA5551, 0}, - {DRM_FORMAT_XBGR1555, 0}, - {DRM_FORMAT_BGRX5551, 0}, - {DRM_FORMAT_ABGR4444, 0}, - {DRM_FORMAT_BGRA4444, 0}, - {DRM_FORMAT_BGRX4444, 0}, - {DRM_FORMAT_XBGR4444, 0}, - - {DRM_FORMAT_YUV420, 0}, - {DRM_FORMAT_NV12, 0}, - {DRM_FORMAT_NV12, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_NV16, 0}, - {DRM_FORMAT_YUYV, 0}, - - {0, 0}, -}; +static const uint32_t plane_formats_yuv[] = { + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_RGBA8888, + DRM_FORMAT_BGRX8888, + DRM_FORMAT_BGRA8888, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_RGBX8888, + DRM_FORMAT_RGB888, + DRM_FORMAT_BGR888, + DRM_FORMAT_RGB565, + DRM_FORMAT_BGR565, + DRM_FORMAT_ARGB1555, + DRM_FORMAT_ABGR1555, + DRM_FORMAT_RGBA5551, + DRM_FORMAT_BGRA5551, + DRM_FORMAT_XRGB1555, + DRM_FORMAT_XBGR1555, + DRM_FORMAT_RGBX5551, + DRM_FORMAT_BGRX5551, + DRM_FORMAT_ARGB4444, + DRM_FORMAT_ABGR4444, + DRM_FORMAT_RGBA4444, + DRM_FORMAT_BGRA4444, + DRM_FORMAT_XRGB4444, + DRM_FORMAT_XBGR4444, + DRM_FORMAT_RGBX4444, + DRM_FORMAT_BGRX4444, -static const struct dpu_format_extended rgb_10bit_formats[] = { - {DRM_FORMAT_BGRA1010102, 0}, - {DRM_FORMAT_BGRX1010102, 0}, - {DRM_FORMAT_RGBA1010102, 0}, - {DRM_FORMAT_RGBX1010102, 0}, - {DRM_FORMAT_ABGR2101010, 0}, - {DRM_FORMAT_ABGR2101010, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_XBGR2101010, 0}, - {DRM_FORMAT_XBGR2101010, DRM_FORMAT_MOD_QCOM_COMPRESSED}, - {DRM_FORMAT_ARGB2101010, 0}, - {DRM_FORMAT_XRGB2101010, 0}, + DRM_FORMAT_NV12, + DRM_FORMAT_NV21, + DRM_FORMAT_NV16, + DRM_FORMAT_NV61, + DRM_FORMAT_VYUY, + DRM_FORMAT_UYVY, + DRM_FORMAT_YUYV, + DRM_FORMAT_YVYU, + DRM_FORMAT_YUV420, + DRM_FORMAT_YVU420, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c index c0b7f0049365..8a28a03ac6a9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c @@ -170,10 +170,6 @@ /** * AD4 interrupt status bit definitions */ -#define DPU_INTR_BRIGHTPR_UPDATED BIT(4) -#define DPU_INTR_DARKENH_UPDATED BIT(3) -#define DPU_INTR_STREN_OUTROI_UPDATED BIT(2) -#define DPU_INTR_STREN_INROI_UPDATED BIT(1) #define DPU_INTR_BACKLIGHT_UPDATED BIT(0) /** * struct dpu_intr_reg - array of DPU register sets @@ -782,18 +778,6 @@ static int dpu_hw_intr_irqidx_lookup(enum dpu_intr_type intr_type, return -EINVAL; } -static void dpu_hw_intr_set_mask(struct dpu_hw_intr *intr, uint32_t reg_off, - uint32_t mask) -{ - if (!intr) - return; - - DPU_REG_WRITE(&intr->hw, reg_off, mask); - - /* ensure register writes go through */ - wmb(); -} - static void dpu_hw_intr_dispatch_irq(struct dpu_hw_intr *intr, void (*cbfunc)(void *, int), void *arg) @@ -1004,18 +988,6 @@ static int dpu_hw_intr_disable_irqs(struct dpu_hw_intr *intr) return 0; } -static int dpu_hw_intr_get_valid_interrupts(struct dpu_hw_intr *intr, - uint32_t *mask) -{ - if (!intr || !mask) - return -EINVAL; - - *mask = IRQ_SOURCE_MDP | IRQ_SOURCE_DSI0 | IRQ_SOURCE_DSI1 - | IRQ_SOURCE_HDMI | IRQ_SOURCE_EDP; - - return 0; -} - static void dpu_hw_intr_get_interrupt_statuses(struct dpu_hw_intr *intr) { int i; @@ -1065,19 +1037,6 @@ static void dpu_hw_intr_clear_intr_status_nolock(struct dpu_hw_intr *intr, wmb(); } -static void dpu_hw_intr_clear_interrupt_status(struct dpu_hw_intr *intr, - int irq_idx) -{ - unsigned long irq_flags; - - if (!intr) - return; - - spin_lock_irqsave(&intr->irq_lock, irq_flags); - dpu_hw_intr_clear_intr_status_nolock(intr, irq_idx); - spin_unlock_irqrestore(&intr->irq_lock, irq_flags); -} - static u32 dpu_hw_intr_get_interrupt_status(struct dpu_hw_intr *intr, int irq_idx, bool clear) { @@ -1113,16 +1072,13 @@ static u32 dpu_hw_intr_get_interrupt_status(struct dpu_hw_intr *intr, static void __setup_intr_ops(struct dpu_hw_intr_ops *ops) { - ops->set_mask = dpu_hw_intr_set_mask; ops->irq_idx_lookup = dpu_hw_intr_irqidx_lookup; ops->enable_irq = dpu_hw_intr_enable_irq; ops->disable_irq = dpu_hw_intr_disable_irq; ops->dispatch_irqs = dpu_hw_intr_dispatch_irq; ops->clear_all_irqs = dpu_hw_intr_clear_irqs; ops->disable_all_irqs = dpu_hw_intr_disable_irqs; - ops->get_valid_interrupts = dpu_hw_intr_get_valid_interrupts; ops->get_interrupt_statuses = dpu_hw_intr_get_interrupt_statuses; - ops->clear_interrupt_status = dpu_hw_intr_clear_interrupt_status; ops->clear_intr_status_nolock = dpu_hw_intr_clear_intr_status_nolock; ops->get_interrupt_status = dpu_hw_intr_get_interrupt_status; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h index 61e4cba36562..4d7a1c727ce2 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h @@ -20,13 +20,6 @@ #include "dpu_hw_util.h" #include "dpu_hw_mdss.h" -#define IRQ_SOURCE_MDP BIT(0) -#define IRQ_SOURCE_DSI0 BIT(4) -#define IRQ_SOURCE_DSI1 BIT(5) -#define IRQ_SOURCE_HDMI BIT(8) -#define IRQ_SOURCE_EDP BIT(12) -#define IRQ_SOURCE_MHL BIT(16) - /** * dpu_intr_type - HW Interrupt Type * @DPU_IRQ_TYPE_WB_ROT_COMP: WB rotator done @@ -96,18 +89,6 @@ struct dpu_hw_intr; */ struct dpu_hw_intr_ops { /** - * set_mask - Programs the given interrupt register with the - * given interrupt mask. Register value will get overwritten. - * @intr: HW interrupt handle - * @reg_off: MDSS HW register offset - * @irqmask: IRQ mask value - */ - void (*set_mask)( - struct dpu_hw_intr *intr, - uint32_t reg, - uint32_t irqmask); - - /** * irq_idx_lookup - Lookup IRQ index on the HW interrupt type * Used for all irq related ops * @intr_type: Interrupt type defined in dpu_intr_type @@ -177,16 +158,6 @@ struct dpu_hw_intr_ops { struct dpu_hw_intr *intr); /** - * clear_interrupt_status - Clears HW interrupt status based on given - * lookup IRQ index. - * @intr: HW interrupt handle - * @irq_idx: Lookup irq index return from irq_idx_lookup - */ - void (*clear_interrupt_status)( - struct dpu_hw_intr *intr, - int irq_idx); - - /** * clear_intr_status_nolock() - clears the HW interrupts without lock * @intr: HW interrupt handle * @irq_idx: Lookup irq index return from irq_idx_lookup @@ -206,21 +177,6 @@ struct dpu_hw_intr_ops { struct dpu_hw_intr *intr, int irq_idx, bool clear); - - /** - * get_valid_interrupts - Gets a mask of all valid interrupt sources - * within DPU. These are actually status bits - * within interrupt registers that specify the - * source of the interrupt in IRQs. For example, - * valid interrupt sources can be MDP, DSI, - * HDMI etc. - * @intr: HW interrupt handle - * @mask: Returning the interrupt source MASK - * @return: 0 for success, otherwise failure - */ - int (*get_valid_interrupts)( - struct dpu_hw_intr *intr, - uint32_t *mask); }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h index 68c54d2c9677..1ab8d4a889f7 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h @@ -258,12 +258,6 @@ enum dpu_vbif { VBIF_NRT = VBIF_1 }; -enum dpu_iommu_domain { - DPU_IOMMU_DOMAIN_UNSECURE, - DPU_IOMMU_DOMAIN_SECURE, - DPU_IOMMU_DOMAIN_MAX -}; - /** * DPU HW,Component order color map */ @@ -358,7 +352,6 @@ enum dpu_3d_blend_mode { * @alpha_enable: whether the format has an alpha channel * @num_planes: number of planes (including meta data planes) * @fetch_mode: linear, tiled, or ubwc hw fetch behavior - * @is_yuv: is format a yuv variant * @flag: usage bit flags * @tile_width: format tile width * @tile_height: format tile height diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 321fc64ddd0e..efe70c508ee0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -18,7 +18,6 @@ #include "dpu_hw_mdss.h" #define REG_MASK(n) ((BIT(n)) - 1) -struct dpu_format_extended; /* * This is the common struct maintained by each sub block diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 4d67b3c96702..885bf88afa3e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -405,35 +405,38 @@ static void dpu_kms_wait_for_commit_done(struct msm_kms *kms, } } -static void _dpu_kms_initialize_dsi(struct drm_device *dev, +static int _dpu_kms_initialize_dsi(struct drm_device *dev, struct msm_drm_private *priv, struct dpu_kms *dpu_kms) { struct drm_encoder *encoder = NULL; - int i, rc; + int i, rc = 0; + + if (!(priv->dsi[0] || priv->dsi[1])) + return rc; /*TODO: Support two independent DSI connectors */ encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DSI); - if (IS_ERR_OR_NULL(encoder)) { + if (IS_ERR(encoder)) { DPU_ERROR("encoder init failed for dsi display\n"); - return; + return PTR_ERR(encoder); } priv->encoders[priv->num_encoders++] = encoder; for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) { - if (!priv->dsi[i]) { - DPU_DEBUG("invalid msm_dsi for ctrl %d\n", i); - return; - } + if (!priv->dsi[i]) + continue; rc = msm_dsi_modeset_init(priv->dsi[i], dev, encoder); if (rc) { DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n", i, rc); - continue; + break; } } + + return rc; } /** @@ -444,16 +447,16 @@ static void _dpu_kms_initialize_dsi(struct drm_device *dev, * @dpu_kms: Pointer to dpu kms structure * Returns: Zero on success */ -static void _dpu_kms_setup_displays(struct drm_device *dev, +static int _dpu_kms_setup_displays(struct drm_device *dev, struct msm_drm_private *priv, struct dpu_kms *dpu_kms) { - _dpu_kms_initialize_dsi(dev, priv, dpu_kms); - /** * Extend this function to initialize other * types of displays */ + + return _dpu_kms_initialize_dsi(dev, priv, dpu_kms); } static void _dpu_kms_drm_obj_destroy(struct dpu_kms *dpu_kms) @@ -516,7 +519,9 @@ static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms) * Create encoder and query display drivers to create * bridges and connectors */ - _dpu_kms_setup_displays(dev, priv, dpu_kms); + ret = _dpu_kms_setup_displays(dev, priv, dpu_kms); + if (ret) + goto fail; max_crtc_count = min(catalog->mixer_count, priv->num_encoders); @@ -627,6 +632,10 @@ static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms) devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_RT]); dpu_kms->vbif[VBIF_RT] = NULL; + if (dpu_kms->hw_mdp) + dpu_hw_mdp_destroy(dpu_kms->hw_mdp); + dpu_kms->hw_mdp = NULL; + if (dpu_kms->mmio) devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->mmio); dpu_kms->mmio = NULL; @@ -877,8 +886,7 @@ static int dpu_kms_hw_init(struct msm_kms *kms) goto power_error; } - rc = dpu_rm_init(&dpu_kms->rm, dpu_kms->catalog, dpu_kms->mmio, - dpu_kms->dev); + rc = dpu_rm_init(&dpu_kms->rm, dpu_kms->catalog, dpu_kms->mmio); if (rc) { DPU_ERROR("rm init failed: %d\n", rc); goto power_error; @@ -886,11 +894,10 @@ static int dpu_kms_hw_init(struct msm_kms *kms) dpu_kms->rm_init = true; - dpu_kms->hw_mdp = dpu_rm_get_mdp(&dpu_kms->rm); - if (IS_ERR_OR_NULL(dpu_kms->hw_mdp)) { + dpu_kms->hw_mdp = dpu_hw_mdptop_init(MDP_TOP, dpu_kms->mmio, + dpu_kms->catalog); + if (IS_ERR(dpu_kms->hw_mdp)) { rc = PTR_ERR(dpu_kms->hw_mdp); - if (!dpu_kms->hw_mdp) - rc = -EINVAL; DPU_ERROR("failed to get hw_mdp: %d\n", rc); dpu_kms->hw_mdp = NULL; goto power_error; @@ -926,16 +933,6 @@ static int dpu_kms_hw_init(struct msm_kms *kms) goto hw_intr_init_err; } - /* - * _dpu_kms_drm_obj_init should create the DRM related objects - * i.e. CRTCs, planes, encoders, connectors and so forth - */ - rc = _dpu_kms_drm_obj_init(dpu_kms); - if (rc) { - DPU_ERROR("modeset init failed: %d\n", rc); - goto drm_obj_init_err; - } - dev->mode_config.min_width = 0; dev->mode_config.min_height = 0; @@ -952,6 +949,16 @@ static int dpu_kms_hw_init(struct msm_kms *kms) */ dev->mode_config.allow_fb_modifiers = true; + /* + * _dpu_kms_drm_obj_init should create the DRM related objects + * i.e. CRTCs, planes, encoders, connectors and so forth + */ + rc = _dpu_kms_drm_obj_init(dpu_kms); + if (rc) { + DPU_ERROR("modeset init failed: %d\n", rc); + goto drm_obj_init_err; + } + dpu_vbif_init_memtypes(dpu_kms); pm_runtime_put_sync(&dpu_kms->pdev->dev); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c index cb307a2abf06..7316b4ab1b85 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c @@ -23,11 +23,14 @@ struct dpu_mdss { struct dpu_irq_controller irq_controller; }; -static irqreturn_t dpu_mdss_irq(int irq, void *arg) +static void dpu_mdss_irq(struct irq_desc *desc) { - struct dpu_mdss *dpu_mdss = arg; + struct dpu_mdss *dpu_mdss = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); u32 interrupts; + chained_irq_enter(chip, desc); + interrupts = readl_relaxed(dpu_mdss->mmio + HW_INTR_STATUS); while (interrupts) { @@ -39,20 +42,20 @@ static irqreturn_t dpu_mdss_irq(int irq, void *arg) hwirq); if (mapping == 0) { DRM_ERROR("couldn't find irq mapping for %lu\n", hwirq); - return IRQ_NONE; + break; } rc = generic_handle_irq(mapping); if (rc < 0) { DRM_ERROR("handle irq fail: irq=%lu mapping=%u rc=%d\n", hwirq, mapping, rc); - return IRQ_NONE; + break; } interrupts &= ~(1 << hwirq); } - return IRQ_HANDLED; + chained_irq_exit(chip, desc); } static void dpu_mdss_irq_mask(struct irq_data *irqd) @@ -83,16 +86,16 @@ static struct irq_chip dpu_mdss_irq_chip = { .irq_unmask = dpu_mdss_irq_unmask, }; +static struct lock_class_key dpu_mdss_lock_key, dpu_mdss_request_key; + static int dpu_mdss_irqdomain_map(struct irq_domain *domain, unsigned int irq, irq_hw_number_t hwirq) { struct dpu_mdss *dpu_mdss = domain->host_data; - int ret; + irq_set_lockdep_class(irq, &dpu_mdss_lock_key, &dpu_mdss_request_key); irq_set_chip_and_handler(irq, &dpu_mdss_irq_chip, handle_level_irq); - ret = irq_set_chip_data(irq, dpu_mdss); - - return ret; + return irq_set_chip_data(irq, dpu_mdss); } static const struct irq_domain_ops dpu_mdss_irqdomain_ops = { @@ -159,11 +162,13 @@ static void dpu_mdss_destroy(struct drm_device *dev) struct msm_drm_private *priv = dev->dev_private; struct dpu_mdss *dpu_mdss = to_dpu_mdss(priv->mdss); struct dss_module_power *mp = &dpu_mdss->mp; + int irq; pm_runtime_suspend(dev->dev); pm_runtime_disable(dev->dev); _dpu_mdss_irq_domain_fini(dpu_mdss); - free_irq(platform_get_irq(pdev, 0), dpu_mdss); + irq = platform_get_irq(pdev, 0); + irq_set_chained_handler_and_data(irq, NULL, NULL); msm_dss_put_clk(mp->clk_config, mp->num_clk); devm_kfree(&pdev->dev, mp->clk_config); @@ -187,6 +192,7 @@ int dpu_mdss_init(struct drm_device *dev) struct dpu_mdss *dpu_mdss; struct dss_module_power *mp; int ret = 0; + int irq; dpu_mdss = devm_kzalloc(dev->dev, sizeof(*dpu_mdss), GFP_KERNEL); if (!dpu_mdss) @@ -219,12 +225,12 @@ int dpu_mdss_init(struct drm_device *dev) if (ret) goto irq_domain_error; - ret = request_irq(platform_get_irq(pdev, 0), - dpu_mdss_irq, 0, "dpu_mdss_isr", dpu_mdss); - if (ret) { - DPU_ERROR("failed to init irq: %d\n", ret); + irq = platform_get_irq(pdev, 0); + if (irq < 0) goto irq_error; - } + + irq_set_chained_handler_and_data(irq, dpu_mdss_irq, + dpu_mdss); pm_runtime_enable(dev->dev); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index fd75870eb17f..b01183b309b9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -95,8 +95,6 @@ struct dpu_plane { enum dpu_sspp pipe; uint32_t features; /* capabilities from catalog */ - uint32_t nformats; - uint32_t formats[64]; struct dpu_hw_pipe *pipe_hw; struct dpu_hw_pipe_cfg pipe_cfg; @@ -121,6 +119,12 @@ struct dpu_plane { bool debugfs_default_scale; }; +static const uint64_t supported_format_modifiers[] = { + DRM_FORMAT_MOD_QCOM_COMPRESSED, + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + #define to_dpu_plane(x) container_of(x, struct dpu_plane, base) static struct dpu_kms *_dpu_plane_get_kms(struct drm_plane *plane) @@ -365,19 +369,6 @@ static void _dpu_plane_set_qos_ctrl(struct drm_plane *plane, &pdpu->pipe_qos_cfg); } -static void dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool enable) -{ - struct dpu_plane *pdpu = to_dpu_plane(plane); - struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); - - if (!pdpu->is_rt_pipe) - return; - - pm_runtime_get_sync(&dpu_kms->pdev->dev); - _dpu_plane_set_qos_ctrl(plane, enable, DPU_PLANE_QOS_PANIC_CTRL); - pm_runtime_put_sync(&dpu_kms->pdev->dev); -} - /** * _dpu_plane_set_ot_limit - set OT limit for the given plane * @plane: Pointer to drm plane @@ -1248,6 +1239,19 @@ static void dpu_plane_reset(struct drm_plane *plane) } #ifdef CONFIG_DEBUG_FS +static void dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool enable) +{ + struct dpu_plane *pdpu = to_dpu_plane(plane); + struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane); + + if (!pdpu->is_rt_pipe) + return; + + pm_runtime_get_sync(&dpu_kms->pdev->dev); + _dpu_plane_set_qos_ctrl(plane, enable, DPU_PLANE_QOS_PANIC_CTRL); + pm_runtime_put_sync(&dpu_kms->pdev->dev); +} + static ssize_t _dpu_plane_danger_read(struct file *file, char __user *buff, size_t count, loff_t *ppos) { @@ -1410,6 +1414,23 @@ static void dpu_plane_early_unregister(struct drm_plane *plane) debugfs_remove_recursive(pdpu->debugfs_root); } +static bool dpu_plane_format_mod_supported(struct drm_plane *plane, + uint32_t format, uint64_t modifier) +{ + if (modifier == DRM_FORMAT_MOD_LINEAR) + return true; + + if (modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED) { + int i; + for (i = 0; i < ARRAY_SIZE(qcom_compressed_supported_formats); i++) { + if (format == qcom_compressed_supported_formats[i]) + return true; + } + } + + return false; +} + static const struct drm_plane_funcs dpu_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, @@ -1419,6 +1440,7 @@ static const struct drm_plane_funcs dpu_plane_funcs = { .atomic_destroy_state = dpu_plane_destroy_state, .late_register = dpu_plane_late_register, .early_unregister = dpu_plane_early_unregister, + .format_mod_supported = dpu_plane_format_mod_supported, }; static const struct drm_plane_helper_funcs dpu_plane_helper_funcs = { @@ -1444,11 +1466,12 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 master_plane_id) { struct drm_plane *plane = NULL, *master_plane = NULL; - const struct dpu_format_extended *format_list; + const uint32_t *format_list; struct dpu_plane *pdpu; struct msm_drm_private *priv = dev->dev_private; struct dpu_kms *kms = to_dpu_kms(priv->kms); int zpos_max = DPU_ZPOS_MAX; + uint32_t num_formats; int ret = -EINVAL; /* create and zero local structure */ @@ -1491,24 +1514,18 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, goto clean_sspp; } - if (!master_plane_id) - format_list = pdpu->pipe_sblk->format_list; - else + if (pdpu->is_virtual) { format_list = pdpu->pipe_sblk->virt_format_list; - - pdpu->nformats = dpu_populate_formats(format_list, - pdpu->formats, - 0, - ARRAY_SIZE(pdpu->formats)); - - if (!pdpu->nformats) { - DPU_ERROR("[%u]no valid formats for plane\n", pipe); - goto clean_sspp; + num_formats = pdpu->pipe_sblk->virt_num_formats; + } + else { + format_list = pdpu->pipe_sblk->format_list; + num_formats = pdpu->pipe_sblk->num_formats; } ret = drm_universal_plane_init(dev, plane, 0xff, &dpu_plane_funcs, - pdpu->formats, pdpu->nformats, - NULL, type, NULL); + format_list, num_formats, + supported_format_modifiers, type, NULL); if (ret) goto clean_sspp; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h index 7fed0b627708..0e6063acd041 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h @@ -28,23 +28,18 @@ /** * struct dpu_plane_state: Define dpu extension of drm plane state object * @base: base drm plane state object - * @property_state: Local storage for msm_prop properties - * @property_values: cached plane property values * @aspace: pointer to address space for input/output buffers - * @input_fence: dereferenced input fence pointer * @stage: assigned by crtc blender * @multirect_index: index of the rectangle of SSPP * @multirect_mode: parallel or time multiplex multirect mode * @pending: whether the current update is still pending * @scaler3_cfg: configuration data for scaler3 * @pixel_ext: configuration data for pixel extensions - * @scaler_check_state: indicates status of user provided pixel extension data * @cdp_cfg: CDP configuration */ struct dpu_plane_state { struct drm_plane_state base; struct msm_gem_address_space *aspace; - void *input_fence; enum dpu_stage stage; uint32_t multirect_index; uint32_t multirect_mode; @@ -107,12 +102,6 @@ void dpu_plane_restore(struct drm_plane *plane); void dpu_plane_flush(struct drm_plane *plane); /** - * dpu_plane_kickoff - final plane operations before commit kickoff - * @plane: Pointer to drm plane structure - */ -void dpu_plane_kickoff(struct drm_plane *plane); - -/** * dpu_plane_set_error: enable/disable error condition * @plane: pointer to drm_plane structure */ @@ -147,14 +136,6 @@ int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane); void dpu_plane_clear_multirect(const struct drm_plane_state *drm_state); /** - * dpu_plane_wait_input_fence - wait for input fence object - * @plane: Pointer to DRM plane object - * @wait_ms: Wait timeout value - * Returns: Zero on success - */ -int dpu_plane_wait_input_fence(struct drm_plane *plane, uint32_t wait_ms); - -/** * dpu_plane_color_fill - enables color fill on plane * @plane: Pointer to DRM plane object * @color: RGB fill color value, [23..16] Blue, [15..8] Green, [7..0] Red @@ -164,12 +145,4 @@ int dpu_plane_wait_input_fence(struct drm_plane *plane, uint32_t wait_ms); int dpu_plane_color_fill(struct drm_plane *plane, uint32_t color, uint32_t alpha); -/** - * dpu_plane_set_revalidate - sets revalidate flag which forces a full - * validation of the plane properties in the next atomic check - * @plane: Pointer to DRM plane object - * @enable: Boolean to set/unset the flag - */ -void dpu_plane_set_revalidate(struct drm_plane *plane, bool enable); - #endif /* _DPU_PLANE_H_ */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c index bdb117709674..037d9f4187f9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c @@ -21,8 +21,8 @@ #include "dpu_encoder.h" #include "dpu_trace.h" -#define RESERVED_BY_OTHER(h, r) \ - ((h)->rsvp && ((h)->rsvp->enc_id != (r)->enc_id)) +#define RESERVED_BY_OTHER(h, r) \ + ((h)->enc_id && (h)->enc_id != r) /** * struct dpu_rm_requirements - Reservation requirements parameter bundle @@ -34,90 +34,21 @@ struct dpu_rm_requirements { struct dpu_encoder_hw_resources hw_res; }; -/** - * struct dpu_rm_rsvp - Use Case Reservation tagging structure - * Used to tag HW blocks as reserved by a CRTC->Encoder->Connector chain - * By using as a tag, rather than lists of pointers to HW blocks used - * we can avoid some list management since we don't know how many blocks - * of each type a given use case may require. - * @list: List head for list of all reservations - * @seq: Global RSVP sequence number for debugging, especially for - * differentiating differenct allocations for same encoder. - * @enc_id: Reservations are tracked by Encoder DRM object ID. - * CRTCs may be connected to multiple Encoders. - * An encoder or connector id identifies the display path. - */ -struct dpu_rm_rsvp { - struct list_head list; - uint32_t seq; - uint32_t enc_id; -}; /** * struct dpu_rm_hw_blk - hardware block tracking list member * @list: List head for list of all hardware blocks tracking items - * @rsvp: Pointer to use case reservation if reserved by a client - * @rsvp_nxt: Temporary pointer used during reservation to the incoming - * request. Will be swapped into rsvp if proposal is accepted - * @type: Type of hardware block this structure tracks * @id: Hardware ID number, within it's own space, ie. LM_X - * @catalog: Pointer to the hardware catalog entry for this block + * @enc_id: Encoder id to which this blk is binded * @hw: Pointer to the hardware register access object for this block */ struct dpu_rm_hw_blk { struct list_head list; - struct dpu_rm_rsvp *rsvp; - struct dpu_rm_rsvp *rsvp_nxt; - enum dpu_hw_blk_type type; uint32_t id; + uint32_t enc_id; struct dpu_hw_blk *hw; }; -/** - * dpu_rm_dbg_rsvp_stage - enum of steps in making reservation for event logging - */ -enum dpu_rm_dbg_rsvp_stage { - DPU_RM_STAGE_BEGIN, - DPU_RM_STAGE_AFTER_CLEAR, - DPU_RM_STAGE_AFTER_RSVPNEXT, - DPU_RM_STAGE_FINAL -}; - -static void _dpu_rm_print_rsvps( - struct dpu_rm *rm, - enum dpu_rm_dbg_rsvp_stage stage) -{ - struct dpu_rm_rsvp *rsvp; - struct dpu_rm_hw_blk *blk; - enum dpu_hw_blk_type type; - - DPU_DEBUG("%d\n", stage); - - list_for_each_entry(rsvp, &rm->rsvps, list) { - DRM_DEBUG_KMS("%d rsvp[s%ue%u]\n", stage, rsvp->seq, - rsvp->enc_id); - } - - for (type = 0; type < DPU_HW_BLK_MAX; type++) { - list_for_each_entry(blk, &rm->hw_blks[type], list) { - if (!blk->rsvp && !blk->rsvp_nxt) - continue; - - DRM_DEBUG_KMS("%d rsvp[s%ue%u->s%ue%u] %d %d\n", stage, - (blk->rsvp) ? blk->rsvp->seq : 0, - (blk->rsvp) ? blk->rsvp->enc_id : 0, - (blk->rsvp_nxt) ? blk->rsvp_nxt->seq : 0, - (blk->rsvp_nxt) ? blk->rsvp_nxt->enc_id : 0, - blk->type, blk->id); - } - } -} - -struct dpu_hw_mdp *dpu_rm_get_mdp(struct dpu_rm *rm) -{ - return rm->hw_mdp; -} - void dpu_rm_init_hw_iter( struct dpu_rm_hw_iter *iter, uint32_t enc_id, @@ -148,15 +79,7 @@ static bool _dpu_rm_get_hw_locked(struct dpu_rm *rm, struct dpu_rm_hw_iter *i) i->blk = list_prepare_entry(i->blk, blk_list, list); list_for_each_entry_continue(i->blk, blk_list, list) { - struct dpu_rm_rsvp *rsvp = i->blk->rsvp; - - if (i->blk->type != i->type) { - DPU_ERROR("found incorrect block type %d on %d list\n", - i->blk->type, i->type); - return false; - } - - if ((i->enc_id == 0) || (rsvp && rsvp->enc_id == i->enc_id)) { + if (i->enc_id == i->blk->enc_id) { i->hw = i->blk->hw; DPU_DEBUG("found type %d id %d for enc %d\n", i->type, i->blk->id, i->enc_id); @@ -208,34 +131,18 @@ static void _dpu_rm_hw_destroy(enum dpu_hw_blk_type type, void *hw) int dpu_rm_destroy(struct dpu_rm *rm) { - - struct dpu_rm_rsvp *rsvp_cur, *rsvp_nxt; struct dpu_rm_hw_blk *hw_cur, *hw_nxt; enum dpu_hw_blk_type type; - if (!rm) { - DPU_ERROR("invalid rm\n"); - return -EINVAL; - } - - list_for_each_entry_safe(rsvp_cur, rsvp_nxt, &rm->rsvps, list) { - list_del(&rsvp_cur->list); - kfree(rsvp_cur); - } - - for (type = 0; type < DPU_HW_BLK_MAX; type++) { list_for_each_entry_safe(hw_cur, hw_nxt, &rm->hw_blks[type], list) { list_del(&hw_cur->list); - _dpu_rm_hw_destroy(hw_cur->type, hw_cur->hw); + _dpu_rm_hw_destroy(type, hw_cur->hw); kfree(hw_cur); } } - dpu_hw_mdp_destroy(rm->hw_mdp); - rm->hw_mdp = NULL; - mutex_destroy(&rm->rm_lock); return 0; @@ -250,11 +157,8 @@ static int _dpu_rm_hw_blk_create( void *hw_catalog_info) { struct dpu_rm_hw_blk *blk; - struct dpu_hw_mdp *hw_mdp; void *hw; - hw_mdp = rm->hw_mdp; - switch (type) { case DPU_HW_BLK_LM: hw = dpu_hw_lm_init(id, mmio, cat); @@ -290,9 +194,9 @@ static int _dpu_rm_hw_blk_create( return -ENOMEM; } - blk->type = type; blk->id = id; blk->hw = hw; + blk->enc_id = 0; list_add_tail(&blk->list, &rm->hw_blks[type]); return 0; @@ -300,13 +204,12 @@ static int _dpu_rm_hw_blk_create( int dpu_rm_init(struct dpu_rm *rm, struct dpu_mdss_cfg *cat, - void __iomem *mmio, - struct drm_device *dev) + void __iomem *mmio) { int rc, i; enum dpu_hw_blk_type type; - if (!rm || !cat || !mmio || !dev) { + if (!rm || !cat || !mmio) { DPU_ERROR("invalid kms\n"); return -EINVAL; } @@ -316,21 +219,9 @@ int dpu_rm_init(struct dpu_rm *rm, mutex_init(&rm->rm_lock); - INIT_LIST_HEAD(&rm->rsvps); for (type = 0; type < DPU_HW_BLK_MAX; type++) INIT_LIST_HEAD(&rm->hw_blks[type]); - rm->dev = dev; - - /* Some of the sub-blocks require an mdptop to be created */ - rm->hw_mdp = dpu_hw_mdptop_init(MDP_TOP, mmio, cat); - if (IS_ERR_OR_NULL(rm->hw_mdp)) { - rc = PTR_ERR(rm->hw_mdp); - rm->hw_mdp = NULL; - DPU_ERROR("failed: mdp hw not available\n"); - goto fail; - } - /* Interrogate HW catalog and create tracking items for hw blocks */ for (i = 0; i < cat->mixer_count; i++) { struct dpu_lm_cfg *lm = &cat->mixer[i]; @@ -410,7 +301,7 @@ static bool _dpu_rm_needs_split_display(const struct msm_display_topology *top) * proposed use case requirements, incl. hardwired dependent blocks like * pingpong * @rm: dpu resource manager handle - * @rsvp: reservation currently being created + * @enc_id: encoder id requesting for allocation * @reqs: proposed use case requirements * @lm: proposed layer mixer, function checks if lm, and all other hardwired * blocks connected to the lm (pp) is available and appropriate @@ -422,7 +313,7 @@ static bool _dpu_rm_needs_split_display(const struct msm_display_topology *top) */ static bool _dpu_rm_check_lm_and_get_connected_blks( struct dpu_rm *rm, - struct dpu_rm_rsvp *rsvp, + uint32_t enc_id, struct dpu_rm_requirements *reqs, struct dpu_rm_hw_blk *lm, struct dpu_rm_hw_blk **pp, @@ -449,7 +340,7 @@ static bool _dpu_rm_check_lm_and_get_connected_blks( } /* Already reserved? */ - if (RESERVED_BY_OTHER(lm, rsvp)) { + if (RESERVED_BY_OTHER(lm, enc_id)) { DPU_DEBUG("lm %d already reserved\n", lm_cfg->id); return false; } @@ -467,7 +358,7 @@ static bool _dpu_rm_check_lm_and_get_connected_blks( return false; } - if (RESERVED_BY_OTHER(*pp, rsvp)) { + if (RESERVED_BY_OTHER(*pp, enc_id)) { DPU_DEBUG("lm %d pp %d already reserved\n", lm->id, (*pp)->id); return false; @@ -476,10 +367,8 @@ static bool _dpu_rm_check_lm_and_get_connected_blks( return true; } -static int _dpu_rm_reserve_lms( - struct dpu_rm *rm, - struct dpu_rm_rsvp *rsvp, - struct dpu_rm_requirements *reqs) +static int _dpu_rm_reserve_lms(struct dpu_rm *rm, uint32_t enc_id, + struct dpu_rm_requirements *reqs) { struct dpu_rm_hw_blk *lm[MAX_BLOCKS]; @@ -504,7 +393,7 @@ static int _dpu_rm_reserve_lms( lm[lm_count] = iter_i.blk; if (!_dpu_rm_check_lm_and_get_connected_blks( - rm, rsvp, reqs, lm[lm_count], + rm, enc_id, reqs, lm[lm_count], &pp[lm_count], NULL)) continue; @@ -519,7 +408,7 @@ static int _dpu_rm_reserve_lms( continue; if (!_dpu_rm_check_lm_and_get_connected_blks( - rm, rsvp, reqs, iter_j.blk, + rm, enc_id, reqs, iter_j.blk, &pp[lm_count], iter_i.blk)) continue; @@ -537,11 +426,10 @@ static int _dpu_rm_reserve_lms( if (!lm[i]) break; - lm[i]->rsvp_nxt = rsvp; - pp[i]->rsvp_nxt = rsvp; + lm[i]->enc_id = enc_id; + pp[i]->enc_id = enc_id; - trace_dpu_rm_reserve_lms(lm[i]->id, lm[i]->type, rsvp->enc_id, - pp[i]->id); + trace_dpu_rm_reserve_lms(lm[i]->id, enc_id, pp[i]->id); } return rc; @@ -549,7 +437,7 @@ static int _dpu_rm_reserve_lms( static int _dpu_rm_reserve_ctls( struct dpu_rm *rm, - struct dpu_rm_rsvp *rsvp, + uint32_t enc_id, const struct msm_display_topology *top) { struct dpu_rm_hw_blk *ctls[MAX_BLOCKS]; @@ -570,7 +458,7 @@ static int _dpu_rm_reserve_ctls( unsigned long features = ctl->caps->features; bool has_split_display; - if (RESERVED_BY_OTHER(iter.blk, rsvp)) + if (RESERVED_BY_OTHER(iter.blk, enc_id)) continue; has_split_display = BIT(DPU_CTL_SPLIT_DISPLAY) & features; @@ -591,9 +479,8 @@ static int _dpu_rm_reserve_ctls( return -ENAVAIL; for (i = 0; i < ARRAY_SIZE(ctls) && i < num_ctls; i++) { - ctls[i]->rsvp_nxt = rsvp; - trace_dpu_rm_reserve_ctls(ctls[i]->id, ctls[i]->type, - rsvp->enc_id); + ctls[i]->enc_id = enc_id; + trace_dpu_rm_reserve_ctls(ctls[i]->id, enc_id); } return 0; @@ -601,7 +488,7 @@ static int _dpu_rm_reserve_ctls( static int _dpu_rm_reserve_intf( struct dpu_rm *rm, - struct dpu_rm_rsvp *rsvp, + uint32_t enc_id, uint32_t id, enum dpu_hw_blk_type type) { @@ -614,14 +501,13 @@ static int _dpu_rm_reserve_intf( if (iter.blk->id != id) continue; - if (RESERVED_BY_OTHER(iter.blk, rsvp)) { + if (RESERVED_BY_OTHER(iter.blk, enc_id)) { DPU_ERROR("type %d id %d already reserved\n", type, id); return -ENAVAIL; } - iter.blk->rsvp_nxt = rsvp; - trace_dpu_rm_reserve_intf(iter.blk->id, iter.blk->type, - rsvp->enc_id); + iter.blk->enc_id = enc_id; + trace_dpu_rm_reserve_intf(iter.blk->id, enc_id); break; } @@ -636,7 +522,7 @@ static int _dpu_rm_reserve_intf( static int _dpu_rm_reserve_intf_related_hw( struct dpu_rm *rm, - struct dpu_rm_rsvp *rsvp, + uint32_t enc_id, struct dpu_encoder_hw_resources *hw_res) { int i, ret = 0; @@ -646,7 +532,7 @@ static int _dpu_rm_reserve_intf_related_hw( if (hw_res->intfs[i] == INTF_MODE_NONE) continue; id = i + INTF_0; - ret = _dpu_rm_reserve_intf(rm, rsvp, id, + ret = _dpu_rm_reserve_intf(rm, enc_id, id, DPU_HW_BLK_INTF); if (ret) return ret; @@ -655,33 +541,27 @@ static int _dpu_rm_reserve_intf_related_hw( return ret; } -static int _dpu_rm_make_next_rsvp( +static int _dpu_rm_make_reservation( struct dpu_rm *rm, struct drm_encoder *enc, struct drm_crtc_state *crtc_state, - struct dpu_rm_rsvp *rsvp, struct dpu_rm_requirements *reqs) { int ret; - /* Create reservation info, tag reserved blocks with it as we go */ - rsvp->seq = ++rm->rsvp_next_seq; - rsvp->enc_id = enc->base.id; - list_add_tail(&rsvp->list, &rm->rsvps); - - ret = _dpu_rm_reserve_lms(rm, rsvp, reqs); + ret = _dpu_rm_reserve_lms(rm, enc->base.id, reqs); if (ret) { DPU_ERROR("unable to find appropriate mixers\n"); return ret; } - ret = _dpu_rm_reserve_ctls(rm, rsvp, &reqs->topology); + ret = _dpu_rm_reserve_ctls(rm, enc->base.id, &reqs->topology); if (ret) { DPU_ERROR("unable to find appropriate CTL\n"); return ret; } - ret = _dpu_rm_reserve_intf_related_hw(rm, rsvp, &reqs->hw_res); + ret = _dpu_rm_reserve_intf_related_hw(rm, enc->base.id, &reqs->hw_res); if (ret) return ret; @@ -706,108 +586,31 @@ static int _dpu_rm_populate_requirements( return 0; } -static struct dpu_rm_rsvp *_dpu_rm_get_rsvp( - struct dpu_rm *rm, - struct drm_encoder *enc) +static void _dpu_rm_release_reservation(struct dpu_rm *rm, uint32_t enc_id) { - struct dpu_rm_rsvp *i; - - if (!rm || !enc) { - DPU_ERROR("invalid params\n"); - return NULL; - } - - if (list_empty(&rm->rsvps)) - return NULL; - - list_for_each_entry(i, &rm->rsvps, list) - if (i->enc_id == enc->base.id) - return i; - - return NULL; -} - -/** - * _dpu_rm_release_rsvp - release resources and release a reservation - * @rm: KMS handle - * @rsvp: RSVP pointer to release and release resources for - */ -static void _dpu_rm_release_rsvp(struct dpu_rm *rm, struct dpu_rm_rsvp *rsvp) -{ - struct dpu_rm_rsvp *rsvp_c, *rsvp_n; struct dpu_rm_hw_blk *blk; enum dpu_hw_blk_type type; - if (!rsvp) - return; - - DPU_DEBUG("rel rsvp %d enc %d\n", rsvp->seq, rsvp->enc_id); - - list_for_each_entry_safe(rsvp_c, rsvp_n, &rm->rsvps, list) { - if (rsvp == rsvp_c) { - list_del(&rsvp_c->list); - break; - } - } - for (type = 0; type < DPU_HW_BLK_MAX; type++) { list_for_each_entry(blk, &rm->hw_blks[type], list) { - if (blk->rsvp == rsvp) { - blk->rsvp = NULL; - DPU_DEBUG("rel rsvp %d enc %d %d %d\n", - rsvp->seq, rsvp->enc_id, - blk->type, blk->id); - } - if (blk->rsvp_nxt == rsvp) { - blk->rsvp_nxt = NULL; - DPU_DEBUG("rel rsvp_nxt %d enc %d %d %d\n", - rsvp->seq, rsvp->enc_id, - blk->type, blk->id); + if (blk->enc_id == enc_id) { + blk->enc_id = 0; + DPU_DEBUG("rel enc %d %d %d\n", enc_id, + type, blk->id); } } } - - kfree(rsvp); } void dpu_rm_release(struct dpu_rm *rm, struct drm_encoder *enc) { - struct dpu_rm_rsvp *rsvp; - - if (!rm || !enc) { - DPU_ERROR("invalid params\n"); - return; - } - mutex_lock(&rm->rm_lock); - rsvp = _dpu_rm_get_rsvp(rm, enc); - if (!rsvp) { - DPU_ERROR("failed to find rsvp for enc %d\n", enc->base.id); - goto end; - } + _dpu_rm_release_reservation(rm, enc->base.id); - _dpu_rm_release_rsvp(rm, rsvp); -end: mutex_unlock(&rm->rm_lock); } -static void _dpu_rm_commit_rsvp(struct dpu_rm *rm, struct dpu_rm_rsvp *rsvp) -{ - struct dpu_rm_hw_blk *blk; - enum dpu_hw_blk_type type; - - /* Swap next rsvp to be the active */ - for (type = 0; type < DPU_HW_BLK_MAX; type++) { - list_for_each_entry(blk, &rm->hw_blks[type], list) { - if (blk->rsvp_nxt) { - blk->rsvp = blk->rsvp_nxt; - blk->rsvp_nxt = NULL; - } - } - } -} - int dpu_rm_reserve( struct dpu_rm *rm, struct drm_encoder *enc, @@ -815,7 +618,6 @@ int dpu_rm_reserve( struct msm_display_topology topology, bool test_only) { - struct dpu_rm_rsvp *rsvp_cur, *rsvp_nxt; struct dpu_rm_requirements reqs; int ret; @@ -828,8 +630,6 @@ int dpu_rm_reserve( mutex_lock(&rm->rm_lock); - _dpu_rm_print_rsvps(rm, DPU_RM_STAGE_BEGIN); - ret = _dpu_rm_populate_requirements(rm, enc, crtc_state, &reqs, topology); if (ret) { @@ -837,50 +637,17 @@ int dpu_rm_reserve( goto end; } - /* - * We only support one active reservation per-hw-block. But to implement - * transactional semantics for test-only, and for allowing failure while - * modifying your existing reservation, over the course of this - * function we can have two reservations: - * Current: Existing reservation - * Next: Proposed reservation. The proposed reservation may fail, or may - * be discarded if in test-only mode. - * If reservation is successful, and we're not in test-only, then we - * replace the current with the next. - */ - rsvp_nxt = kzalloc(sizeof(*rsvp_nxt), GFP_KERNEL); - if (!rsvp_nxt) { - ret = -ENOMEM; - goto end; - } - - rsvp_cur = _dpu_rm_get_rsvp(rm, enc); - - /* Check the proposed reservation, store it in hw's "next" field */ - ret = _dpu_rm_make_next_rsvp(rm, enc, crtc_state, rsvp_nxt, &reqs); - - _dpu_rm_print_rsvps(rm, DPU_RM_STAGE_AFTER_RSVPNEXT); - + ret = _dpu_rm_make_reservation(rm, enc, crtc_state, &reqs); if (ret) { DPU_ERROR("failed to reserve hw resources: %d\n", ret); - _dpu_rm_release_rsvp(rm, rsvp_nxt); + _dpu_rm_release_reservation(rm, enc->base.id); } else if (test_only) { - /* - * Normally, if test_only, test the reservation and then undo - * However, if the user requests LOCK, then keep the reservation - * made during the atomic_check phase. - */ - DPU_DEBUG("test_only: discard test rsvp[s%de%d]\n", - rsvp_nxt->seq, rsvp_nxt->enc_id); - _dpu_rm_release_rsvp(rm, rsvp_nxt); - } else { - _dpu_rm_release_rsvp(rm, rsvp_cur); - - _dpu_rm_commit_rsvp(rm, rsvp_nxt); + /* test_only: test the reservation and then undo */ + DPU_DEBUG("test_only: discard test [enc: %d]\n", + enc->base.id); + _dpu_rm_release_reservation(rm, enc->base.id); } - _dpu_rm_print_rsvps(rm, DPU_RM_STAGE_FINAL); - end: mutex_unlock(&rm->rm_lock); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h index b8273bd23801..381611fc5877 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h @@ -22,22 +22,14 @@ /** * struct dpu_rm - DPU dynamic hardware resource manager - * @dev: device handle for event logging purposes - * @rsvps: list of hardware reservations by each crtc->encoder->connector * @hw_blks: array of lists of hardware resources present in the system, one * list per type of hardware block - * @hw_mdp: hardware object for mdp_top * @lm_max_width: cached layer mixer maximum width - * @rsvp_next_seq: sequence number for next reservation for debugging purposes * @rm_lock: resource manager mutex */ struct dpu_rm { - struct drm_device *dev; - struct list_head rsvps; struct list_head hw_blks[DPU_HW_BLK_MAX]; - struct dpu_hw_mdp *hw_mdp; uint32_t lm_max_width; - uint32_t rsvp_next_seq; struct mutex rm_lock; }; @@ -67,13 +59,11 @@ struct dpu_rm_hw_iter { * @rm: DPU Resource Manager handle * @cat: Pointer to hardware catalog * @mmio: mapped register io address of MDP - * @dev: device handle for event logging purposes * @Return: 0 on Success otherwise -ERROR */ int dpu_rm_init(struct dpu_rm *rm, struct dpu_mdss_cfg *cat, - void __iomem *mmio, - struct drm_device *dev); + void __iomem *mmio); /** * dpu_rm_destroy - Free all memory allocated by dpu_rm_init @@ -112,14 +102,6 @@ int dpu_rm_reserve(struct dpu_rm *rm, void dpu_rm_release(struct dpu_rm *rm, struct drm_encoder *enc); /** - * dpu_rm_get_mdp - Retrieve HW block for MDP TOP. - * This is never reserved, and is usable by any display. - * @rm: DPU Resource Manager handle - * @Return: Pointer to hw block or NULL - */ -struct dpu_hw_mdp *dpu_rm_get_mdp(struct dpu_rm *rm); - -/** * dpu_rm_init_hw_iter - setup given iterator for new iteration over hw list * using dpu_rm_get_hw * @iter: iter object to initialize @@ -144,12 +126,4 @@ void dpu_rm_init_hw_iter( * @Return: true on match found, false on no match found */ bool dpu_rm_get_hw(struct dpu_rm *rm, struct dpu_rm_hw_iter *iter); - -/** - * dpu_rm_check_property_topctl - validate property bitmask before it is set - * @val: user's proposed topology control bitmask - * @Return: 0 on success or error - */ -int dpu_rm_check_property_topctl(uint64_t val); - #endif /* __DPU_RM_H__ */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h index c78b521ceda1..8bb46090bd16 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h @@ -831,48 +831,42 @@ TRACE_EVENT(dpu_plane_disable, ); DECLARE_EVENT_CLASS(dpu_rm_iter_template, - TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id), - TP_ARGS(id, type, enc_id), + TP_PROTO(uint32_t id, uint32_t enc_id), + TP_ARGS(id, enc_id), TP_STRUCT__entry( __field( uint32_t, id ) - __field( enum dpu_hw_blk_type, type ) __field( uint32_t, enc_id ) ), TP_fast_assign( __entry->id = id; - __entry->type = type; __entry->enc_id = enc_id; ), - TP_printk("id:%d type:%d enc_id:%u", __entry->id, __entry->type, - __entry->enc_id) + TP_printk("id:%d enc_id:%u", __entry->id, __entry->enc_id) ); DEFINE_EVENT(dpu_rm_iter_template, dpu_rm_reserve_intf, - TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id), - TP_ARGS(id, type, enc_id) + TP_PROTO(uint32_t id, uint32_t enc_id), + TP_ARGS(id, enc_id) ); DEFINE_EVENT(dpu_rm_iter_template, dpu_rm_reserve_ctls, - TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id), - TP_ARGS(id, type, enc_id) + TP_PROTO(uint32_t id, uint32_t enc_id), + TP_ARGS(id, enc_id) ); TRACE_EVENT(dpu_rm_reserve_lms, - TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id, - uint32_t pp_id), - TP_ARGS(id, type, enc_id, pp_id), + TP_PROTO(uint32_t id, uint32_t enc_id, uint32_t pp_id), + TP_ARGS(id, enc_id, pp_id), TP_STRUCT__entry( __field( uint32_t, id ) - __field( enum dpu_hw_blk_type, type ) __field( uint32_t, enc_id ) __field( uint32_t, pp_id ) ), TP_fast_assign( __entry->id = id; - __entry->type = type; __entry->enc_id = enc_id; __entry->pp_id = pp_id; ), - TP_printk("id:%d type:%d enc_id:%u pp_id:%u", __entry->id, - __entry->type, __entry->enc_id, __entry->pp_id) + TP_printk("id:%d enc_id:%u pp_id:%u", __entry->id, + __entry->enc_id, __entry->pp_id) ); TRACE_EVENT(dpu_vbif_wait_xin_halt_fail, diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 8747fb32a106..0bdd93648761 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -207,62 +207,44 @@ u32 msm_readl(const void __iomem *addr) return val; } -struct vblank_event { - struct list_head node; +struct msm_vblank_work { + struct work_struct work; int crtc_id; bool enable; + struct msm_drm_private *priv; }; -static void vblank_ctrl_worker(struct kthread_work *work) +static void vblank_ctrl_worker(struct work_struct *work) { - struct msm_vblank_ctrl *vbl_ctrl = container_of(work, - struct msm_vblank_ctrl, work); - struct msm_drm_private *priv = container_of(vbl_ctrl, - struct msm_drm_private, vblank_ctrl); + struct msm_vblank_work *vbl_work = container_of(work, + struct msm_vblank_work, work); + struct msm_drm_private *priv = vbl_work->priv; struct msm_kms *kms = priv->kms; - struct vblank_event *vbl_ev, *tmp; - unsigned long flags; - - spin_lock_irqsave(&vbl_ctrl->lock, flags); - list_for_each_entry_safe(vbl_ev, tmp, &vbl_ctrl->event_list, node) { - list_del(&vbl_ev->node); - spin_unlock_irqrestore(&vbl_ctrl->lock, flags); - - if (vbl_ev->enable) - kms->funcs->enable_vblank(kms, - priv->crtcs[vbl_ev->crtc_id]); - else - kms->funcs->disable_vblank(kms, - priv->crtcs[vbl_ev->crtc_id]); - - kfree(vbl_ev); - spin_lock_irqsave(&vbl_ctrl->lock, flags); - } + if (vbl_work->enable) + kms->funcs->enable_vblank(kms, priv->crtcs[vbl_work->crtc_id]); + else + kms->funcs->disable_vblank(kms, priv->crtcs[vbl_work->crtc_id]); - spin_unlock_irqrestore(&vbl_ctrl->lock, flags); + kfree(vbl_work); } static int vblank_ctrl_queue_work(struct msm_drm_private *priv, int crtc_id, bool enable) { - struct msm_vblank_ctrl *vbl_ctrl = &priv->vblank_ctrl; - struct vblank_event *vbl_ev; - unsigned long flags; + struct msm_vblank_work *vbl_work; - vbl_ev = kzalloc(sizeof(*vbl_ev), GFP_ATOMIC); - if (!vbl_ev) + vbl_work = kzalloc(sizeof(*vbl_work), GFP_ATOMIC); + if (!vbl_work) return -ENOMEM; - vbl_ev->crtc_id = crtc_id; - vbl_ev->enable = enable; + INIT_WORK(&vbl_work->work, vblank_ctrl_worker); - spin_lock_irqsave(&vbl_ctrl->lock, flags); - list_add_tail(&vbl_ev->node, &vbl_ctrl->event_list); - spin_unlock_irqrestore(&vbl_ctrl->lock, flags); + vbl_work->crtc_id = crtc_id; + vbl_work->enable = enable; + vbl_work->priv = priv; - kthread_queue_work(&priv->disp_thread[crtc_id].worker, - &vbl_ctrl->work); + queue_work(priv->wq, &vbl_work->work); return 0; } @@ -274,31 +256,20 @@ static int msm_drm_uninit(struct device *dev) struct msm_drm_private *priv = ddev->dev_private; struct msm_kms *kms = priv->kms; struct msm_mdss *mdss = priv->mdss; - struct msm_vblank_ctrl *vbl_ctrl = &priv->vblank_ctrl; - struct vblank_event *vbl_ev, *tmp; int i; /* We must cancel and cleanup any pending vblank enable/disable * work before drm_irq_uninstall() to avoid work re-enabling an * irq after uninstall has disabled it. */ - kthread_flush_work(&vbl_ctrl->work); - list_for_each_entry_safe(vbl_ev, tmp, &vbl_ctrl->event_list, node) { - list_del(&vbl_ev->node); - kfree(vbl_ev); - } - /* clean up display commit/event worker threads */ - for (i = 0; i < priv->num_crtcs; i++) { - if (priv->disp_thread[i].thread) { - kthread_flush_worker(&priv->disp_thread[i].worker); - kthread_stop(priv->disp_thread[i].thread); - priv->disp_thread[i].thread = NULL; - } + flush_workqueue(priv->wq); + destroy_workqueue(priv->wq); + /* clean up event worker threads */ + for (i = 0; i < priv->num_crtcs; i++) { if (priv->event_thread[i].thread) { - kthread_flush_worker(&priv->event_thread[i].worker); - kthread_stop(priv->event_thread[i].thread); + kthread_destroy_worker(&priv->event_thread[i].worker); priv->event_thread[i].thread = NULL; } } @@ -323,9 +294,6 @@ static int msm_drm_uninit(struct device *dev) drm_irq_uninstall(ddev); pm_runtime_put_sync(dev); - flush_workqueue(priv->wq); - destroy_workqueue(priv->wq); - if (kms && kms->funcs) kms->funcs->destroy(kms); @@ -490,9 +458,6 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv) priv->wq = alloc_ordered_workqueue("msm", 0); INIT_LIST_HEAD(&priv->inactive_list); - INIT_LIST_HEAD(&priv->vblank_ctrl.event_list); - kthread_init_work(&priv->vblank_ctrl.work, vblank_ctrl_worker); - spin_lock_init(&priv->vblank_ctrl.lock); drm_mode_config_init(ddev); @@ -554,27 +519,6 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv) */ param.sched_priority = 16; for (i = 0; i < priv->num_crtcs; i++) { - - /* initialize display thread */ - priv->disp_thread[i].crtc_id = priv->crtcs[i]->base.id; - kthread_init_worker(&priv->disp_thread[i].worker); - priv->disp_thread[i].dev = ddev; - priv->disp_thread[i].thread = - kthread_run(kthread_worker_fn, - &priv->disp_thread[i].worker, - "crtc_commit:%d", priv->disp_thread[i].crtc_id); - if (IS_ERR(priv->disp_thread[i].thread)) { - DRM_DEV_ERROR(dev, "failed to create crtc_commit kthread\n"); - priv->disp_thread[i].thread = NULL; - goto err_msm_uninit; - } - - ret = sched_setscheduler(priv->disp_thread[i].thread, - SCHED_FIFO, ¶m); - if (ret) - dev_warn(dev, "disp_thread set priority failed: %d\n", - ret); - /* initialize event thread */ priv->event_thread[i].crtc_id = priv->crtcs[i]->base.id; kthread_init_worker(&priv->event_thread[i].worker); @@ -589,13 +533,6 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv) goto err_msm_uninit; } - /** - * event thread should also run at same priority as disp_thread - * because it is handling frame_done events. A lower priority - * event thread and higher priority disp_thread can causes - * frame_pending counters beyond 2. This can lead to commit - * failure at crtc commit level. - */ ret = sched_setscheduler(priv->event_thread[i].thread, SCHED_FIFO, ¶m); if (ret) @@ -914,8 +851,12 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data, ret = -EINVAL; break; } - ret = copy_from_user(msm_obj->name, - u64_to_user_ptr(args->value), args->len); + if (copy_from_user(msm_obj->name, u64_to_user_ptr(args->value), + args->len)) { + msm_obj->name[0] = '\0'; + ret = -EFAULT; + break; + } msm_obj->name[args->len] = '\0'; for (i = 0; i < args->len; i++) { if (!isprint(msm_obj->name[i])) { @@ -931,8 +872,9 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data, } args->len = strlen(msm_obj->name); if (args->value) { - ret = copy_to_user(u64_to_user_ptr(args->value), - msm_obj->name, args->len); + if (copy_to_user(u64_to_user_ptr(args->value), + msm_obj->name, args->len)) + ret = -EFAULT; } break; } diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 4e0c6c2f9a86..c56dade2c1dc 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -77,12 +77,6 @@ enum msm_mdp_plane_property { PLANE_PROP_MAX_NUM }; -struct msm_vblank_ctrl { - struct kthread_work work; - struct list_head event_list; - spinlock_t lock; -}; - #define MSM_GPU_MAX_RINGS 4 #define MAX_H_TILES_PER_DISPLAY 2 @@ -126,7 +120,7 @@ struct msm_display_topology { /** * struct msm_display_info - defines display properties - * @intf_type: DRM_MODE_CONNECTOR_ display type + * @intf_type: DRM_MODE_ENCODER_ type * @capabilities: Bitmask of display flags * @num_of_h_tiles: Number of horizontal tiles in case of split interface * @h_tile_instance: Controller instance used per tile. Number of elements is @@ -199,7 +193,6 @@ struct msm_drm_private { unsigned int num_crtcs; struct drm_crtc *crtcs[MAX_CRTCS]; - struct msm_drm_thread disp_thread[MAX_CRTCS]; struct msm_drm_thread event_thread[MAX_CRTCS]; unsigned int num_encoders; @@ -228,7 +221,6 @@ struct msm_drm_private { struct notifier_block vmap_notifier; struct shrinker shrinker; - struct msm_vblank_ctrl vblank_ctrl; struct drm_atomic_state *pm_state; }; @@ -250,7 +242,8 @@ void msm_gem_purge_vma(struct msm_gem_address_space *aspace, void msm_gem_unmap_vma(struct msm_gem_address_space *aspace, struct msm_gem_vma *vma); int msm_gem_map_vma(struct msm_gem_address_space *aspace, - struct msm_gem_vma *vma, struct sg_table *sgt, int npages); + struct msm_gem_vma *vma, int prot, + struct sg_table *sgt, int npages); void msm_gem_close_vma(struct msm_gem_address_space *aspace, struct msm_gem_vma *vma); @@ -333,6 +326,7 @@ void msm_gem_kernel_put(struct drm_gem_object *bo, struct drm_gem_object *msm_gem_import(struct drm_device *dev, struct dma_buf *dmabuf, struct sg_table *sgt); +__printf(2, 3) void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...); int msm_framebuffer_prepare(struct drm_framebuffer *fb, @@ -396,12 +390,14 @@ void msm_framebuffer_describe(struct drm_framebuffer *fb, struct seq_file *m); int msm_debugfs_late_init(struct drm_device *dev); int msm_rd_debugfs_init(struct drm_minor *minor); void msm_rd_debugfs_cleanup(struct msm_drm_private *priv); +__printf(3, 4) void msm_rd_dump_submit(struct msm_rd_state *rd, struct msm_gem_submit *submit, const char *fmt, ...); int msm_perf_debugfs_init(struct drm_minor *minor); void msm_perf_debugfs_cleanup(struct msm_drm_private *priv); #else static inline int msm_debugfs_late_init(struct drm_device *dev) { return 0; } +__printf(3, 4) static inline void msm_rd_dump_submit(struct msm_rd_state *rd, struct msm_gem_submit *submit, const char *fmt, ...) {} static inline void msm_rd_debugfs_cleanup(struct msm_drm_private *priv) {} diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 20c979a7fc9c..18ca651ab942 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -391,6 +391,10 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; struct page **pages; + int prot = IOMMU_READ; + + if (!(msm_obj->flags & MSM_BO_GPU_READONLY)) + prot |= IOMMU_WRITE; WARN_ON(!mutex_is_locked(&msm_obj->lock)); @@ -405,8 +409,8 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, if (IS_ERR(pages)) return PTR_ERR(pages); - return msm_gem_map_vma(aspace, vma, msm_obj->sgt, - obj->size >> PAGE_SHIFT); + return msm_gem_map_vma(aspace, vma, prot, + msm_obj->sgt, obj->size >> PAGE_SHIFT); } /* get iova and pin it. Should have a matching put */ diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index 557360788084..49c04829cf34 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -68,7 +68,8 @@ void msm_gem_unmap_vma(struct msm_gem_address_space *aspace, int msm_gem_map_vma(struct msm_gem_address_space *aspace, - struct msm_gem_vma *vma, struct sg_table *sgt, int npages) + struct msm_gem_vma *vma, int prot, + struct sg_table *sgt, int npages) { unsigned size = npages << PAGE_SHIFT; int ret = 0; @@ -86,7 +87,7 @@ msm_gem_map_vma(struct msm_gem_address_space *aspace, if (aspace->mmu) ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt, - size, IOMMU_READ | IOMMU_WRITE); + size, prot); if (ret) vma->mapped = false; diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 5f3eff304355..10babd18e286 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -900,7 +900,7 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, } /* Get Interrupt: */ - gpu->irq = platform_get_irq_byname(pdev, config->irqname); + gpu->irq = platform_get_irq(pdev, 0); if (gpu->irq < 0) { ret = gpu->irq; DRM_DEV_ERROR(drm->dev, "failed to get irq: %d\n", ret); diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index efb49bb64191..ca17086f72c9 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -31,7 +31,6 @@ struct msm_gpu_state; struct msm_gpu_config { const char *ioname; - const char *irqname; uint64_t va_start; uint64_t va_end; unsigned int nr_rings; @@ -63,7 +62,7 @@ struct msm_gpu_funcs { struct msm_ringbuffer *(*active_ring)(struct msm_gpu *gpu); void (*recover)(struct msm_gpu *gpu); void (*destroy)(struct msm_gpu *gpu); -#ifdef CONFIG_DEBUG_FS +#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) /* show GPU status in debugfs: */ void (*show)(struct msm_gpu *gpu, struct msm_gpu_state *state, struct drm_printer *p); diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c index 90e9d0a48dc0..d21172933d92 100644 --- a/drivers/gpu/drm/msm/msm_rd.c +++ b/drivers/gpu/drm/msm/msm_rd.c @@ -115,7 +115,9 @@ static void rd_write(struct msm_rd_state *rd, const void *buf, int sz) char *fptr = &fifo->buf[fifo->head]; int n; - wait_event(rd->fifo_event, circ_space(&rd->fifo) > 0); + wait_event(rd->fifo_event, circ_space(&rd->fifo) > 0 || !rd->open); + if (!rd->open) + return; /* Note that smp_load_acquire() is not strictly required * as CIRC_SPACE_TO_END() does not access the tail more @@ -213,7 +215,10 @@ out: static int rd_release(struct inode *inode, struct file *file) { struct msm_rd_state *rd = inode->i_private; + rd->open = false; + wake_up_all(&rd->fifo_event); + return 0; } diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index b17843dd050d..581404e6544d 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -30,6 +30,8 @@ nouveau-y += nouveau_vga.o # DRM - memory management nouveau-y += nouveau_bo.o nouveau-y += nouveau_gem.o +nouveau-$(CONFIG_DRM_NOUVEAU_SVM) += nouveau_svm.o +nouveau-$(CONFIG_DRM_NOUVEAU_SVM) += nouveau_dmem.o nouveau-y += nouveau_mem.o nouveau-y += nouveau_prime.o nouveau-y += nouveau_sgdma.o diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig index 432c440223bb..00cd9ab8948d 100644 --- a/drivers/gpu/drm/nouveau/Kconfig +++ b/drivers/gpu/drm/nouveau/Kconfig @@ -71,3 +71,15 @@ config DRM_NOUVEAU_BACKLIGHT help Say Y here if you want to control the backlight of your display (e.g. a laptop panel). + +config DRM_NOUVEAU_SVM + bool "(EXPERIMENTAL) Enable SVM (Shared Virtual Memory) support" + depends on ARCH_HAS_HMM + depends on DRM_NOUVEAU + depends on STAGING + select HMM_MIRROR + select DEVICE_PRIVATE + default n + help + Say Y here if you want to enable experimental support for + Shared Virtual Memory (SVM). diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c index 2c569e264df3..f22f01020625 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c +++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c @@ -40,6 +40,7 @@ #include "nvreg.h" #include "nouveau_fbcon.h" #include "disp.h" +#include "nouveau_dma.h" #include <subdev/bios/pll.h> #include <subdev/clk.h> @@ -1077,12 +1078,223 @@ nouveau_crtc_set_config(struct drm_mode_set *set, return ret; } +struct nv04_page_flip_state { + struct list_head head; + struct drm_pending_vblank_event *event; + struct drm_crtc *crtc; + int bpp, pitch; + u64 offset; +}; + +static int +nv04_finish_page_flip(struct nouveau_channel *chan, + struct nv04_page_flip_state *ps) +{ + struct nouveau_fence_chan *fctx = chan->fence; + struct nouveau_drm *drm = chan->drm; + struct drm_device *dev = drm->dev; + struct nv04_page_flip_state *s; + unsigned long flags; + + spin_lock_irqsave(&dev->event_lock, flags); + + if (list_empty(&fctx->flip)) { + NV_ERROR(drm, "unexpected pageflip\n"); + spin_unlock_irqrestore(&dev->event_lock, flags); + return -EINVAL; + } + + s = list_first_entry(&fctx->flip, struct nv04_page_flip_state, head); + if (s->event) { + drm_crtc_arm_vblank_event(s->crtc, s->event); + } else { + /* Give up ownership of vblank for page-flipped crtc */ + drm_crtc_vblank_put(s->crtc); + } + + list_del(&s->head); + if (ps) + *ps = *s; + kfree(s); + + spin_unlock_irqrestore(&dev->event_lock, flags); + return 0; +} + +int +nv04_flip_complete(struct nvif_notify *notify) +{ + struct nouveau_cli *cli = (void *)notify->object->client; + struct nouveau_drm *drm = cli->drm; + struct nouveau_channel *chan = drm->channel; + struct nv04_page_flip_state state; + + if (!nv04_finish_page_flip(chan, &state)) { + nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc), + state.offset + state.crtc->y * + state.pitch + state.crtc->x * + state.bpp / 8); + } + + return NVIF_NOTIFY_KEEP; +} + +static int +nv04_page_flip_emit(struct nouveau_channel *chan, + struct nouveau_bo *old_bo, + struct nouveau_bo *new_bo, + struct nv04_page_flip_state *s, + struct nouveau_fence **pfence) +{ + struct nouveau_fence_chan *fctx = chan->fence; + struct nouveau_drm *drm = chan->drm; + struct drm_device *dev = drm->dev; + unsigned long flags; + int ret; + + /* Queue it to the pending list */ + spin_lock_irqsave(&dev->event_lock, flags); + list_add_tail(&s->head, &fctx->flip); + spin_unlock_irqrestore(&dev->event_lock, flags); + + /* Synchronize with the old framebuffer */ + ret = nouveau_fence_sync(old_bo, chan, false, false); + if (ret) + goto fail; + + /* Emit the pageflip */ + ret = RING_SPACE(chan, 2); + if (ret) + goto fail; + + BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1); + OUT_RING (chan, 0x00000000); + FIRE_RING (chan); + + ret = nouveau_fence_new(chan, false, pfence); + if (ret) + goto fail; + + return 0; +fail: + spin_lock_irqsave(&dev->event_lock, flags); + list_del(&s->head); + spin_unlock_irqrestore(&dev->event_lock, flags); + return ret; +} + +static int +nv04_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, + struct drm_pending_vblank_event *event, u32 flags, + struct drm_modeset_acquire_ctx *ctx) +{ + const int swap_interval = (flags & DRM_MODE_PAGE_FLIP_ASYNC) ? 0 : 1; + struct drm_device *dev = crtc->dev; + struct nouveau_drm *drm = nouveau_drm(dev); + struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->primary->fb)->nvbo; + struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo; + struct nv04_page_flip_state *s; + struct nouveau_channel *chan; + struct nouveau_cli *cli; + struct nouveau_fence *fence; + struct nv04_display *dispnv04 = nv04_display(dev); + int head = nouveau_crtc(crtc)->index; + int ret; + + chan = drm->channel; + if (!chan) + return -ENODEV; + cli = (void *)chan->user.client; + + s = kzalloc(sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + + if (new_bo != old_bo) { + ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM, true); + if (ret) + goto fail_free; + } + + mutex_lock(&cli->mutex); + ret = ttm_bo_reserve(&new_bo->bo, true, false, NULL); + if (ret) + goto fail_unpin; + + /* synchronise rendering channel with the kernel's channel */ + ret = nouveau_fence_sync(new_bo, chan, false, true); + if (ret) { + ttm_bo_unreserve(&new_bo->bo); + goto fail_unpin; + } + + if (new_bo != old_bo) { + ttm_bo_unreserve(&new_bo->bo); + + ret = ttm_bo_reserve(&old_bo->bo, true, false, NULL); + if (ret) + goto fail_unpin; + } + + /* Initialize a page flip struct */ + *s = (struct nv04_page_flip_state) + { { }, event, crtc, fb->format->cpp[0] * 8, fb->pitches[0], + new_bo->bo.offset }; + + /* Keep vblanks on during flip, for the target crtc of this flip */ + drm_crtc_vblank_get(crtc); + + /* Emit a page flip */ + if (swap_interval) { + ret = RING_SPACE(chan, 8); + if (ret) + goto fail_unreserve; + + BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1); + OUT_RING (chan, 0); + BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1); + OUT_RING (chan, head); + BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1); + OUT_RING (chan, 0); + BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1); + OUT_RING (chan, 0); + } + + nouveau_bo_ref(new_bo, &dispnv04->image[head]); + + ret = nv04_page_flip_emit(chan, old_bo, new_bo, s, &fence); + if (ret) + goto fail_unreserve; + mutex_unlock(&cli->mutex); + + /* Update the crtc struct and cleanup */ + crtc->primary->fb = fb; + + nouveau_bo_fence(old_bo, fence, false); + ttm_bo_unreserve(&old_bo->bo); + if (old_bo != new_bo) + nouveau_bo_unpin(old_bo); + nouveau_fence_unref(&fence); + return 0; + +fail_unreserve: + drm_crtc_vblank_put(crtc); + ttm_bo_unreserve(&old_bo->bo); +fail_unpin: + mutex_unlock(&cli->mutex); + if (old_bo != new_bo) + nouveau_bo_unpin(new_bo); +fail_free: + kfree(s); + return ret; +} + static const struct drm_crtc_funcs nv04_crtc_funcs = { .cursor_set = nv04_crtc_cursor_set, .cursor_move = nv04_crtc_cursor_move, .gamma_set = nv_crtc_gamma_set, .set_config = nouveau_crtc_set_config, - .page_flip = nouveau_crtc_page_flip, + .page_flip = nv04_crtc_page_flip, .destroy = nv_crtc_destroy, }; diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c index 1727d399833c..5713bacaee80 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c @@ -30,6 +30,160 @@ #include "hw.h" #include "nouveau_encoder.h" #include "nouveau_connector.h" +#include "nouveau_bo.h" + +#include <nvif/if0004.h> + +static void +nv04_display_fini(struct drm_device *dev, bool suspend) +{ + struct nv04_display *disp = nv04_display(dev); + struct drm_crtc *crtc; + + /* Disable flip completion events. */ + nvif_notify_put(&disp->flip); + + /* Disable vblank interrupts. */ + NVWriteCRTC(dev, 0, NV_PCRTC_INTR_EN_0, 0); + if (nv_two_heads(dev)) + NVWriteCRTC(dev, 1, NV_PCRTC_INTR_EN_0, 0); + + if (!suspend) + return; + + /* Un-pin FB and cursors so they'll be evicted to system memory. */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_framebuffer *nouveau_fb; + + nouveau_fb = nouveau_framebuffer(crtc->primary->fb); + if (!nouveau_fb || !nouveau_fb->nvbo) + continue; + + nouveau_bo_unpin(nouveau_fb->nvbo); + } + + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); + if (nv_crtc->cursor.nvbo) { + if (nv_crtc->cursor.set_offset) + nouveau_bo_unmap(nv_crtc->cursor.nvbo); + nouveau_bo_unpin(nv_crtc->cursor.nvbo); + } + } +} + +static int +nv04_display_init(struct drm_device *dev, bool resume, bool runtime) +{ + struct nv04_display *disp = nv04_display(dev); + struct nouveau_drm *drm = nouveau_drm(dev); + struct nouveau_encoder *encoder; + struct drm_crtc *crtc; + int ret; + + /* meh.. modeset apparently doesn't setup all the regs and depends + * on pre-existing state, for now load the state of the card *before* + * nouveau was loaded, and then do a modeset. + * + * best thing to do probably is to make save/restore routines not + * save/restore "pre-load" state, but more general so we can save + * on suspend too. + */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); + nv_crtc->save(&nv_crtc->base); + } + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head) + encoder->enc_save(&encoder->base.base); + + /* Enable flip completion events. */ + nvif_notify_get(&disp->flip); + + if (!resume) + return 0; + + /* Re-pin FB/cursors. */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_framebuffer *nouveau_fb; + + nouveau_fb = nouveau_framebuffer(crtc->primary->fb); + if (!nouveau_fb || !nouveau_fb->nvbo) + continue; + + ret = nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM, true); + if (ret) + NV_ERROR(drm, "Could not pin framebuffer\n"); + } + + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); + if (!nv_crtc->cursor.nvbo) + continue; + + ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM, true); + if (!ret && nv_crtc->cursor.set_offset) + ret = nouveau_bo_map(nv_crtc->cursor.nvbo); + if (ret) + NV_ERROR(drm, "Could not pin/map cursor.\n"); + } + + /* Force CLUT to get re-loaded during modeset. */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); + + nv_crtc->lut.depth = 0; + } + + /* This should ensure we don't hit a locking problem when someone + * wakes us up via a connector. We should never go into suspend + * while the display is on anyways. + */ + if (runtime) + return 0; + + /* Restore mode. */ + drm_helper_resume_force_mode(dev); + + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); + + if (!nv_crtc->cursor.nvbo) + continue; + + if (nv_crtc->cursor.set_offset) + nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset); + nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x, + nv_crtc->cursor_saved_y); + } + + return 0; +} + +static void +nv04_display_destroy(struct drm_device *dev) +{ + struct nv04_display *disp = nv04_display(dev); + struct nouveau_drm *drm = nouveau_drm(dev); + struct nouveau_encoder *encoder; + struct nouveau_crtc *nv_crtc; + + /* Restore state */ + list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head) + encoder->enc_restore(&encoder->base.base); + + list_for_each_entry(nv_crtc, &dev->mode_config.crtc_list, base.head) + nv_crtc->restore(&nv_crtc->base); + + nouveau_hw_save_vga_fonts(dev, 0); + + nvif_notify_fini(&disp->flip); + + nouveau_display(dev)->priv = NULL; + kfree(disp); + + nvif_object_unmap(&drm->client.device.object); +} int nv04_display_create(struct drm_device *dev) @@ -58,6 +212,13 @@ nv04_display_create(struct drm_device *dev) /* Pre-nv50 doesn't support atomic, so don't expose the ioctls */ dev->driver->driver_features &= ~DRIVER_ATOMIC; + /* Request page flip completion event. */ + if (drm->nvsw.client) { + nvif_notify_init(&drm->nvsw, nv04_flip_complete, + false, NV04_NVSW_NTFY_UEVENT, + NULL, 0, 0, &disp->flip); + } + nouveau_hw_save_vga_fonts(dev, 1); nv04_crtc_create(dev, 0); @@ -121,58 +282,3 @@ nv04_display_create(struct drm_device *dev) return 0; } - -void -nv04_display_destroy(struct drm_device *dev) -{ - struct nv04_display *disp = nv04_display(dev); - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_encoder *encoder; - struct nouveau_crtc *nv_crtc; - - /* Restore state */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head) - encoder->enc_restore(&encoder->base.base); - - list_for_each_entry(nv_crtc, &dev->mode_config.crtc_list, base.head) - nv_crtc->restore(&nv_crtc->base); - - nouveau_hw_save_vga_fonts(dev, 0); - - nouveau_display(dev)->priv = NULL; - kfree(disp); - - nvif_object_unmap(&drm->client.device.object); -} - -int -nv04_display_init(struct drm_device *dev) -{ - struct nouveau_encoder *encoder; - struct nouveau_crtc *crtc; - - /* meh.. modeset apparently doesn't setup all the regs and depends - * on pre-existing state, for now load the state of the card *before* - * nouveau was loaded, and then do a modeset. - * - * best thing to do probably is to make save/restore routines not - * save/restore "pre-load" state, but more general so we can save - * on suspend too. - */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) - crtc->save(&crtc->base); - - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head) - encoder->enc_save(&encoder->base.base); - - return 0; -} - -void -nv04_display_fini(struct drm_device *dev) -{ - /* disable vblank interrupts */ - NVWriteCRTC(dev, 0, NV_PCRTC_INTR_EN_0, 0); - if (nv_two_heads(dev)) - NVWriteCRTC(dev, 1, NV_PCRTC_INTR_EN_0, 0); -} diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.h b/drivers/gpu/drm/nouveau/dispnv04/disp.h index f74f1f2b186e..c6ed20a09f4a 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.h +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.h @@ -82,6 +82,7 @@ struct nv04_display { uint32_t saved_vga_font[4][16384]; uint32_t dac_users[4]; struct nouveau_bo *image[2]; + struct nvif_notify flip; }; static inline struct nv04_display * @@ -92,9 +93,6 @@ nv04_display(struct drm_device *dev) /* nv04_display.c */ int nv04_display_create(struct drm_device *); -void nv04_display_destroy(struct drm_device *); -int nv04_display_init(struct drm_device *); -void nv04_display_fini(struct drm_device *); /* nv04_crtc.c */ int nv04_crtc_create(struct drm_device *, int index); @@ -176,4 +174,5 @@ nouveau_bios_run_init_table(struct drm_device *dev, u16 table, ); } +int nv04_flip_complete(struct nvif_notify *); #endif diff --git a/drivers/gpu/drm/nouveau/dispnv50/core.c b/drivers/gpu/drm/nouveau/dispnv50/core.c index c25e0ebe3c92..27ea3f34706d 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/core.c +++ b/drivers/gpu/drm/nouveau/dispnv50/core.c @@ -42,7 +42,7 @@ nv50_core_new(struct nouveau_drm *drm, struct nv50_core **pcore) int version; int (*new)(struct nouveau_drm *, s32, struct nv50_core **); } cores[] = { - { TU104_DISP_CORE_CHANNEL_DMA, 0, corec57d_new }, + { TU102_DISP_CORE_CHANNEL_DMA, 0, corec57d_new }, { GV100_DISP_CORE_CHANNEL_DMA, 0, corec37d_new }, { GP102_DISP_CORE_CHANNEL_DMA, 0, core917d_new }, { GP100_DISP_CORE_CHANNEL_DMA, 0, core917d_new }, diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs.c b/drivers/gpu/drm/nouveau/dispnv50/curs.c index cb6e4d2b1b45..121c24a18f11 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/curs.c +++ b/drivers/gpu/drm/nouveau/dispnv50/curs.c @@ -31,7 +31,7 @@ nv50_curs_new(struct nouveau_drm *drm, int head, struct nv50_wndw **pwndw) int version; int (*new)(struct nouveau_drm *, int, s32, struct nv50_wndw **); } curses[] = { - { TU104_DISP_CURSOR, 0, cursc37a_new }, + { TU102_DISP_CURSOR, 0, cursc37a_new }, { GV100_DISP_CURSOR, 0, cursc37a_new }, { GK104_DISP_CURSOR, 0, curs907a_new }, { GF110_DISP_CURSOR, 0, curs907a_new }, diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index e8bb35f6d015..4b1650f51955 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -817,7 +817,8 @@ nv50_msto_enable(struct drm_encoder *encoder) r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, armh->dp.pbn, armh->dp.tu); - WARN_ON(!r); + if (!r) + DRM_DEBUG_KMS("Failed to allocate VCPI\n"); if (!mstm->links++) nv50_outp_acquire(mstm->outp); @@ -2220,8 +2221,8 @@ nv50_disp_func = { * Init *****************************************************************************/ -void -nv50_display_fini(struct drm_device *dev) +static void +nv50_display_fini(struct drm_device *dev, bool suspend) { struct nouveau_encoder *nv_encoder; struct drm_encoder *encoder; @@ -2242,8 +2243,8 @@ nv50_display_fini(struct drm_device *dev) } } -int -nv50_display_init(struct drm_device *dev) +static int +nv50_display_init(struct drm_device *dev, bool resume, bool runtime) { struct nv50_core *core = nv50_disp(dev)->core; struct drm_encoder *encoder; @@ -2269,7 +2270,7 @@ nv50_display_init(struct drm_device *dev) return 0; } -void +static void nv50_display_destroy(struct drm_device *dev) { struct nv50_disp *disp = nv50_disp(dev); diff --git a/drivers/gpu/drm/nouveau/dispnv50/wimm.c b/drivers/gpu/drm/nouveau/dispnv50/wimm.c index bc9eeaf212ae..a1ac153d5e98 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wimm.c +++ b/drivers/gpu/drm/nouveau/dispnv50/wimm.c @@ -31,7 +31,7 @@ nv50_wimm_init(struct nouveau_drm *drm, struct nv50_wndw *wndw) int version; int (*init)(struct nouveau_drm *, s32, struct nv50_wndw *); } wimms[] = { - { TU104_DISP_WINDOW_IMM_CHANNEL_DMA, 0, wimmc37b_init }, + { TU102_DISP_WINDOW_IMM_CHANNEL_DMA, 0, wimmc37b_init }, { GV100_DISP_WINDOW_IMM_CHANNEL_DMA, 0, wimmc37b_init }, {} }; diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c index ba9eea2ff16b..b95181027b31 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c +++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c @@ -626,7 +626,7 @@ nv50_wndw_new(struct nouveau_drm *drm, enum drm_plane_type type, int index, int (*new)(struct nouveau_drm *, enum drm_plane_type, int, s32, struct nv50_wndw **); } wndws[] = { - { TU104_DISP_WINDOW_CHANNEL_DMA, 0, wndwc57e_new }, + { TU102_DISP_WINDOW_CHANNEL_DMA, 0, wndwc57e_new }, { GV100_DISP_WINDOW_CHANNEL_DMA, 0, wndwc37e_new }, {} }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 1d82cbf70cf4..7d556a1c92fa 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -54,6 +54,9 @@ #define VOLTA_USERMODE_A 0x0000c361 +#define MAXWELL_FAULT_BUFFER_A /* clb069.h */ 0x0000b069 +#define VOLTA_FAULT_BUFFER_A /* clb069.h */ 0x0000c369 + #define NV03_CHANNEL_DMA /* cl506b.h */ 0x0000006b #define NV10_CHANNEL_DMA /* cl506b.h */ 0x0000006e #define NV17_CHANNEL_DMA /* cl506b.h */ 0x0000176e @@ -84,7 +87,7 @@ #define GP100_DISP /* cl5070.h */ 0x00009770 #define GP102_DISP /* cl5070.h */ 0x00009870 #define GV100_DISP /* cl5070.h */ 0x0000c370 -#define TU104_DISP /* cl5070.h */ 0x0000c570 +#define TU102_DISP /* cl5070.h */ 0x0000c570 #define NV31_MPEG 0x00003174 #define G82_MPEG 0x00008274 @@ -97,7 +100,7 @@ #define GF110_DISP_CURSOR /* cl507a.h */ 0x0000907a #define GK104_DISP_CURSOR /* cl507a.h */ 0x0000917a #define GV100_DISP_CURSOR /* cl507a.h */ 0x0000c37a -#define TU104_DISP_CURSOR /* cl507a.h */ 0x0000c57a +#define TU102_DISP_CURSOR /* cl507a.h */ 0x0000c57a #define NV50_DISP_OVERLAY /* cl507b.h */ 0x0000507b #define G82_DISP_OVERLAY /* cl507b.h */ 0x0000827b @@ -106,7 +109,7 @@ #define GK104_DISP_OVERLAY /* cl507b.h */ 0x0000917b #define GV100_DISP_WINDOW_IMM_CHANNEL_DMA /* clc37b.h */ 0x0000c37b -#define TU104_DISP_WINDOW_IMM_CHANNEL_DMA /* clc37b.h */ 0x0000c57b +#define TU102_DISP_WINDOW_IMM_CHANNEL_DMA /* clc37b.h */ 0x0000c57b #define NV50_DISP_BASE_CHANNEL_DMA /* cl507c.h */ 0x0000507c #define G82_DISP_BASE_CHANNEL_DMA /* cl507c.h */ 0x0000827c @@ -129,7 +132,7 @@ #define GP100_DISP_CORE_CHANNEL_DMA /* cl507d.h */ 0x0000977d #define GP102_DISP_CORE_CHANNEL_DMA /* cl507d.h */ 0x0000987d #define GV100_DISP_CORE_CHANNEL_DMA /* cl507d.h */ 0x0000c37d -#define TU104_DISP_CORE_CHANNEL_DMA /* cl507d.h */ 0x0000c57d +#define TU102_DISP_CORE_CHANNEL_DMA /* cl507d.h */ 0x0000c57d #define NV50_DISP_OVERLAY_CHANNEL_DMA /* cl507e.h */ 0x0000507e #define G82_DISP_OVERLAY_CHANNEL_DMA /* cl507e.h */ 0x0000827e @@ -139,7 +142,7 @@ #define GK104_DISP_OVERLAY_CONTROL_DMA /* cl507e.h */ 0x0000917e #define GV100_DISP_WINDOW_CHANNEL_DMA /* clc37e.h */ 0x0000c37e -#define TU104_DISP_WINDOW_CHANNEL_DMA /* clc37e.h */ 0x0000c57e +#define TU102_DISP_WINDOW_CHANNEL_DMA /* clc37e.h */ 0x0000c57e #define NV50_TESLA 0x00005097 #define G82_TESLA 0x00008297 diff --git a/drivers/gpu/drm/nouveau/include/nvif/clb069.h b/drivers/gpu/drm/nouveau/include/nvif/clb069.h new file mode 100644 index 000000000000..eef5d0227bab --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/clb069.h @@ -0,0 +1,12 @@ +#ifndef __NVIF_CLB069_H__ +#define __NVIF_CLB069_H__ +struct nvif_clb069_v0 { + __u8 version; + __u8 pad01[3]; + __u32 entries; + __u32 get; + __u32 put; +}; + +#define NVB069_V0_NTFY_FAULT 0x00 +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000c.h b/drivers/gpu/drm/nouveau/include/nvif/if000c.h index 2928ecd989ad..d6dd40f21eed 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if000c.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if000c.h @@ -3,7 +3,8 @@ struct nvif_vmm_v0 { __u8 version; __u8 page_nr; - __u8 pad02[6]; + __u8 managed; + __u8 pad03[5]; __u64 addr; __u64 size; __u8 data[]; @@ -14,6 +15,9 @@ struct nvif_vmm_v0 { #define NVIF_VMM_V0_PUT 0x02 #define NVIF_VMM_V0_MAP 0x03 #define NVIF_VMM_V0_UNMAP 0x04 +#define NVIF_VMM_V0_PFNMAP 0x05 +#define NVIF_VMM_V0_PFNCLR 0x06 +#define NVIF_VMM_V0_MTHD(i) ((i) + 0x80) struct nvif_vmm_page_v0 { __u8 version; @@ -61,4 +65,28 @@ struct nvif_vmm_unmap_v0 { __u8 pad01[7]; __u64 addr; }; + +struct nvif_vmm_pfnmap_v0 { + __u8 version; + __u8 page; + __u8 pad02[6]; + __u64 addr; + __u64 size; +#define NVIF_VMM_PFNMAP_V0_ADDR 0xfffffffffffff000ULL +#define NVIF_VMM_PFNMAP_V0_ADDR_SHIFT 12 +#define NVIF_VMM_PFNMAP_V0_APER 0x00000000000000f0ULL +#define NVIF_VMM_PFNMAP_V0_HOST 0x0000000000000000ULL +#define NVIF_VMM_PFNMAP_V0_VRAM 0x0000000000000010ULL +#define NVIF_VMM_PFNMAP_V0_W 0x0000000000000002ULL +#define NVIF_VMM_PFNMAP_V0_V 0x0000000000000001ULL +#define NVIF_VMM_PFNMAP_V0_NONE 0x0000000000000000ULL + __u64 phys[]; +}; + +struct nvif_vmm_pfnclr_v0 { + __u8 version; + __u8 pad01[7]; + __u64 addr; + __u64 size; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h index 1d9c637859f3..4cabd613a280 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h @@ -6,6 +6,12 @@ struct gp100_vmm_vn { /* nvif_vmm_vX ... */ }; +struct gp100_vmm_v0 { + /* nvif_vmm_vX ... */ + __u8 version; + __u8 fault_replay; +}; + struct gp100_vmm_map_vn { /* nvif_vmm_map_vX ... */ }; @@ -18,4 +24,19 @@ struct gp100_vmm_map_v0 { __u8 priv; __u8 kind; }; + +#define GP100_VMM_VN_FAULT_REPLAY NVIF_VMM_V0_MTHD(0x00) +#define GP100_VMM_VN_FAULT_CANCEL NVIF_VMM_V0_MTHD(0x01) + +struct gp100_vmm_fault_replay_vn { +}; + +struct gp100_vmm_fault_cancel_v0 { + __u8 version; + __u8 hub; + __u8 gpc; + __u8 client; + __u8 pad04[4]; + __u64 inst; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/vmm.h b/drivers/gpu/drm/nouveau/include/nvif/vmm.h index c5db8a2e82df..79bf85d2f43a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/vmm.h +++ b/drivers/gpu/drm/nouveau/include/nvif/vmm.h @@ -30,8 +30,8 @@ struct nvif_vmm { int page_nr; }; -int nvif_vmm_init(struct nvif_mmu *, s32 oclass, u64 addr, u64 size, - void *argv, u32 argc, struct nvif_vmm *); +int nvif_vmm_init(struct nvif_mmu *, s32 oclass, bool managed, u64 addr, + u64 size, void *argv, u32 argc, struct nvif_vmm *); void nvif_vmm_fini(struct nvif_vmm *); int nvif_vmm_get(struct nvif_vmm *, enum nvif_vmm_get, bool sparse, u8 page, u8 align, u64 size, struct nvif_vma *); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index 72e4dc1f0236..642492344196 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -28,6 +28,7 @@ enum nvkm_devidx { NVKM_SUBDEV_ICCSENSE, NVKM_SUBDEV_THERM, NVKM_SUBDEV_CLK, + NVKM_SUBDEV_GSP, NVKM_SUBDEV_SECBOOT, NVKM_ENGINE_BSP, @@ -137,6 +138,7 @@ struct nvkm_device { struct nvkm_fb *fb; struct nvkm_fuse *fuse; struct nvkm_gpio *gpio; + struct nvkm_gsp *gsp; struct nvkm_i2c *i2c; struct nvkm_subdev *ibus; struct nvkm_iccsense *iccsense; @@ -209,6 +211,7 @@ struct nvkm_device_chip { int (*fb )(struct nvkm_device *, int idx, struct nvkm_fb **); int (*fuse )(struct nvkm_device *, int idx, struct nvkm_fuse **); int (*gpio )(struct nvkm_device *, int idx, struct nvkm_gpio **); + int (*gsp )(struct nvkm_device *, int idx, struct nvkm_gsp **); int (*i2c )(struct nvkm_device *, int idx, struct nvkm_i2c **); int (*ibus )(struct nvkm_device *, int idx, struct nvkm_subdev **); int (*iccsense)(struct nvkm_device *, int idx, struct nvkm_iccsense **); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h index 86abe76023c2..5f3650692e4d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h @@ -11,5 +11,5 @@ int gm200_ce_new(struct nvkm_device *, int, struct nvkm_engine **); int gp100_ce_new(struct nvkm_device *, int, struct nvkm_engine **); int gp102_ce_new(struct nvkm_device *, int, struct nvkm_engine **); int gv100_ce_new(struct nvkm_device *, int, struct nvkm_engine **); -int tu104_ce_new(struct nvkm_device *, int, struct nvkm_engine **); +int tu102_ce_new(struct nvkm_device *, int, struct nvkm_engine **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h index 5ca86e178bb9..3026b22d44fb 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h @@ -36,5 +36,5 @@ int gm200_disp_new(struct nvkm_device *, int, struct nvkm_disp **); int gp100_disp_new(struct nvkm_device *, int, struct nvkm_disp **); int gp102_disp_new(struct nvkm_device *, int, struct nvkm_disp **); int gv100_disp_new(struct nvkm_device *, int, struct nvkm_disp **); -int tu104_disp_new(struct nvkm_device *, int, struct nvkm_disp **); +int tu102_disp_new(struct nvkm_device *, int, struct nvkm_disp **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 3b2b685778eb..b7fc04dd1628 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -74,5 +74,5 @@ int gm20b_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); int gp100_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); int gp10b_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); int gv100_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); -int tu104_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); +int tu102_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h index ba1518ff8b66..1e924c7f7ba7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h @@ -10,6 +10,9 @@ struct nvkm_gr { u64 nvkm_gr_units(struct nvkm_gr *); int nvkm_gr_tlb_flush(struct nvkm_gr *); +int nvkm_gr_ctxsw_pause(struct nvkm_device *); +int nvkm_gr_ctxsw_resume(struct nvkm_device *); +u32 nvkm_gr_ctxsw_inst(struct nvkm_device *); int nv04_gr_new(struct nvkm_device *, int, struct nvkm_gr **); int nv10_gr_new(struct nvkm_device *, int, struct nvkm_gr **); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h index fe716859d4a9..b72a4844c5f7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h @@ -6,6 +6,8 @@ struct nvkm_nvdec { struct nvkm_engine engine; + u32 addr; + struct nvkm_falcon *falcon; }; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h index f7d89822b905..c93ad332461a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h @@ -5,10 +5,13 @@ struct nvkm_sec2 { struct nvkm_engine engine; + u32 addr; + struct nvkm_falcon *falcon; struct nvkm_msgqueue *queue; struct work_struct work; }; int gp102_sec2_new(struct nvkm_device *, int, struct nvkm_sec2 **); +int tu102_sec2_new(struct nvkm_device *, int, struct nvkm_sec2 **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index fd9d713b611c..da14486317ca 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -29,5 +29,5 @@ int gf100_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int gk20a_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int gm107_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int gm20b_bar_new(struct nvkm_device *, int, struct nvkm_bar **); -int tu104_bar_new(struct nvkm_device *, int, struct nvkm_bar **); +int tu102_bar_new(struct nvkm_device *, int, struct nvkm_bar **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h index 1b71812a790b..8ba982c2fdfb 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h @@ -31,5 +31,5 @@ int gf100_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); int gm107_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); int gm200_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); int gv100_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); -int tu104_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); +int tu102_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h index 127f48066026..97322f95b3ee 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h @@ -13,6 +13,8 @@ struct nvkm_fault { struct nvkm_event event; struct nvkm_notify nrpfb; + + struct nvkm_device_oclass user; }; struct nvkm_fault_data { @@ -30,5 +32,5 @@ struct nvkm_fault_data { int gp100_fault_new(struct nvkm_device *, int, struct nvkm_fault **); int gv100_fault_new(struct nvkm_device *, int, struct nvkm_fault **); -int tu104_fault_new(struct nvkm_device *, int, struct nvkm_fault **); +int tu102_fault_new(struct nvkm_device *, int, struct nvkm_fault **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h new file mode 100644 index 000000000000..4c672a5c4cd5 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h @@ -0,0 +1,14 @@ +#ifndef __NVKM_GSP_H__ +#define __NVKM_GSP_H__ +#define nvkm_gsp(p) container_of((p), struct nvkm_gsp, subdev) +#include <core/subdev.h> + +struct nvkm_gsp { + struct nvkm_subdev subdev; + u32 addr; + + struct nvkm_falcon *falcon; +}; + +int gv100_gsp_new(struct nvkm_device *, int, struct nvkm_gsp **); +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h index b66dedd8abb6..e38f4958dea2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h @@ -31,5 +31,5 @@ int gk104_mc_new(struct nvkm_device *, int, struct nvkm_mc **); int gk20a_mc_new(struct nvkm_device *, int, struct nvkm_mc **); int gp100_mc_new(struct nvkm_device *, int, struct nvkm_mc **); int gp10b_mc_new(struct nvkm_device *, int, struct nvkm_mc **); -int tu104_mc_new(struct nvkm_device *, int, struct nvkm_mc **); +int tu102_mc_new(struct nvkm_device *, int, struct nvkm_mc **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 0a0e064f22e5..28ade86f74c5 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -17,6 +17,7 @@ struct nvkm_vma { bool part:1; /* Region was split from an allocated region by map(). */ bool user:1; /* Region user-allocated. */ bool busy:1; /* Region busy (for temporarily preventing user access). */ + bool mapped:1; /* Region contains valid pages. */ struct nvkm_memory *memory; /* Memory currently mapped into VMA. */ struct nvkm_tags *tags; /* Compression tag reference. */ }; @@ -44,6 +45,8 @@ struct nvkm_vmm { dma_addr_t null; void *nullp; + + bool replay; }; int nvkm_vmm_new(struct nvkm_device *, u64 addr, u64 size, void *argv, u32 argc, @@ -63,6 +66,7 @@ struct nvkm_vmm_map { struct nvkm_mm_node *mem; struct scatterlist *sgl; dma_addr_t *dma; + u64 *pfn; u64 off; const struct nvkm_vmm_page *page; @@ -130,5 +134,5 @@ int gm20b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gp100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gp10b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gv100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); -int tu104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int tu102_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h index f7d3eb647e2e..2904e67d79d2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h @@ -9,6 +9,7 @@ struct nvkm_top { struct list_head device; }; +u32 nvkm_top_addr(struct nvkm_device *, enum nvkm_devidx); u32 nvkm_top_reset(struct nvkm_device *, enum nvkm_devidx); u32 nvkm_top_intr(struct nvkm_device *, u32 intr, u64 *subdevs); u32 nvkm_top_intr_mask(struct nvkm_device *, enum nvkm_devidx); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h index 8a0f85f5fc1a..6a765682fbfa 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h @@ -38,6 +38,7 @@ int nvkm_volt_set_id(struct nvkm_volt *, u8 id, u8 min_id, u8 temp, int nv40_volt_new(struct nvkm_device *, int, struct nvkm_volt **); int gf100_volt_new(struct nvkm_device *, int, struct nvkm_volt **); +int gf117_volt_new(struct nvkm_device *, int, struct nvkm_volt **); int gk104_volt_new(struct nvkm_device *, int, struct nvkm_volt **); int gk20a_volt_new(struct nvkm_device *, int, struct nvkm_volt **); int gm20b_volt_new(struct nvkm_device *, int, struct nvkm_volt **); diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index b06cdac8f3a2..c3fd5dd39ed9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -214,6 +214,7 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS) WARN_ON(1); break; } + break; case NOUVEAU_GETPARAM_FB_SIZE: getparam->value = drm->gem.vram_available; break; @@ -338,7 +339,8 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) goto done; if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_vma_new(chan->ntfy, &cli->vmm, &chan->ntfy_vma); + ret = nouveau_vma_new(chan->ntfy, chan->chan->vmm, + &chan->ntfy_vma); if (ret) goto done; } diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c index 5f5be6368aed..c7a94c94dbf3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_backlight.c +++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c @@ -253,6 +253,9 @@ nouveau_backlight_init(struct drm_connector *connector) case NV_DEVICE_INFO_V0_FERMI: case NV_DEVICE_INFO_V0_KEPLER: case NV_DEVICE_INFO_V0_MAXWELL: + case NV_DEVICE_INFO_V0_PASCAL: + case NV_DEVICE_INFO_V0_VOLTA: + case NV_DEVICE_INFO_V0_TURING: ret = nv50_backlight_init(nv_encoder, &props, &ops); break; default: diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 73eff52036d2..34a998012bf6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -194,7 +194,7 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, struct nouveau_drm *drm = cli->drm; struct nouveau_bo *nvbo; struct nvif_mmu *mmu = &cli->mmu; - struct nvif_vmm *vmm = &cli->vmm.vmm; + struct nvif_vmm *vmm = cli->svm.cli ? &cli->svm.vmm : &cli->vmm.vmm; size_t acc_size; int type = ttm_bo_type_device; int ret, i, pi = -1; @@ -1434,7 +1434,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || !mem->kind) /* untiled */ break; - /* fallthrough, tiled memory */ + /* fall through - tiled memory */ case TTM_PL_VRAM: reg->bus.offset = reg->start << PAGE_SHIFT; reg->bus.base = device->func->resource_addr(device, 1); diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 668afbc29c3e..282fd90b65e1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -42,6 +42,7 @@ #include "nouveau_fence.h" #include "nouveau_abi16.h" #include "nouveau_vmm.h" +#include "nouveau_svm.h" MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM"); int nouveau_vram_pushbuf; @@ -95,6 +96,10 @@ nouveau_channel_del(struct nouveau_channel **pchan) if (chan->fence) nouveau_fence(chan->drm)->context_del(chan); + + if (cli) + nouveau_svmm_part(chan->vmm->svmm, chan->inst); + nvif_object_fini(&chan->nvsw); nvif_object_fini(&chan->gart); nvif_object_fini(&chan->vram); @@ -130,6 +135,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, chan->device = device; chan->drm = drm; + chan->vmm = cli->svm.cli ? &cli->svm : &cli->vmm; atomic_set(&chan->killed, 0); /* allocate memory for dma push buffer */ @@ -157,7 +163,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, chan->push.addr = chan->push.buffer->bo.offset; if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_vma_new(chan->push.buffer, &cli->vmm, + ret = nouveau_vma_new(chan->push.buffer, chan->vmm, &chan->push.vma); if (ret) { nouveau_channel_del(pchan); @@ -172,7 +178,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vmm.vmm.limit - 1; + args.limit = chan->vmm->vmm.limit - 1; } else if (chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM) { if (device->info.family == NV_DEVICE_INFO_V0_TNT) { @@ -202,7 +208,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_RDWR; args.start = 0; - args.limit = cli->vmm.vmm.limit - 1; + args.limit = chan->vmm->vmm.limit - 1; } } @@ -220,7 +226,6 @@ static int nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, u64 runlist, bool priv, struct nouveau_channel **pchan) { - struct nouveau_cli *cli = (void *)device->object.client; static const u16 oclasses[] = { TURING_CHANNEL_GPFIFO_A, VOLTA_CHANNEL_GPFIFO_A, PASCAL_CHANNEL_GPFIFO_A, @@ -255,7 +260,7 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, args.volta.ilength = 0x02000; args.volta.ioffset = 0x10000 + chan->push.addr; args.volta.runlist = runlist; - args.volta.vmm = nvif_handle(&cli->vmm.vmm.object); + args.volta.vmm = nvif_handle(&chan->vmm->vmm.object); args.volta.priv = priv; size = sizeof(args.volta); } else @@ -264,7 +269,7 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, args.kepler.ilength = 0x02000; args.kepler.ioffset = 0x10000 + chan->push.addr; args.kepler.runlist = runlist; - args.kepler.vmm = nvif_handle(&cli->vmm.vmm.object); + args.kepler.vmm = nvif_handle(&chan->vmm->vmm.object); args.kepler.priv = priv; size = sizeof(args.kepler); } else @@ -272,14 +277,14 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, args.fermi.version = 0; args.fermi.ilength = 0x02000; args.fermi.ioffset = 0x10000 + chan->push.addr; - args.fermi.vmm = nvif_handle(&cli->vmm.vmm.object); + args.fermi.vmm = nvif_handle(&chan->vmm->vmm.object); size = sizeof(args.fermi); } else { args.nv50.version = 0; args.nv50.ilength = 0x02000; args.nv50.ioffset = 0x10000 + chan->push.addr; args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); - args.nv50.vmm = nvif_handle(&cli->vmm.vmm.object); + args.nv50.vmm = nvif_handle(&chan->vmm->vmm.object); size = sizeof(args.nv50); } @@ -350,7 +355,6 @@ static int nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) { struct nvif_device *device = chan->device; - struct nouveau_cli *cli = (void *)chan->user.client; struct nouveau_drm *drm = chan->drm; struct nv_dma_v0 args = {}; int ret, i; @@ -376,7 +380,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vmm.vmm.limit - 1; + args.limit = chan->vmm->vmm.limit - 1; } else { args.target = NV_DMA_V0_TARGET_VRAM; args.access = NV_DMA_V0_ACCESS_RDWR; @@ -393,7 +397,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vmm.vmm.limit - 1; + args.limit = chan->vmm->vmm.limit - 1; } else if (chan->drm->agp.bridge) { args.target = NV_DMA_V0_TARGET_AGP; @@ -405,7 +409,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_RDWR; args.start = 0; - args.limit = cli->vmm.vmm.limit - 1; + args.limit = chan->vmm->vmm.limit - 1; } ret = nvif_object_init(&chan->user, gart, NV_DMA_IN_MEMORY, @@ -495,6 +499,10 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, nouveau_channel_del(pchan); } + ret = nouveau_svmm_join((*pchan)->vmm->svmm, (*pchan)->inst); + if (ret) + nouveau_channel_del(pchan); + done: cli->base.super = super; return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 28418f4e5748..93814d1d31e4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -8,6 +8,7 @@ struct nvif_device; struct nouveau_channel { struct nvif_device *device; struct nouveau_drm *drm; + struct nouveau_vmm *vmm; int chid; u64 inst; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 56b6ac1b8edd..55c0fa451163 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -32,18 +32,13 @@ #include <drm/drm_fb_helper.h> #include <drm/drm_probe_helper.h> -#include <nvif/class.h> - #include "nouveau_fbcon.h" -#include "dispnv04/hw.h" #include "nouveau_crtc.h" -#include "nouveau_dma.h" #include "nouveau_gem.h" #include "nouveau_connector.h" #include "nv50_display.h" -#include "nouveau_fence.h" - +#include <nvif/class.h> #include <nvif/cl0046.h> #include <nvif/event.h> @@ -412,15 +407,14 @@ nouveau_display_acpi_ntfy(struct notifier_block *nb, unsigned long val, #endif int -nouveau_display_init(struct drm_device *dev) +nouveau_display_init(struct drm_device *dev, bool resume, bool runtime) { struct nouveau_display *disp = nouveau_display(dev); - struct nouveau_drm *drm = nouveau_drm(dev); struct drm_connector *connector; struct drm_connector_list_iter conn_iter; int ret; - ret = disp->init(dev); + ret = disp->init(dev, resume, runtime); if (ret) return ret; @@ -437,8 +431,6 @@ nouveau_display_init(struct drm_device *dev) } drm_connector_list_iter_end(&conn_iter); - /* enable flip completion events */ - nvif_notify_get(&drm->flip); return ret; } @@ -457,9 +449,6 @@ nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime) drm_helper_force_disable_all(dev); } - /* disable flip completion events */ - nvif_notify_put(&drm->flip); - /* disable hotplug interrupts */ drm_connector_list_iter_begin(dev, &conn_iter); nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) { @@ -472,7 +461,7 @@ nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime) cancel_work_sync(&drm->hpd_work); drm_kms_helper_poll_disable(dev); - disp->fini(dev); + disp->fini(dev, suspend); } static void @@ -625,7 +614,6 @@ int nouveau_display_suspend(struct drm_device *dev, bool runtime) { struct nouveau_display *disp = nouveau_display(dev); - struct drm_crtc *crtc; if (drm_drv_uses_atomic_modeset(dev)) { if (!runtime) { @@ -636,32 +624,9 @@ nouveau_display_suspend(struct drm_device *dev, bool runtime) return ret; } } - - nouveau_display_fini(dev, true, runtime); - return 0; } nouveau_display_fini(dev, true, runtime); - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_framebuffer *nouveau_fb; - - nouveau_fb = nouveau_framebuffer(crtc->primary->fb); - if (!nouveau_fb || !nouveau_fb->nvbo) - continue; - - nouveau_bo_unpin(nouveau_fb->nvbo); - } - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - if (nv_crtc->cursor.nvbo) { - if (nv_crtc->cursor.set_offset) - nouveau_bo_unmap(nv_crtc->cursor.nvbo); - nouveau_bo_unpin(nv_crtc->cursor.nvbo); - } - } - return 0; } @@ -669,275 +634,16 @@ void nouveau_display_resume(struct drm_device *dev, bool runtime) { struct nouveau_display *disp = nouveau_display(dev); - struct nouveau_drm *drm = nouveau_drm(dev); - struct drm_crtc *crtc; - int ret; + + nouveau_display_init(dev, true, runtime); if (drm_drv_uses_atomic_modeset(dev)) { - nouveau_display_init(dev); if (disp->suspend) { drm_atomic_helper_resume(dev, disp->suspend); disp->suspend = NULL; } return; } - - /* re-pin fb/cursors */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_framebuffer *nouveau_fb; - - nouveau_fb = nouveau_framebuffer(crtc->primary->fb); - if (!nouveau_fb || !nouveau_fb->nvbo) - continue; - - ret = nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM, true); - if (ret) - NV_ERROR(drm, "Could not pin framebuffer\n"); - } - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - if (!nv_crtc->cursor.nvbo) - continue; - - ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM, true); - if (!ret && nv_crtc->cursor.set_offset) - ret = nouveau_bo_map(nv_crtc->cursor.nvbo); - if (ret) - NV_ERROR(drm, "Could not pin/map cursor.\n"); - } - - nouveau_display_init(dev); - - /* Force CLUT to get re-loaded during modeset */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - - nv_crtc->lut.depth = 0; - } - - /* This should ensure we don't hit a locking problem when someone - * wakes us up via a connector. We should never go into suspend - * while the display is on anyways. - */ - if (runtime) - return; - - drm_helper_resume_force_mode(dev); - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - - if (!nv_crtc->cursor.nvbo) - continue; - - if (nv_crtc->cursor.set_offset) - nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset); - nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x, - nv_crtc->cursor_saved_y); - } -} - -static int -nouveau_page_flip_emit(struct nouveau_channel *chan, - struct nouveau_bo *old_bo, - struct nouveau_bo *new_bo, - struct nouveau_page_flip_state *s, - struct nouveau_fence **pfence) -{ - struct nouveau_fence_chan *fctx = chan->fence; - struct nouveau_drm *drm = chan->drm; - struct drm_device *dev = drm->dev; - unsigned long flags; - int ret; - - /* Queue it to the pending list */ - spin_lock_irqsave(&dev->event_lock, flags); - list_add_tail(&s->head, &fctx->flip); - spin_unlock_irqrestore(&dev->event_lock, flags); - - /* Synchronize with the old framebuffer */ - ret = nouveau_fence_sync(old_bo, chan, false, false); - if (ret) - goto fail; - - /* Emit the pageflip */ - ret = RING_SPACE(chan, 2); - if (ret) - goto fail; - - BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1); - OUT_RING (chan, 0x00000000); - FIRE_RING (chan); - - ret = nouveau_fence_new(chan, false, pfence); - if (ret) - goto fail; - - return 0; -fail: - spin_lock_irqsave(&dev->event_lock, flags); - list_del(&s->head); - spin_unlock_irqrestore(&dev->event_lock, flags); - return ret; -} - -int -nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, - struct drm_pending_vblank_event *event, u32 flags, - struct drm_modeset_acquire_ctx *ctx) -{ - const int swap_interval = (flags & DRM_MODE_PAGE_FLIP_ASYNC) ? 0 : 1; - struct drm_device *dev = crtc->dev; - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->primary->fb)->nvbo; - struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo; - struct nouveau_page_flip_state *s; - struct nouveau_channel *chan; - struct nouveau_cli *cli; - struct nouveau_fence *fence; - struct nv04_display *dispnv04 = nv04_display(dev); - int head = nouveau_crtc(crtc)->index; - int ret; - - chan = drm->channel; - if (!chan) - return -ENODEV; - cli = (void *)chan->user.client; - - s = kzalloc(sizeof(*s), GFP_KERNEL); - if (!s) - return -ENOMEM; - - if (new_bo != old_bo) { - ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM, true); - if (ret) - goto fail_free; - } - - mutex_lock(&cli->mutex); - ret = ttm_bo_reserve(&new_bo->bo, true, false, NULL); - if (ret) - goto fail_unpin; - - /* synchronise rendering channel with the kernel's channel */ - ret = nouveau_fence_sync(new_bo, chan, false, true); - if (ret) { - ttm_bo_unreserve(&new_bo->bo); - goto fail_unpin; - } - - if (new_bo != old_bo) { - ttm_bo_unreserve(&new_bo->bo); - - ret = ttm_bo_reserve(&old_bo->bo, true, false, NULL); - if (ret) - goto fail_unpin; - } - - /* Initialize a page flip struct */ - *s = (struct nouveau_page_flip_state) - { { }, event, crtc, fb->format->cpp[0] * 8, fb->pitches[0], - new_bo->bo.offset }; - - /* Keep vblanks on during flip, for the target crtc of this flip */ - drm_crtc_vblank_get(crtc); - - /* Emit a page flip */ - if (swap_interval) { - ret = RING_SPACE(chan, 8); - if (ret) - goto fail_unreserve; - - BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1); - OUT_RING (chan, 0); - BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1); - OUT_RING (chan, head); - BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1); - OUT_RING (chan, 0); - BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1); - OUT_RING (chan, 0); - } - - nouveau_bo_ref(new_bo, &dispnv04->image[head]); - - ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence); - if (ret) - goto fail_unreserve; - mutex_unlock(&cli->mutex); - - /* Update the crtc struct and cleanup */ - crtc->primary->fb = fb; - - nouveau_bo_fence(old_bo, fence, false); - ttm_bo_unreserve(&old_bo->bo); - if (old_bo != new_bo) - nouveau_bo_unpin(old_bo); - nouveau_fence_unref(&fence); - return 0; - -fail_unreserve: - drm_crtc_vblank_put(crtc); - ttm_bo_unreserve(&old_bo->bo); -fail_unpin: - mutex_unlock(&cli->mutex); - if (old_bo != new_bo) - nouveau_bo_unpin(new_bo); -fail_free: - kfree(s); - return ret; -} - -int -nouveau_finish_page_flip(struct nouveau_channel *chan, - struct nouveau_page_flip_state *ps) -{ - struct nouveau_fence_chan *fctx = chan->fence; - struct nouveau_drm *drm = chan->drm; - struct drm_device *dev = drm->dev; - struct nouveau_page_flip_state *s; - unsigned long flags; - - spin_lock_irqsave(&dev->event_lock, flags); - - if (list_empty(&fctx->flip)) { - NV_ERROR(drm, "unexpected pageflip\n"); - spin_unlock_irqrestore(&dev->event_lock, flags); - return -EINVAL; - } - - s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head); - if (s->event) { - drm_crtc_arm_vblank_event(s->crtc, s->event); - } else { - /* Give up ownership of vblank for page-flipped crtc */ - drm_crtc_vblank_put(s->crtc); - } - - list_del(&s->head); - if (ps) - *ps = *s; - kfree(s); - - spin_unlock_irqrestore(&dev->event_lock, flags); - return 0; -} - -int -nouveau_flip_complete(struct nvif_notify *notify) -{ - struct nouveau_drm *drm = container_of(notify, typeof(*drm), flip); - struct nouveau_channel *chan = drm->channel; - struct nouveau_page_flip_state state; - - if (!nouveau_finish_page_flip(chan, &state)) { - nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc), - state.offset + state.crtc->y * - state.pitch + state.crtc->x * - state.bpp / 8); - } - - return NVIF_NOTIFY_KEEP; } int diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h index eb77e41c2d4e..311e175f0513 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.h +++ b/drivers/gpu/drm/nouveau/nouveau_display.h @@ -25,19 +25,11 @@ int nouveau_framebuffer_new(struct drm_device *, const struct drm_mode_fb_cmd2 *, struct nouveau_bo *, struct nouveau_framebuffer **); -struct nouveau_page_flip_state { - struct list_head head; - struct drm_pending_vblank_event *event; - struct drm_crtc *crtc; - int bpp, pitch; - u64 offset; -}; - struct nouveau_display { void *priv; void (*dtor)(struct drm_device *); - int (*init)(struct drm_device *); - void (*fini)(struct drm_device *); + int (*init)(struct drm_device *, bool resume, bool runtime); + void (*fini)(struct drm_device *, bool suspend); struct nvif_disp disp; @@ -61,7 +53,7 @@ nouveau_display(struct drm_device *dev) int nouveau_display_create(struct drm_device *dev); void nouveau_display_destroy(struct drm_device *dev); -int nouveau_display_init(struct drm_device *dev); +int nouveau_display_init(struct drm_device *dev, bool resume, bool runtime); void nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime); int nouveau_display_suspend(struct drm_device *dev, bool runtime); void nouveau_display_resume(struct drm_device *dev, bool runtime); @@ -71,13 +63,6 @@ bool nouveau_display_scanoutpos(struct drm_device *, unsigned int, bool, int *, int *, ktime_t *, ktime_t *, const struct drm_display_mode *); -int nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, - struct drm_pending_vblank_event *event, - uint32_t page_flip_flags, - struct drm_modeset_acquire_ctx *ctx); -int nouveau_finish_page_flip(struct nouveau_channel *, - struct nouveau_page_flip_state *); - int nouveau_display_dumb_create(struct drm_file *, struct drm_device *, struct drm_mode_create_dumb *args); int nouveau_display_dumb_map_offset(struct drm_file *, struct drm_device *, diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c new file mode 100644 index 000000000000..8be7a83ced9b --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -0,0 +1,887 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nouveau_dmem.h" +#include "nouveau_drv.h" +#include "nouveau_chan.h" +#include "nouveau_dma.h" +#include "nouveau_mem.h" +#include "nouveau_bo.h" + +#include <nvif/class.h> +#include <nvif/object.h> +#include <nvif/if500b.h> +#include <nvif/if900b.h> + +#include <linux/sched/mm.h> +#include <linux/hmm.h> + +/* + * FIXME: this is ugly right now we are using TTM to allocate vram and we pin + * it in vram while in use. We likely want to overhaul memory management for + * nouveau to be more page like (not necessarily with system page size but a + * bigger page size) at lowest level and have some shim layer on top that would + * provide the same functionality as TTM. + */ +#define DMEM_CHUNK_SIZE (2UL << 20) +#define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) + +struct nouveau_migrate; + +enum nouveau_aper { + NOUVEAU_APER_VIRT, + NOUVEAU_APER_VRAM, + NOUVEAU_APER_HOST, +}; + +typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, + enum nouveau_aper, u64 dst_addr, + enum nouveau_aper, u64 src_addr); + +struct nouveau_dmem_chunk { + struct list_head list; + struct nouveau_bo *bo; + struct nouveau_drm *drm; + unsigned long pfn_first; + unsigned long callocated; + unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)]; + spinlock_t lock; +}; + +struct nouveau_dmem_migrate { + nouveau_migrate_copy_t copy_func; + struct nouveau_channel *chan; +}; + +struct nouveau_dmem { + struct hmm_devmem *devmem; + struct nouveau_dmem_migrate migrate; + struct list_head chunk_free; + struct list_head chunk_full; + struct list_head chunk_empty; + struct mutex mutex; +}; + +struct nouveau_dmem_fault { + struct nouveau_drm *drm; + struct nouveau_fence *fence; + dma_addr_t *dma; + unsigned long npages; +}; + +struct nouveau_migrate { + struct vm_area_struct *vma; + struct nouveau_drm *drm; + struct nouveau_fence *fence; + unsigned long npages; + dma_addr_t *dma; + unsigned long dma_nr; +}; + +static void +nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page) +{ + struct nouveau_dmem_chunk *chunk; + struct nouveau_drm *drm; + unsigned long idx; + + chunk = (void *)hmm_devmem_page_get_drvdata(page); + idx = page_to_pfn(page) - chunk->pfn_first; + drm = chunk->drm; + + /* + * FIXME: + * + * This is really a bad example, we need to overhaul nouveau memory + * management to be more page focus and allow lighter locking scheme + * to be use in the process. + */ + spin_lock(&chunk->lock); + clear_bit(idx, chunk->bitmap); + WARN_ON(!chunk->callocated); + chunk->callocated--; + /* + * FIXME when chunk->callocated reach 0 we should add the chunk to + * a reclaim list so that it can be freed in case of memory pressure. + */ + spin_unlock(&chunk->lock); +} + +static void +nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, + const unsigned long *src_pfns, + unsigned long *dst_pfns, + unsigned long start, + unsigned long end, + void *private) +{ + struct nouveau_dmem_fault *fault = private; + struct nouveau_drm *drm = fault->drm; + struct device *dev = drm->dev->dev; + unsigned long addr, i, npages = 0; + nouveau_migrate_copy_t copy; + int ret; + + + /* First allocate new memory */ + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { + struct page *dpage, *spage; + + dst_pfns[i] = 0; + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + + dpage = hmm_vma_alloc_locked_page(vma, addr); + if (!dpage) { + dst_pfns[i] = MIGRATE_PFN_ERROR; + continue; + } + + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | + MIGRATE_PFN_LOCKED; + npages++; + } + + /* Allocate storage for DMA addresses, so we can unmap later. */ + fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL); + if (!fault->dma) + goto error; + + /* Copy things over */ + copy = drm->dmem->migrate.copy_func; + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { + struct nouveau_dmem_chunk *chunk; + struct page *spage, *dpage; + u64 src_addr, dst_addr; + + dpage = migrate_pfn_to_page(dst_pfns[i]); + if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) + continue; + + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { + dst_pfns[i] = MIGRATE_PFN_ERROR; + __free_page(dpage); + continue; + } + + fault->dma[fault->npages] = + dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(dev, fault->dma[fault->npages])) { + dst_pfns[i] = MIGRATE_PFN_ERROR; + __free_page(dpage); + continue; + } + + dst_addr = fault->dma[fault->npages++]; + + chunk = (void *)hmm_devmem_page_get_drvdata(spage); + src_addr = page_to_pfn(spage) - chunk->pfn_first; + src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset; + + ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr, + NOUVEAU_APER_VRAM, src_addr); + if (ret) { + dst_pfns[i] = MIGRATE_PFN_ERROR; + __free_page(dpage); + continue; + } + } + + nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence); + + return; + +error: + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { + struct page *page; + + if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) + continue; + + page = migrate_pfn_to_page(dst_pfns[i]); + dst_pfns[i] = MIGRATE_PFN_ERROR; + if (page == NULL) + continue; + + __free_page(page); + } +} + +void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma, + const unsigned long *src_pfns, + const unsigned long *dst_pfns, + unsigned long start, + unsigned long end, + void *private) +{ + struct nouveau_dmem_fault *fault = private; + struct nouveau_drm *drm = fault->drm; + + if (fault->fence) { + nouveau_fence_wait(fault->fence, true, false); + nouveau_fence_unref(&fault->fence); + } else { + /* + * FIXME wait for channel to be IDLE before calling finalizing + * the hmem object below (nouveau_migrate_hmem_fini()). + */ + } + + while (fault->npages--) { + dma_unmap_page(drm->dev->dev, fault->dma[fault->npages], + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + } + kfree(fault->dma); +} + +static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { + .alloc_and_copy = nouveau_dmem_fault_alloc_and_copy, + .finalize_and_map = nouveau_dmem_fault_finalize_and_map, +}; + +static int +nouveau_dmem_fault(struct hmm_devmem *devmem, + struct vm_area_struct *vma, + unsigned long addr, + const struct page *page, + unsigned int flags, + pmd_t *pmdp) +{ + struct drm_device *drm_dev = dev_get_drvdata(devmem->device); + unsigned long src[1] = {0}, dst[1] = {0}; + struct nouveau_dmem_fault fault = {0}; + int ret; + + + + /* + * FIXME what we really want is to find some heuristic to migrate more + * than just one page on CPU fault. When such fault happens it is very + * likely that more surrounding page will CPU fault too. + */ + fault.drm = nouveau_drm(drm_dev); + ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr, + addr + PAGE_SIZE, src, dst, &fault); + if (ret) + return VM_FAULT_SIGBUS; + + if (dst[0] == MIGRATE_PFN_ERROR) + return VM_FAULT_SIGBUS; + + return 0; +} + +static const struct hmm_devmem_ops +nouveau_dmem_devmem_ops = { + .free = nouveau_dmem_free, + .fault = nouveau_dmem_fault, +}; + +static int +nouveau_dmem_chunk_alloc(struct nouveau_drm *drm) +{ + struct nouveau_dmem_chunk *chunk; + int ret; + + if (drm->dmem == NULL) + return -EINVAL; + + mutex_lock(&drm->dmem->mutex); + chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, + struct nouveau_dmem_chunk, + list); + if (chunk == NULL) { + mutex_unlock(&drm->dmem->mutex); + return -ENOMEM; + } + + list_del(&chunk->list); + mutex_unlock(&drm->dmem->mutex); + + ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, + TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, + &chunk->bo); + if (ret) + goto out; + + ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); + if (ret) { + nouveau_bo_ref(NULL, &chunk->bo); + goto out; + } + + bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES); + spin_lock_init(&chunk->lock); + +out: + mutex_lock(&drm->dmem->mutex); + if (chunk->bo) + list_add(&chunk->list, &drm->dmem->chunk_empty); + else + list_add_tail(&chunk->list, &drm->dmem->chunk_empty); + mutex_unlock(&drm->dmem->mutex); + + return ret; +} + +static struct nouveau_dmem_chunk * +nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm) +{ + struct nouveau_dmem_chunk *chunk; + + chunk = list_first_entry_or_null(&drm->dmem->chunk_free, + struct nouveau_dmem_chunk, + list); + if (chunk) + return chunk; + + chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, + struct nouveau_dmem_chunk, + list); + if (chunk->bo) + return chunk; + + return NULL; +} + +static int +nouveau_dmem_pages_alloc(struct nouveau_drm *drm, + unsigned long npages, + unsigned long *pages) +{ + struct nouveau_dmem_chunk *chunk; + unsigned long c; + int ret; + + memset(pages, 0xff, npages * sizeof(*pages)); + + mutex_lock(&drm->dmem->mutex); + for (c = 0; c < npages;) { + unsigned long i; + + chunk = nouveau_dmem_chunk_first_free_locked(drm); + if (chunk == NULL) { + mutex_unlock(&drm->dmem->mutex); + ret = nouveau_dmem_chunk_alloc(drm); + if (ret) { + if (c) + break; + return ret; + } + continue; + } + + spin_lock(&chunk->lock); + i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES); + while (i < DMEM_CHUNK_NPAGES && c < npages) { + pages[c] = chunk->pfn_first + i; + set_bit(i, chunk->bitmap); + chunk->callocated++; + c++; + + i = find_next_zero_bit(chunk->bitmap, + DMEM_CHUNK_NPAGES, i); + } + spin_unlock(&chunk->lock); + } + mutex_unlock(&drm->dmem->mutex); + + return 0; +} + +static struct page * +nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) +{ + unsigned long pfns[1]; + struct page *page; + int ret; + + /* FIXME stop all the miss-match API ... */ + ret = nouveau_dmem_pages_alloc(drm, 1, pfns); + if (ret) + return NULL; + + page = pfn_to_page(pfns[0]); + get_page(page); + lock_page(page); + return page; +} + +static void +nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) +{ + unlock_page(page); + put_page(page); +} + +void +nouveau_dmem_resume(struct nouveau_drm *drm) +{ + struct nouveau_dmem_chunk *chunk; + int ret; + + if (drm->dmem == NULL) + return; + + mutex_lock(&drm->dmem->mutex); + list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { + ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); + /* FIXME handle pin failure */ + WARN_ON(ret); + } + list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { + ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); + /* FIXME handle pin failure */ + WARN_ON(ret); + } + list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) { + ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); + /* FIXME handle pin failure */ + WARN_ON(ret); + } + mutex_unlock(&drm->dmem->mutex); +} + +void +nouveau_dmem_suspend(struct nouveau_drm *drm) +{ + struct nouveau_dmem_chunk *chunk; + + if (drm->dmem == NULL) + return; + + mutex_lock(&drm->dmem->mutex); + list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { + nouveau_bo_unpin(chunk->bo); + } + list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { + nouveau_bo_unpin(chunk->bo); + } + list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) { + nouveau_bo_unpin(chunk->bo); + } + mutex_unlock(&drm->dmem->mutex); +} + +void +nouveau_dmem_fini(struct nouveau_drm *drm) +{ + struct nouveau_dmem_chunk *chunk, *tmp; + + if (drm->dmem == NULL) + return; + + mutex_lock(&drm->dmem->mutex); + + WARN_ON(!list_empty(&drm->dmem->chunk_free)); + WARN_ON(!list_empty(&drm->dmem->chunk_full)); + + list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) { + if (chunk->bo) { + nouveau_bo_unpin(chunk->bo); + nouveau_bo_ref(NULL, &chunk->bo); + } + list_del(&chunk->list); + kfree(chunk); + } + + mutex_unlock(&drm->dmem->mutex); +} + +static int +nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, + enum nouveau_aper dst_aper, u64 dst_addr, + enum nouveau_aper src_aper, u64 src_addr) +{ + struct nouveau_channel *chan = drm->dmem->migrate.chan; + u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | + (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | + (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | + (1 << 2) /* FLUSH_ENABLE_TRUE. */ | + (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; + int ret; + + ret = RING_SPACE(chan, 13); + if (ret) + return ret; + + if (src_aper != NOUVEAU_APER_VIRT) { + switch (src_aper) { + case NOUVEAU_APER_VRAM: + BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); + break; + case NOUVEAU_APER_HOST: + BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); + break; + default: + return -EINVAL; + } + launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ + } + + if (dst_aper != NOUVEAU_APER_VIRT) { + switch (dst_aper) { + case NOUVEAU_APER_VRAM: + BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); + break; + case NOUVEAU_APER_HOST: + BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); + break; + default: + return -EINVAL; + } + launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ + } + + BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); + OUT_RING (chan, upper_32_bits(src_addr)); + OUT_RING (chan, lower_32_bits(src_addr)); + OUT_RING (chan, upper_32_bits(dst_addr)); + OUT_RING (chan, lower_32_bits(dst_addr)); + OUT_RING (chan, PAGE_SIZE); + OUT_RING (chan, PAGE_SIZE); + OUT_RING (chan, PAGE_SIZE); + OUT_RING (chan, npages); + BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); + OUT_RING (chan, launch_dma); + return 0; +} + +static int +nouveau_dmem_migrate_init(struct nouveau_drm *drm) +{ + switch (drm->ttm.copy.oclass) { + case PASCAL_DMA_COPY_A: + case PASCAL_DMA_COPY_B: + case VOLTA_DMA_COPY_A: + case TURING_DMA_COPY_A: + drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; + drm->dmem->migrate.chan = drm->ttm.chan; + return 0; + default: + break; + } + return -ENODEV; +} + +void +nouveau_dmem_init(struct nouveau_drm *drm) +{ + struct device *device = drm->dev->dev; + unsigned long i, size; + int ret; + + /* This only make sense on PASCAL or newer */ + if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) + return; + + if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) + return; + + mutex_init(&drm->dmem->mutex); + INIT_LIST_HEAD(&drm->dmem->chunk_free); + INIT_LIST_HEAD(&drm->dmem->chunk_full); + INIT_LIST_HEAD(&drm->dmem->chunk_empty); + + size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE); + + /* Initialize migration dma helpers before registering memory */ + ret = nouveau_dmem_migrate_init(drm); + if (ret) { + kfree(drm->dmem); + drm->dmem = NULL; + return; + } + + /* + * FIXME we need some kind of policy to decide how much VRAM we + * want to register with HMM. For now just register everything + * and latter if we want to do thing like over commit then we + * could revisit this. + */ + drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops, + device, size); + if (drm->dmem->devmem == NULL) { + kfree(drm->dmem); + drm->dmem = NULL; + return; + } + + for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) { + struct nouveau_dmem_chunk *chunk; + struct page *page; + unsigned long j; + + chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); + if (chunk == NULL) { + nouveau_dmem_fini(drm); + return; + } + + chunk->drm = drm; + chunk->pfn_first = drm->dmem->devmem->pfn_first; + chunk->pfn_first += (i * DMEM_CHUNK_NPAGES); + list_add_tail(&chunk->list, &drm->dmem->chunk_empty); + + page = pfn_to_page(chunk->pfn_first); + for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) { + hmm_devmem_page_set_drvdata(page, (long)chunk); + } + } + + NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20); +} + +static void +nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, + const unsigned long *src_pfns, + unsigned long *dst_pfns, + unsigned long start, + unsigned long end, + void *private) +{ + struct nouveau_migrate *migrate = private; + struct nouveau_drm *drm = migrate->drm; + struct device *dev = drm->dev->dev; + unsigned long addr, i, npages = 0; + nouveau_migrate_copy_t copy; + int ret; + + /* First allocate new memory */ + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { + struct page *dpage, *spage; + + dst_pfns[i] = 0; + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + + dpage = nouveau_dmem_page_alloc_locked(drm); + if (!dpage) + continue; + + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | + MIGRATE_PFN_LOCKED | + MIGRATE_PFN_DEVICE; + npages++; + } + + if (!npages) + return; + + /* Allocate storage for DMA addresses, so we can unmap later. */ + migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL); + if (!migrate->dma) + goto error; + + /* Copy things over */ + copy = drm->dmem->migrate.copy_func; + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { + struct nouveau_dmem_chunk *chunk; + struct page *spage, *dpage; + u64 src_addr, dst_addr; + + dpage = migrate_pfn_to_page(dst_pfns[i]); + if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) + continue; + + chunk = (void *)hmm_devmem_page_get_drvdata(dpage); + dst_addr = page_to_pfn(dpage) - chunk->pfn_first; + dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset; + + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { + nouveau_dmem_page_free_locked(drm, dpage); + dst_pfns[i] = 0; + continue; + } + + migrate->dma[migrate->dma_nr] = + dma_map_page_attrs(dev, spage, 0, PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) { + nouveau_dmem_page_free_locked(drm, dpage); + dst_pfns[i] = 0; + continue; + } + + src_addr = migrate->dma[migrate->dma_nr++]; + + ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr, + NOUVEAU_APER_HOST, src_addr); + if (ret) { + nouveau_dmem_page_free_locked(drm, dpage); + dst_pfns[i] = 0; + continue; + } + } + + nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence); + + return; + +error: + for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { + struct page *page; + + if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) + continue; + + page = migrate_pfn_to_page(dst_pfns[i]); + dst_pfns[i] = MIGRATE_PFN_ERROR; + if (page == NULL) + continue; + + __free_page(page); + } +} + +void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma, + const unsigned long *src_pfns, + const unsigned long *dst_pfns, + unsigned long start, + unsigned long end, + void *private) +{ + struct nouveau_migrate *migrate = private; + struct nouveau_drm *drm = migrate->drm; + + if (migrate->fence) { + nouveau_fence_wait(migrate->fence, true, false); + nouveau_fence_unref(&migrate->fence); + } else { + /* + * FIXME wait for channel to be IDLE before finalizing + * the hmem object below (nouveau_migrate_hmem_fini()) ? + */ + } + + while (migrate->dma_nr--) { + dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr], + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + } + kfree(migrate->dma); + + /* + * FIXME optimization: update GPU page table to point to newly + * migrated memory. + */ +} + +static const struct migrate_vma_ops nouveau_dmem_migrate_ops = { + .alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy, + .finalize_and_map = nouveau_dmem_migrate_finalize_and_map, +}; + +int +nouveau_dmem_migrate_vma(struct nouveau_drm *drm, + struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + unsigned long *src_pfns, *dst_pfns, npages; + struct nouveau_migrate migrate = {0}; + unsigned long i, c, max; + int ret = 0; + + npages = (end - start) >> PAGE_SHIFT; + max = min(SG_MAX_SINGLE_ALLOC, npages); + src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); + if (src_pfns == NULL) + return -ENOMEM; + dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); + if (dst_pfns == NULL) { + kfree(src_pfns); + return -ENOMEM; + } + + migrate.drm = drm; + migrate.vma = vma; + migrate.npages = npages; + for (i = 0; i < npages; i += c) { + unsigned long next; + + c = min(SG_MAX_SINGLE_ALLOC, npages); + next = start + (c << PAGE_SHIFT); + ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start, + next, src_pfns, dst_pfns, &migrate); + if (ret) + goto out; + start = next; + } + +out: + kfree(dst_pfns); + kfree(src_pfns); + return ret; +} + +static inline bool +nouveau_dmem_page(struct nouveau_drm *drm, struct page *page) +{ + if (!is_device_private_page(page)) + return false; + + if (drm->dmem->devmem != page->pgmap->data) + return false; + + return true; +} + +void +nouveau_dmem_convert_pfn(struct nouveau_drm *drm, + struct hmm_range *range) +{ + unsigned long i, npages; + + npages = (range->end - range->start) >> PAGE_SHIFT; + for (i = 0; i < npages; ++i) { + struct nouveau_dmem_chunk *chunk; + struct page *page; + uint64_t addr; + + page = hmm_pfn_to_page(range, range->pfns[i]); + if (page == NULL) + continue; + + if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) { + continue; + } + + if (!nouveau_dmem_page(drm, page)) { + WARN(1, "Some unknown device memory !\n"); + range->pfns[i] = 0; + continue; + } + + chunk = (void *)hmm_devmem_page_get_drvdata(page); + addr = page_to_pfn(page) - chunk->pfn_first; + addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT; + + range->pfns[i] &= ((1UL << range->pfn_shift) - 1); + range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift; + } +} diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.h b/drivers/gpu/drm/nouveau/nouveau_dmem.h new file mode 100644 index 000000000000..9d97d756fb7d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.h @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __NOUVEAU_DMEM_H__ +#define __NOUVEAU_DMEM_H__ +#include <nvif/os.h> +struct drm_device; +struct drm_file; +struct nouveau_drm; +struct hmm_range; + +#if IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) +void nouveau_dmem_init(struct nouveau_drm *); +void nouveau_dmem_fini(struct nouveau_drm *); +void nouveau_dmem_suspend(struct nouveau_drm *); +void nouveau_dmem_resume(struct nouveau_drm *); + +int nouveau_dmem_migrate_vma(struct nouveau_drm *drm, + struct vm_area_struct *vma, + unsigned long start, + unsigned long end); + +void nouveau_dmem_convert_pfn(struct nouveau_drm *drm, + struct hmm_range *range); +#else /* IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) */ +static inline void nouveau_dmem_init(struct nouveau_drm *drm) {} +static inline void nouveau_dmem_fini(struct nouveau_drm *drm) {} +static inline void nouveau_dmem_suspend(struct nouveau_drm *drm) {} +static inline void nouveau_dmem_resume(struct nouveau_drm *drm) {} + +static inline int nouveau_dmem_migrate_vma(struct nouveau_drm *drm, + struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + return 0; +} + +static inline void nouveau_dmem_convert_pfn(struct nouveau_drm *drm, + struct hmm_range *range) {} +#endif /* IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) */ +#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index f900e94592f8..5020265bfbd9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -44,7 +44,6 @@ #include <nvif/class.h> #include <nvif/cl0002.h> #include <nvif/cla06f.h> -#include <nvif/if0004.h> #include "nouveau_drv.h" #include "nouveau_dma.h" @@ -63,6 +62,8 @@ #include "nouveau_usif.h" #include "nouveau_connector.h" #include "nouveau_platform.h" +#include "nouveau_svm.h" +#include "nouveau_dmem.h" MODULE_PARM_DESC(config, "option string to pass to driver core"); static char *nouveau_config; @@ -173,6 +174,7 @@ nouveau_cli_fini(struct nouveau_cli *cli) WARN_ON(!list_empty(&cli->worker)); usif_client_fini(cli); + nouveau_vmm_fini(&cli->svm); nouveau_vmm_fini(&cli->vmm); nvif_mmu_fini(&cli->mmu); nvif_device_fini(&cli->device); @@ -283,19 +285,134 @@ done: } static void -nouveau_accel_fini(struct nouveau_drm *drm) +nouveau_accel_ce_fini(struct nouveau_drm *drm) +{ + nouveau_channel_idle(drm->cechan); + nvif_object_fini(&drm->ttm.copy); + nouveau_channel_del(&drm->cechan); +} + +static void +nouveau_accel_ce_init(struct nouveau_drm *drm) +{ + struct nvif_device *device = &drm->client.device; + int ret = 0; + + /* Allocate channel that has access to a (preferably async) copy + * engine, to use for TTM buffer moves. + */ + if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { + ret = nouveau_channel_new(drm, device, + nvif_fifo_runlist_ce(device), 0, + true, &drm->cechan); + } else + if (device->info.chipset >= 0xa3 && + device->info.chipset != 0xaa && + device->info.chipset != 0xac) { + /* Prior to Kepler, there's only a single runlist, so all + * engines can be accessed from any channel. + * + * We still want to use a separate channel though. + */ + ret = nouveau_channel_new(drm, device, NvDmaFB, NvDmaTT, false, + &drm->cechan); + } + + if (ret) + NV_ERROR(drm, "failed to create ce channel, %d\n", ret); +} + +static void +nouveau_accel_gr_fini(struct nouveau_drm *drm) { nouveau_channel_idle(drm->channel); nvif_object_fini(&drm->ntfy); nvkm_gpuobj_del(&drm->notify); - nvif_notify_fini(&drm->flip); nvif_object_fini(&drm->nvsw); nouveau_channel_del(&drm->channel); +} - nouveau_channel_idle(drm->cechan); - nvif_object_fini(&drm->ttm.copy); - nouveau_channel_del(&drm->cechan); +static void +nouveau_accel_gr_init(struct nouveau_drm *drm) +{ + struct nvif_device *device = &drm->client.device; + u32 arg0, arg1; + int ret; + + /* Allocate channel that has access to the graphics engine. */ + if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { + arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR); + arg1 = 1; + } else { + arg0 = NvDmaFB; + arg1 = NvDmaTT; + } + ret = nouveau_channel_new(drm, device, arg0, arg1, false, + &drm->channel); + if (ret) { + NV_ERROR(drm, "failed to create kernel channel, %d\n", ret); + nouveau_accel_gr_fini(drm); + return; + } + + /* A SW class is used on pre-NV50 HW to assist with handling the + * synchronisation of page flips, as well as to implement fences + * on TNT/TNT2 HW that lacks any kind of support in host. + */ + if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { + ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW, + nouveau_abi16_swclass(drm), NULL, 0, + &drm->nvsw); + if (ret == 0) { + ret = RING_SPACE(drm->channel, 2); + if (ret == 0) { + BEGIN_NV04(drm->channel, NvSubSw, 0, 1); + OUT_RING (drm->channel, drm->nvsw.handle); + } + } + + if (ret) { + NV_ERROR(drm, "failed to allocate sw class, %d\n", ret); + nouveau_accel_gr_fini(drm); + return; + } + } + + /* NvMemoryToMemoryFormat requires a notifier ctxdma for some reason, + * even if notification is never requested, so, allocate a ctxdma on + * any GPU where it's possible we'll end up using M2MF for BO moves. + */ + if (device->info.family < NV_DEVICE_INFO_V0_FERMI) { + ret = nvkm_gpuobj_new(nvxx_device(device), 32, 0, false, NULL, + &drm->notify); + if (ret) { + NV_ERROR(drm, "failed to allocate notifier, %d\n", ret); + nouveau_accel_gr_fini(drm); + return; + } + + ret = nvif_object_init(&drm->channel->user, NvNotify0, + NV_DMA_IN_MEMORY, + &(struct nv_dma_v0) { + .target = NV_DMA_V0_TARGET_VRAM, + .access = NV_DMA_V0_ACCESS_RDWR, + .start = drm->notify->addr, + .limit = drm->notify->addr + 31 + }, sizeof(struct nv_dma_v0), + &drm->ntfy); + if (ret) { + nouveau_accel_gr_fini(drm); + return; + } + } +} + +static void +nouveau_accel_fini(struct nouveau_drm *drm) +{ + nouveau_accel_ce_fini(drm); + nouveau_accel_gr_fini(drm); if (drm->fence) nouveau_fence(drm)->dtor(drm); } @@ -305,23 +422,16 @@ nouveau_accel_init(struct nouveau_drm *drm) { struct nvif_device *device = &drm->client.device; struct nvif_sclass *sclass; - u32 arg0, arg1; int ret, i, n; if (nouveau_noaccel) return; + /* Initialise global support for channels, and synchronisation. */ ret = nouveau_channels_init(drm); if (ret) return; - if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) { - ret = nvif_user_init(device); - if (ret) - return; - } - - /* initialise synchronisation routines */ /*XXX: this is crap, but the fence/channel stuff is a little * backwards in some places. this will be fixed. */ @@ -368,95 +478,18 @@ nouveau_accel_init(struct nouveau_drm *drm) return; } - if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { - ret = nouveau_channel_new(drm, &drm->client.device, - nvif_fifo_runlist_ce(device), 0, - true, &drm->cechan); - if (ret) - NV_ERROR(drm, "failed to create ce channel, %d\n", ret); - - arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR); - arg1 = 1; - } else - if (device->info.chipset >= 0xa3 && - device->info.chipset != 0xaa && - device->info.chipset != 0xac) { - ret = nouveau_channel_new(drm, &drm->client.device, - NvDmaFB, NvDmaTT, false, - &drm->cechan); + /* Volta requires access to a doorbell register for kickoff. */ + if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) { + ret = nvif_user_init(device); if (ret) - NV_ERROR(drm, "failed to create ce channel, %d\n", ret); - - arg0 = NvDmaFB; - arg1 = NvDmaTT; - } else { - arg0 = NvDmaFB; - arg1 = NvDmaTT; - } - - ret = nouveau_channel_new(drm, &drm->client.device, - arg0, arg1, false, &drm->channel); - if (ret) { - NV_ERROR(drm, "failed to create kernel channel, %d\n", ret); - nouveau_accel_fini(drm); - return; - } - - if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { - ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW, - nouveau_abi16_swclass(drm), NULL, 0, - &drm->nvsw); - if (ret == 0) { - ret = RING_SPACE(drm->channel, 2); - if (ret == 0) { - BEGIN_NV04(drm->channel, NvSubSw, 0, 1); - OUT_RING (drm->channel, drm->nvsw.handle); - } - - ret = nvif_notify_init(&drm->nvsw, - nouveau_flip_complete, - false, NV04_NVSW_NTFY_UEVENT, - NULL, 0, 0, &drm->flip); - if (ret == 0) - ret = nvif_notify_get(&drm->flip); - if (ret) { - nouveau_accel_fini(drm); - return; - } - } - - if (ret) { - NV_ERROR(drm, "failed to allocate sw class, %d\n", ret); - nouveau_accel_fini(drm); - return; - } - } - - if (device->info.family < NV_DEVICE_INFO_V0_FERMI) { - ret = nvkm_gpuobj_new(nvxx_device(&drm->client.device), 32, 0, - false, NULL, &drm->notify); - if (ret) { - NV_ERROR(drm, "failed to allocate notifier, %d\n", ret); - nouveau_accel_fini(drm); return; - } - - ret = nvif_object_init(&drm->channel->user, NvNotify0, - NV_DMA_IN_MEMORY, - &(struct nv_dma_v0) { - .target = NV_DMA_V0_TARGET_VRAM, - .access = NV_DMA_V0_ACCESS_RDWR, - .start = drm->notify->addr, - .limit = drm->notify->addr + 31 - }, sizeof(struct nv_dma_v0), - &drm->ntfy); - if (ret) { - nouveau_accel_fini(drm); - return; - } } + /* Allocate channels we need to support various functions. */ + nouveau_accel_gr_init(drm); + nouveau_accel_ce_init(drm); + /* Initialise accelerated TTM buffer moves. */ nouveau_bo_move_init(drm); } @@ -504,19 +537,22 @@ nouveau_drm_device_init(struct drm_device *dev) if (ret) goto fail_bios; + nouveau_accel_init(drm); + ret = nouveau_display_create(dev); if (ret) goto fail_dispctor; if (dev->mode_config.num_crtc) { - ret = nouveau_display_init(dev); + ret = nouveau_display_init(dev, false, false); if (ret) goto fail_dispinit; } nouveau_debugfs_init(drm); nouveau_hwmon_init(dev); - nouveau_accel_init(drm); + nouveau_svm_init(drm); + nouveau_dmem_init(drm); nouveau_fbcon_init(dev); nouveau_led_init(dev); @@ -534,6 +570,7 @@ nouveau_drm_device_init(struct drm_device *dev) fail_dispinit: nouveau_display_destroy(dev); fail_dispctor: + nouveau_accel_fini(drm); nouveau_bios_takedown(dev); fail_bios: nouveau_ttm_fini(drm); @@ -559,7 +596,8 @@ nouveau_drm_device_fini(struct drm_device *dev) nouveau_led_fini(dev); nouveau_fbcon_fini(dev); - nouveau_accel_fini(drm); + nouveau_dmem_fini(drm); + nouveau_svm_fini(drm); nouveau_hwmon_fini(dev); nouveau_debugfs_fini(drm); @@ -567,6 +605,7 @@ nouveau_drm_device_fini(struct drm_device *dev) nouveau_display_fini(dev, false, false); nouveau_display_destroy(dev); + nouveau_accel_fini(drm); nouveau_bios_takedown(dev); nouveau_ttm_fini(drm); @@ -704,6 +743,8 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime) struct nouveau_drm *drm = nouveau_drm(dev); int ret; + nouveau_svm_suspend(drm); + nouveau_dmem_suspend(drm); nouveau_led_suspend(dev); if (dev->mode_config.num_crtc) { @@ -780,7 +821,8 @@ nouveau_do_resume(struct drm_device *dev, bool runtime) } nouveau_led_resume(dev); - + nouveau_dmem_resume(drm); + nouveau_svm_resume(drm); return 0; } @@ -1000,6 +1042,8 @@ nouveau_ioctls[] = { DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_INIT, nouveau_svmm_init, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_BIND, nouveau_svmm_bind, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW), diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index d20b9ba4b1c1..da847244479d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -96,6 +96,7 @@ struct nouveau_cli { struct nvif_device device; struct nvif_mmu mmu; struct nouveau_vmm vmm; + struct nouveau_vmm svm; const struct nvif_mclass *mem; struct list_head head; @@ -181,7 +182,6 @@ struct nouveau_drm { struct nouveau_fbdev *fbcon; struct nvif_object nvsw; struct nvif_object ntfy; - struct nvif_notify flip; /* nv10-nv40 tiling regions */ struct { @@ -210,6 +210,10 @@ struct nouveau_drm { bool have_disp_power_ref; struct dev_pm_domain vga_pm_domain; + + struct nouveau_svm *svm; + + struct nouveau_dmem *dmem; }; static inline struct nouveau_drm * diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c index d275418edd24..0d3cd4e05728 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c +++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c @@ -353,7 +353,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, chan = nouveau_nofbaccel ? NULL : drm->channel; if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_vma_new(nvbo, &drm->client.vmm, &fb->vma); + ret = nouveau_vma_new(nvbo, chan->vmm, &fb->vma); if (ret) { NV_ERROR(drm, "failed to map fb into chan: %d\n", ret); chan = NULL; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index b999e6058046..ad27caeca0fd 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -82,8 +82,6 @@ int nv50_fence_create(struct nouveau_drm *); int nv84_fence_create(struct nouveau_drm *); int nvc0_fence_create(struct nouveau_drm *); -int nouveau_flip_complete(struct nvif_notify *); - struct nv84_fence_chan { struct nouveau_fence_chan base; struct nouveau_vma *vma; diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index fb028e3b5f51..b4bda716564d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -68,10 +68,11 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_bo *nvbo = nouveau_gem_object(gem); struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct device *dev = drm->dev->dev; + struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; struct nouveau_vma *vma; int ret; - if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) + if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) return 0; ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); @@ -82,7 +83,7 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) if (ret < 0 && ret != -EACCES) goto out; - ret = nouveau_vma_new(nvbo, &cli->vmm, &vma); + ret = nouveau_vma_new(nvbo, vmm, &vma); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); out: @@ -142,17 +143,18 @@ nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_bo *nvbo = nouveau_gem_object(gem); struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct device *dev = drm->dev->dev; + struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; struct nouveau_vma *vma; int ret; - if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) + if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) return; ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); if (ret) return; - vma = nouveau_vma_find(nvbo, &cli->vmm); + vma = nouveau_vma_find(nvbo, vmm); if (vma) { if (--vma->refs == 0) { ret = pm_runtime_get_sync(dev); @@ -219,6 +221,7 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, { struct nouveau_cli *cli = nouveau_cli(file_priv); struct nouveau_bo *nvbo = nouveau_gem_object(gem); + struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; struct nouveau_vma *vma; if (is_power_of_2(nvbo->valid_domains)) @@ -228,8 +231,8 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, else rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; rep->offset = nvbo->bo.offset; - if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { - vma = nouveau_vma_find(nvbo, &cli->vmm); + if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { + vma = nouveau_vma_find(nvbo, vmm); if (!vma) return -EINVAL; @@ -321,7 +324,8 @@ struct validate_op { }; static void -validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, +validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, + struct nouveau_fence *fence, struct drm_nouveau_gem_pushbuf_bo *pbbo) { struct nouveau_bo *nvbo; @@ -332,13 +336,11 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, b = &pbbo[nvbo->pbbo_index]; if (likely(fence)) { - struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nouveau_vma *vma; - nouveau_bo_fence(nvbo, fence, !!b->write_domains); - if (drm->client.vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { - vma = (void *)(unsigned long)b->user_priv; + if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { + struct nouveau_vma *vma = + (void *)(unsigned long)b->user_priv; nouveau_fence_unref(&vma->fence); dma_fence_get(&fence->base); vma->fence = fence; @@ -358,10 +360,11 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, } static void -validate_fini(struct validate_op *op, struct nouveau_fence *fence, +validate_fini(struct validate_op *op, struct nouveau_channel *chan, + struct nouveau_fence *fence, struct drm_nouveau_gem_pushbuf_bo *pbbo) { - validate_fini_no_ticket(op, fence, pbbo); + validate_fini_no_ticket(op, chan, fence, pbbo); ww_acquire_fini(&op->ticket); } @@ -416,7 +419,7 @@ retry: list_splice_tail_init(&vram_list, &op->list); list_splice_tail_init(&gart_list, &op->list); list_splice_tail_init(&both_list, &op->list); - validate_fini_no_ticket(op, NULL, NULL); + validate_fini_no_ticket(op, chan, NULL, NULL); if (unlikely(ret == -EDEADLK)) { ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, &op->ticket); @@ -430,8 +433,8 @@ retry: } } - if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { - struct nouveau_vmm *vmm = &cli->vmm; + if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { + struct nouveau_vmm *vmm = chan->vmm; struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); if (!vma) { NV_PRINTK(err, cli, "vma not found!\n"); @@ -471,7 +474,7 @@ retry: list_splice_tail(&gart_list, &op->list); list_splice_tail(&both_list, &op->list); if (ret) - validate_fini(op, NULL, NULL); + validate_fini(op, chan, NULL, NULL); return ret; } @@ -563,7 +566,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, if (unlikely(ret < 0)) { if (ret != -ERESTARTSYS) NV_PRINTK(err, cli, "validating bo list\n"); - validate_fini(op, NULL, NULL); + validate_fini(op, chan, NULL, NULL); return ret; } *apply_relocs = ret; @@ -842,7 +845,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, } out: - validate_fini(&op, fence, bo); + validate_fini(&op, chan, fence, bo); nouveau_fence_unref(&fence); out_prevalid: diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c new file mode 100644 index 000000000000..93ed43c413f0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -0,0 +1,835 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nouveau_svm.h" +#include "nouveau_drv.h" +#include "nouveau_chan.h" +#include "nouveau_dmem.h" + +#include <nvif/notify.h> +#include <nvif/object.h> +#include <nvif/vmm.h> + +#include <nvif/class.h> +#include <nvif/clb069.h> +#include <nvif/ifc00d.h> + +#include <linux/sched/mm.h> +#include <linux/sort.h> +#include <linux/hmm.h> + +struct nouveau_svm { + struct nouveau_drm *drm; + struct mutex mutex; + struct list_head inst; + + struct nouveau_svm_fault_buffer { + int id; + struct nvif_object object; + u32 entries; + u32 getaddr; + u32 putaddr; + u32 get; + u32 put; + struct nvif_notify notify; + + struct nouveau_svm_fault { + u64 inst; + u64 addr; + u64 time; + u32 engine; + u8 gpc; + u8 hub; + u8 access; + u8 client; + u8 fault; + struct nouveau_svmm *svmm; + } **fault; + int fault_nr; + } buffer[1]; +}; + +#define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a) +#define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a) + +struct nouveau_ivmm { + struct nouveau_svmm *svmm; + u64 inst; + struct list_head head; +}; + +static struct nouveau_ivmm * +nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) +{ + struct nouveau_ivmm *ivmm; + list_for_each_entry(ivmm, &svm->inst, head) { + if (ivmm->inst == inst) + return ivmm; + } + return NULL; +} + +struct nouveau_svmm { + struct nouveau_vmm *vmm; + struct { + unsigned long start; + unsigned long limit; + } unmanaged; + + struct mutex mutex; + + struct mm_struct *mm; + struct hmm_mirror mirror; +}; + +#define SVMM_DBG(s,f,a...) \ + NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) +#define SVMM_ERR(s,f,a...) \ + NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) + +int +nouveau_svmm_bind(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct nouveau_cli *cli = nouveau_cli(file_priv); + struct drm_nouveau_svm_bind *args = data; + unsigned target, cmd, priority; + unsigned long addr, end, size; + struct mm_struct *mm; + + args->va_start &= PAGE_MASK; + args->va_end &= PAGE_MASK; + + /* Sanity check arguments */ + if (args->reserved0 || args->reserved1) + return -EINVAL; + if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK)) + return -EINVAL; + if (args->va_start >= args->va_end) + return -EINVAL; + if (!args->npages) + return -EINVAL; + + cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT; + cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK; + switch (cmd) { + case NOUVEAU_SVM_BIND_COMMAND__MIGRATE: + break; + default: + return -EINVAL; + } + + priority = args->header >> NOUVEAU_SVM_BIND_PRIORITY_SHIFT; + priority &= NOUVEAU_SVM_BIND_PRIORITY_MASK; + + /* FIXME support CPU target ie all target value < GPU_VRAM */ + target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT; + target &= NOUVEAU_SVM_BIND_TARGET_MASK; + switch (target) { + case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM: + break; + default: + return -EINVAL; + } + + /* + * FIXME: For now refuse non 0 stride, we need to change the migrate + * kernel function to handle stride to avoid to create a mess within + * each device driver. + */ + if (args->stride) + return -EINVAL; + + size = ((unsigned long)args->npages) << PAGE_SHIFT; + if ((args->va_start + size) <= args->va_start) + return -EINVAL; + if ((args->va_start + size) > args->va_end) + return -EINVAL; + + /* + * Ok we are ask to do something sane, for now we only support migrate + * commands but we will add things like memory policy (what to do on + * page fault) and maybe some other commands. + */ + + mm = get_task_mm(current); + down_read(&mm->mmap_sem); + + for (addr = args->va_start, end = args->va_start + size; addr < end;) { + struct vm_area_struct *vma; + unsigned long next; + + vma = find_vma_intersection(mm, addr, end); + if (!vma) + break; + + next = min(vma->vm_end, end); + /* This is a best effort so we ignore errors */ + nouveau_dmem_migrate_vma(cli->drm, vma, addr, next); + addr = next; + } + + /* + * FIXME Return the number of page we have migrated, again we need to + * update the migrate API to return that information so that we can + * report it to user space. + */ + args->result = 0; + + up_read(&mm->mmap_sem); + mmput(mm); + + return 0; +} + +/* Unlink channel instance from SVMM. */ +void +nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst) +{ + struct nouveau_ivmm *ivmm; + if (svmm) { + mutex_lock(&svmm->vmm->cli->drm->svm->mutex); + ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst); + if (ivmm) { + list_del(&ivmm->head); + kfree(ivmm); + } + mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); + } +} + +/* Link channel instance to SVMM. */ +int +nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) +{ + struct nouveau_ivmm *ivmm; + if (svmm) { + if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL))) + return -ENOMEM; + ivmm->svmm = svmm; + ivmm->inst = inst; + + mutex_lock(&svmm->vmm->cli->drm->svm->mutex); + list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst); + mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); + } + return 0; +} + +/* Invalidate SVMM address-range on GPU. */ +static void +nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit) +{ + if (limit > start) { + bool super = svmm->vmm->vmm.object.client->super; + svmm->vmm->vmm.object.client->super = true; + nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR, + &(struct nvif_vmm_pfnclr_v0) { + .addr = start, + .size = limit - start, + }, sizeof(struct nvif_vmm_pfnclr_v0)); + svmm->vmm->vmm.object.client->super = super; + } +} + +static int +nouveau_svmm_sync_cpu_device_pagetables(struct hmm_mirror *mirror, + const struct hmm_update *update) +{ + struct nouveau_svmm *svmm = container_of(mirror, typeof(*svmm), mirror); + unsigned long start = update->start; + unsigned long limit = update->end; + + if (!update->blockable) + return -EAGAIN; + + SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit); + + mutex_lock(&svmm->mutex); + if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) { + if (start < svmm->unmanaged.start) { + nouveau_svmm_invalidate(svmm, start, + svmm->unmanaged.limit); + } + start = svmm->unmanaged.limit; + } + + nouveau_svmm_invalidate(svmm, start, limit); + mutex_unlock(&svmm->mutex); + return 0; +} + +static void +nouveau_svmm_release(struct hmm_mirror *mirror) +{ +} + +static const struct hmm_mirror_ops +nouveau_svmm = { + .sync_cpu_device_pagetables = nouveau_svmm_sync_cpu_device_pagetables, + .release = nouveau_svmm_release, +}; + +void +nouveau_svmm_fini(struct nouveau_svmm **psvmm) +{ + struct nouveau_svmm *svmm = *psvmm; + if (svmm) { + hmm_mirror_unregister(&svmm->mirror); + kfree(*psvmm); + *psvmm = NULL; + } +} + +int +nouveau_svmm_init(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct nouveau_cli *cli = nouveau_cli(file_priv); + struct nouveau_svmm *svmm; + struct drm_nouveau_svm_init *args = data; + int ret; + + /* Allocate tracking for SVM-enabled VMM. */ + if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL))) + return -ENOMEM; + svmm->vmm = &cli->svm; + svmm->unmanaged.start = args->unmanaged_addr; + svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size; + mutex_init(&svmm->mutex); + + /* Check that SVM isn't already enabled for the client. */ + mutex_lock(&cli->mutex); + if (cli->svm.cli) { + ret = -EBUSY; + goto done; + } + + /* Allocate a new GPU VMM that can support SVM (managed by the + * client, with replayable faults enabled). + * + * All future channel/memory allocations will make use of this + * VMM instead of the standard one. + */ + ret = nvif_vmm_init(&cli->mmu, cli->vmm.vmm.object.oclass, true, + args->unmanaged_addr, args->unmanaged_size, + &(struct gp100_vmm_v0) { + .fault_replay = true, + }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm); + if (ret) + goto done; + + /* Enable HMM mirroring of CPU address-space to VMM. */ + svmm->mm = get_task_mm(current); + down_write(&svmm->mm->mmap_sem); + svmm->mirror.ops = &nouveau_svmm; + ret = hmm_mirror_register(&svmm->mirror, svmm->mm); + if (ret == 0) { + cli->svm.svmm = svmm; + cli->svm.cli = cli; + } + up_write(&svmm->mm->mmap_sem); + mmput(svmm->mm); + +done: + if (ret) + nouveau_svmm_fini(&svmm); + mutex_unlock(&cli->mutex); + return ret; +} + +static const u64 +nouveau_svm_pfn_flags[HMM_PFN_FLAG_MAX] = { + [HMM_PFN_VALID ] = NVIF_VMM_PFNMAP_V0_V, + [HMM_PFN_WRITE ] = NVIF_VMM_PFNMAP_V0_W, + [HMM_PFN_DEVICE_PRIVATE] = NVIF_VMM_PFNMAP_V0_VRAM, +}; + +static const u64 +nouveau_svm_pfn_values[HMM_PFN_VALUE_MAX] = { + [HMM_PFN_ERROR ] = ~NVIF_VMM_PFNMAP_V0_V, + [HMM_PFN_NONE ] = NVIF_VMM_PFNMAP_V0_NONE, + [HMM_PFN_SPECIAL] = ~NVIF_VMM_PFNMAP_V0_V, +}; + +/* Issue fault replay for GPU to retry accesses that faulted previously. */ +static void +nouveau_svm_fault_replay(struct nouveau_svm *svm) +{ + SVM_DBG(svm, "replay"); + WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, + GP100_VMM_VN_FAULT_REPLAY, + &(struct gp100_vmm_fault_replay_vn) {}, + sizeof(struct gp100_vmm_fault_replay_vn))); +} + +/* Cancel a replayable fault that could not be handled. + * + * Cancelling the fault will trigger recovery to reset the engine + * and kill the offending channel (ie. GPU SIGSEGV). + */ +static void +nouveau_svm_fault_cancel(struct nouveau_svm *svm, + u64 inst, u8 hub, u8 gpc, u8 client) +{ + SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client); + WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, + GP100_VMM_VN_FAULT_CANCEL, + &(struct gp100_vmm_fault_cancel_v0) { + .hub = hub, + .gpc = gpc, + .client = client, + .inst = inst, + }, sizeof(struct gp100_vmm_fault_cancel_v0))); +} + +static void +nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm, + struct nouveau_svm_fault *fault) +{ + nouveau_svm_fault_cancel(svm, fault->inst, + fault->hub, + fault->gpc, + fault->client); +} + +static int +nouveau_svm_fault_cmp(const void *a, const void *b) +{ + const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a; + const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b; + int ret; + if ((ret = (s64)fa->inst - fb->inst)) + return ret; + if ((ret = (s64)fa->addr - fb->addr)) + return ret; + /*XXX: atomic? */ + return (fa->access == 0 || fa->access == 3) - + (fb->access == 0 || fb->access == 3); +} + +static void +nouveau_svm_fault_cache(struct nouveau_svm *svm, + struct nouveau_svm_fault_buffer *buffer, u32 offset) +{ + struct nvif_object *memory = &buffer->object; + const u32 instlo = nvif_rd32(memory, offset + 0x00); + const u32 insthi = nvif_rd32(memory, offset + 0x04); + const u32 addrlo = nvif_rd32(memory, offset + 0x08); + const u32 addrhi = nvif_rd32(memory, offset + 0x0c); + const u32 timelo = nvif_rd32(memory, offset + 0x10); + const u32 timehi = nvif_rd32(memory, offset + 0x14); + const u32 engine = nvif_rd32(memory, offset + 0x18); + const u32 info = nvif_rd32(memory, offset + 0x1c); + const u64 inst = (u64)insthi << 32 | instlo; + const u8 gpc = (info & 0x1f000000) >> 24; + const u8 hub = (info & 0x00100000) >> 20; + const u8 client = (info & 0x00007f00) >> 8; + struct nouveau_svm_fault *fault; + + //XXX: i think we're supposed to spin waiting */ + if (WARN_ON(!(info & 0x80000000))) + return; + + nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000); + + if (!buffer->fault[buffer->fault_nr]) { + fault = kmalloc(sizeof(*fault), GFP_KERNEL); + if (WARN_ON(!fault)) { + nouveau_svm_fault_cancel(svm, inst, hub, gpc, client); + return; + } + buffer->fault[buffer->fault_nr] = fault; + } + + fault = buffer->fault[buffer->fault_nr++]; + fault->inst = inst; + fault->addr = (u64)addrhi << 32 | addrlo; + fault->time = (u64)timehi << 32 | timelo; + fault->engine = engine; + fault->gpc = gpc; + fault->hub = hub; + fault->access = (info & 0x000f0000) >> 16; + fault->client = client; + fault->fault = (info & 0x0000001f); + + SVM_DBG(svm, "fault %016llx %016llx %02x", + fault->inst, fault->addr, fault->access); +} + +static int +nouveau_svm_fault(struct nvif_notify *notify) +{ + struct nouveau_svm_fault_buffer *buffer = + container_of(notify, typeof(*buffer), notify); + struct nouveau_svm *svm = + container_of(buffer, typeof(*svm), buffer[buffer->id]); + struct nvif_object *device = &svm->drm->client.device.object; + struct nouveau_svmm *svmm; + struct { + struct { + struct nvif_ioctl_v0 i; + struct nvif_ioctl_mthd_v0 m; + struct nvif_vmm_pfnmap_v0 p; + } i; + u64 phys[16]; + } args; + struct hmm_range range; + struct vm_area_struct *vma; + u64 inst, start, limit; + int fi, fn, pi, fill; + int replay = 0, ret; + + /* Parse available fault buffer entries into a cache, and update + * the GET pointer so HW can reuse the entries. + */ + SVM_DBG(svm, "fault handler"); + if (buffer->get == buffer->put) { + buffer->put = nvif_rd32(device, buffer->putaddr); + buffer->get = nvif_rd32(device, buffer->getaddr); + if (buffer->get == buffer->put) + return NVIF_NOTIFY_KEEP; + } + buffer->fault_nr = 0; + + SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put); + while (buffer->get != buffer->put) { + nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20); + if (++buffer->get == buffer->entries) + buffer->get = 0; + } + nvif_wr32(device, buffer->getaddr, buffer->get); + SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr); + + /* Sort parsed faults by instance pointer to prevent unnecessary + * instance to SVMM translations, followed by address and access + * type to reduce the amount of work when handling the faults. + */ + sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault), + nouveau_svm_fault_cmp, NULL); + + /* Lookup SVMM structure for each unique instance pointer. */ + mutex_lock(&svm->mutex); + for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) { + if (!svmm || buffer->fault[fi]->inst != inst) { + struct nouveau_ivmm *ivmm = + nouveau_ivmm_find(svm, buffer->fault[fi]->inst); + svmm = ivmm ? ivmm->svmm : NULL; + inst = buffer->fault[fi]->inst; + SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm); + } + buffer->fault[fi]->svmm = svmm; + } + mutex_unlock(&svm->mutex); + + /* Process list of faults. */ + args.i.i.version = 0; + args.i.i.type = NVIF_IOCTL_V0_MTHD; + args.i.m.version = 0; + args.i.m.method = NVIF_VMM_V0_PFNMAP; + args.i.p.version = 0; + + for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) { + /* Cancel any faults from non-SVM channels. */ + if (!(svmm = buffer->fault[fi]->svmm)) { + nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); + continue; + } + SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr); + + /* We try and group handling of faults within a small + * window into a single update. + */ + start = buffer->fault[fi]->addr; + limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT); + if (start < svmm->unmanaged.limit) + limit = min_t(u64, limit, svmm->unmanaged.start); + else + if (limit > svmm->unmanaged.start) + start = max_t(u64, start, svmm->unmanaged.limit); + SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); + + /* Intersect fault window with the CPU VMA, cancelling + * the fault if the address is invalid. + */ + down_read(&svmm->mm->mmap_sem); + vma = find_vma_intersection(svmm->mm, start, limit); + if (!vma) { + SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit); + up_read(&svmm->mm->mmap_sem); + nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); + continue; + } + start = max_t(u64, start, vma->vm_start); + limit = min_t(u64, limit, vma->vm_end); + SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); + + if (buffer->fault[fi]->addr != start) { + SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr); + up_read(&svmm->mm->mmap_sem); + nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); + continue; + } + + /* Prepare the GPU-side update of all pages within the + * fault window, determining required pages and access + * permissions based on pending faults. + */ + args.i.p.page = PAGE_SHIFT; + args.i.p.addr = start; + for (fn = fi, pi = 0;;) { + /* Determine required permissions based on GPU fault + * access flags. + *XXX: atomic? + */ + if (buffer->fault[fn]->access != 0 /* READ. */ && + buffer->fault[fn]->access != 3 /* PREFETCH. */) { + args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V | + NVIF_VMM_PFNMAP_V0_W; + } else { + args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V; + } + args.i.p.size = pi << PAGE_SHIFT; + + /* It's okay to skip over duplicate addresses from the + * same SVMM as faults are ordered by access type such + * that only the first one needs to be handled. + * + * ie. WRITE faults appear first, thus any handling of + * pending READ faults will already be satisfied. + */ + while (++fn < buffer->fault_nr && + buffer->fault[fn]->svmm == svmm && + buffer->fault[fn ]->addr == + buffer->fault[fn - 1]->addr); + + /* If the next fault is outside the window, or all GPU + * faults have been dealt with, we're done here. + */ + if (fn >= buffer->fault_nr || + buffer->fault[fn]->svmm != svmm || + buffer->fault[fn]->addr >= limit) + break; + + /* Fill in the gap between this fault and the next. */ + fill = (buffer->fault[fn ]->addr - + buffer->fault[fn - 1]->addr) >> PAGE_SHIFT; + while (--fill) + args.phys[pi++] = NVIF_VMM_PFNMAP_V0_NONE; + } + + SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)", + args.i.p.addr, + args.i.p.addr + args.i.p.size, fn - fi); + + /* Have HMM fault pages within the fault window to the GPU. */ + range.vma = vma; + range.start = args.i.p.addr; + range.end = args.i.p.addr + args.i.p.size; + range.pfns = args.phys; + range.flags = nouveau_svm_pfn_flags; + range.values = nouveau_svm_pfn_values; + range.pfn_shift = NVIF_VMM_PFNMAP_V0_ADDR_SHIFT; +again: + ret = hmm_vma_fault(&range, true); + if (ret == 0) { + mutex_lock(&svmm->mutex); + if (!hmm_vma_range_done(&range)) { + mutex_unlock(&svmm->mutex); + goto again; + } + + nouveau_dmem_convert_pfn(svm->drm, &range); + + svmm->vmm->vmm.object.client->super = true; + ret = nvif_object_ioctl(&svmm->vmm->vmm.object, + &args, sizeof(args.i) + + pi * sizeof(args.phys[0]), + NULL); + svmm->vmm->vmm.object.client->super = false; + mutex_unlock(&svmm->mutex); + } + up_read(&svmm->mm->mmap_sem); + + /* Cancel any faults in the window whose pages didn't manage + * to keep their valid bit, or stay writeable when required. + * + * If handling failed completely, cancel all faults. + */ + while (fi < fn) { + struct nouveau_svm_fault *fault = buffer->fault[fi++]; + pi = (fault->addr - range.start) >> PAGE_SHIFT; + if (ret || + !(range.pfns[pi] & NVIF_VMM_PFNMAP_V0_V) || + (!(range.pfns[pi] & NVIF_VMM_PFNMAP_V0_W) && + fault->access != 0 && fault->access != 3)) { + nouveau_svm_fault_cancel_fault(svm, fault); + continue; + } + replay++; + } + } + + /* Issue fault replay to the GPU. */ + if (replay) + nouveau_svm_fault_replay(svm); + return NVIF_NOTIFY_KEEP; +} + +static void +nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id) +{ + struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; + nvif_notify_put(&buffer->notify); +} + +static int +nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id) +{ + struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; + struct nvif_object *device = &svm->drm->client.device.object; + buffer->get = nvif_rd32(device, buffer->getaddr); + buffer->put = nvif_rd32(device, buffer->putaddr); + SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put); + return nvif_notify_get(&buffer->notify); +} + +static void +nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id) +{ + struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; + int i; + + if (buffer->fault) { + for (i = 0; buffer->fault[i] && i < buffer->entries; i++) + kfree(buffer->fault[i]); + kvfree(buffer->fault); + } + + nouveau_svm_fault_buffer_fini(svm, id); + + nvif_notify_fini(&buffer->notify); + nvif_object_fini(&buffer->object); +} + +static int +nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id) +{ + struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; + struct nouveau_drm *drm = svm->drm; + struct nvif_object *device = &drm->client.device.object; + struct nvif_clb069_v0 args = {}; + int ret; + + buffer->id = id; + + ret = nvif_object_init(device, 0, oclass, &args, sizeof(args), + &buffer->object); + if (ret < 0) { + SVM_ERR(svm, "Fault buffer allocation failed: %d", ret); + return ret; + } + + nvif_object_map(&buffer->object, NULL, 0); + buffer->entries = args.entries; + buffer->getaddr = args.get; + buffer->putaddr = args.put; + + ret = nvif_notify_init(&buffer->object, nouveau_svm_fault, true, + NVB069_V0_NTFY_FAULT, NULL, 0, 0, + &buffer->notify); + if (ret) + return ret; + + buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL); + if (!buffer->fault) + return -ENOMEM; + + return nouveau_svm_fault_buffer_init(svm, id); +} + +void +nouveau_svm_resume(struct nouveau_drm *drm) +{ + struct nouveau_svm *svm = drm->svm; + if (svm) + nouveau_svm_fault_buffer_init(svm, 0); +} + +void +nouveau_svm_suspend(struct nouveau_drm *drm) +{ + struct nouveau_svm *svm = drm->svm; + if (svm) + nouveau_svm_fault_buffer_fini(svm, 0); +} + +void +nouveau_svm_fini(struct nouveau_drm *drm) +{ + struct nouveau_svm *svm = drm->svm; + if (svm) { + nouveau_svm_fault_buffer_dtor(svm, 0); + kfree(drm->svm); + drm->svm = NULL; + } +} + +void +nouveau_svm_init(struct nouveau_drm *drm) +{ + static const struct nvif_mclass buffers[] = { + { VOLTA_FAULT_BUFFER_A, 0 }, + { MAXWELL_FAULT_BUFFER_A, 0 }, + {} + }; + struct nouveau_svm *svm; + int ret; + + /* Disable on Volta and newer until channel recovery is fixed, + * otherwise clients will have a trivial way to trash the GPU + * for everyone. + */ + if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL) + return; + + if (!(drm->svm = svm = kzalloc(sizeof(*drm->svm), GFP_KERNEL))) + return; + + drm->svm->drm = drm; + mutex_init(&drm->svm->mutex); + INIT_LIST_HEAD(&drm->svm->inst); + + ret = nvif_mclass(&drm->client.device.object, buffers); + if (ret < 0) { + SVM_DBG(svm, "No supported fault buffer class"); + nouveau_svm_fini(drm); + return; + } + + ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0); + if (ret) { + nouveau_svm_fini(drm); + return; + } + + SVM_DBG(svm, "Initialised"); +} diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.h b/drivers/gpu/drm/nouveau/nouveau_svm.h new file mode 100644 index 000000000000..e839d8189461 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_svm.h @@ -0,0 +1,48 @@ +#ifndef __NOUVEAU_SVM_H__ +#define __NOUVEAU_SVM_H__ +#include <nvif/os.h> +struct drm_device; +struct drm_file; +struct nouveau_drm; + +struct nouveau_svmm; + +#if IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) +void nouveau_svm_init(struct nouveau_drm *); +void nouveau_svm_fini(struct nouveau_drm *); +void nouveau_svm_suspend(struct nouveau_drm *); +void nouveau_svm_resume(struct nouveau_drm *); + +int nouveau_svmm_init(struct drm_device *, void *, struct drm_file *); +void nouveau_svmm_fini(struct nouveau_svmm **); +int nouveau_svmm_join(struct nouveau_svmm *, u64 inst); +void nouveau_svmm_part(struct nouveau_svmm *, u64 inst); +int nouveau_svmm_bind(struct drm_device *, void *, struct drm_file *); +#else /* IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) */ +static inline void nouveau_svm_init(struct nouveau_drm *drm) {} +static inline void nouveau_svm_fini(struct nouveau_drm *drm) {} +static inline void nouveau_svm_suspend(struct nouveau_drm *drm) {} +static inline void nouveau_svm_resume(struct nouveau_drm *drm) {} + +static inline int nouveau_svmm_init(struct drm_device *device, void *p, + struct drm_file *file) +{ + return -ENOSYS; +} + +static inline void nouveau_svmm_fini(struct nouveau_svmm **svmmp) {} + +static inline int nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) +{ + return 0; +} + +static inline void nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst) {} + +static inline int nouveau_svmm_bind(struct drm_device *device, void *p, + struct drm_file *file) +{ + return -ENOSYS; +} +#endif /* IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM) */ +#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index 2032c3e4f6e5..77061182a1cf 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -22,6 +22,7 @@ #include "nouveau_vmm.h" #include "nouveau_drv.h" #include "nouveau_bo.h" +#include "nouveau_svm.h" #include "nouveau_mem.h" void @@ -119,6 +120,7 @@ done: void nouveau_vmm_fini(struct nouveau_vmm *vmm) { + nouveau_svmm_fini(&vmm->svmm); nvif_vmm_fini(&vmm->vmm); vmm->cli = NULL; } @@ -126,7 +128,7 @@ nouveau_vmm_fini(struct nouveau_vmm *vmm) int nouveau_vmm_init(struct nouveau_cli *cli, s32 oclass, struct nouveau_vmm *vmm) { - int ret = nvif_vmm_init(&cli->mmu, oclass, PAGE_SIZE, 0, NULL, 0, + int ret = nvif_vmm_init(&cli->mmu, oclass, false, PAGE_SIZE, 0, NULL, 0, &vmm->vmm); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h index ede872f6f668..2b98d975f37e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.h +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h @@ -25,6 +25,7 @@ void nouveau_vma_unmap(struct nouveau_vma *); struct nouveau_vmm { struct nouveau_cli *cli; struct nvif_vmm vmm; + struct nouveau_svmm *svmm; }; int nouveau_vmm_init(struct nouveau_cli *, s32 oclass, struct nouveau_vmm *); diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index e721bb2163a0..f07da00f285f 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -109,7 +109,6 @@ nv84_fence_context_del(struct nouveau_channel *chan) int nv84_fence_context_new(struct nouveau_channel *chan) { - struct nouveau_cli *cli = (void *)chan->user.client; struct nv84_fence_priv *priv = chan->drm->fence; struct nv84_fence_chan *fctx; int ret; @@ -127,7 +126,7 @@ nv84_fence_context_new(struct nouveau_channel *chan) fctx->base.sequence = nv84_fence_read(chan); mutex_lock(&priv->mutex); - ret = nouveau_vma_new(priv->bo, &cli->vmm, &fctx->vma); + ret = nouveau_vma_new(priv->bo, chan->vmm, &fctx->vma); mutex_unlock(&priv->mutex); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvif/disp.c b/drivers/gpu/drm/nouveau/nvif/disp.c index ef97dd223a32..61638b3b9d3d 100644 --- a/drivers/gpu/drm/nouveau/nvif/disp.c +++ b/drivers/gpu/drm/nouveau/nvif/disp.c @@ -34,7 +34,7 @@ int nvif_disp_ctor(struct nvif_device *device, s32 oclass, struct nvif_disp *disp) { static const struct nvif_mclass disps[] = { - { TU104_DISP, -1 }, + { TU102_DISP, -1 }, { GV100_DISP, -1 }, { GP102_DISP, -1 }, { GP100_DISP, -1 }, diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c index 6b9c5776547f..11487c00b909 100644 --- a/drivers/gpu/drm/nouveau/nvif/vmm.c +++ b/drivers/gpu/drm/nouveau/nvif/vmm.c @@ -112,8 +112,8 @@ nvif_vmm_fini(struct nvif_vmm *vmm) } int -nvif_vmm_init(struct nvif_mmu *mmu, s32 oclass, u64 addr, u64 size, - void *argv, u32 argc, struct nvif_vmm *vmm) +nvif_vmm_init(struct nvif_mmu *mmu, s32 oclass, bool managed, u64 addr, + u64 size, void *argv, u32 argc, struct nvif_vmm *vmm) { struct nvif_vmm_v0 *args; u32 argn = sizeof(*args) + argc; @@ -125,6 +125,7 @@ nvif_vmm_init(struct nvif_mmu *mmu, s32 oclass, u64 addr, u64 size, if (!(args = kmalloc(argn, GFP_KERNEL))) return -ENOMEM; args->version = 0; + args->managed = managed; args->addr = addr; args->size = size; memcpy(args->data, argv, argc); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c index c61b467cf45e..245990de1e90 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c @@ -39,6 +39,7 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = { [NVKM_SUBDEV_FB ] = "fb", [NVKM_SUBDEV_FUSE ] = "fuse", [NVKM_SUBDEV_GPIO ] = "gpio", + [NVKM_SUBDEV_GSP ] = "gsp", [NVKM_SUBDEV_I2C ] = "i2c", [NVKM_SUBDEV_IBUS ] = "priv", [NVKM_SUBDEV_ICCSENSE] = "iccsense", diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild index 177a23301d6a..9211663239af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild @@ -6,4 +6,4 @@ nvkm-y += nvkm/engine/ce/gm200.o nvkm-y += nvkm/engine/ce/gp100.o nvkm-y += nvkm/engine/ce/gp102.o nvkm-y += nvkm/engine/ce/gv100.o -nvkm-y += nvkm/engine/ce/tu104.o +nvkm-y += nvkm/engine/ce/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu102.c index 3c25043bbb33..b4308e2d8c75 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu102.c @@ -24,7 +24,7 @@ #include <nvif/class.h> static const struct nvkm_engine_func -tu104_ce = { +tu102_ce = { .intr = gp100_ce_intr, .sclass = { { -1, -1, TURING_DMA_COPY_A }, @@ -33,8 +33,8 @@ tu104_ce = { }; int -tu104_ce_new(struct nvkm_device *device, int index, +tu102_ce_new(struct nvkm_device *device, int index, struct nvkm_engine **pengine) { - return nvkm_engine_new_(&tu104_ce, device, index, true, pengine); + return nvkm_engine_new_(&tu102_ce, device, index, true, pengine); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index bfbc9341e0c2..7971096b6767 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1613,7 +1613,7 @@ nvd7_chipset = { .pci = gf106_pci_new, .therm = gf119_therm_new, .timer = nv41_timer_new, - .volt = gf100_volt_new, + .volt = gf117_volt_new, .ce[0] = gf100_ce_new, .disp = gf119_disp_new, .dma = gf119_dma_new, @@ -2405,6 +2405,7 @@ nv140_chipset = { .fb = gv100_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, .i2c = gm200_i2c_new, .ibus = gm200_ibus_new, .imem = nv50_instmem_new, @@ -2435,67 +2436,108 @@ nv140_chipset = { }; static const struct nvkm_device_chip +nv162_chipset = { + .name = "TU102", + .bar = tu102_bar_new, + .bios = nvkm_bios_new, + .bus = gf100_bus_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, + .fb = gv100_fb_new, + .fuse = gm107_fuse_new, + .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, + .i2c = gm200_i2c_new, + .ibus = gm200_ibus_new, + .imem = nv50_instmem_new, + .ltc = gp102_ltc_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, + .pci = gp100_pci_new, + .pmu = gp102_pmu_new, + .therm = gp100_therm_new, + .timer = gk20a_timer_new, + .top = gk104_top_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, + .dma = gv100_dma_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, +}; + +static const struct nvkm_device_chip nv164_chipset = { .name = "TU104", - .bar = tu104_bar_new, + .bar = tu102_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, - .devinit = tu104_devinit_new, - .fault = tu104_fault_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, .fb = gv100_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, .i2c = gm200_i2c_new, .ibus = gm200_ibus_new, .imem = nv50_instmem_new, .ltc = gp102_ltc_new, - .mc = tu104_mc_new, - .mmu = tu104_mmu_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, .therm = gp100_therm_new, .timer = gk20a_timer_new, .top = gk104_top_new, - .ce[0] = tu104_ce_new, - .ce[1] = tu104_ce_new, - .ce[2] = tu104_ce_new, - .ce[3] = tu104_ce_new, - .ce[4] = tu104_ce_new, - .disp = tu104_disp_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, .dma = gv100_dma_new, - .fifo = tu104_fifo_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, }; static const struct nvkm_device_chip nv166_chipset = { .name = "TU106", - .bar = tu104_bar_new, + .bar = tu102_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, - .devinit = tu104_devinit_new, - .fault = tu104_fault_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, .fb = gv100_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, .i2c = gm200_i2c_new, .ibus = gm200_ibus_new, .imem = nv50_instmem_new, .ltc = gp102_ltc_new, - .mc = tu104_mc_new, - .mmu = tu104_mmu_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, .therm = gp100_therm_new, .timer = gk20a_timer_new, .top = gk104_top_new, - .ce[0] = tu104_ce_new, - .ce[1] = tu104_ce_new, - .ce[2] = tu104_ce_new, - .ce[3] = tu104_ce_new, - .ce[4] = tu104_ce_new, - .disp = tu104_disp_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, .dma = gv100_dma_new, - .fifo = tu104_fifo_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, }; static int @@ -2535,6 +2577,7 @@ nvkm_device_subdev(struct nvkm_device *device, int index) _(FB , device->fb , &device->fb->subdev); _(FUSE , device->fuse , &device->fuse->subdev); _(GPIO , device->gpio , &device->gpio->subdev); + _(GSP , device->gsp , &device->gsp->subdev); _(I2C , device->i2c , &device->i2c->subdev); _(IBUS , device->ibus , device->ibus); _(ICCSENSE, device->iccsense, &device->iccsense->subdev); @@ -2950,6 +2993,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, case 0x138: device->chip = &nv138_chipset; break; case 0x13b: device->chip = &nv13b_chipset; break; case 0x140: device->chip = &nv140_chipset; break; + case 0x162: device->chip = &nv162_chipset; break; case 0x164: device->chip = &nv164_chipset; break; case 0x166: device->chip = &nv166_chipset; break; default: @@ -3017,6 +3061,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, _(NVKM_SUBDEV_FB , fb); _(NVKM_SUBDEV_FUSE , fuse); _(NVKM_SUBDEV_GPIO , gpio); + _(NVKM_SUBDEV_GSP , gsp); _(NVKM_SUBDEV_I2C , i2c); _(NVKM_SUBDEV_IBUS , ibus); _(NVKM_SUBDEV_ICCSENSE, iccsense); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h index 253ab914a8ef..2a53e37dfa7a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h @@ -12,6 +12,7 @@ #include <subdev/fb.h> #include <subdev/fuse.h> #include <subdev/gpio.h> +#include <subdev/gsp.h> #include <subdev/i2c.h> #include <subdev/ibus.h> #include <subdev/iccsense.h> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 092ddc4ffefa..03c6d9aef075 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -365,16 +365,15 @@ nvkm_udevice_child_get(struct nvkm_object *object, int index, } if (!sclass) { - switch (index) { - case 0: sclass = &nvkm_control_oclass; break; - case 1: - if (!device->mmu) - return -EINVAL; + if (index-- == 0) + sclass = &nvkm_control_oclass; + else if (device->mmu && index-- == 0) sclass = &device->mmu->user; - break; - default: + else if (device->fault && index-- == 0) + sclass = &device->fault->user; + else return -EINVAL; - } + oclass->base = sclass->base; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index c6a257ba4347..2c28a5e747cc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -15,7 +15,7 @@ nvkm-y += nvkm/engine/disp/gm200.o nvkm-y += nvkm/engine/disp/gp100.o nvkm-y += nvkm/engine/disp/gp102.o nvkm-y += nvkm/engine/disp/gv100.o -nvkm-y += nvkm/engine/disp/tu104.o +nvkm-y += nvkm/engine/disp/tu102.o nvkm-y += nvkm/engine/disp/vga.o nvkm-y += nvkm/engine/disp/head.o @@ -39,7 +39,7 @@ nvkm-y += nvkm/engine/disp/sorgk104.o nvkm-y += nvkm/engine/disp/sorgm107.o nvkm-y += nvkm/engine/disp/sorgm200.o nvkm-y += nvkm/engine/disp/sorgv100.o -nvkm-y += nvkm/engine/disp/sortu104.o +nvkm-y += nvkm/engine/disp/sortu102.o nvkm-y += nvkm/engine/disp/outp.o nvkm-y += nvkm/engine/disp/dp.o @@ -71,7 +71,7 @@ nvkm-y += nvkm/engine/disp/rootgm200.o nvkm-y += nvkm/engine/disp/rootgp100.o nvkm-y += nvkm/engine/disp/rootgp102.o nvkm-y += nvkm/engine/disp/rootgv100.o -nvkm-y += nvkm/engine/disp/roottu104.o +nvkm-y += nvkm/engine/disp/roottu102.o nvkm-y += nvkm/engine/disp/channv50.o nvkm-y += nvkm/engine/disp/changf119.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c index 794e90982641..e675d9b9d5d7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c @@ -91,15 +91,21 @@ gf119_disp_intr_error(struct nv50_disp *disp, int chid) { struct nvkm_subdev *subdev = &disp->base.engine.subdev; struct nvkm_device *device = subdev->device; - u32 mthd = nvkm_rd32(device, 0x6101f0 + (chid * 12)); + u32 stat = nvkm_rd32(device, 0x6101f0 + (chid * 12)); + u32 type = (stat & 0x00007000) >> 12; + u32 mthd = (stat & 0x00000ffc); u32 data = nvkm_rd32(device, 0x6101f4 + (chid * 12)); - u32 unkn = nvkm_rd32(device, 0x6101f8 + (chid * 12)); + u32 code = nvkm_rd32(device, 0x6101f8 + (chid * 12)); + const struct nvkm_enum *reason = + nvkm_enum_find(nv50_disp_intr_error_type, type); - nvkm_error(subdev, "chid %d mthd %04x data %08x %08x %08x\n", - chid, (mthd & 0x0000ffc), data, mthd, unkn); + nvkm_error(subdev, "chid %d stat %08x reason %d [%s] mthd %04x " + "data %08x code %08x\n", + chid, stat, type, reason ? reason->name : "", + mthd, data, code); if (chid < ARRAY_SIZE(disp->chan)) { - switch (mthd & 0xffc) { + switch (mthd) { case 0x0080: nv50_disp_chan_mthd(disp->chan[chid], NV_DBG_ERROR); break; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index 47be0ba4aebe..892be6c9b76c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -103,10 +103,13 @@ gv100_disp_exception(struct nv50_disp *disp, int chid) u32 mthd = (stat & 0x00000fff) << 2; u32 data = nvkm_rd32(device, 0x611024 + (chid * 12)); u32 code = nvkm_rd32(device, 0x611028 + (chid * 12)); + const struct nvkm_enum *reason = + nvkm_enum_find(nv50_disp_intr_error_type, type); - nvkm_error(subdev, "chid %d %08x [type %d mthd %04x] " + nvkm_error(subdev, "chid %d stat %08x reason %d [%s] mthd %04x " "data %08x code %08x\n", - chid, stat, type, mthd, data, code); + chid, stat, type, reason ? reason->name : "", + mthd, data, code); if (chid < ARRAY_SIZE(disp->chan) && disp->chan[chid]) { switch (mthd) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index 790e42f460fd..1681ddccd298 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -201,5 +201,5 @@ int gm200_sor_new(struct nvkm_disp *, int); int gv100_sor_cnt(struct nvkm_disp *, unsigned long *); int gv100_sor_new(struct nvkm_disp *, int); -int tu104_sor_new(struct nvkm_disp *, int); +int tu102_sor_new(struct nvkm_disp *, int); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c index def005dd5fda..e21556bf2cb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c @@ -28,7 +28,6 @@ #include "rootnv50.h" #include <core/client.h> -#include <core/enum.h> #include <core/ramht.h> #include <subdev/bios.h> #include <subdev/bios/disp.h> @@ -593,12 +592,15 @@ nv50_disp_super(struct work_struct *work) nvkm_wr32(device, 0x610030, 0x80000000); } -static const struct nvkm_enum +const struct nvkm_enum nv50_disp_intr_error_type[] = { - { 3, "ILLEGAL_MTHD" }, - { 4, "INVALID_VALUE" }, + { 0, "NONE" }, + { 1, "PUSHBUFFER_ERR" }, + { 2, "TRAP" }, + { 3, "RESERVED_METHOD" }, + { 4, "INVALID_ARG" }, { 5, "INVALID_STATE" }, - { 7, "INVALID_HANDLE" }, + { 7, "UNRESOLVABLE_HANDLE" }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h index c36a8a7cafa1..e5d00f478bb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h @@ -5,6 +5,8 @@ #include "priv.h" struct nvkm_head; +#include <core/enum.h> + struct nv50_disp { const struct nv50_disp_func *func; struct nvkm_disp base; @@ -71,6 +73,7 @@ int nv50_disp_init(struct nv50_disp *); void nv50_disp_fini(struct nv50_disp *); void nv50_disp_intr(struct nv50_disp *); void nv50_disp_super(struct work_struct *); +extern const struct nvkm_enum nv50_disp_intr_error_type[]; int gf119_disp_init(struct nv50_disp *); void gf119_disp_fini(struct nv50_disp *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h index 97de928cbde1..aee9822a7a87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h @@ -37,5 +37,5 @@ extern const struct nvkm_disp_oclass gm200_disp_root_oclass; extern const struct nvkm_disp_oclass gp100_disp_root_oclass; extern const struct nvkm_disp_oclass gp102_disp_root_oclass; extern const struct nvkm_disp_oclass gv100_disp_root_oclass; -extern const struct nvkm_disp_oclass tu104_disp_root_oclass; +extern const struct nvkm_disp_oclass tu102_disp_root_oclass; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu102.c index ad438c62f66c..579a5d02308a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu102.c @@ -25,28 +25,28 @@ #include <nvif/class.h> static const struct nv50_disp_root_func -tu104_disp_root = { +tu102_disp_root = { .user = { - {{0,0,TU104_DISP_CURSOR }, gv100_disp_curs_new }, - {{0,0,TU104_DISP_WINDOW_IMM_CHANNEL_DMA}, gv100_disp_wimm_new }, - {{0,0,TU104_DISP_CORE_CHANNEL_DMA }, gv100_disp_core_new }, - {{0,0,TU104_DISP_WINDOW_CHANNEL_DMA }, gv100_disp_wndw_new }, + {{0,0,TU102_DISP_CURSOR }, gv100_disp_curs_new }, + {{0,0,TU102_DISP_WINDOW_IMM_CHANNEL_DMA}, gv100_disp_wimm_new }, + {{0,0,TU102_DISP_CORE_CHANNEL_DMA }, gv100_disp_core_new }, + {{0,0,TU102_DISP_WINDOW_CHANNEL_DMA }, gv100_disp_wndw_new }, {} }, }; static int -tu104_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass, +tu102_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass, void *data, u32 size, struct nvkm_object **pobject) { - return nv50_disp_root_new_(&tu104_disp_root, disp, oclass, + return nv50_disp_root_new_(&tu102_disp_root, disp, oclass, data, size, pobject); } const struct nvkm_disp_oclass -tu104_disp_root_oclass = { - .base.oclass = TU104_DISP, +tu102_disp_root_oclass = { + .base.oclass = TU102_DISP, .base.minver = -1, .base.maxver = -1, - .ctor = tu104_disp_root_new, + .ctor = tu102_disp_root_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c index df026a525ef1..d57b73ada89e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c @@ -24,7 +24,7 @@ #include <subdev/timer.h> static void -tu104_sor_dp_vcpi(struct nvkm_ior *sor, int head, +tu102_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned) { struct nvkm_device *device = sor->disp->engine.subdev.device; @@ -35,7 +35,7 @@ tu104_sor_dp_vcpi(struct nvkm_ior *sor, int head, } static int -tu104_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux) +tu102_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux) { struct nvkm_device *device = sor->disp->engine.subdev.device; const u32 soff = nv50_ior_base(sor); @@ -62,7 +62,7 @@ tu104_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux) } static const struct nvkm_ior_func -tu104_sor = { +tu102_sor = { .route = { .get = gm200_sor_route_get, .set = gm200_sor_route_set, @@ -75,11 +75,11 @@ tu104_sor = { }, .dp = { .lanes = { 0, 1, 2, 3 }, - .links = tu104_sor_dp_links, + .links = tu102_sor_dp_links, .power = g94_sor_dp_power, .pattern = gm107_sor_dp_pattern, .drive = gm200_sor_dp_drive, - .vcpi = tu104_sor_dp_vcpi, + .vcpi = tu102_sor_dp_vcpi, .audio = gv100_sor_dp_audio, .audio_sym = gv100_sor_dp_audio_sym, .watermark = gv100_sor_dp_watermark, @@ -91,7 +91,7 @@ tu104_sor = { }; int -tu104_sor_new(struct nvkm_disp *disp, int id) +tu102_sor_new(struct nvkm_disp *disp, int id) { - return nvkm_ior_new_(&tu104_sor, disp, SOR, id); + return nvkm_ior_new_(&tu102_sor, disp, SOR, id); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c index 13fa21459d38..883ae4151ff8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c @@ -29,7 +29,7 @@ #include <subdev/timer.h> static int -tu104_disp_init(struct nv50_disp *disp) +tu102_disp_init(struct nv50_disp *disp) { struct nvkm_device *device = disp->base.engine.subdev.device; struct nvkm_head *head; @@ -132,21 +132,21 @@ tu104_disp_init(struct nv50_disp *disp) } static const struct nv50_disp_func -tu104_disp = { - .init = tu104_disp_init, +tu102_disp = { + .init = tu102_disp_init, .fini = gv100_disp_fini, .intr = gv100_disp_intr, .uevent = &gv100_disp_chan_uevent, .super = gv100_disp_super, - .root = &tu104_disp_root_oclass, + .root = &tu102_disp_root_oclass, .wndw = { .cnt = gv100_disp_wndw_cnt }, .head = { .cnt = gv100_head_cnt, .new = gv100_head_new }, - .sor = { .cnt = gv100_sor_cnt, .new = tu104_sor_new }, + .sor = { .cnt = gv100_sor_cnt, .new = tu102_sor_new }, .ramht_size = 0x2000, }; int -tu104_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp) +tu102_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp) { - return nv50_disp_new_(&tu104_disp, device, index, pdisp); + return nv50_disp_new_(&tu102_disp, device, index, pdisp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c index 816ccaedfc73..8675613e142b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c @@ -22,6 +22,7 @@ #include <engine/falcon.h> #include <core/gpuobj.h> +#include <subdev/mc.h> #include <subdev/timer.h> #include <engine/fifo.h> @@ -107,8 +108,10 @@ nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend) } } - nvkm_mask(device, base + 0x048, 0x00000003, 0x00000000); - nvkm_wr32(device, base + 0x014, 0xffffffff); + if (nvkm_mc_enabled(device, engine->subdev.index)) { + nvkm_mask(device, base + 0x048, 0x00000003, 0x00000000); + nvkm_wr32(device, base + 0x014, 0xffffffff); + } return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 87d8e054e40a..05aada541ea5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -16,7 +16,7 @@ nvkm-y += nvkm/engine/fifo/gm20b.o nvkm-y += nvkm/engine/fifo/gp100.o nvkm-y += nvkm/engine/fifo/gp10b.o nvkm-y += nvkm/engine/fifo/gv100.o -nvkm-y += nvkm/engine/fifo/tu104.o +nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/channv50.o @@ -34,7 +34,7 @@ nvkm-y += nvkm/engine/fifo/gpfifog84.o nvkm-y += nvkm/engine/fifo/gpfifogf100.o nvkm-y += nvkm/engine/fifo/gpfifogk104.o nvkm-y += nvkm/engine/fifo/gpfifogv100.o -nvkm-y += nvkm/engine/fifo/gpfifotu104.o +nvkm-y += nvkm/engine/fifo/gpfifotu102.o nvkm-y += nvkm/engine/fifo/usergv100.o -nvkm-y += nvkm/engine/fifo/usertu104.o +nvkm-y += nvkm/engine/fifo/usertu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index a14545d871d8..f8557cdfbd81 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -47,6 +47,6 @@ int gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *, int gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *, struct nvkm_engine *, bool); -int tu104_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, +int tu102_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index ff70484dd01a..abef7fb6e2d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -29,14 +29,14 @@ #include <nvif/unpack.h> static u32 -tu104_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *base) +tu102_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); return (chan->runl << 16) | chan->base.chid; } static const struct nvkm_fifo_chan_func -tu104_fifo_gpfifo = { +tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, .init = gk104_fifo_gpfifo_init, .fini = gk104_fifo_gpfifo_fini, @@ -45,11 +45,11 @@ tu104_fifo_gpfifo = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, - .submit_token = tu104_fifo_gpfifo_submit_token, + .submit_token = tu102_fifo_gpfifo_submit_token, }; int -tu104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, +tu102_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, void *data, u32 size, struct nvkm_object **pobject) { struct nvkm_object *parent = oclass->parent; @@ -67,7 +67,7 @@ tu104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, args->v0.ilength, args->v0.runlist, args->v0.priv); if (args->v0.priv && !oclass->client->super) return -EINVAL; - return gv100_fifo_gpfifo_new_(&tu104_fifo_gpfifo, fifo, + return gv100_fifo_gpfifo_new_(&tu102_fifo_gpfifo, fifo, &args->v0.runlist, &args->v0.chid, args->v0.vmm, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 98c80705bc61..005f3e1729b9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -29,7 +29,7 @@ #include <nvif/class.h> static void -tu104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, +tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) { struct nvkm_device *device = fifo->base.engine.subdev.device; @@ -44,15 +44,15 @@ tu104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, } const struct gk104_fifo_runlist_func -tu104_fifo_runlist = { +tu102_fifo_runlist = { .size = 16, .cgrp = gv100_fifo_runlist_cgrp, .chan = gv100_fifo_runlist_chan, - .commit = tu104_fifo_runlist_commit, + .commit = tu102_fifo_runlist_commit, }; static const struct nvkm_enum -tu104_fifo_fault_engine[] = { +tu102_fifo_fault_engine[] = { { 0x01, "DISPLAY" }, { 0x03, "PTP" }, { 0x06, "PWR_PMU" }, @@ -80,7 +80,7 @@ tu104_fifo_fault_engine[] = { }; static void -tu104_fifo_pbdma_init(struct gk104_fifo *fifo) +tu102_fifo_pbdma_init(struct gk104_fifo *fifo) { struct nvkm_device *device = fifo->base.engine.subdev.device; const u32 mask = (1 << fifo->pbdma_nr) - 1; @@ -89,28 +89,28 @@ tu104_fifo_pbdma_init(struct gk104_fifo *fifo) } static const struct gk104_fifo_pbdma_func -tu104_fifo_pbdma = { +tu102_fifo_pbdma = { .nr = gm200_fifo_pbdma_nr, - .init = tu104_fifo_pbdma_init, + .init = tu102_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; static const struct gk104_fifo_func -tu104_fifo = { - .pbdma = &tu104_fifo_pbdma, +tu102_fifo = { + .pbdma = &tu102_fifo_pbdma, .fault.access = gv100_fifo_fault_access, - .fault.engine = tu104_fifo_fault_engine, + .fault.engine = tu102_fifo_fault_engine, .fault.reason = gv100_fifo_fault_reason, .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, - .runlist = &tu104_fifo_runlist, - .user = {{-1,-1,VOLTA_USERMODE_A }, tu104_fifo_user_new }, - .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu104_fifo_gpfifo_new }, + .runlist = &tu102_fifo_runlist, + .user = {{-1,-1,VOLTA_USERMODE_A }, tu102_fifo_user_new }, + .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, .cgrp_force = true, }; int -tu104_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo) +tu102_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&tu104_fifo, device, index, 4096, pfifo); + return gk104_fifo_new_(&tu102_fifo, device, index, 4096, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h index 14b0c6bde8eb..54a3a3092cc0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h @@ -3,6 +3,6 @@ #include "priv.h" int gv100_fifo_user_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); -int tu104_fifo_user_new(const struct nvkm_oclass *, void *, u32, +int tu102_fifo_user_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c index 8f98548a21f6..217268f8ccad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c @@ -22,7 +22,7 @@ #include "user.h" static int -tu104_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, +tu102_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) { struct nvkm_device *device = object->engine->subdev.device; @@ -33,13 +33,13 @@ tu104_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, } static const struct nvkm_object_func -tu104_fifo_user = { - .map = tu104_fifo_user_map, +tu102_fifo_user = { + .map = tu102_fifo_user_map, }; int -tu104_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, +tu102_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) { - return nvkm_object_new_(&tu104_fifo_user, oclass, argv, argc, pobject); + return nvkm_object_new_(&tu102_fifo_user, oclass, argv, argc, pobject); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c index cd8cf6f7024c..d41fb94524e9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c @@ -25,6 +25,33 @@ #include <engine/fifo.h> +u32 +nvkm_gr_ctxsw_inst(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.inst) + return gr->func->ctxsw.inst(gr); + return 0; +} + +int +nvkm_gr_ctxsw_resume(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.resume) + return gr->func->ctxsw.resume(gr); + return 0; +} + +int +nvkm_gr_ctxsw_pause(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.pause) + return gr->func->ctxsw.pause(gr); + return 0; +} + static bool nvkm_gr_chsw_load(struct nvkm_engine *engine) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index e813a3f8ea93..85f2d1e950e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1523,13 +1523,9 @@ gf100_grctx_generate(struct gf100_gr *gr) /* Make channel current. */ addr = nvkm_memory_addr(inst) >> 12; if (gr->firmware) { - nvkm_wr32(device, 0x409840, 0x00000030); - nvkm_wr32(device, 0x409500, 0x80000000 | addr); - nvkm_wr32(device, 0x409504, 0x00000003); - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800) & 0x00000010) - break; - ); + ret = gf100_gr_fecs_bind_pointer(gr, 0x80000000 | addr); + if (ret) + goto done; nvkm_kmap(data); nvkm_wo32(data, 0x1c, 1); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 70d3d41e616c..81a13cf9a292 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -715,6 +715,211 @@ gf100_gr_pack_mmio[] = { * PGRAPH engine/subdev functions ******************************************************************************/ +static u32 +gf100_gr_ctxsw_inst(struct nvkm_gr *gr) +{ + return nvkm_rd32(gr->engine.subdev.device, 0x409b00); +} + +static int +gf100_gr_fecs_ctrl_ctxsw(struct gf100_gr *gr, u32 mthd) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409804, 0xffffffff); + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0xffffffff); + nvkm_wr32(device, 0x409504, mthd); + nvkm_msec(device, 2000, + u32 stat = nvkm_rd32(device, 0x409804); + if (stat == 0x00000002) + return -EIO; + if (stat == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +int +gf100_gr_fecs_start_ctxsw(struct nvkm_gr *base) +{ + struct gf100_gr *gr = gf100_gr(base); + int ret = 0; + + mutex_lock(&gr->fecs.mutex); + if (!--gr->fecs.disable) { + if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x39))) + gr->fecs.disable++; + } + mutex_unlock(&gr->fecs.mutex); + return ret; +} + +int +gf100_gr_fecs_stop_ctxsw(struct nvkm_gr *base) +{ + struct gf100_gr *gr = gf100_gr(base); + int ret = 0; + + mutex_lock(&gr->fecs.mutex); + if (!gr->fecs.disable++) { + if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x38))) + gr->fecs.disable--; + } + mutex_unlock(&gr->fecs.mutex); + return ret; +} + +int +gf100_gr_fecs_bind_pointer(struct gf100_gr *gr, u32 inst) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0x00000030); + nvkm_wr32(device, 0x409500, inst); + nvkm_wr32(device, 0x409504, 0x00000003); + nvkm_msec(device, 2000, + u32 stat = nvkm_rd32(device, 0x409800); + if (stat & 0x00000020) + return -EIO; + if (stat & 0x00000010) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_set_reglist_virtual_address(struct gf100_gr *gr, u64 addr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409810, addr >> 8); + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000032); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x409800) == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_set_reglist_bind_instance(struct gf100_gr *gr, u32 inst) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409810, inst); + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000031); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x409800) == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_reglist_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000030); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_elpg_bind(struct gf100_gr *gr) +{ + u32 size; + int ret; + + ret = gf100_gr_fecs_discover_reglist_image_size(gr, &size); + if (ret) + return ret; + + /*XXX: We need to allocate + map the above into PMU's inst block, + * which which means we probably need a proper PMU before we + * even bother. + */ + + ret = gf100_gr_fecs_set_reglist_bind_instance(gr, 0); + if (ret) + return ret; + + return gf100_gr_fecs_set_reglist_virtual_address(gr, 0); +} + +static int +gf100_gr_fecs_discover_pm_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000025); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_zcull_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000016); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000010); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static void +gf100_gr_fecs_set_watchdog_timeout(struct gf100_gr *gr, u32 timeout) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, timeout); + nvkm_wr32(device, 0x409504, 0x00000021); +} + static bool gf100_gr_chsw_load(struct nvkm_gr *base) { @@ -1487,6 +1692,7 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) struct nvkm_device *device = subdev->device; struct nvkm_secboot *sb = device->secboot; u32 secboot_mask = 0; + int ret; /* load fuc microcode */ nvkm_mc_unk260(device, 0); @@ -1495,12 +1701,12 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_FECS)) secboot_mask |= BIT(NVKM_SECBOOT_FALCON_FECS); else - gf100_gr_init_fw(gr->fecs, &gr->fuc409c, &gr->fuc409d); + gf100_gr_init_fw(gr->fecs.falcon, &gr->fuc409c, &gr->fuc409d); if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_GPCCS)) secboot_mask |= BIT(NVKM_SECBOOT_FALCON_GPCCS); else - gf100_gr_init_fw(gr->gpccs, &gr->fuc41ac, &gr->fuc41ad); + gf100_gr_init_fw(gr->gpccs.falcon, &gr->fuc41ac, &gr->fuc41ad); if (secboot_mask != 0) { int ret = nvkm_secboot_reset(sb, secboot_mask); @@ -1515,8 +1721,8 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) nvkm_wr32(device, 0x41a10c, 0x00000000); nvkm_wr32(device, 0x40910c, 0x00000000); - nvkm_falcon_start(gr->gpccs); - nvkm_falcon_start(gr->fecs); + nvkm_falcon_start(gr->gpccs.falcon); + nvkm_falcon_start(gr->fecs.falcon); if (nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x409800) & 0x00000001) @@ -1524,72 +1730,36 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) ) < 0) return -EBUSY; - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x7fffffff); - nvkm_wr32(device, 0x409504, 0x00000021); + gf100_gr_fecs_set_watchdog_timeout(gr, 0x7fffffff); - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000010); - if (nvkm_msec(device, 2000, - if ((gr->size = nvkm_rd32(device, 0x409800))) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000016); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store main context image. */ + ret = gf100_gr_fecs_discover_image_size(gr, &gr->size); + if (ret) + return ret; - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000025); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store ZCULL image. */ + ret = gf100_gr_fecs_discover_zcull_image_size(gr, &gr->size_zcull); + if (ret) + return ret; - if (device->chipset >= 0xe0) { - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000030); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409810, 0xb00095c8); - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000031); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409810, 0x00080420); - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000032); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store PerfMon image. */ + ret = gf100_gr_fecs_discover_pm_image_size(gr, &gr->size_pm); + if (ret) + return ret; - nvkm_wr32(device, 0x409614, 0x00000070); - nvkm_wr32(device, 0x409614, 0x00000770); - nvkm_wr32(device, 0x40802c, 0x00000001); + /*XXX: We (likely) require PMU support to even bother with this. + * + * Also, it seems like not all GPUs support ELPG. Traces I + * have here show RM enabling it on Kepler/Turing, but none + * of the GPUs between those. NVGPU decides this by PCIID. + */ + if (0) { + ret = gf100_gr_fecs_elpg_bind(gr); + if (ret) + return ret; } + /* Generate golden context image. */ if (gr->data == NULL) { int ret = gf100_grctx_generate(gr); if (ret) { @@ -1614,15 +1784,19 @@ gf100_gr_init_ctxctl_int(struct gf100_gr *gr) /* load HUB microcode */ nvkm_mc_unk260(device, 0); - nvkm_falcon_load_dmem(gr->fecs, gr->func->fecs.ucode->data.data, 0x0, + nvkm_falcon_load_dmem(gr->fecs.falcon, + gr->func->fecs.ucode->data.data, 0x0, gr->func->fecs.ucode->data.size, 0); - nvkm_falcon_load_imem(gr->fecs, gr->func->fecs.ucode->code.data, 0x0, + nvkm_falcon_load_imem(gr->fecs.falcon, + gr->func->fecs.ucode->code.data, 0x0, gr->func->fecs.ucode->code.size, 0, 0, false); /* load GPC microcode */ - nvkm_falcon_load_dmem(gr->gpccs, gr->func->gpccs.ucode->data.data, 0x0, + nvkm_falcon_load_dmem(gr->gpccs.falcon, + gr->func->gpccs.ucode->data.data, 0x0, gr->func->gpccs.ucode->data.size, 0); - nvkm_falcon_load_imem(gr->gpccs, gr->func->gpccs.ucode->code.data, 0x0, + nvkm_falcon_load_imem(gr->gpccs.falcon, + gr->func->gpccs.ucode->code.data, 0x0, gr->func->gpccs.ucode->code.size, 0, 0, false); nvkm_mc_unk260(device, 1); @@ -1769,11 +1943,13 @@ gf100_gr_oneinit(struct nvkm_gr *base) int i, j; int ret; - ret = nvkm_falcon_v1_new(subdev, "FECS", 0x409000, &gr->fecs); + ret = nvkm_falcon_v1_new(subdev, "FECS", 0x409000, &gr->fecs.falcon); if (ret) return ret; - ret = nvkm_falcon_v1_new(subdev, "GPCCS", 0x41a000, &gr->gpccs); + mutex_init(&gr->fecs.mutex); + + ret = nvkm_falcon_v1_new(subdev, "GPCCS", 0x41a000, &gr->gpccs.falcon); if (ret) return ret; @@ -1816,11 +1992,11 @@ gf100_gr_init_(struct nvkm_gr *base) nvkm_pmu_pgob(gr->base.engine.subdev.device->pmu, false); - ret = nvkm_falcon_get(gr->fecs, subdev); + ret = nvkm_falcon_get(gr->fecs.falcon, subdev); if (ret) return ret; - ret = nvkm_falcon_get(gr->gpccs, subdev); + ret = nvkm_falcon_get(gr->gpccs.falcon, subdev); if (ret) return ret; @@ -1832,8 +2008,8 @@ gf100_gr_fini_(struct nvkm_gr *base, bool suspend) { struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; - nvkm_falcon_put(gr->gpccs, subdev); - nvkm_falcon_put(gr->fecs, subdev); + nvkm_falcon_put(gr->gpccs.falcon, subdev); + nvkm_falcon_put(gr->fecs.falcon, subdev); return 0; } @@ -1859,8 +2035,8 @@ gf100_gr_dtor(struct nvkm_gr *base) gr->func->dtor(gr); kfree(gr->data); - nvkm_falcon_del(&gr->gpccs); - nvkm_falcon_del(&gr->fecs); + nvkm_falcon_del(&gr->gpccs.falcon); + nvkm_falcon_del(&gr->fecs.falcon); gf100_gr_dtor_fw(&gr->fuc409c); gf100_gr_dtor_fw(&gr->fuc409d); @@ -1886,6 +2062,9 @@ gf100_gr_ = { .chan_new = gf100_gr_chan_new, .object_get = gf100_gr_object_get, .chsw_load = gf100_gr_chsw_load, + .ctxsw.pause = gf100_gr_fecs_stop_ctxsw, + .ctxsw.resume = gf100_gr_fecs_start_ctxsw, + .ctxsw.inst = gf100_gr_ctxsw_inst, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index dc46cf0131db..fafdd0bbea9b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -82,8 +82,16 @@ struct gf100_gr { const struct gf100_gr_func *func; struct nvkm_gr base; - struct nvkm_falcon *fecs; - struct nvkm_falcon *gpccs; + struct { + struct nvkm_falcon *falcon; + struct mutex mutex; + u32 disable; + } fecs; + + struct { + struct nvkm_falcon *falcon; + } gpccs; + struct gf100_gr_fuc fuc409c; struct gf100_gr_fuc fuc409d; struct gf100_gr_fuc fuc41ac; @@ -128,6 +136,8 @@ struct gf100_gr { struct gf100_gr_mmio mmio_list[4096/8]; u32 size; u32 *data; + u32 size_zcull; + u32 size_pm; }; int gf100_gr_ctor(const struct gf100_gr_func *, struct nvkm_device *, @@ -136,6 +146,8 @@ int gf100_gr_new_(const struct gf100_gr_func *, struct nvkm_device *, int, struct nvkm_gr **); void *gf100_gr_dtor(struct nvkm_gr *); +int gf100_gr_fecs_bind_pointer(struct gf100_gr *, u32 inst); + struct gf100_gr_func_zbc { void (*clear_color)(struct gf100_gr *, int zbc); void (*clear_depth)(struct gf100_gr *, int zbc); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h index 66359c23cbce..d4d5601c51e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h @@ -27,6 +27,11 @@ struct nvkm_gr_func { */ u64 (*units)(struct nvkm_gr *); bool (*chsw_load)(struct nvkm_gr *); + struct { + int (*pause)(struct nvkm_gr *); + int (*resume)(struct nvkm_gr *); + u32 (*inst)(struct nvkm_gr *); + } ctxsw; struct nvkm_sclass sclass[]; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c index 4807021fd990..4a63581bdd5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c @@ -21,13 +21,21 @@ */ #include "priv.h" +#include <subdev/top.h> #include <engine/falcon.h> static int nvkm_nvdec_oneinit(struct nvkm_engine *engine) { struct nvkm_nvdec *nvdec = nvkm_nvdec(engine); - return nvkm_falcon_v1_new(&nvdec->engine.subdev, "NVDEC", 0x84000, + struct nvkm_subdev *subdev = &nvdec->engine.subdev; + + nvdec->addr = nvkm_top_addr(subdev->device, subdev->index); + if (!nvdec->addr) + return -EINVAL; + + /*XXX: fix naming of this when adding support for multiple-NVDEC */ + return nvkm_falcon_v1_new(subdev, "NVDEC", nvdec->addr, &nvdec->falcon); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild index 4b17254cfbd0..d9cdea7d9353 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild @@ -1,2 +1,3 @@ nvkm-y += nvkm/engine/sec2/base.o nvkm-y += nvkm/engine/sec2/gp102.o +nvkm-y += nvkm/engine/sec2/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index f865d2a3e184..1b49e5b6717f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -22,6 +22,7 @@ #include "priv.h" #include <core/msgqueue.h> +#include <subdev/top.h> #include <engine/falcon.h> static void * @@ -39,18 +40,18 @@ nvkm_sec2_intr(struct nvkm_engine *engine) struct nvkm_sec2 *sec2 = nvkm_sec2(engine); struct nvkm_subdev *subdev = &engine->subdev; struct nvkm_device *device = subdev->device; - u32 disp = nvkm_rd32(device, 0x8701c); - u32 intr = nvkm_rd32(device, 0x87008) & disp & ~(disp >> 16); + u32 disp = nvkm_rd32(device, sec2->addr + 0x01c); + u32 intr = nvkm_rd32(device, sec2->addr + 0x008) & disp & ~(disp >> 16); if (intr & 0x00000040) { schedule_work(&sec2->work); - nvkm_wr32(device, 0x87004, 0x00000040); + nvkm_wr32(device, sec2->addr + 0x004, 0x00000040); intr &= ~0x00000040; } if (intr) { nvkm_error(subdev, "unhandled intr %08x\n", intr); - nvkm_wr32(device, 0x87004, intr); + nvkm_wr32(device, sec2->addr + 0x004, intr); } } @@ -74,8 +75,15 @@ static int nvkm_sec2_oneinit(struct nvkm_engine *engine) { struct nvkm_sec2 *sec2 = nvkm_sec2(engine); - return nvkm_falcon_v1_new(&sec2->engine.subdev, "SEC2", 0x87000, - &sec2->falcon); + struct nvkm_subdev *subdev = &sec2->engine.subdev; + + if (!sec2->addr) { + sec2->addr = nvkm_top_addr(subdev->device, subdev->index); + if (WARN_ON(!sec2->addr)) + return -EINVAL; + } + + return nvkm_falcon_v1_new(subdev, "SEC2", sec2->addr, &sec2->falcon); } static int @@ -95,13 +103,14 @@ nvkm_sec2 = { }; int -nvkm_sec2_new_(struct nvkm_device *device, int index, +nvkm_sec2_new_(struct nvkm_device *device, int index, u32 addr, struct nvkm_sec2 **psec2) { struct nvkm_sec2 *sec2; if (!(sec2 = *psec2 = kzalloc(sizeof(*sec2), GFP_KERNEL))) return -ENOMEM; + sec2->addr = addr; INIT_WORK(&sec2->work, nvkm_sec2_recv); return nvkm_engine_ctor(&nvkm_sec2, device, index, true, &sec2->engine); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index 9be1524c08f5..858cf27fa010 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -26,5 +26,5 @@ int gp102_sec2_new(struct nvkm_device *device, int index, struct nvkm_sec2 **psec2) { - return nvkm_sec2_new_(device, index, psec2); + return nvkm_sec2_new_(device, index, 0, psec2); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h index 2f97c806a79d..ab0165e2d1a3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h @@ -5,6 +5,5 @@ #define nvkm_sec2(p) container_of((p), struct nvkm_sec2, engine) -int nvkm_sec2_new_(struct nvkm_device *, int, struct nvkm_sec2 **); - +int nvkm_sec2_new_(struct nvkm_device *, int, u32 addr, struct nvkm_sec2 **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c new file mode 100644 index 000000000000..d655576164b1 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "priv.h" + +int +tu102_sec2_new(struct nvkm_device *device, int index, + struct nvkm_sec2 **psec2) +{ + /* TOP info wasn't updated on Turing to reflect the PRI + * address change for some reason. We override it here. + */ + return nvkm_sec2_new_(device, index, 0x840000, psec2); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index 427340153640..366c87de6e72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -204,6 +204,9 @@ nvkm_falcon_ctor(const struct nvkm_falcon_func *func, debug_reg = 0x408; falcon->has_emem = true; break; + case NVKM_SUBDEV_GSP: + debug_reg = 0x0; /*XXX*/ + break; default: nvkm_warn(subdev, "unsupported falcon %s!\n", nvkm_subdev_name[subdev->index]); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c index 771e16a16267..a8bee1e046aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c @@ -269,7 +269,7 @@ cmd_write(struct nvkm_msgqueue *priv, struct nvkm_msgqueue_hdr *cmd, commit = false; } - cmd_queue_close(priv, queue, commit); + cmd_queue_close(priv, queue, commit); return ret; } @@ -347,7 +347,7 @@ nvkm_msgqueue_post(struct nvkm_msgqueue *priv, enum msgqueue_msg_priority prio, ret = cmd_write(priv, cmd, queue); if (ret) { seq->state = SEQ_STATE_PENDING; - msgqueue_seq_release(priv, seq); + msgqueue_seq_release(priv, seq); } return ret; @@ -373,7 +373,7 @@ msgqueue_msg_handle(struct nvkm_msgqueue *priv, struct nvkm_msgqueue_hdr *hdr) if (seq->completion) complete(seq->completion); - msgqueue_seq_release(priv, seq); + msgqueue_seq_release(priv, seq); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild index cfdffef1afb9..a339fe03d423 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild @@ -7,6 +7,7 @@ include $(src)/nvkm/subdev/fault/Kbuild include $(src)/nvkm/subdev/fb/Kbuild include $(src)/nvkm/subdev/fuse/Kbuild include $(src)/nvkm/subdev/gpio/Kbuild +include $(src)/nvkm/subdev/gsp/Kbuild include $(src)/nvkm/subdev/i2c/Kbuild include $(src)/nvkm/subdev/ibus/Kbuild include $(src)/nvkm/subdev/iccsense/Kbuild diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild index ab0282dc0736..dc300600c019 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild @@ -5,4 +5,4 @@ nvkm-y += nvkm/subdev/bar/gf100.o nvkm-y += nvkm/subdev/bar/gk20a.o nvkm-y += nvkm/subdev/bar/gm107.o nvkm-y += nvkm/subdev/bar/gm20b.o -nvkm-y += nvkm/subdev/bar/tu104.o +nvkm-y += nvkm/subdev/bar/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c index ecaead156e9b..798f65ec3a86 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c @@ -25,7 +25,7 @@ #include <subdev/timer.h> static void -tu104_bar_bar2_wait(struct nvkm_bar *bar) +tu102_bar_bar2_wait(struct nvkm_bar *bar) { struct nvkm_device *device = bar->subdev.device; nvkm_msec(device, 2000, @@ -35,13 +35,13 @@ tu104_bar_bar2_wait(struct nvkm_bar *bar) } static void -tu104_bar_bar2_fini(struct nvkm_bar *bar) +tu102_bar_bar2_fini(struct nvkm_bar *bar) { nvkm_mask(bar->subdev.device, 0xb80f48, 0x80000000, 0x00000000); } static void -tu104_bar_bar2_init(struct nvkm_bar *base) +tu102_bar_bar2_init(struct nvkm_bar *base) { struct nvkm_device *device = base->subdev.device; struct gf100_bar *bar = gf100_bar(base); @@ -52,7 +52,7 @@ tu104_bar_bar2_init(struct nvkm_bar *base) } static void -tu104_bar_bar1_wait(struct nvkm_bar *bar) +tu102_bar_bar1_wait(struct nvkm_bar *bar) { struct nvkm_device *device = bar->subdev.device; nvkm_msec(device, 2000, @@ -62,13 +62,13 @@ tu104_bar_bar1_wait(struct nvkm_bar *bar) } static void -tu104_bar_bar1_fini(struct nvkm_bar *bar) +tu102_bar_bar1_fini(struct nvkm_bar *bar) { nvkm_mask(bar->subdev.device, 0xb80f40, 0x80000000, 0x00000000); } static void -tu104_bar_bar1_init(struct nvkm_bar *base) +tu102_bar_bar1_init(struct nvkm_bar *base) { struct nvkm_device *device = base->subdev.device; struct gf100_bar *bar = gf100_bar(base); @@ -77,22 +77,22 @@ tu104_bar_bar1_init(struct nvkm_bar *base) } static const struct nvkm_bar_func -tu104_bar = { +tu102_bar = { .dtor = gf100_bar_dtor, .oneinit = gf100_bar_oneinit, - .bar1.init = tu104_bar_bar1_init, - .bar1.fini = tu104_bar_bar1_fini, - .bar1.wait = tu104_bar_bar1_wait, + .bar1.init = tu102_bar_bar1_init, + .bar1.fini = tu102_bar_bar1_fini, + .bar1.wait = tu102_bar_bar1_wait, .bar1.vmm = gf100_bar_bar1_vmm, - .bar2.init = tu104_bar_bar2_init, - .bar2.fini = tu104_bar_bar2_fini, - .bar2.wait = tu104_bar_bar2_wait, + .bar2.init = tu102_bar_bar2_init, + .bar2.fini = tu102_bar_bar2_fini, + .bar2.wait = tu102_bar_bar2_wait, .bar2.vmm = gf100_bar_bar2_vmm, .flush = g84_bar_flush, }; int -tu104_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) +tu102_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) { - return gf100_bar_new_(&tu104_bar, device, index, pbar); + return gf100_bar_new_(&tu102_bar, device, index, pbar); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c index 3133b28f849c..b099d1209be8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c @@ -212,7 +212,7 @@ nvbios_dpcfg_match(struct nvkm_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe, u16 data; if (*ver >= 0x30) { - const u8 vsoff[] = { 0, 4, 7, 9 }; + static const u8 vsoff[] = { 0, 4, 7, 9 }; idx = (pc * 10) + vsoff[vs] + pe; if (*ver >= 0x40 && *ver <= 0x41 && *hdr >= 0x12) idx += nvbios_rd08(bios, outp + 0x11) * 40; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c index 9cc10e438b3d..ec0e9f7224b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c @@ -806,12 +806,12 @@ init_generic_condition(struct nvbios_init *init) init->offset += 3; switch (cond) { - case 0: + case 0: /* CONDITION_ID_INT_DP. */ if (init_conn(init) != DCB_CONNECTOR_eDP) init_exec_set(init, false); break; - case 1: - case 2: + case 1: /* CONDITION_ID_USE_SPPLL0. */ + case 2: /* CONDITION_ID_USE_SPPLL1. */ if ( init->outp && (data = nvbios_dpout_match(bios, DCB_OUTPUT_DP, (init->outp->or << 0) | @@ -826,10 +826,13 @@ init_generic_condition(struct nvbios_init *init) if (init_exec(init)) warn("script needs dp output table data\n"); break; - case 5: + case 5: /* CONDITION_ID_ASSR_SUPPORT. */ if (!(init_rdauxr(init, 0x0d) & 1)) init_exec_set(init, false); break; + case 7: /* CONDITION_ID_NO_PANEL_SEQ_DELAYS. */ + init_exec_set(init, false); + break; default: warn("INIT_GENERIC_CONDITON: unknown 0x%02x\n", cond); init->offset += size; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild index 3ef505a5c01b..f3c388932b6f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild @@ -13,4 +13,4 @@ nvkm-y += nvkm/subdev/devinit/gf100.o nvkm-y += nvkm/subdev/devinit/gm107.o nvkm-y += nvkm/subdev/devinit/gm200.o nvkm-y += nvkm/subdev/devinit/gv100.o -nvkm-y += nvkm/subdev/devinit/tu104.o +nvkm-y += nvkm/subdev/devinit/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c index aae87b3fc429..397670e72fff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c @@ -26,7 +26,7 @@ #include <subdev/clk/pll.h> static int -tu104_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq) +tu102_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq) { struct nvkm_subdev *subdev = &init->subdev; struct nvkm_device *device = subdev->device; @@ -66,7 +66,7 @@ tu104_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq) } static int -tu104_devinit_post(struct nvkm_devinit *base, bool post) +tu102_devinit_post(struct nvkm_devinit *base, bool post) { struct nv50_devinit *init = nv50_devinit(base); gm200_devinit_preos(init, post); @@ -74,16 +74,16 @@ tu104_devinit_post(struct nvkm_devinit *base, bool post) } static const struct nvkm_devinit_func -tu104_devinit = { +tu102_devinit = { .init = nv50_devinit_init, - .post = tu104_devinit_post, - .pll_set = tu104_devinit_pll_set, + .post = tu102_devinit_post, + .pll_set = tu102_devinit_pll_set, .disable = gm107_devinit_disable, }; int -tu104_devinit_new(struct nvkm_device *device, int index, +tu102_devinit_new(struct nvkm_device *device, int index, struct nvkm_devinit **pinit) { - return nv50_devinit_new_(&tu104_devinit, device, index, pinit); + return nv50_devinit_new_(&tu102_devinit, device, index, pinit); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild index 794eb1745b2f..42586267fc08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild @@ -1,4 +1,5 @@ nvkm-y += nvkm/subdev/fault/base.o +nvkm-y += nvkm/subdev/fault/user.o nvkm-y += nvkm/subdev/fault/gp100.o nvkm-y += nvkm/subdev/fault/gv100.o -nvkm-y += nvkm/subdev/fault/tu104.o +nvkm-y += nvkm/subdev/fault/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index 4ba1e21e8fda..ca251560d3e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -176,5 +176,7 @@ nvkm_fault_new_(const struct nvkm_fault_func *func, struct nvkm_device *device, return -ENOMEM; nvkm_subdev_ctor(&nvkm_fault, device, index, &fault->subdev); fault->func = func; + fault->user.ctor = nvkm_ufault_new; + fault->user.base = func->user.base; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c index 8fb96fe614f9..4f3c4e091117 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c @@ -23,6 +23,8 @@ #include <subdev/mc.h> +#include <nvif/class.h> + static void gp100_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) { @@ -69,6 +71,7 @@ gp100_fault = { .buffer.init = gp100_fault_buffer_init, .buffer.fini = gp100_fault_buffer_fini, .buffer.intr = gp100_fault_buffer_intr, + .user = { { 0, 0, MAXWELL_FAULT_BUFFER_A }, 0 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c index 6fc54e17c935..6747f09c2dc3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c @@ -25,6 +25,8 @@ #include <subdev/mmu.h> #include <engine/fifo.h> +#include <nvif/class.h> + static void gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) { @@ -166,6 +168,13 @@ gv100_fault_intr(struct nvkm_fault *fault) } } + if (stat & 0x08000000) { + if (fault->buffer[1]) { + nvkm_event_send(&fault->event, 1, 1, NULL, 0); + stat &= ~0x08000000; + } + } + if (stat) { nvkm_debug(subdev, "intr %08x\n", stat); } @@ -208,6 +217,13 @@ gv100_fault = { .buffer.init = gv100_fault_buffer_init, .buffer.fini = gv100_fault_buffer_fini, .buffer.intr = gv100_fault_buffer_intr, + /*TODO: Figure out how to expose non-replayable fault buffer, which, + * for some reason, is where recoverable CE faults appear... + * + * It's a bit tricky, as both NVKM and SVM will need access to + * the non-replayable fault buffer. + */ + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h index 8ca8b2876dad..975e66ac6344 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h @@ -34,7 +34,14 @@ struct nvkm_fault_func { void (*fini)(struct nvkm_fault_buffer *); void (*intr)(struct nvkm_fault_buffer *, bool enable); } buffer; + struct { + struct nvkm_sclass base; + int rp; + } user; }; int gv100_fault_oneinit(struct nvkm_fault *); + +int nvkm_ufault_new(struct nvkm_device *, const struct nvkm_oclass *, + void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c index 9c8a3adf99d7..fa1dfe5692b0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -28,7 +28,7 @@ #include <nvif/class.h> static void -tu104_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) +tu102_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) { /*XXX: Earlier versions of RM touched the old regs on Turing, * which don't appear to actually work anymore, but newer @@ -37,7 +37,7 @@ tu104_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) } static void -tu104_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +tu102_fault_buffer_fini(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x20; @@ -45,7 +45,7 @@ tu104_fault_buffer_fini(struct nvkm_fault_buffer *buffer) } static void -tu104_fault_buffer_init(struct nvkm_fault_buffer *buffer) +tu102_fault_buffer_init(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x20; @@ -57,7 +57,7 @@ tu104_fault_buffer_init(struct nvkm_fault_buffer *buffer) } static void -tu104_fault_buffer_info(struct nvkm_fault_buffer *buffer) +tu102_fault_buffer_info(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x20; @@ -70,7 +70,7 @@ tu104_fault_buffer_info(struct nvkm_fault_buffer *buffer) } static void -tu104_fault_intr_fault(struct nvkm_fault *fault) +tu102_fault_intr_fault(struct nvkm_fault *fault) { struct nvkm_subdev *subdev = &fault->subdev; struct nvkm_device *device = subdev->device; @@ -96,14 +96,14 @@ tu104_fault_intr_fault(struct nvkm_fault *fault) } static void -tu104_fault_intr(struct nvkm_fault *fault) +tu102_fault_intr(struct nvkm_fault *fault) { struct nvkm_subdev *subdev = &fault->subdev; struct nvkm_device *device = subdev->device; u32 stat = nvkm_rd32(device, 0xb83094); if (stat & 0x80000000) { - tu104_fault_intr_fault(fault); + tu102_fault_intr_fault(fault); nvkm_wr32(device, 0xb83094, 0x80000000); stat &= ~0x80000000; } @@ -129,7 +129,7 @@ tu104_fault_intr(struct nvkm_fault *fault) } static void -tu104_fault_fini(struct nvkm_fault *fault) +tu102_fault_fini(struct nvkm_fault *fault) { nvkm_notify_put(&fault->nrpfb); if (fault->buffer[0]) @@ -138,7 +138,7 @@ tu104_fault_fini(struct nvkm_fault *fault) } static void -tu104_fault_init(struct nvkm_fault *fault) +tu102_fault_init(struct nvkm_fault *fault) { /*XXX: enable priv faults */ fault->func->buffer.init(fault->buffer[0]); @@ -146,22 +146,23 @@ tu104_fault_init(struct nvkm_fault *fault) } static const struct nvkm_fault_func -tu104_fault = { +tu102_fault = { .oneinit = gv100_fault_oneinit, - .init = tu104_fault_init, - .fini = tu104_fault_fini, - .intr = tu104_fault_intr, + .init = tu102_fault_init, + .fini = tu102_fault_fini, + .intr = tu102_fault_intr, .buffer.nr = 2, .buffer.entry_size = 32, - .buffer.info = tu104_fault_buffer_info, - .buffer.init = tu104_fault_buffer_init, - .buffer.fini = tu104_fault_buffer_fini, - .buffer.intr = tu104_fault_buffer_intr, + .buffer.info = tu102_fault_buffer_info, + .buffer.init = tu102_fault_buffer_init, + .buffer.fini = tu102_fault_buffer_fini, + .buffer.intr = tu102_fault_buffer_intr, + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, }; int -tu104_fault_new(struct nvkm_device *device, int index, +tu102_fault_new(struct nvkm_device *device, int index, struct nvkm_fault **pfault) { - return nvkm_fault_new_(&tu104_fault, device, index, pfault); + return nvkm_fault_new_(&tu102_fault, device, index, pfault); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c new file mode 100644 index 000000000000..ac835c9582fd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c @@ -0,0 +1,106 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <core/memory.h> +#include <subdev/mmu.h> + +#include <nvif/clb069.h> +#include <nvif/unpack.h> + +static int +nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + struct nvkm_device *device = buffer->fault->subdev.device; + *type = NVKM_OBJECT_MAP_IO; + *addr = device->func->resource_addr(device, 3) + buffer->addr; + *size = nvkm_memory_size(buffer->mem); + return 0; +} + +static int +nvkm_ufault_ntfy(struct nvkm_object *object, u32 type, + struct nvkm_event **pevent) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + if (type == NVB069_V0_NTFY_FAULT) { + *pevent = &buffer->fault->event; + return 0; + } + return -EINVAL; +} + +static int +nvkm_ufault_fini(struct nvkm_object *object, bool suspend) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.fini(buffer); + return 0; +} + +static int +nvkm_ufault_init(struct nvkm_object *object) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.init(buffer); + return 0; +} + +static void * +nvkm_ufault_dtor(struct nvkm_object *object) +{ + return NULL; +} + +static const struct nvkm_object_func +nvkm_ufault = { + .dtor = nvkm_ufault_dtor, + .init = nvkm_ufault_init, + .fini = nvkm_ufault_fini, + .ntfy = nvkm_ufault_ntfy, + .map = nvkm_ufault_map, +}; + +int +nvkm_ufault_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) +{ + union { + struct nvif_clb069_v0 v0; + } *args = argv; + struct nvkm_fault *fault = device->fault; + struct nvkm_fault_buffer *buffer = fault->buffer[fault->func->user.rp]; + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + args->v0.entries = buffer->entries; + args->v0.get = buffer->get; + args->v0.put = buffer->put; + } else + return ret; + + nvkm_object_ctor(&nvkm_ufault, oclass, &buffer->object); + *pobject = &buffer->object; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c index 60ece0a8a2e1..1d2d6bae73cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c @@ -87,7 +87,7 @@ nvkm_gddr3_calc(struct nvkm_ram *ram) WR = (ram->next->bios.timing[2] & 0x007f0000) >> 16; /* XXX: Get these values from the VBIOS instead */ DLL = !(ram->mr[1] & 0x1); - RON = !(ram->mr[1] & 0x300) >> 8; + RON = !((ram->mr[1] & 0x300) >> 8); break; default: return -ENOSYS; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild new file mode 100644 index 000000000000..26fc6feb807e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild @@ -0,0 +1 @@ +nvkm-y += nvkm/subdev/gsp/gv100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c new file mode 100644 index 000000000000..dccfaf1162e2 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include <subdev/gsp.h> +#include <subdev/top.h> +#include <engine/falcon.h> + +static int +gv100_gsp_oneinit(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + + gsp->addr = nvkm_top_addr(subdev->device, subdev->index); + if (!gsp->addr) + return -EINVAL; + + return nvkm_falcon_v1_new(subdev, "GSP", gsp->addr, &gsp->falcon); +} + +static void * +gv100_gsp_dtor(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + nvkm_falcon_del(&gsp->falcon); + return gsp; +} + +static const struct nvkm_subdev_func +gv100_gsp = { + .dtor = gv100_gsp_dtor, + .oneinit = gv100_gsp_oneinit, +}; + +int +gv100_gsp_new(struct nvkm_device *device, int index, struct nvkm_gsp **pgsp) +{ + struct nvkm_gsp *gsp; + + if (!(gsp = *pgsp = kzalloc(sizeof(*gsp), GFP_KERNEL))) + return -ENOMEM; + + nvkm_subdev_ctor(&gv100_gsp, device, index, &gsp->subdev); + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild index f3b06329c338..c64e399326b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild @@ -12,4 +12,4 @@ nvkm-y += nvkm/subdev/mc/gk104.o nvkm-y += nvkm/subdev/mc/gk20a.o nvkm-y += nvkm/subdev/mc/gp100.o nvkm-y += nvkm/subdev/mc/gp10b.o -nvkm-y += nvkm/subdev/mc/tu104.o +nvkm-y += nvkm/subdev/mc/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c index b7165bd18999..d098c44a4fcb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c @@ -22,7 +22,7 @@ #include "priv.h" static void -tu104_mc_intr_hack(struct nvkm_mc *mc, bool *handled) +tu102_mc_intr_hack(struct nvkm_mc *mc, bool *handled) { struct nvkm_device *device = mc->subdev.device; u32 stat = nvkm_rd32(device, 0xb81010); @@ -37,19 +37,19 @@ tu104_mc_intr_hack(struct nvkm_mc *mc, bool *handled) } static const struct nvkm_mc_func -tu104_mc = { +tu102_mc = { .init = nv50_mc_init, .intr = gp100_mc_intr, .intr_unarm = gp100_mc_intr_unarm, .intr_rearm = gp100_mc_intr_rearm, .intr_mask = gp100_mc_intr_mask, .intr_stat = gf100_mc_intr_stat, - .intr_hack = tu104_mc_intr_hack, + .intr_hack = tu102_mc_intr_hack, .reset = gk104_mc_reset, }; int -tu104_mc_new(struct nvkm_device *device, int index, struct nvkm_mc **pmc) +tu102_mc_new(struct nvkm_device *device, int index, struct nvkm_mc **pmc) { - return gp100_mc_new_(&tu104_mc, device, index, pmc); + return gp100_mc_new_(&tu102_mc, device, index, pmc); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 8966180b36cc..db9c56028f21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -13,7 +13,7 @@ nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/gv100.o -nvkm-y += nvkm/subdev/mmu/tu104.o +nvkm-y += nvkm/subdev/mmu/tu102.o nvkm-y += nvkm/subdev/mmu/mem.o nvkm-y += nvkm/subdev/mmu/memnv04.o @@ -34,7 +34,7 @@ nvkm-y += nvkm/subdev/mmu/vmmgm20b.o nvkm-y += nvkm/subdev/mmu/vmmgp100.o nvkm-y += nvkm/subdev/mmu/vmmgp10b.o nvkm-y += nvkm/subdev/mmu/vmmgv100.o -nvkm-y += nvkm/subdev/mmu/vmmtu104.o +nvkm-y += nvkm/subdev/mmu/vmmtu102.o nvkm-y += nvkm/subdev/mmu/umem.o nvkm-y += nvkm/subdev/mmu/ummu.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 651b8805c67c..65cb9d28e60e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -31,7 +31,7 @@ gp100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 3bd3db31e0bb..0a50be9a785a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -31,7 +31,7 @@ gp10b_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c index f666cb57f69e..e0997eedd6d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c @@ -31,7 +31,7 @@ gv100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 948a48c21be4..2ad1102a4e31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -28,7 +28,7 @@ struct nvkm_mmu_func { struct { struct nvkm_sclass user; - int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, + int (*ctor)(struct nvkm_mmu *, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); bool global; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c index 8e6f4096170d..c0db0ce10cba 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c @@ -27,17 +27,17 @@ #include <nvif/class.h> static const struct nvkm_mmu_func -tu104_mmu = { +tu102_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, tu104_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, tu102_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; int -tu104_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +tu102_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { - return nvkm_mmu_new_(&tu104_mmu, device, index, pmmu); + return nvkm_mmu_new_(&tu102_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c index 6889076097ec..c43b8248c682 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -43,6 +43,69 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle) } static int +nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnclr_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + addr = args->v0.addr; + size = args->v0.size; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_unmap(vmm, addr, size); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int +nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnmap_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size, *phys; + u8 page; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + page = args->v0.page; + addr = args->v0.addr; + size = args->v0.size; + phys = args->v0.phys; + if (argc != (size >> page) * sizeof(args->v0.phys[0])) + return -EINVAL; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { struct nvkm_client *client = uvmm->object.client; @@ -78,7 +141,7 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto done; } - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); ret = 0; done: mutex_unlock(&vmm->mutex); @@ -124,6 +187,11 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } + if (ret = -EINVAL, vma->mapped && !vma->memory) { + VMM_DEBUG(vmm, "pfnmap %016llx", addr); + goto fail; + } + if (ret = -EINVAL, vma->addr != addr || vma->size != size) { if (addr + size > vma->addr + vma->size || vma->memory || (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { @@ -271,6 +339,15 @@ nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); + case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc); + case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc); + case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f): + if (uvmm->vmm->func->mthd) { + return uvmm->vmm->func->mthd(uvmm->vmm, + uvmm->object.client, + mthd, argv, argc); + } + break; default: break; } @@ -304,8 +381,10 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_uvmm *uvmm; int ret = -ENOSYS; u64 addr, size; + bool managed; if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { + managed = args->v0.managed != 0; addr = args->v0.addr; size = args->v0.size; } else @@ -317,7 +396,7 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, *pobject = &uvmm->object; if (!mmu->vmm) { - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, + ret = mmu->func->vmm.ctor(mmu, managed, addr, size, argv, argc, NULL, "user", &uvmm->vmm); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 6b87fff014b3..fa93f964e6a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -255,11 +255,23 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; struct nvkm_vmm_pt *pgt = it->pt[0]; + bool dma; + + if (pfn) { + /* Need to clear PTE valid bits before we dma_unmap_page(). */ + dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes); + if (dma) { + /* GPU may have cached the PT, flush before unmap. */ + nvkm_vmm_flush_mark(it); + nvkm_vmm_flush(it); + desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes); + } + } /* Drop PTE references. */ pgt->refs[type] -= ptes; @@ -349,7 +361,7 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -379,7 +391,7 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, } static bool -nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { struct nvkm_vmm_pt *pt = it->pt[0]; if (it->desc->type == PGD) @@ -387,14 +399,14 @@ nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) else if (it->desc->type == LPT) memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); - return nvkm_vmm_unref_ptes(it, ptei, ptes); + return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes); } static bool -nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); - return nvkm_vmm_ref_ptes(it, ptei, ptes); + return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes); } static bool @@ -487,8 +499,8 @@ nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) static inline u64 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, const char *name, bool ref, - bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32), + u64 addr, u64 size, const char *name, bool ref, bool pfn, + bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32), nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map, nvkm_vmm_pxe_func CLR_PTES) { @@ -548,7 +560,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, } /* Handle PTE updates. */ - if (!REF_PTES || REF_PTES(&it, ptei, ptes)) { + if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) { struct nvkm_mmu_pt *pt = pgt->pt[type]; if (MAP_PTES || CLR_PTES) { if (MAP_PTES) @@ -590,7 +602,7 @@ static void nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false, nvkm_vmm_sparse_unref_ptes, NULL, NULL, page->desc->func->invalid ? page->desc->func->invalid : page->desc->func->unmap); @@ -602,8 +614,8 @@ nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, { if ((page->type & NVKM_VMM_PAGE_SPARSE)) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref", - true, nvkm_vmm_sparse_ref_ptes, NULL, - NULL, page->desc->func->sparse); + true, false, nvkm_vmm_sparse_ref_ptes, + NULL, NULL, page->desc->func->sparse); if (fail != ~0ULL) { if ((size = fail - addr)) nvkm_vmm_ptes_sparse_put(vmm, page, addr, size); @@ -666,11 +678,11 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref) static void nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref", - false, nvkm_vmm_unref_ptes, NULL, NULL, + false, pfn, nvkm_vmm_unref_ptes, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -681,10 +693,10 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, nvkm_vmm_pte_func func) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true, - nvkm_vmm_ref_ptes, func, map, NULL); + false, nvkm_vmm_ref_ptes, func, map, NULL); if (fail != ~0ULL) { if ((size = fail - addr)) - nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false); + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false); return -ENOMEM; } return 0; @@ -692,10 +704,11 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, static void nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; - nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL, + nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn, + NULL, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -705,7 +718,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size, struct nvkm_vmm_map *map, nvkm_vmm_pte_func func) { - nvkm_vmm_iter(vmm, page, addr, size, "map", false, + nvkm_vmm_iter(vmm, page, addr, size, "map", false, false, NULL, func, map, NULL); } @@ -713,7 +726,7 @@ static void nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false, nvkm_vmm_unref_ptes, NULL, NULL, NULL); } @@ -721,7 +734,7 @@ static int nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false, nvkm_vmm_ref_ptes, NULL, NULL, NULL); if (fail != ~0ULL) { if (fail != addr) @@ -763,6 +776,7 @@ nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) new->part = vma->part; new->user = vma->user; new->busy = vma->busy; + new->mapped = vma->mapped; list_add(&new->head, &vma->head); return new; } @@ -935,11 +949,40 @@ nvkm_vmm_node_split(struct nvkm_vmm *vmm, } static void +nvkm_vma_dump(struct nvkm_vma *vma) +{ + printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n", + vma->addr, (u64)vma->size, + vma->used ? '-' : 'F', + vma->mapref ? 'R' : '-', + vma->sparse ? 'S' : '-', + vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-', + vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-', + vma->part ? 'P' : '-', + vma->user ? 'U' : '-', + vma->busy ? 'B' : '-', + vma->mapped ? 'M' : '-', + vma->memory); +} + +static void +nvkm_vmm_dump(struct nvkm_vmm *vmm) +{ + struct nvkm_vma *vma; + list_for_each_entry(vma, &vmm->list, head) { + nvkm_vma_dump(vma); + } +} + +static void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { struct nvkm_vma *vma; struct rb_node *node; + if (0) + nvkm_vmm_dump(vmm); + while ((node = rb_first(&vmm->root))) { struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); nvkm_vmm_put(vmm, &vma); @@ -972,16 +1015,32 @@ nvkm_vmm_dtor(struct nvkm_vmm *vmm) } } +static int +nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size) +{ + struct nvkm_vma *vma; + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + vma->mapref = true; + vma->sparse = false; + vma->used = true; + vma->user = true; + nvkm_vmm_node_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + return 0; +} + int nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm *vmm) + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm *vmm) { static struct lock_class_key _key; const struct nvkm_vmm_page *page = func->page; const struct nvkm_vmm_desc *desc; struct nvkm_vma *vma; - int levels, bits = 0; + int levels, bits = 0, ret; vmm->func = func; vmm->mmu = mmu; @@ -1009,11 +1068,6 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) return -EINVAL; - vmm->start = addr; - vmm->limit = size ? (addr + size) : (1ULL << bits); - if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) - return -EINVAL; - /* Allocate top-level page table. */ vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); if (!vmm->pd) @@ -1036,50 +1090,273 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, vmm->free = RB_ROOT; vmm->root = RB_ROOT; - if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) - return -ENOMEM; + if (managed) { + /* Address-space will be managed by the client for the most + * part, except for a specified area where NVKM allocations + * are allowed to be placed. + */ + vmm->start = 0; + vmm->limit = 1ULL << bits; + if (addr + size < addr || addr + size > vmm->limit) + return -EINVAL; + + /* Client-managed area before the NVKM-managed area. */ + if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr))) + return ret; + + /* NVKM-managed area. */ + if (size) { + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + nvkm_vmm_free_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + } + + /* Client-managed area after the NVKM-managed area. */ + addr = addr + size; + size = vmm->limit - addr; + if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size))) + return ret; + } else { + /* Address-space fully managed by NVKM, requiring calls to + * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space. + */ + vmm->start = addr; + vmm->limit = size ? (addr + size) : (1ULL << bits); + if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) + return -EINVAL; + + if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) + return -ENOMEM; + + nvkm_vmm_free_insert(vmm, vma); + list_add(&vma->head, &vmm->list); + } - nvkm_vmm_free_insert(vmm, vma); - list_add(&vma->head, &vmm->list); return 0; } int nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 hdr, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm **pvmm) + u32 hdr, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) { if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) return -ENOMEM; - return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); + return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm); +} + +static struct nvkm_vma * +nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + u64 addr, u64 size, u8 page, bool map) +{ + struct nvkm_vma *prev = NULL; + struct nvkm_vma *next = NULL; + + if (vma->addr == addr && vma->part && (prev = node(vma, prev))) { + if (prev->memory || prev->mapped != map) + prev = NULL; + } + + if (vma->addr + vma->size == addr + size && (next = node(vma, next))) { + if (!next->part || + next->memory || next->mapped != map) + next = NULL; + } + + if (prev || next) + return nvkm_vmm_node_merge(vmm, prev, vma, next, size); + return nvkm_vmm_node_split(vmm, vma, addr, size); +} + +int +nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) +{ + struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr); + struct nvkm_vma *next; + u64 limit = addr + size; + u64 start = addr; + + if (!vma) + return -EINVAL; + + do { + if (!vma->mapped || vma->memory) + continue; + + size = min(limit - start, vma->size - (start - vma->addr)); + + nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd], + start, size, false, true); + + next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false); + if (!WARN_ON(!next)) { + vma = next; + vma->refd = NVKM_VMA_PAGE_NONE; + vma->mapped = false; + } + } while ((vma = node(vma, next)) && (start = vma->addr) < limit); + + return 0; +} + +/*TODO: + * - Avoid PT readback (for dma_unmap etc), this might end up being dealt + * with inside HMM, which would be a lot nicer for us to deal with. + * - Multiple page sizes (particularly for huge page support). + * - Support for systems without a 4KiB page size. + */ +int +nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + struct nvkm_vma *vma, *tmp; + u64 limit = addr + size; + u64 start = addr; + int pm = size >> shift; + int pi = 0; + + /* Only support mapping where the page size of the incoming page + * array matches a page size available for direct mapping. + */ + while (page->shift && page->shift != shift && + page->desc->func->pfn == NULL) + page++; + + if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) || + !IS_ALIGNED(size, 1ULL << shift) || + addr + size < addr || addr + size > vmm->limit) { + VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n", + shift, page->shift, addr, size); + return -EINVAL; + } + + if (!(vma = nvkm_vmm_node_search(vmm, addr))) + return -ENOENT; + + do { + bool map = !!(pfn[pi] & NVKM_VMM_PFN_V); + bool mapped = vma->mapped; + u64 size = limit - start; + u64 addr = start; + int pn, ret = 0; + + /* Narrow the operation window to cover a single action (page + * should be mapped or not) within a single VMA. + */ + for (pn = 0; pi + pn < pm; pn++) { + if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V)) + break; + } + size = min_t(u64, size, pn << page->shift); + size = min_t(u64, size, vma->size + vma->addr - addr); + + /* Reject any operation to unmanaged regions, and areas that + * have nvkm_memory objects mapped in them already. + */ + if (!vma->mapref || vma->memory) { + ret = -EINVAL; + goto next; + } + + /* In order to both properly refcount GPU page tables, and + * prevent "normal" mappings and these direct mappings from + * interfering with each other, we need to track contiguous + * ranges that have been mapped with this interface. + * + * Here we attempt to either split an existing VMA so we're + * able to flag the region as either unmapped/mapped, or to + * merge with adjacent VMAs that are already compatible. + * + * If the region is already compatible, nothing is required. + */ + if (map != mapped) { + tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size, + page - + vmm->func->page, map); + if (WARN_ON(!tmp)) { + ret = -ENOMEM; + goto next; + } + + if ((tmp->mapped = map)) + tmp->refd = page - vmm->func->page; + else + tmp->refd = NVKM_VMA_PAGE_NONE; + vma = tmp; + } + + /* Update HW page tables. */ + if (map) { + struct nvkm_vmm_map args; + args.page = page; + args.pfn = &pfn[pi]; + + if (!mapped) { + ret = nvkm_vmm_ptes_get_map(vmm, page, addr, + size, &args, page-> + desc->func->pfn); + } else { + nvkm_vmm_ptes_map(vmm, page, addr, size, &args, + page->desc->func->pfn); + } + } else { + if (mapped) { + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, + false, true); + } + } + +next: + /* Iterate to next operation. */ + if (vma->addr + vma->size == addr + size) + vma = node(vma, next); + start += size; + + if (ret) { + /* Failure is signalled by clearing the valid bit on + * any PFN that couldn't be modified as requested. + */ + while (size) { + pfn[pi++] = NVKM_VMM_PFN_NONE; + size -= 1 << page->shift; + } + } else { + pi += size >> page->shift; + } + } while (vma && start < limit); + + return 0; } void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { - struct nvkm_vma *next = node(vma, next); struct nvkm_vma *prev = NULL; + struct nvkm_vma *next; nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); nvkm_memory_unref(&vma->memory); + vma->mapped = false; - if (!vma->part || ((prev = node(vma, prev)), prev->memory)) + if (vma->part && (prev = node(vma, prev)) && prev->mapped) prev = NULL; - if (!next->part || next->memory) + if ((next = node(vma, next)) && (!next->part || next->mapped)) next = NULL; nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size); } void -nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn) { const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd]; if (vma->mapref) { - nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn); vma->refd = NVKM_VMA_PAGE_NONE; } else { - nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn); } nvkm_vmm_unmap_region(vmm, vma); @@ -1090,7 +1367,7 @@ nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { if (vma->memory) { mutex_lock(&vmm->mutex); - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); mutex_unlock(&vmm->mutex); } } @@ -1224,6 +1501,7 @@ nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); nvkm_memory_unref(&vma->memory); vma->memory = nvkm_memory_ref(map->memory); + vma->mapped = true; vma->tags = map->tags; return 0; } @@ -1269,14 +1547,16 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) if (vma->mapref || !vma->sparse) { do { - const bool map = next->memory != NULL; + const bool mem = next->memory != NULL; + const bool map = next->mapped; const u8 refd = next->refd; const u64 addr = next->addr; u64 size = next->size; /* Merge regions that are in the same state. */ while ((next = node(next, next)) && next->part && - (next->memory != NULL) == map && + (next->mapped == map) && + (next->memory != NULL) == mem && (next->refd == refd)) size += next->size; @@ -1286,7 +1566,8 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) * the page tree. */ nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr, - size, vma->sparse); + size, vma->sparse, + !mem); } else if (refd != NVKM_VMA_PAGE_NONE) { /* Drop allocation-time PTE references. */ @@ -1301,7 +1582,7 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) */ next = vma; do { - if (next->memory) + if (next->mapped) nvkm_vmm_unmap_region(vmm, next); } while ((next = node(vma, next)) && next->part); @@ -1522,7 +1803,7 @@ nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) } static bool -nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -1544,7 +1825,7 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm) if (ret) return ret; - nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, + nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false, nvkm_vmm_boot_ptes, NULL, NULL, NULL); vmm->bootstrapped = true; return 0; @@ -1584,7 +1865,8 @@ nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv, struct nvkm_mmu *mmu = device->mmu; struct nvkm_vmm *vmm = NULL; int ret; - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm); + ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc, + key, name, &vmm); if (ret) nvkm_vmm_unref(&vmm); *pvmm = vmm; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 42ad326521a3..5e55ecbd8005 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -67,6 +67,10 @@ struct nvkm_vmm_desc_func { nvkm_vmm_pte_func mem; nvkm_vmm_pte_func dma; nvkm_vmm_pte_func sgl; + + nvkm_vmm_pte_func pfn; + bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes); + nvkm_vmm_pxe_func pfn_unmap; }; extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; @@ -141,6 +145,11 @@ struct nvkm_vmm_func { struct nvkm_vmm_map *); void (*flush)(struct nvkm_vmm *, int depth); + int (*mthd)(struct nvkm_vmm *, struct nvkm_client *, + u32 mthd, void *argv, u32 argc); + + void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr); + u64 page_block; const struct nvkm_vmm_page page[]; }; @@ -151,11 +160,12 @@ struct nvkm_vmm_join { }; int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm **); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, + struct nvkm_vmm **); int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm *); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, struct nvkm_vmm *); struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *, u64 addr, u64 size); @@ -163,13 +173,25 @@ int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, bool sparse, u8 page, u8 align, u64 size, struct nvkm_vma **pvma); void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); +void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn); +void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *); + +#define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL +#define NVKM_VMM_PFN_ADDR_SHIFT 12 +#define NVKM_VMM_PFN_APER 0x00000000000000f0ULL +#define NVKM_VMM_PFN_HOST 0x0000000000000000ULL +#define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL +#define NVKM_VMM_PFN_W 0x0000000000000002ULL +#define NVKM_VMM_PFN_V 0x0000000000000001ULL +#define NVKM_VMM_PFN_NONE 0x0000000000000000ULL + +int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn); +int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size); struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, - u64, u64, void *, u32, struct lock_class_key *, + bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); @@ -179,70 +201,76 @@ int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void nv50_vmm_flush(struct nvkm_vmm *, int); int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); int gf100_vmm_aper(enum nvkm_memory_target); int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); -void gf100_vmm_flush_(struct nvkm_vmm *, int); void gf100_vmm_flush(struct nvkm_vmm *, int); +void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); +void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); int gk20a_vmm_aper(enum nvkm_memory_target); int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +int gp100_vmm_new_(const struct nvkm_vmm_func *, + struct nvkm_mmu *, bool, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void gp100_vmm_flush(struct nvkm_vmm *, int); +int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32); +void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); -int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int tu104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c index faf5a7e9265e..ab6424faf84c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -178,15 +178,19 @@ gf100_vmm_desc_16_16[] = { }; void -gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) +gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, addr); +} + +void +gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type) { struct nvkm_subdev *subdev = &vmm->mmu->subdev; struct nvkm_device *device = subdev->device; - u32 type = depth << 24; - - type = 0x00000001; /* PAGE_ALL */ - if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) - type |= 0x00000004; /* HUB_ONLY */ + struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; + u64 addr = 0; mutex_lock(&subdev->mutex); /* Looks like maybe a "free flush slots" counter, the @@ -197,7 +201,20 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) break; ); - nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8); + if (!(type & 0x00000002) /* ALL_PDB. */) { + switch (nvkm_memory_target(pd->memory)) { + case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break; + case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break; + case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break; + default: + WARN_ON(1); + break; + } + addr |= (vmm->pd->pt[0]->addr >> 12) << 4; + + vmm->func->invalidate_pdb(vmm, addr); + } + nvkm_wr32(device, 0x100cbc, 0x80000000 | type); /* Wait for flush to be queued? */ @@ -211,7 +228,10 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) void gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 0); + u32 type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + gf100_vmm_invalidate(vmm, type); } int @@ -354,6 +374,7 @@ gf100_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -368,6 +389,7 @@ gf100_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -378,14 +400,14 @@ gf100_vmm_16 = { int gf100_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { switch (mmu->subdev.device->fb->page) { - case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size, + case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); - case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size, + case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); default: WARN_ON(1); @@ -394,10 +416,10 @@ gf100_vmm_new_(const struct nvkm_vmm_func *func_16, } int -gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr, + return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c index 0ebb7bccfcd2..0b59c01fd146 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c @@ -71,6 +71,7 @@ gk104_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -85,6 +86,7 @@ gk104_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -93,10 +95,10 @@ gk104_vmm_16 = { }; int -gk104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk104_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c index 8086994a0446..5a9582dce970 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -40,6 +40,7 @@ gk20a_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx }, @@ -54,6 +55,7 @@ gk20a_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx }, @@ -62,10 +64,10 @@ gk20a_vmm_16 = { }; int -gk20a_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk20a_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c index a1676a4644fe..2e61af02d4d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c @@ -113,6 +113,7 @@ gm200_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC }, @@ -128,6 +129,7 @@ gm200_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC }, @@ -139,9 +141,9 @@ gm200_vmm_16 = { int gm200_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { const struct nvkm_vmm_func *func; union { @@ -163,23 +165,23 @@ gm200_vmm_new_(const struct nvkm_vmm_func *func_16, } else return ret; - return nvkm_vmm_new_(func, mmu, 0, addr, size, key, name, pvmm); + return nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); } int -gm200_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm200_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm200_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm200_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c index 64d4b6cff8dd..96b759695dd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -28,6 +28,7 @@ gm20b_vmm_17 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC }, @@ -43,6 +44,7 @@ gm20b_vmm_16 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC }, @@ -52,19 +54,19 @@ gm20b_vmm_16 = { }; int -gm20b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm20b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c index 059fafe0e771..b4f519768d5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -21,12 +21,90 @@ */ #include "vmm.h" +#include <core/client.h> #include <subdev/fb.h> #include <subdev/ltc.h> +#include <subdev/timer.h> +#include <engine/gr.h> #include <nvif/ifc00d.h> #include <nvif/unpack.h> +static void +gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & (3ULL << 1)) != 0) { + addr = (data >> 8) << 12; + dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + ptei++; + } + nvkm_done(pt->memory); +} + +static bool +gp100_vmm_pfn_clear(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + bool dma = false; + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) { + VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0)); + dma = true; + } + ptei++; + } + nvkm_done(pt->memory); + return dma; +} + +static void +gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u64 data = 0; + if (!(*map->pfn & NVKM_VMM_PFN_W)) + data |= BIT_ULL(6); /* RO. */ + + if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) { + addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT; + addr = dma_map_page(dev, pfn_to_page(addr), 0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + if (!WARN_ON(dma_mapping_error(dev, addr))) { + data |= addr >> 4; + data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */ + data |= BIT_ULL(3); /* VOL. */ + data |= BIT_ULL(0); /* VALID. */ + } + } else { + data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4; + data |= BIT_ULL(0); /* VALID. */ + } + + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->pfn++; + } + nvkm_done(pt->memory); +} + static inline void gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) @@ -89,6 +167,9 @@ gp100_vmm_desc_spt = { .mem = gp100_vmm_pgt_mem, .dma = gp100_vmm_pgt_dma, .sgl = gp100_vmm_pgt_sgl, + .pfn = gp100_vmm_pgt_pfn, + .pfn_clear = gp100_vmm_pfn_clear, + .pfn_unmap = gp100_vmm_pfn_unmap, }; static void @@ -306,16 +387,100 @@ gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, return 0; } +static int +gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + union { + struct gp100_vmm_fault_cancel_v0 v0; + } *args = argv; + int ret = -ENOSYS; + u32 inst, aper; + + if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) + return ret; + + /* Translate MaxwellFaultBufferA instance pointer to the same + * format as the NV_GR_FECS_CURRENT_CTX register. + */ + aper = (args->v0.inst >> 8) & 3; + args->v0.inst >>= 12; + args->v0.inst |= aper << 28; + args->v0.inst |= 0x80000000; + + if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) { + if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) { + gf100_vmm_invalidate(vmm, 0x0000001b + /* CANCEL_TARGETED. */ | + (args->v0.hub << 20) | + (args->v0.gpc << 15) | + (args->v0.client << 9)); + } + WARN_ON(nvkm_gr_ctxsw_resume(device)); + } + + return 0; +} + +static int +gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + union { + struct gp100_vmm_fault_replay_vn vn; + } *args = argv; + int ret = -ENOSYS; + + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */ + } + + return ret; +} + +int +gp100_vmm_mthd(struct nvkm_vmm *vmm, + struct nvkm_client *client, u32 mthd, void *argv, u32 argc) +{ + if (client->super) { + switch (mthd) { + case GP100_VMM_VN_FAULT_REPLAY: + return gp100_vmm_fault_replay(vmm, argv, argc); + case GP100_VMM_VN_FAULT_CANCEL: + return gp100_vmm_fault_cancel(vmm, argv, argc); + default: + break; + } + } + return -EINVAL; +} + +void +gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, lower_32_bits(addr)); + nvkm_wr32(device, 0x100cec, upper_32_bits(addr)); +} + void gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth); + u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24; + type = 0; /*XXX: need to confirm stuff works with depth enabled... */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + type |= 0x00000001; /* PAGE_ALL */ + gf100_vmm_invalidate(vmm, type); } int gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { - const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */ + u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */; + if (vmm->replay) { + base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */ + base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */ + } return gf100_vmm_join_(vmm, inst, base); } @@ -326,6 +491,8 @@ gp100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -338,10 +505,39 @@ gp100_vmm = { }; int -gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp100_vmm_new_(const struct nvkm_vmm_func *func, + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + union { + struct gp100_vmm_vn vn; + struct gp100_vmm_v0 v0; + } *args = argv; + int ret = -ENOSYS; + bool replay; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + replay = args->v0.fault_replay != 0; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + replay = false; + } else + return ret; + + ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); + if (ret) + return ret; + + (*pvmm)->replay = replay; + return 0; +} + +int +gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c index 3dcc6bddb32f..e081239afe58 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -28,6 +28,8 @@ gp10b_vmm = { .aper = gk20a_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -40,10 +42,10 @@ gp10b_vmm = { }; int -gp10b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp10b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp10b_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp10b_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c index 2fa40c16e6d2..f0e21f63253a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c @@ -66,6 +66,8 @@ gv100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -78,10 +80,10 @@ gv100_vmm = { }; int -gv100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gv100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gv100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gv100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c index e63d984cbfd4..bdddd99f5877 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c @@ -36,10 +36,10 @@ mcp77_vmm = { }; int -mcp77_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +mcp77_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&mcp77_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&mcp77_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c index 0cab1ffc9f64..4c6b3b7d221f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c @@ -100,16 +100,17 @@ nv04_vmm = { int nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + u32 pd_header, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { union { struct nv04_vmm_vn vn; } *args = argv; int ret; - ret = nvkm_vmm_new_(func, mmu, pd_header, addr, size, key, name, pvmm); + ret = nvkm_vmm_new_(func, mmu, pd_header, managed, addr, size, + key, name, pvmm); if (ret) return ret; @@ -117,15 +118,15 @@ nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, } int -nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv04_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_memory *mem; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size, + ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c index b595f130e573..1d3369683a21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c @@ -104,10 +104,10 @@ nv41_vmm = { }; int -nv41_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv41_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv41_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv41_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c index b834e4352334..a82936ba9890 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c @@ -205,15 +205,15 @@ nv44_vmm = { }; int -nv44_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv44_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_subdev *subdev = &mmu->subdev; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, addr, size, + ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c index 64f75d906202..c98afe3134ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c @@ -376,10 +376,10 @@ nv50_vmm = { }; int -nv50_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv50_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv50_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv50_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c index adaadd92110f..be91cffc3b52 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c @@ -24,7 +24,7 @@ #include <subdev/timer.h> static void -tu104_vmm_flush(struct nvkm_vmm *vmm, int depth) +tu102_vmm_flush(struct nvkm_vmm *vmm, int depth) { struct nvkm_subdev *subdev = &vmm->mmu->subdev; struct nvkm_device *device = subdev->device; @@ -50,12 +50,13 @@ tu104_vmm_flush(struct nvkm_vmm *vmm, int depth) } static const struct nvkm_vmm_func -tu104_vmm = { +tu102_vmm = { .join = gv100_vmm_join, .part = gf100_vmm_part, .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, - .flush = tu104_vmm_flush, + .flush = tu102_vmm_flush, + .mthd = gp100_vmm_mthd, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -68,10 +69,10 @@ tu104_vmm = { }; int -tu104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, +tu102_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&tu104_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&tu102_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c index 11b28b086a06..7b052879af72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c @@ -88,10 +88,10 @@ nvkm_memx_fini(struct nvkm_memx **pmemx, bool exec) if (exec) { nvkm_pmu_send(pmu, reply, PROC_MEMX, MEMX_MSG_EXEC, memx->base, finish); + nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", + reply[0], reply[1]); } - nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", - reply[0], reply[1]); kfree(memx); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c index 5c14d6ac855d..1df09ed6fe6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c @@ -853,7 +853,7 @@ acr_r352_shutdown(struct acr_r352 *acr, struct nvkm_secboot *sb) * and the expected behavior on RM as well */ if (ret && ret != 0x1d) { - nvkm_error(subdev, "HS unload failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS unload failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS unload blob completed\n"); @@ -922,7 +922,7 @@ acr_r352_bootstrap(struct acr_r352 *acr, struct nvkm_secboot *sb) if (ret < 0) { return ret; } else if (ret > 0) { - nvkm_error(subdev, "HS load failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS load failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS load blob completed\n"); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c index 3695cde669f8..07914e36939e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c @@ -132,11 +132,12 @@ nvkm_therm_update(struct nvkm_therm *therm, int mode) duty = nvkm_therm_update_linear(therm); break; case NVBIOS_THERM_FAN_OTHER: - if (therm->cstate) + if (therm->cstate) { duty = therm->cstate; - else + poll = false; + } else { duty = nvkm_therm_update_linear_fallback(therm); - poll = false; + } break; } immd = false; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c index 67ada1d9a28c..cce6e4e90ebf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c @@ -41,6 +41,22 @@ nvkm_top_device_new(struct nvkm_top *top) } u32 +nvkm_top_addr(struct nvkm_device *device, enum nvkm_devidx index) +{ + struct nvkm_top *top = device->top; + struct nvkm_top_device *info; + + if (top) { + list_for_each_entry(info, &top->device, head) { + if (info->index == index) + return info->addr; + } + } + + return 0; +} + +u32 nvkm_top_reset(struct nvkm_device *device, enum nvkm_devidx index) { struct nvkm_top *top = device->top; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c index 39081eadfd84..e01746ce9fc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c @@ -73,6 +73,7 @@ gk104_top_oneinit(struct nvkm_top *top) #define A_(A) if (inst == 0) info->index = NVKM_ENGINE_##A #define B_(A) if (inst + NVKM_ENGINE_##A##0 < NVKM_ENGINE_##A##_LAST + 1) \ info->index = NVKM_ENGINE_##A##0 + inst +#define C_(A) if (inst == 0) info->index = NVKM_SUBDEV_##A switch (type) { case 0x00000000: A_(GR ); break; case 0x00000001: A_(CE0 ); break; @@ -88,6 +89,7 @@ gk104_top_oneinit(struct nvkm_top *top) case 0x0000000f: A_(NVENC1); break; case 0x00000010: B_(NVDEC ); break; case 0x00000013: B_(CE ); break; + case 0x00000014: C_(GSP ); break; break; default: break; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild index bcd179ba11d0..146adcdd316a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild @@ -2,6 +2,7 @@ nvkm-y += nvkm/subdev/volt/base.o nvkm-y += nvkm/subdev/volt/gpio.o nvkm-y += nvkm/subdev/volt/nv40.o nvkm-y += nvkm/subdev/volt/gf100.o +nvkm-y += nvkm/subdev/volt/gf117.o nvkm-y += nvkm/subdev/volt/gk104.o nvkm-y += nvkm/subdev/volt/gk20a.o nvkm-y += nvkm/subdev/volt/gm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c new file mode 100644 index 000000000000..547a58f0aeac --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Ilia Mirkin + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ilia Mirkin + */ +#include "priv.h" + +#include <subdev/fuse.h> + +static int +gf117_volt_speedo_read(struct nvkm_volt *volt) +{ + struct nvkm_device *device = volt->subdev.device; + struct nvkm_fuse *fuse = device->fuse; + + if (!fuse) + return -EINVAL; + + return nvkm_fuse_read(fuse, 0x3a8); +} + +static const struct nvkm_volt_func +gf117_volt = { + .oneinit = gf100_volt_oneinit, + .vid_get = nvkm_voltgpio_get, + .vid_set = nvkm_voltgpio_set, + .speedo_read = gf117_volt_speedo_read, +}; + +int +gf117_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt) +{ + struct nvkm_volt *volt; + int ret; + + ret = nvkm_volt_new_(&gf117_volt, device, index, &volt); + *pvolt = volt; + if (ret) + return ret; + + return nvkm_voltgpio_init(volt); +} diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index 00a9c2ab9e6c..64fb788b6647 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -1406,7 +1406,7 @@ static void dsi_pll_disable(struct dss_pll *pll) static int dsi_dump_dsi_clocks(struct seq_file *s, void *p) { - struct dsi_data *dsi = p; + struct dsi_data *dsi = s->private; struct dss_pll_clock_info *cinfo = &dsi->pll.cinfo; enum dss_clk_source dispc_clk_src, dsi_clk_src; int dsi_module = dsi->module_id; @@ -1467,7 +1467,7 @@ static int dsi_dump_dsi_clocks(struct seq_file *s, void *p) #ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS static int dsi_dump_dsi_irqs(struct seq_file *s, void *p) { - struct dsi_data *dsi = p; + struct dsi_data *dsi = s->private; unsigned long flags; struct dsi_irq_stats stats; @@ -1558,7 +1558,7 @@ static int dsi_dump_dsi_irqs(struct seq_file *s, void *p) static int dsi_dump_dsi_regs(struct seq_file *s, void *p) { - struct dsi_data *dsi = p; + struct dsi_data *dsi = s->private; if (dsi_runtime_get(dsi)) return 0; @@ -4751,6 +4751,17 @@ static int dsi_set_config(struct omap_dss_device *dssdev, dsi->vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH; dsi->vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW; dsi->vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH; + /* + * HACK: These flags should be handled through the omap_dss_device bus + * flags, but this will only be possible when the DSI encoder will be + * converted to the omapdrm-managed encoder model. + */ + dsi->vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE; + dsi->vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE; + dsi->vm.flags &= ~DISPLAY_FLAGS_DE_LOW; + dsi->vm.flags |= DISPLAY_FLAGS_DE_HIGH; + dsi->vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE; + dsi->vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE; dss_mgr_set_timings(&dsi->output, &dsi->vm); @@ -5083,15 +5094,15 @@ static int dsi_bind(struct device *dev, struct device *master, void *data) snprintf(name, sizeof(name), "dsi%u_regs", dsi->module_id + 1); dsi->debugfs.regs = dss_debugfs_create_file(dss, name, - dsi_dump_dsi_regs, &dsi); + dsi_dump_dsi_regs, dsi); #ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS snprintf(name, sizeof(name), "dsi%u_irqs", dsi->module_id + 1); dsi->debugfs.irqs = dss_debugfs_create_file(dss, name, - dsi_dump_dsi_irqs, &dsi); + dsi_dump_dsi_irqs, dsi); #endif snprintf(name, sizeof(name), "dsi%u_clks", dsi->module_id + 1); dsi->debugfs.clks = dss_debugfs_create_file(dss, name, - dsi_dump_dsi_clocks, &dsi); + dsi_dump_dsi_clocks, dsi); return 0; } @@ -5104,8 +5115,6 @@ static void dsi_unbind(struct device *dev, struct device *master, void *data) dss_debugfs_remove_file(dsi->debugfs.irqs); dss_debugfs_remove_file(dsi->debugfs.regs); - of_platform_depopulate(dev); - WARN_ON(dsi->scp_clk_refcount > 0); dss_pll_unregister(&dsi->pll); @@ -5457,6 +5466,8 @@ static int dsi_remove(struct platform_device *pdev) dsi_uninit_output(dsi); + of_platform_depopulate(&pdev->dev); + pm_runtime_disable(&pdev->dev); if (dsi->vdds_dsi_reg != NULL && dsi->vdds_dsi_enabled) { diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c index f33e349c4ec5..578d867a81d5 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.c +++ b/drivers/gpu/drm/qxl/qxl_drv.c @@ -255,14 +255,10 @@ static struct drm_driver qxl_driver = { #if defined(CONFIG_DEBUG_FS) .debugfs_init = qxl_debugfs_init, #endif - .prime_handle_to_fd = drm_gem_prime_handle_to_fd, - .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_export = drm_gem_prime_export, .gem_prime_import = drm_gem_prime_import, .gem_prime_pin = qxl_gem_prime_pin, .gem_prime_unpin = qxl_gem_prime_unpin, - .gem_prime_get_sg_table = qxl_gem_prime_get_sg_table, - .gem_prime_import_sg_table = qxl_gem_prime_import_sg_table, .gem_prime_vmap = qxl_gem_prime_vmap, .gem_prime_vunmap = qxl_gem_prime_vunmap, .gem_prime_mmap = qxl_gem_prime_mmap, diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c index 22e1faf0473a..8b448eca1cd9 100644 --- a/drivers/gpu/drm/qxl/qxl_prime.c +++ b/drivers/gpu/drm/qxl/qxl_prime.c @@ -42,20 +42,6 @@ void qxl_gem_prime_unpin(struct drm_gem_object *obj) qxl_bo_unpin(bo); } -struct sg_table *qxl_gem_prime_get_sg_table(struct drm_gem_object *obj) -{ - WARN_ONCE(1, "not implemented"); - return ERR_PTR(-ENOSYS); -} - -struct drm_gem_object *qxl_gem_prime_import_sg_table( - struct drm_device *dev, struct dma_buf_attachment *attach, - struct sg_table *table) -{ - WARN_ONCE(1, "not implemented"); - return ERR_PTR(-ENOSYS); -} - void *qxl_gem_prime_vmap(struct drm_gem_object *obj) { struct qxl_bo *bo = gem_to_qxl_bo(obj); diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index a97294ac96d5..a12439266bb0 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -4869,10 +4869,12 @@ static void ci_request_link_speed_change_before_state_change(struct radeon_devic pi->force_pcie_gen = RADEON_PCIE_GEN2; if (current_link_speed == RADEON_PCIE_GEN2) break; + /* fall through */ case RADEON_PCIE_GEN2: if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; #endif + /* fall through */ default: pi->force_pcie_gen = ci_get_current_pcie_speed(rdev); break; diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c index f471537c852f..1e14c6921454 100644 --- a/drivers/gpu/drm/radeon/evergreen_cs.c +++ b/drivers/gpu/drm/radeon/evergreen_cs.c @@ -1299,6 +1299,7 @@ static int evergreen_cs_handle_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) return -EINVAL; } ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); + break; case CB_TARGET_MASK: track->cb_target_mask = radeon_get_ib_value(p, idx); track->cb_dirty = true; diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c index dec1e081f529..6a8fb6fd183c 100644 --- a/drivers/gpu/drm/radeon/radeon_kms.c +++ b/drivers/gpu/drm/radeon/radeon_kms.c @@ -172,6 +172,7 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) } if (radeon_is_px(dev)) { + dev_pm_set_driver_flags(dev->dev, DPM_FLAG_NEVER_SKIP); pm_runtime_use_autosuspend(dev->dev); pm_runtime_set_autosuspend_delay(dev->dev, 5000); pm_runtime_set_active(dev->dev); diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 0a785ef0ab66..c9f6cb77e857 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -5762,10 +5762,12 @@ static void si_request_link_speed_change_before_state_change(struct radeon_devic si_pi->force_pcie_gen = RADEON_PCIE_GEN2; if (current_link_speed == RADEON_PCIE_GEN2) break; + /* fall through */ case RADEON_PCIE_GEN2: if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; #endif + /* fall through */ default: si_pi->force_pcie_gen = si_get_current_pcie_speed(rdev); break; diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c index c3dd750c7189..ce4d82d293e4 100644 --- a/drivers/gpu/drm/rockchip/rockchip_rgb.c +++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c @@ -1,17 +1,8 @@ -//SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd * Author: * Sandy Huang <hjc@rock-chips.com> - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <drm/drmP.h> @@ -113,8 +104,10 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev, child_count++; ret = drm_of_find_panel_or_bridge(dev->of_node, 0, endpoint_id, &panel, &bridge); - if (!ret) + if (!ret) { + of_node_put(endpoint); break; + } } of_node_put(port); diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.h b/drivers/gpu/drm/rockchip/rockchip_rgb.h index 38b52e63b2b0..27b9635124bc 100644 --- a/drivers/gpu/drm/rockchip/rockchip_rgb.h +++ b/drivers/gpu/drm/rockchip/rockchip_rgb.h @@ -1,17 +1,8 @@ -//SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd * Author: * Sandy Huang <hjc@rock-chips.com> - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifdef CONFIG_ROCKCHIP_RGB diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 4463d3826ecb..35ddbec1375a 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -52,12 +52,12 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, { int i; - if (!(entity && rq_list && num_rq_list > 0 && rq_list[0])) + if (!(entity && rq_list && (num_rq_list == 0 || rq_list[0]))) return -EINVAL; memset(entity, 0, sizeof(struct drm_sched_entity)); INIT_LIST_HEAD(&entity->list); - entity->rq = rq_list[0]; + entity->rq = NULL; entity->guilty = guilty; entity->num_rq_list = num_rq_list; entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *), @@ -67,6 +67,10 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, for (i = 0; i < num_rq_list; ++i) entity->rq_list[i] = rq_list[i]; + + if (num_rq_list) + entity->rq = rq_list[0]; + entity->last_scheduled = NULL; spin_lock_init(&entity->rq_lock); @@ -165,6 +169,9 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) struct task_struct *last_user; long ret = timeout; + if (!entity->rq) + return 0; + sched = entity->rq->sched; /** * The client will not queue more IBs during this fini, consume existing @@ -264,20 +271,24 @@ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) */ void drm_sched_entity_fini(struct drm_sched_entity *entity) { - struct drm_gpu_scheduler *sched; + struct drm_gpu_scheduler *sched = NULL; - sched = entity->rq->sched; - drm_sched_rq_remove_entity(entity->rq, entity); + if (entity->rq) { + sched = entity->rq->sched; + drm_sched_rq_remove_entity(entity->rq, entity); + } /* Consumption of existing IBs wasn't completed. Forcefully * remove them here. */ if (spsc_queue_peek(&entity->job_queue)) { - /* Park the kernel for a moment to make sure it isn't processing - * our enity. - */ - kthread_park(sched->thread); - kthread_unpark(sched->thread); + if (sched) { + /* Park the kernel for a moment to make sure it isn't processing + * our enity. + */ + kthread_park(sched->thread); + kthread_unpark(sched->thread); + } if (entity->dependency) { dma_fence_remove_callback(entity->dependency, &entity->cb); @@ -362,9 +373,11 @@ void drm_sched_entity_set_priority(struct drm_sched_entity *entity, for (i = 0; i < entity->num_rq_list; ++i) drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); - drm_sched_rq_remove_entity(entity->rq, entity); - drm_sched_entity_set_rq_priority(&entity->rq, priority); - drm_sched_rq_add_entity(entity->rq, entity); + if (entity->rq) { + drm_sched_rq_remove_entity(entity->rq, entity); + drm_sched_entity_set_rq_priority(&entity->rq, priority); + drm_sched_rq_add_entity(entity->rq, entity); + } spin_unlock(&entity->rq_lock); } @@ -440,13 +453,10 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) while ((entity->dependency = sched->ops->dependency(sched_job, entity))) { + trace_drm_sched_job_wait_dep(sched_job, entity->dependency); - if (drm_sched_entity_add_dependency_cb(entity)) { - - trace_drm_sched_job_wait_dep(sched_job, - entity->dependency); + if (drm_sched_entity_add_dependency_cb(entity)) return NULL; - } } /* skip jobs from entity that marked guilty */ diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index 990847cb40f6..4c0d51f73237 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -764,17 +764,18 @@ static struct sun4i_frontend *sun4i_backend_find_frontend(struct sun4i_drv *drv, remote = of_graph_get_remote_port_parent(ep); if (!remote) continue; + of_node_put(remote); /* does this node match any registered engines? */ list_for_each_entry(frontend, &drv->frontend_list, list) { if (remote == frontend->node) { - of_node_put(remote); of_node_put(port); + of_node_put(ep); return frontend; } } } - + of_node_put(port); return ERR_PTR(-EINVAL); } diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c index d95c6e224bd9..d18862629301 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c @@ -93,6 +93,8 @@ static void sun4i_hdmi_disable(struct drm_encoder *encoder) val = readl(hdmi->base + SUN4I_HDMI_VID_CTRL_REG); val &= ~SUN4I_HDMI_VID_CTRL_ENABLE; writel(val, hdmi->base + SUN4I_HDMI_VID_CTRL_REG); + + clk_disable_unprepare(hdmi->tmds_clk); } static void sun4i_hdmi_enable(struct drm_encoder *encoder) @@ -103,6 +105,8 @@ static void sun4i_hdmi_enable(struct drm_encoder *encoder) DRM_DEBUG_DRIVER("Enabling the HDMI Output\n"); + clk_prepare_enable(hdmi->tmds_clk); + sun4i_hdmi_setup_avi_infoframes(hdmi, mode); val |= SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); val |= SUN4I_HDMI_PKT_CTRL_TYPE(1, SUN4I_HDMI_PKT_END); diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 2bd2eda6480a..7136fc91c603 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -761,6 +761,7 @@ static int sun4i_tcon_init_clocks(struct device *dev, return PTR_ERR(tcon->sclk0); } } + clk_prepare_enable(tcon->sclk0); if (tcon->quirks->has_channel_1) { tcon->sclk1 = devm_clk_get(dev, "tcon-ch1"); @@ -775,6 +776,7 @@ static int sun4i_tcon_init_clocks(struct device *dev, static void sun4i_tcon_free_clocks(struct sun4i_tcon *tcon) { + clk_disable_unprepare(tcon->sclk0); clk_disable_unprepare(tcon->clk); } diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c index af92964b6889..b996ac1d4fcc 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.c +++ b/drivers/gpu/drm/virtio/virtgpu_drv.c @@ -205,14 +205,10 @@ static struct drm_driver driver = { #if defined(CONFIG_DEBUG_FS) .debugfs_init = virtio_gpu_debugfs_init, #endif - .prime_handle_to_fd = drm_gem_prime_handle_to_fd, - .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_export = drm_gem_prime_export, .gem_prime_import = drm_gem_prime_import, .gem_prime_pin = virtgpu_gem_prime_pin, .gem_prime_unpin = virtgpu_gem_prime_unpin, - .gem_prime_get_sg_table = virtgpu_gem_prime_get_sg_table, - .gem_prime_import_sg_table = virtgpu_gem_prime_import_sg_table, .gem_prime_vmap = virtgpu_gem_prime_vmap, .gem_prime_vunmap = virtgpu_gem_prime_vunmap, .gem_prime_mmap = virtgpu_gem_prime_mmap, diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h index d577cb76f5ad..3238fdf58eb4 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.h +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h @@ -354,10 +354,6 @@ int virtio_gpu_object_wait(struct virtio_gpu_object *bo, bool no_wait); /* virtgpu_prime.c */ int virtgpu_gem_prime_pin(struct drm_gem_object *obj); void virtgpu_gem_prime_unpin(struct drm_gem_object *obj); -struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj); -struct drm_gem_object *virtgpu_gem_prime_import_sg_table( - struct drm_device *dev, struct dma_buf_attachment *attach, - struct sg_table *sgt); void *virtgpu_gem_prime_vmap(struct drm_gem_object *obj); void virtgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); int virtgpu_gem_prime_mmap(struct drm_gem_object *obj, diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c index 86ce0ae93f59..c59ec34c80a5 100644 --- a/drivers/gpu/drm/virtio/virtgpu_prime.c +++ b/drivers/gpu/drm/virtio/virtgpu_prime.c @@ -39,20 +39,6 @@ void virtgpu_gem_prime_unpin(struct drm_gem_object *obj) WARN_ONCE(1, "not implemented"); } -struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj) -{ - WARN_ONCE(1, "not implemented"); - return ERR_PTR(-ENODEV); -} - -struct drm_gem_object *virtgpu_gem_prime_import_sg_table( - struct drm_device *dev, struct dma_buf_attachment *attach, - struct sg_table *table) -{ - WARN_ONCE(1, "not implemented"); - return ERR_PTR(-ENODEV); -} - void *virtgpu_gem_prime_vmap(struct drm_gem_object *obj) { struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj); diff --git a/drivers/gpu/drm/vkms/vkms_crc.c b/drivers/gpu/drm/vkms/vkms_crc.c index 9d9e8146db90..d7b409a3c0f8 100644 --- a/drivers/gpu/drm/vkms/vkms_crc.c +++ b/drivers/gpu/drm/vkms/vkms_crc.c @@ -1,4 +1,5 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0+ + #include "vkms_drv.h" #include <linux/crc32.h> #include <drm/drm_atomic.h> diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c index 734a3e197df8..8a9aeb0a9ea8 100644 --- a/drivers/gpu/drm/vkms/vkms_crtc.c +++ b/drivers/gpu/drm/vkms/vkms_crtc.c @@ -1,10 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ +// SPDX-License-Identifier: GPL-2.0+ #include "vkms_drv.h" #include <drm/drm_atomic_helper.h> diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c index b13f99a5c849..738dd6206d85 100644 --- a/drivers/gpu/drm/vkms/vkms_drv.c +++ b/drivers/gpu/drm/vkms/vkms_drv.c @@ -1,9 +1,4 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ +// SPDX-License-Identifier: GPL-2.0+ /** * DOC: vkms (Virtual Kernel Modesetting) diff --git a/drivers/gpu/drm/vkms/vkms_drv.h b/drivers/gpu/drm/vkms/vkms_drv.h index e4469cd3d254..81f1cfbeb936 100644 --- a/drivers/gpu/drm/vkms/vkms_drv.h +++ b/drivers/gpu/drm/vkms/vkms_drv.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + #ifndef _VKMS_DRV_H_ #define _VKMS_DRV_H_ diff --git a/drivers/gpu/drm/vkms/vkms_gem.c b/drivers/gpu/drm/vkms/vkms_gem.c index 80311daed47a..138b0bb325cf 100644 --- a/drivers/gpu/drm/vkms/vkms_gem.c +++ b/drivers/gpu/drm/vkms/vkms_gem.c @@ -1,10 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ +// SPDX-License-Identifier: GPL-2.0+ #include <linux/shmem_fs.h> diff --git a/drivers/gpu/drm/vkms/vkms_output.c b/drivers/gpu/drm/vkms/vkms_output.c index c5b16efed51a..3b162b25312e 100644 --- a/drivers/gpu/drm/vkms/vkms_output.c +++ b/drivers/gpu/drm/vkms/vkms_output.c @@ -1,10 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ +// SPDX-License-Identifier: GPL-2.0+ #include "vkms_drv.h" #include <drm/drm_atomic_helper.h> diff --git a/drivers/gpu/drm/vkms/vkms_plane.c b/drivers/gpu/drm/vkms/vkms_plane.c index 418817600ad1..0e67d2d42f0c 100644 --- a/drivers/gpu/drm/vkms/vkms_plane.c +++ b/drivers/gpu/drm/vkms/vkms_plane.c @@ -1,10 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ +// SPDX-License-Identifier: GPL-2.0+ #include "vkms_drv.h" #include <drm/drm_plane_helper.h> diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 4638f6791cda..6165fe2c4504 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -26,6 +26,7 @@ **************************************************************************/ #include <linux/module.h> #include <linux/console.h> +#include <linux/dma-mapping.h> #include <drm/drmP.h> #include "vmwgfx_drv.h" @@ -34,7 +35,6 @@ #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_module.h> -#include <linux/intel-iommu.h> #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices" #define VMWGFX_CHIP_SVGAII 0 @@ -546,6 +546,21 @@ static void vmw_get_initial_size(struct vmw_private *dev_priv) } /** + * vmw_assume_iommu - Figure out whether coherent dma-remapping might be + * taking place. + * @dev: Pointer to the struct drm_device. + * + * Return: true if iommu present, false otherwise. + */ +static bool vmw_assume_iommu(struct drm_device *dev) +{ + const struct dma_map_ops *ops = get_dma_ops(dev->dev); + + return !dma_is_direct(ops) && ops && + ops->map_page != dma_direct_map_page; +} + +/** * vmw_dma_select_mode - Determine how DMA mappings should be set up for this * system. * @@ -565,55 +580,27 @@ static int vmw_dma_select_mode(struct vmw_private *dev_priv) [vmw_dma_alloc_coherent] = "Using coherent TTM pages.", [vmw_dma_map_populate] = "Keeping DMA mappings.", [vmw_dma_map_bind] = "Giving up DMA mappings early."}; -#ifdef CONFIG_X86 - const struct dma_map_ops *dma_ops = get_dma_ops(dev_priv->dev->dev); -#ifdef CONFIG_INTEL_IOMMU - if (intel_iommu_enabled) { + if (vmw_force_coherent) + dev_priv->map_mode = vmw_dma_alloc_coherent; + else if (vmw_assume_iommu(dev_priv->dev)) dev_priv->map_mode = vmw_dma_map_populate; - goto out_fixup; - } -#endif - - if (!(vmw_force_iommu || vmw_force_coherent)) { + else if (!vmw_force_iommu) dev_priv->map_mode = vmw_dma_phys; - DRM_INFO("DMA map mode: %s\n", names[dev_priv->map_mode]); - return 0; - } - - dev_priv->map_mode = vmw_dma_map_populate; - - if (dma_ops && dma_ops->sync_single_for_cpu) + else if (IS_ENABLED(CONFIG_SWIOTLB) && swiotlb_nr_tbl()) dev_priv->map_mode = vmw_dma_alloc_coherent; -#ifdef CONFIG_SWIOTLB - if (swiotlb_nr_tbl() == 0) + else dev_priv->map_mode = vmw_dma_map_populate; -#endif -#ifdef CONFIG_INTEL_IOMMU -out_fixup: -#endif - if (dev_priv->map_mode == vmw_dma_map_populate && - vmw_restrict_iommu) + if (dev_priv->map_mode == vmw_dma_map_populate && vmw_restrict_iommu) dev_priv->map_mode = vmw_dma_map_bind; - if (vmw_force_coherent) - dev_priv->map_mode = vmw_dma_alloc_coherent; - -#if !defined(CONFIG_SWIOTLB) && !defined(CONFIG_INTEL_IOMMU) - /* - * No coherent page pool - */ - if (dev_priv->map_mode == vmw_dma_alloc_coherent) + /* No TTM coherent page pool? FIXME: Ask TTM instead! */ + if (!(IS_ENABLED(CONFIG_SWIOTLB) || IS_ENABLED(CONFIG_INTEL_IOMMU)) && + (dev_priv->map_mode == vmw_dma_alloc_coherent)) return -EINVAL; -#endif - -#else /* CONFIG_X86 */ - dev_priv->map_mode = vmw_dma_map_populate; -#endif /* CONFIG_X86 */ DRM_INFO("DMA map mode: %s\n", names[dev_priv->map_mode]); - return 0; } @@ -625,24 +612,20 @@ out_fixup: * With 32-bit we can only handle 32 bit PFNs. Optionally set that * restriction also for 64-bit systems. */ -#ifdef CONFIG_INTEL_IOMMU static int vmw_dma_masks(struct vmw_private *dev_priv) { struct drm_device *dev = dev_priv->dev; + int ret = 0; - if (intel_iommu_enabled && + ret = dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)); + if (dev_priv->map_mode != vmw_dma_phys && (sizeof(unsigned long) == 4 || vmw_restrict_dma_mask)) { DRM_INFO("Restricting DMA addresses to 44 bits.\n"); - return dma_set_mask(dev->dev, DMA_BIT_MASK(44)); + return dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(44)); } - return 0; -} -#else -static int vmw_dma_masks(struct vmw_private *dev_priv) -{ - return 0; + + return ret; } -#endif static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index f2d13a72c05d..88b8178d4687 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -3570,7 +3570,7 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv, *p_fence = NULL; } - return 0; + return ret; } /** diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index b351fb5214d3..ed2f67822f45 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -1646,7 +1646,7 @@ static int vmw_kms_check_topology(struct drm_device *dev, struct drm_connector_state *conn_state; struct vmw_connector_state *vmw_conn_state; - if (!du->pref_active) { + if (!du->pref_active && new_crtc_state->enable) { ret = -EINVAL; goto clean; } @@ -2554,8 +2554,8 @@ void vmw_kms_helper_validation_finish(struct vmw_private *dev_priv, user_fence_rep) { struct vmw_fence_obj *fence = NULL; - uint32_t handle; - int ret; + uint32_t handle = 0; + int ret = 0; if (file_priv || user_fence_rep || vmw_validation_has_bos(ctx) || out_fence) |