diff options
| author |   Maxime Ripard <maxime@cerno.tech> | 2020-09-14 18:11:40 +0200 |
|---|---|---|
| committer | Maxime Ripard <maxime@cerno.tech> | 2020-09-14 18:11:40 +0200 |
| commit | 00af6729b52ede86a08173c8d5f2c8cd9fa3390d (patch) | |
| tree | 6a8cec3575d5cd49fffadb372b85da5475a266b0 /drivers/gpu | |
| parent | drm: fix double free for gbo in drm_gem_vram_init and drm_gem_vram_create (diff) | |
| parent | Merge v5.9-rc5 into drm-next (diff) | |
| download | linux-dev-00af6729b52ede86a08173c8d5f2c8cd9fa3390d.tar.xz linux-dev-00af6729b52ede86a08173c8d5f2c8cd9fa3390d.zip | |
Merge drm/drm-next into drm-misc-next
Paul Cercueil needs some patches in -rc5 to apply new patches for ingenic
properly.
Signed-off-by: Maxime Ripard <maxime@cerno.tech>
Diffstat (limited to 'drivers/gpu')
688 files changed, 20374 insertions, 6452 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 403ec3db29df..39976c7b100c 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -30,7 +30,7 @@ FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME) ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \ -I$(FULL_AMD_PATH)/include \ -I$(FULL_AMD_PATH)/amdgpu \ - -I$(FULL_AMD_PATH)/powerplay/inc \ + -I$(FULL_AMD_PATH)/pm/inc \ -I$(FULL_AMD_PATH)/acp/include \ -I$(FULL_AMD_DISPLAY_PATH) \ -I$(FULL_AMD_DISPLAY_PATH)/include \ @@ -47,7 +47,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \ amdgpu_fb.o amdgpu_gem.o amdgpu_ring.o \ amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o amdgpu_test.o \ - amdgpu_pm.o atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \ + atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \ atombios_encoders.o amdgpu_sa.o atombios_i2c.o \ amdgpu_dma_buf.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \ amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \ @@ -55,15 +55,15 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \ amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \ amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \ - amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o + amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o amdgpu-$(CONFIG_PERF_EVENTS) += amdgpu_pmu.o # add asic specific block -amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ +amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o \ dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o -amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o \ +amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o \ uvd_v3_1.o amdgpu-y += \ @@ -85,7 +85,7 @@ amdgpu-y += \ # add UMC block amdgpu-y += \ - umc_v6_1.o umc_v6_0.o + umc_v6_1.o umc_v6_0.o umc_v8_7.o # add IH block amdgpu-y += \ @@ -105,10 +105,6 @@ amdgpu-y += \ psp_v11_0.o \ psp_v12_0.o -# add SMC block -amdgpu-y += \ - amdgpu_dpm.o - # add DCE block amdgpu-y += \ dce_v10_0.o \ @@ -212,7 +208,7 @@ amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o -include $(FULL_AMD_PATH)/powerplay/Makefile +include $(FULL_AMD_PATH)/pm/Makefile amdgpu-y += $(AMD_POWERPLAY_FILES) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 327a0daf4a1d..4009d2e30727 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -178,6 +178,7 @@ extern uint amdgpu_dm_abm_level; extern struct amdgpu_mgpu_info mgpu_info; extern int amdgpu_ras_enable; extern uint amdgpu_ras_mask; +extern int amdgpu_bad_page_threshold; extern int amdgpu_async_gfx_ring; extern int amdgpu_mcbp; extern int amdgpu_discovery; @@ -187,9 +188,11 @@ extern int amdgpu_force_asic_type; #ifdef CONFIG_HSA_AMD extern int sched_policy; extern bool debug_evictions; +extern bool no_system_mem_limit; #else static const int sched_policy = KFD_SCHED_POLICY_HWS; static const bool debug_evictions; /* = false */ +static const bool no_system_mem_limit; #endif extern int amdgpu_tmz; @@ -201,6 +204,7 @@ extern int amdgpu_si_support; #ifdef CONFIG_DRM_AMDGPU_CIK extern int amdgpu_cik_support; #endif +extern int amdgpu_num_kcq; #define AMDGPU_VM_MAX_NUM_CTX 4096 #define AMDGPU_SG_THRESHOLD (256*1024*1024) @@ -212,6 +216,8 @@ extern int amdgpu_cik_support; #define AMDGPUFB_CONN_LIMIT 4 #define AMDGPU_BIOS_NUM_SCRATCH 16 +#define AMDGPU_VBIOS_VGA_ALLOCATION (9 * 1024 * 1024) /* reserve 8MB for vga emulator and 1 MB for FB */ + /* hard reset data */ #define AMDGPU_ASIC_RESET_DATA 0x39d5e86b @@ -245,6 +251,7 @@ struct amdgpu_fpriv; struct amdgpu_bo_va_mapping; struct amdgpu_atif; struct kfd_vm_fault_info; +struct amdgpu_hive_info; enum amdgpu_cp_irq { AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0, @@ -611,6 +618,8 @@ struct amdgpu_asic_funcs { uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev); /* device supports BACO */ bool (*supports_baco)(struct amdgpu_device *adev); + /* pre asic_init quirks */ + void (*pre_asic_init)(struct amdgpu_device *adev); }; /* @@ -648,16 +657,6 @@ struct amdgpu_atcs { }; /* - * Firmware VRAM reservation - */ -struct amdgpu_fw_vram_usage { - u64 start_offset; - u64 size; - struct amdgpu_bo *reserved_bo; - void *va; -}; - -/* * CGS */ struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev); @@ -725,13 +724,13 @@ struct amd_powerplay { #define AMDGPU_MAX_DF_PERFMONS 4 struct amdgpu_device { struct device *dev; - struct drm_device *ddev; struct pci_dev *pdev; + struct drm_device ddev; #ifdef CONFIG_DRM_AMD_ACP struct amdgpu_acp acp; #endif - + struct amdgpu_hive_info *hive; /* ASIC */ enum amd_asic_type asic_type; uint32_t family; @@ -765,7 +764,6 @@ struct amdgpu_device { bool is_atom_fw; uint8_t *bios; uint32_t bios_size; - struct amdgpu_bo *stolen_vga_memory; uint32_t bios_scratch_reg_offset; uint32_t bios_scratch[AMDGPU_BIOS_NUM_SCRATCH]; @@ -917,11 +915,6 @@ struct amdgpu_device { /* display related functionality */ struct amdgpu_display_manager dm; - /* discovery */ - uint8_t *discovery_bin; - uint32_t discovery_tmr_size; - struct amdgpu_bo *discovery_memory; - /* mes */ bool enable_mes; struct amdgpu_mes mes; @@ -946,8 +939,6 @@ struct amdgpu_device { struct delayed_work delayed_init_work; struct amdgpu_virt virt; - /* firmware VRAM reservation */ - struct amdgpu_fw_vram_usage fw_vram_usage; /* link all shadow bo */ struct list_head shadow_list; @@ -961,9 +952,9 @@ struct amdgpu_device { bool in_suspend; bool in_hibernate; - bool in_gpu_reset; + atomic_t in_gpu_reset; enum pp_mp1_state mp1_state; - struct mutex lock_reset; + struct rw_semaphore reset_sem; struct amdgpu_doorbell_index doorbell_index; struct mutex notifier_lock; @@ -995,16 +986,25 @@ struct amdgpu_device { atomic_t throttling_logging_enabled; struct ratelimit_state throttling_logging_rs; + uint32_t ras_features; }; +static inline struct amdgpu_device *drm_to_adev(struct drm_device *ddev) +{ + return container_of(ddev, struct amdgpu_device, ddev); +} + +static inline struct drm_device *adev_to_drm(struct amdgpu_device *adev) +{ + return &adev->ddev; +} + static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) { return container_of(bdev, struct amdgpu_device, mman.bdev); } int amdgpu_device_init(struct amdgpu_device *adev, - struct drm_device *ddev, - struct pci_dev *pdev, uint32_t flags); void amdgpu_device_fini(struct amdgpu_device *adev); int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev); @@ -1141,10 +1141,12 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev)) #define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev))) #define amdgpu_asic_supports_baco(adev) (adev)->asic_funcs->supports_baco((adev)) +#define amdgpu_asic_pre_asic_init(adev) (adev)->asic_funcs->pre_asic_init((adev)) #define amdgpu_inc_vram_lost(adev) atomic_inc(&((adev)->vram_lost_counter)); /* Common functions */ +bool amdgpu_device_has_job_running(struct amdgpu_device *adev); bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); int amdgpu_device_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job* job); @@ -1194,7 +1196,7 @@ static inline void *amdgpu_atpx_get_dhandle(void) { return NULL; } extern const struct drm_ioctl_desc amdgpu_ioctls_kms[]; extern const int amdgpu_max_kms_ioctl; -int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags); +int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags); void amdgpu_driver_unload_kms(struct drm_device *dev); void amdgpu_driver_lastclose_kms(struct drm_device *dev); int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv); @@ -1278,4 +1280,8 @@ static inline bool amdgpu_is_tmz(struct amdgpu_device *adev) return adev->gmc.tmz_enabled; } +static inline int amdgpu_in_reset(struct amdgpu_device *adev) +{ + return atomic_read(&adev->in_gpu_reset); +} #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 12247a32f9ef..d3e51d361179 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -136,9 +136,7 @@ static int acp_poweroff(struct generic_pm_domain *genpd) * 2. power off the acp tiles * 3. check and enter ulv state */ - if (adev->powerplay.pp_funcs && - adev->powerplay.pp_funcs->set_powergating_by_smu) - amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, true); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, true); } return 0; } @@ -157,8 +155,7 @@ static int acp_poweron(struct generic_pm_domain *genpd) * 2. turn on acp clock * 3. power on acp tiles */ - if (adev->powerplay.pp_funcs->set_powergating_by_smu) - amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, false); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, false); } return 0; } @@ -529,9 +526,7 @@ static int acp_set_powergating_state(void *handle, struct amdgpu_device *adev = (struct amdgpu_device *)handle; bool enable = (state == AMD_PG_STATE_GATE); - if (adev->powerplay.pp_funcs && - adev->powerplay.pp_funcs->set_powergating_by_smu) - amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, enable); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, enable); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 913c8f0513bd..4a93b880c6bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -463,11 +463,11 @@ static int amdgpu_atif_handler(struct amdgpu_device *adev, if (req.pending & ATIF_DGPU_DISPLAY_EVENT) { if (adev->flags & AMD_IS_PX) { - pm_runtime_get_sync(adev->ddev->dev); + pm_runtime_get_sync(adev_to_drm(adev)->dev); /* Just fire off a uevent and let userspace tell us what to do */ - drm_helper_hpd_irq_event(adev->ddev); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + drm_helper_hpd_irq_event(adev_to_drm(adev)); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); } } /* TODO: check other events */ @@ -817,7 +817,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev) struct drm_encoder *tmp; /* Find the encoder controlling the brightness */ - list_for_each_entry(tmp, &adev->ddev->mode_config.encoder_list, + list_for_each_entry(tmp, &adev_to_drm(adev)->mode_config.encoder_list, head) { struct amdgpu_encoder *enc = to_amdgpu_encoder(tmp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 478f67498a17..edff1b7f282a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -119,7 +119,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) .gpuvm_size = min(adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GMC_HOLE_START), - .drm_render_minor = adev->ddev->render->index, + .drm_render_minor = adev_to_drm(adev)->render->index, .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, }; @@ -160,7 +160,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) adev->doorbell_index.last_non_cp; } - kgd2kfd_device_init(adev->kfd.dev, adev->ddev, &gpu_resources); + kgd2kfd_device_init(adev->kfd.dev, adev_to_drm(adev), &gpu_resources); } } @@ -479,11 +479,11 @@ int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd, goto out_put; obj = dma_buf->priv; - if (obj->dev->driver != adev->ddev->driver) + if (obj->dev->driver != adev_to_drm(adev)->driver) /* Can't handle buffers from different drivers */ goto out_put; - adev = obj->dev->dev_private; + adev = drm_to_adev(obj->dev); bo = gem_to_amdgpu_bo(obj); if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT))) @@ -613,6 +613,7 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, job->vmid = vmid; ret = amdgpu_ib_schedule(ring, 1, ib, job, &f); + if (ret) { DRM_ERROR("amdgpu: failed to schedule IB.\n"); goto err_ib_sched; @@ -756,4 +757,8 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) { } + +void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) +{ +} #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index ffe149aafc39..a10507ecb750 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -270,5 +270,6 @@ int kgd2kfd_resume_mm(struct mm_struct *mm); int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, struct dma_fence *fence); void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd); +void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask); #endif /* AMDGPU_AMDKFD_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c index 35d4a5ab0228..1afa8f122e7d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c @@ -283,22 +283,6 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, return 0; } -static void kgd_set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, - uint64_t page_table_base) -{ - struct amdgpu_device *adev = get_amdgpu_device(kgd); - - if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { - pr_err("trying to set page table base for wrong VMID %u\n", - vmid); - return; - } - - mmhub_v9_4_setup_vm_pt_regs(adev, vmid, page_table_base); - - gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); -} - const struct kfd2kgd_calls arcturus_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping, @@ -317,7 +301,7 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { .wave_control_execute = kgd_gfx_v9_wave_control_execute, .address_watch_get_offset = kgd_gfx_v9_address_watch_get_offset, .get_atc_vmid_pasid_mapping_info = - kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, - .set_vm_context_page_table_base = kgd_set_vm_context_page_table_base, - .get_hive_id = amdgpu_amdkfd_get_hive_id, + kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, + .set_vm_context_page_table_base = + kgd_gfx_v9_set_vm_context_page_table_base, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index bf927f432506..df0aab0fc67e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -542,7 +542,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t temp; struct v10_compute_mqd *m = get_mqd(mqd); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; #if 0 @@ -776,6 +776,4 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = set_vm_context_page_table_base, - .get_hive_id = amdgpu_amdkfd_get_hive_id, - .get_unique_id = amdgpu_amdkfd_get_unique_id, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c index 7e59e473a190..5b38f848b772 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c @@ -152,7 +152,7 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, dev_warn(adev->dev, "Invalid sdma engine id (%d), using engine id 0\n", engine_id); - /* fall through */ + fallthrough; case 0: sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL; @@ -822,7 +822,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .address_watch_get_offset = address_watch_get_offset_v10_3, .get_atc_vmid_pasid_mapping_info = NULL, .set_vm_context_page_table_base = set_vm_context_page_table_base_v10_3, - .get_hive_id = amdgpu_amdkfd_get_hive_id, #if 0 .enable_debug_trap = enable_debug_trap_v10_3, .disable_debug_trap = disable_debug_trap_v10_3, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 744366c7ee85..275f20399373 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -423,7 +423,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, unsigned long flags, end_jiffies; int retry; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; acquire_queue(kgd, pipe_id, queue_id); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index feab4cc6e836..4997189d8b36 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -419,7 +419,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, int retry; struct vi_mqd *m = get_mqd(mqd); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; acquire_queue(kgd, pipe_id, queue_id); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index c7fd0c47b254..e6aede725197 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -195,19 +195,32 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, unsigned int engine_id, unsigned int queue_id) { - uint32_t sdma_engine_reg_base[2] = { - SOC15_REG_OFFSET(SDMA0, 0, - mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL, - SOC15_REG_OFFSET(SDMA1, 0, - mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL - }; - uint32_t retval = sdma_engine_reg_base[engine_id] + uint32_t sdma_engine_reg_base = 0; + uint32_t sdma_rlc_reg_offset; + + switch (engine_id) { + default: + dev_warn(adev->dev, + "Invalid sdma engine id (%d), using engine id 0\n", + engine_id); + fallthrough; + case 0: + sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0, + mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL; + break; + case 1: + sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, 0, + mmSDMA1_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL; + break; + } + + sdma_rlc_reg_offset = sdma_engine_reg_base + queue_id * (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL); pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id, - queue_id, retval); + queue_id, sdma_rlc_reg_offset); - return retval; + return sdma_rlc_reg_offset; } static inline struct v9_mqd *get_mqd(void *mqd) @@ -539,7 +552,7 @@ int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t temp; struct v9_mqd *m = get_mqd(mqd); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; acquire_queue(kgd, pipe_id, queue_id); @@ -677,7 +690,7 @@ uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd, return 0; } -static void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, +void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, uint64_t page_table_base) { struct amdgpu_device *adev = get_amdgpu_device(kgd); @@ -688,7 +701,7 @@ static void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, return; } - mmhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); + adev->mmhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); } @@ -713,6 +726,4 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = kgd_gfx_v9_set_vm_context_page_table_base, - .get_hive_id = amdgpu_amdkfd_get_hive_id, - .get_unique_id = amdgpu_amdkfd_get_unique_id, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h index aedf67d57449..f098e88d3a0d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h @@ -60,3 +60,6 @@ uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd, bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd, uint8_t vmid, uint16_t *p_pasid); + +void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, + uint32_t vmid, uint64_t page_table_base); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index aa2b328c6202..b6b821500d30 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -148,8 +148,12 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, spin_lock(&kfd_mem_limit.mem_limit_lock); + if (kfd_mem_limit.system_mem_used + system_mem_needed > + kfd_mem_limit.max_system_mem_limit) + pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); + if ((kfd_mem_limit.system_mem_used + system_mem_needed > - kfd_mem_limit.max_system_mem_limit) || + kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev->kfd.vram_used + vram_needed > @@ -1668,7 +1672,7 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, return -EINVAL; obj = dma_buf->priv; - if (obj->dev->dev_private != adev) + if (drm_to_adev(obj->dev) != adev) /* Can't handle buffers from other devices */ return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 29f767e026e4..469352e2d6ec 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -148,7 +148,7 @@ void amdgpu_atombios_i2c_init(struct amdgpu_device *adev) if (i2c.valid) { sprintf(stmp, "0x%x", i2c.i2c_id); - adev->i2c_bus[i] = amdgpu_i2c_create(adev->ddev, &i2c, stmp); + adev->i2c_bus[i] = amdgpu_i2c_create(adev_to_drm(adev), &i2c, stmp); } gpio = (ATOM_GPIO_I2C_ASSIGMENT *) ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT)); @@ -541,7 +541,7 @@ bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device * } } - amdgpu_link_encoder_connector(adev->ddev); + amdgpu_link_encoder_connector(adev_to_drm(adev)); return true; } @@ -1786,9 +1786,9 @@ static int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev) (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { /* Firmware request VRAM reservation for SR-IOV */ - adev->fw_vram_usage.start_offset = (start_addr & + adev->mman.fw_vram_usage_start_offset = (start_addr & (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; - adev->fw_vram_usage.size = size << 10; + adev->mman.fw_vram_usage_size = size << 10; /* Use the default scratch size */ usage_bytes = 0; } else { @@ -1882,7 +1882,7 @@ static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val) */ static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val) { - struct amdgpu_device *adev = info->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(info->dev); WREG32(reg, val); } @@ -1898,7 +1898,7 @@ static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val) */ static uint32_t cail_reg_read(struct card_info *info, uint32_t reg) { - struct amdgpu_device *adev = info->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(info->dev); uint32_t r; r = RREG32(reg); @@ -1916,7 +1916,7 @@ static uint32_t cail_reg_read(struct card_info *info, uint32_t reg) */ static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val) { - struct amdgpu_device *adev = info->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(info->dev); WREG32_IO(reg, val); } @@ -1932,7 +1932,7 @@ static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val) */ static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg) { - struct amdgpu_device *adev = info->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(info->dev); uint32_t r; r = RREG32_IO(reg); @@ -1944,7 +1944,7 @@ static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct atom_context *ctx = adev->mode_info.atom_context; return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version); @@ -1995,7 +1995,7 @@ int amdgpu_atombios_init(struct amdgpu_device *adev) return -ENOMEM; adev->mode_info.atom_card_info = atom_card_info; - atom_card_info->dev = adev->ddev; + atom_card_info->dev = adev_to_drm(adev); atom_card_info->reg_read = cail_reg_read; atom_card_info->reg_write = cail_reg_write; /* needed for iio ops */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c index 1279053324f9..17c010d0431f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c @@ -89,9 +89,9 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { /* Firmware request VRAM reservation for SR-IOV */ - adev->fw_vram_usage.start_offset = (start_addr & + adev->mman.fw_vram_usage_start_offset = (start_addr & (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; - adev->fw_vram_usage.size = size << 10; + adev->mman.fw_vram_usage_size = size << 10; /* Use the default scratch size */ usage_bytes = 0; } else { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index b1172d93c99c..6333cada1e09 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -417,26 +417,40 @@ static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) bool amdgpu_get_bios(struct amdgpu_device *adev) { - if (amdgpu_atrm_get_bios(adev)) + if (amdgpu_atrm_get_bios(adev)) { + dev_info(adev->dev, "Fetched VBIOS from ATRM\n"); goto success; + } - if (amdgpu_acpi_vfct_bios(adev)) + if (amdgpu_acpi_vfct_bios(adev)) { + dev_info(adev->dev, "Fetched VBIOS from VFCT\n"); goto success; + } - if (igp_read_bios_from_vram(adev)) + if (igp_read_bios_from_vram(adev)) { + dev_info(adev->dev, "Fetched VBIOS from VRAM BAR\n"); goto success; + } - if (amdgpu_read_bios(adev)) + if (amdgpu_read_bios(adev)) { + dev_info(adev->dev, "Fetched VBIOS from ROM BAR\n"); goto success; + } - if (amdgpu_read_bios_from_rom(adev)) + if (amdgpu_read_bios_from_rom(adev)) { + dev_info(adev->dev, "Fetched VBIOS from ROM\n"); goto success; + } - if (amdgpu_read_disabled_bios(adev)) + if (amdgpu_read_disabled_bios(adev)) { + dev_info(adev->dev, "Fetched VBIOS from disabled ROM BAR\n"); goto success; + } - if (amdgpu_read_platform_bios(adev)) + if (amdgpu_read_platform_bios(adev)) { + dev_info(adev->dev, "Fetched VBIOS from platform\n"); goto success; + } DRM_ERROR("Unable to locate a BIOS ROM\n"); return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 4053597b3af2..15c45b2a3983 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -265,7 +265,7 @@ error_free: int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; union drm_amdgpu_bo_list *args = data; uint32_t handle = args->in.list_handle; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index 93160a849af4..65d1b23d7e74 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -42,7 +42,7 @@ void amdgpu_connector_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); /* bail if the connector does not have hpd pin, e.g., @@ -280,7 +280,7 @@ amdgpu_connector_get_hardcoded_edid(struct amdgpu_device *adev) static void amdgpu_connector_get_edid(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); if (amdgpu_connector->edid) @@ -464,7 +464,7 @@ static int amdgpu_connector_set_property(struct drm_connector *connector, uint64_t val) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_encoder *encoder; struct amdgpu_encoder *amdgpu_encoder; @@ -835,7 +835,7 @@ static enum drm_mode_status amdgpu_connector_vga_mode_valid(struct drm_connector struct drm_display_mode *mode) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* XXX check mode bandwidth */ @@ -942,7 +942,7 @@ static bool amdgpu_connector_check_hpd_status_unchanged(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); enum drm_connector_status status; @@ -973,7 +973,7 @@ static enum drm_connector_status amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); const struct drm_encoder_helper_funcs *encoder_funcs; int r; @@ -1160,7 +1160,7 @@ static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector struct drm_display_mode *mode) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); /* XXX check mode bandwidth */ @@ -1312,7 +1312,7 @@ static bool amdgpu_connector_encoder_is_hbr2(struct drm_connector *connector) bool amdgpu_connector_is_dp12_capable(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if ((adev->clock.default_dispclk >= 53900) && amdgpu_connector_encoder_is_hbr2(connector)) { @@ -1326,7 +1326,7 @@ static enum drm_connector_status amdgpu_connector_dp_detect(struct drm_connector *connector, bool force) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); enum drm_connector_status ret = connector_status_disconnected; struct amdgpu_connector_atom_dig *amdgpu_dig_connector = amdgpu_connector->con_priv; @@ -1526,7 +1526,7 @@ amdgpu_connector_add(struct amdgpu_device *adev, struct amdgpu_hpd *hpd, struct amdgpu_router *router) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; struct amdgpu_connector *amdgpu_connector; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index ecd051976bce..12598a4b5c78 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -1275,13 +1275,24 @@ error_unlock: return r; } +static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser) +{ + int i; + + if (!trace_amdgpu_cs_enabled()) + return; + + for (i = 0; i < parser->job->num_ibs; i++) + trace_amdgpu_cs(parser, i); +} + int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_cs *cs = data; struct amdgpu_cs_parser parser = {}; bool reserved_buffers = false; - int i, r; + int r; if (amdgpu_ras_intr_triggered()) return -EHWPOISON; @@ -1294,7 +1305,8 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) r = amdgpu_cs_parser_init(&parser, data); if (r) { - DRM_ERROR("Failed to initialize parser %d!\n", r); + if (printk_ratelimit()) + DRM_ERROR("Failed to initialize parser %d!\n", r); goto out; } @@ -1319,8 +1331,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) reserved_buffers = true; - for (i = 0; i < parser.job->num_ibs; i++) - trace_amdgpu_cs(&parser, i); + trace_amdgpu_cs_ibs(&parser); r = amdgpu_cs_vm_handling(&parser); if (r) @@ -1421,7 +1432,7 @@ static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_fence_to_handle *info = data; struct dma_fence *fence; struct drm_syncobj *syncobj; @@ -1597,7 +1608,7 @@ err_free_fence_array: int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_wait_fences *wait = data; uint32_t fence_count = wait->in.fence_count; struct drm_amdgpu_fence *fences_user; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 8842c55d4490..c80d8339f58c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -46,7 +46,7 @@ const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { static int amdgpu_ctx_priority_permit(struct drm_file *filp, enum drm_sched_priority priority) { - if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) + if (priority < 0 || priority >= DRM_SCHED_PRIORITY_COUNT) return -EINVAL; /* NORMAL and below are accessible by everyone */ @@ -65,7 +65,7 @@ static int amdgpu_ctx_priority_permit(struct drm_file *filp, static enum gfx_pipe_priority amdgpu_ctx_sched_prio_to_compute_prio(enum drm_sched_priority prio) { switch (prio) { - case DRM_SCHED_PRIORITY_HIGH_HW: + case DRM_SCHED_PRIORITY_HIGH: case DRM_SCHED_PRIORITY_KERNEL: return AMDGPU_GFX_PIPE_PRIO_HIGH; default: @@ -114,7 +114,11 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip, scheds = adev->gpu_sched[hw_ip][hw_prio].sched; num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; - if (hw_ip == AMDGPU_HW_IP_VCN_ENC || hw_ip == AMDGPU_HW_IP_VCN_DEC) { + /* disable load balance if the hw engine retains context among dependent jobs */ + if (hw_ip == AMDGPU_HW_IP_VCN_ENC || + hw_ip == AMDGPU_HW_IP_VCN_DEC || + hw_ip == AMDGPU_HW_IP_UVD_ENC || + hw_ip == AMDGPU_HW_IP_UVD) { sched = drm_sched_pick_best(scheds, num_scheds); scheds = &sched; num_scheds = 1; @@ -385,16 +389,15 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, enum drm_sched_priority priority; union drm_amdgpu_ctx *args = data; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; - r = 0; id = args->in.ctx_id; - priority = amdgpu_to_sched_priority(args->in.priority); + r = amdgpu_to_sched_priority(args->in.priority, &priority); /* For backwards compatibility reasons, we need to accept * ioctls with garbage in the priority field */ - if (priority == DRM_SCHED_PRIORITY_INVALID) + if (r == -EINVAL) priority = DRM_SCHED_PRIORITY_NORMAL; switch (args->in.op) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 193ffdb957b6..abe0c2729e1c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -34,6 +34,7 @@ #include "amdgpu_pm.h" #include "amdgpu_dm_debugfs.h" #include "amdgpu_ras.h" +#include "amdgpu_rap.h" /** * amdgpu_debugfs_add_files - Add simple debugfs entries @@ -68,8 +69,8 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev, adev->debugfs_count = i; #if defined(CONFIG_DEBUG_FS) drm_debugfs_create_files(files, nfiles, - adev->ddev->primary->debugfs_root, - adev->ddev->primary); + adev_to_drm(adev)->primary->debugfs_root, + adev_to_drm(adev)->primary); #endif return 0; } @@ -100,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; - mutex_lock(&adev->lock_reset); + ret = down_read_killable(&adev->reset_sem); + if (ret) + return ret; + if (adev->autodump.dumping.done) { reinit_completion(&adev->autodump.dumping); ret = 0; } else { ret = -EBUSY; } - mutex_unlock(&adev->lock_reset); + + up_read(&adev->reset_sem); return ret; } @@ -126,7 +131,7 @@ static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_ poll_wait(file, &adev->autodump.gpu_hang, poll_table); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return POLLIN | POLLRDNORM | POLLWRNORM; return 0; @@ -146,7 +151,7 @@ static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev) init_waitqueue_head(&adev->autodump.gpu_hang); debugfs_create_file("amdgpu_autodump", 0600, - adev->ddev->primary->debugfs_root, + adev_to_drm(adev)->primary->debugfs_root, adev, &autodump_debug_fops); } @@ -222,23 +227,23 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, *pos &= (1UL << 22) - 1; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } if (use_bank) { if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) || (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return -EINVAL; } @@ -287,8 +292,8 @@ end: if (pm_pg_lock) mutex_unlock(&adev->pm.mutex); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -335,15 +340,15 @@ static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -353,8 +358,8 @@ static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, value = RREG32_PCIE(*pos >> 2); r = put_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -365,8 +370,8 @@ static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -394,15 +399,15 @@ static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -411,8 +416,8 @@ static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user r = get_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -425,8 +430,8 @@ static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -454,15 +459,15 @@ static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -472,8 +477,8 @@ static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, value = RREG32_DIDT(*pos >> 2); r = put_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -484,8 +489,8 @@ static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -513,15 +518,15 @@ static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -530,8 +535,8 @@ static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user r = get_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -544,8 +549,8 @@ static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -573,15 +578,15 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -591,8 +596,8 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, value = RREG32_SMC(*pos); r = put_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -603,8 +608,8 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -632,15 +637,15 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user * if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -649,8 +654,8 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user * r = get_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return r; } @@ -663,8 +668,8 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user * size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); return result; @@ -791,22 +796,22 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf, valuesize = sizeof(values); - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) { amdgpu_virt_disable_access_debugfs(adev); @@ -873,15 +878,15 @@ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, wave = (*pos & GENMASK_ULL(36, 31)) >> 31; simd = (*pos & GENMASK_ULL(44, 37)) >> 37; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_virt_enable_access_debugfs(adev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -896,8 +901,8 @@ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); mutex_unlock(&adev->grbm_idx_mutex); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (!x) { amdgpu_virt_disable_access_debugfs(adev); @@ -971,7 +976,7 @@ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, if (!data) return -ENOMEM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) goto err; @@ -994,8 +999,8 @@ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); mutex_unlock(&adev->grbm_idx_mutex); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); while (size) { uint32_t value; @@ -1017,7 +1022,7 @@ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, return result; err: - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); kfree(data); return r; } @@ -1042,9 +1047,9 @@ static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *bu if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -1053,8 +1058,8 @@ static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *bu r = get_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -1066,8 +1071,8 @@ static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *bu size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return result; } @@ -1091,7 +1096,7 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, if (size & 0x3 || *pos & 0x3) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) return r; @@ -1100,15 +1105,15 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, r = amdgpu_get_gfx_off_status(adev, &value); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = put_user(value, (uint32_t *)buf); if (r) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -1118,8 +1123,8 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, size -= 4; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return result; } @@ -1211,7 +1216,7 @@ static const char *debugfs_regs_names[] = { */ int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) { - struct drm_minor *minor = adev->ddev->primary; + struct drm_minor *minor = adev_to_drm(adev)->primary; struct dentry *ent, *root = minor->debugfs_root; unsigned int i; @@ -1231,17 +1236,19 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int r = 0, i; r = pm_runtime_get_sync(dev->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } /* Avoid accidently unparking the sched thread during GPU reset */ - mutex_lock(&adev->lock_reset); + r = down_read_killable(&adev->reset_sem); + if (r) + return r; /* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1268,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) kthread_unpark(ring->sched.thread); } - mutex_unlock(&adev->lock_reset); + up_read(&adev->reset_sem); pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); @@ -1280,7 +1287,7 @@ static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); seq_write(m, adev->bios, adev->bios_size); return 0; @@ -1290,12 +1297,12 @@ static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int r; r = pm_runtime_get_sync(dev->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -1311,12 +1318,12 @@ static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int r; r = pm_runtime_get_sync(dev->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -1458,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) return -ENOMEM; /* Avoid accidently unparking the sched thread during GPU reset */ - mutex_lock(&adev->lock_reset); + r = down_read_killable(&adev->reset_sem); + if (r) + goto pro_end; /* stop the scheduler */ kthread_park(ring->sched.thread); @@ -1499,13 +1508,14 @@ failure: /* restart the scheduler */ kthread_unpark(ring->sched.thread); - mutex_unlock(&adev->lock_reset); + up_read(&adev->reset_sem); ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched); +pro_end: kfree(fences); - return 0; + return r; } static int amdgpu_debugfs_sclk_set(void *data, u64 val) @@ -1517,9 +1527,9 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val) if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev)) return -EINVAL; - ret = pm_runtime_get_sync(adev->ddev->dev); + ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (ret < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return ret; } @@ -1532,8 +1542,8 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val) return 0; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (ret) return -EINVAL; @@ -1553,7 +1563,7 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) adev->debugfs_preempt = debugfs_create_file("amdgpu_preempt_ib", 0600, - adev->ddev->primary->debugfs_root, adev, + adev_to_drm(adev)->primary->debugfs_root, adev, &fops_ib_preempt); if (!(adev->debugfs_preempt)) { DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n"); @@ -1562,7 +1572,7 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) adev->smu.debugfs_sclk = debugfs_create_file("amdgpu_force_sclk", 0200, - adev->ddev->primary->debugfs_root, adev, + adev_to_drm(adev)->primary->debugfs_root, adev, &fops_sclk_set); if (!(adev->smu.debugfs_sclk)) { DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n"); @@ -1623,6 +1633,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) amdgpu_debugfs_autodump_init(adev); + amdgpu_rap_debugfs_init(adev); + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list, ARRAY_SIZE(amdgpu_debugfs_list)); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 4204cda680f5..f7307af76452 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -132,7 +132,7 @@ static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); return snprintf(buf, PAGE_SIZE, "%llu\n", cnt); @@ -157,7 +157,7 @@ static ssize_t amdgpu_device_get_product_name(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_name); } @@ -179,7 +179,7 @@ static ssize_t amdgpu_device_get_product_number(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_number); } @@ -201,7 +201,7 @@ static ssize_t amdgpu_device_get_serial_number(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%s\n", adev->serial); } @@ -219,7 +219,7 @@ static DEVICE_ATTR(serial_number, S_IRUGO, */ bool amdgpu_device_supports_boco(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (adev->flags & AMD_IS_PX) return true; @@ -236,7 +236,7 @@ bool amdgpu_device_supports_boco(struct drm_device *dev) */ bool amdgpu_device_supports_baco(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); return amdgpu_asic_supports_baco(adev); } @@ -319,8 +319,12 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, { uint32_t ret; - if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) - return amdgpu_kiq_rreg(adev, reg); + if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && + down_read_trylock(&adev->reset_sem)) { + ret = amdgpu_kiq_rreg(adev, reg); + up_read(&adev->reset_sem); + return ret; + } if ((reg * 4) < adev->rmmio_size) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -332,6 +336,7 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); } + trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); return ret; } @@ -378,7 +383,9 @@ void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) BUG(); } -void static inline amdgpu_mm_wreg_mmio(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags) +static inline void amdgpu_mm_wreg_mmio(struct amdgpu_device *adev, + uint32_t reg, uint32_t v, + uint32_t acc_flags) { trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); @@ -407,8 +414,12 @@ void static inline amdgpu_mm_wreg_mmio(struct amdgpu_device *adev, uint32_t reg, void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags) { - if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) - return amdgpu_kiq_wreg(adev, reg, v); + if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && + down_read_trylock(&adev->reset_sem)) { + amdgpu_kiq_wreg(adev, reg, v); + up_read(&adev->reset_sem); + return; + } amdgpu_mm_wreg_mmio(adev, reg, v, acc_flags); } @@ -654,6 +665,20 @@ static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, } /** + * amdgpu_device_asic_init - Wrapper for atom asic_init + * + * @dev: drm_device pointer + * + * Does any asic specific work and then calls atom asic init. + */ +static int amdgpu_device_asic_init(struct amdgpu_device *adev) +{ + amdgpu_asic_pre_asic_init(adev); + + return amdgpu_atom_asic_init(adev->mode_info.atom_context); +} + +/** * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page * * @adev: amdgpu device pointer @@ -1199,6 +1224,11 @@ static int amdgpu_device_check_arguments(struct amdgpu_device *adev) amdgpu_gmc_tmz_set(adev); + if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) { + amdgpu_num_kcq = 8; + dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n"); + } + return 0; } @@ -1211,7 +1241,8 @@ static int amdgpu_device_check_arguments(struct amdgpu_device *adev) * Callback for the switcheroo driver. Suspends or resumes the * the asics before or after it is powered up using ACPI methods. */ -static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) +static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, + enum vga_switcheroo_state state) { struct drm_device *dev = pci_get_drvdata(pdev); int r; @@ -1504,7 +1535,7 @@ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) adev->enable_virtual_display = false; if (amdgpu_virtual_display) { - struct drm_device *ddev = adev->ddev; + struct drm_device *ddev = adev_to_drm(adev); const char *pci_address_name = pci_name(ddev->pdev); char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; @@ -1563,7 +1594,7 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) adev->firmware.gpu_info_fw = NULL; - if (adev->discovery_bin) { + if (adev->mman.discovery_bin) { amdgpu_discovery_get_gfx_info(adev); /* @@ -1935,7 +1966,7 @@ static int amdgpu_device_fw_loading(struct amdgpu_device *adev) if (adev->ip_blocks[i].status.hw == true) break; - if (adev->in_gpu_reset || adev->in_suspend) { + if (amdgpu_in_reset(adev) || adev->in_suspend) { r = adev->ip_blocks[i].version->funcs->resume(adev); if (r) { DRM_ERROR("resume of IP block <%s> failed %d\n", @@ -2055,13 +2086,19 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) * it should be called after amdgpu_device_ip_hw_init_phase2 since * for some ASICs the RAS EEPROM code relies on SMU fully functioning * for I2C communication which only true at this point. - * recovery_init may fail, but it can free all resources allocated by - * itself and its failure should not stop amdgpu init process. + * + * amdgpu_ras_recovery_init may fail, but the upper only cares the + * failure from bad gpu situation and stop amdgpu init process + * accordingly. For other failed cases, it will still release all + * the resource and print error message, rather than returning one + * negative value to upper level. * * Note: theoretically, this should be called before all vram allocations * to protect retired page from abusing */ - amdgpu_ras_recovery_init(adev); + r = amdgpu_ras_recovery_init(adev); + if (r) + goto init_failed; if (adev->gmc.xgmi.num_physical_nodes > 1) amdgpu_xgmi_add_device(adev); @@ -2106,7 +2143,7 @@ static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) AMDGPU_RESET_MAGIC_NUM)) return true; - if (!adev->in_gpu_reset) + if (!amdgpu_in_reset(adev)) return false; /* @@ -2217,9 +2254,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void) gpu_ins = &(mgpu_info.gpu_ins[i]); adev = gpu_ins->adev; if (!(adev->flags & AMD_IS_APU) && - !gpu_ins->mgpu_fan_enabled && - adev->powerplay.pp_funcs && - adev->powerplay.pp_funcs->enable_mgpu_fan_boost) { + !gpu_ins->mgpu_fan_enabled) { ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); if (ret) break; @@ -2574,17 +2609,16 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) AMD_IP_BLOCK_TYPE_IH, }; - for (i = 0; i < adev->num_ip_blocks; i++) - adev->ip_blocks[i].status.hw = false; - for (i = 0; i < ARRAY_SIZE(ip_order); i++) { int j; struct amdgpu_ip_block *block; - for (j = 0; j < adev->num_ip_blocks; j++) { - block = &adev->ip_blocks[j]; + block = &adev->ip_blocks[i]; + block->status.hw = false; - if (block->version->type != ip_order[i] || + for (j = 0; j < ARRAY_SIZE(ip_order); j++) { + + if (block->version->type != ip_order[j] || !block->status.valid) continue; @@ -2777,6 +2811,12 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) { switch (asic_type) { #if defined(CONFIG_DRM_AMD_DC) +#if defined(CONFIG_DRM_AMD_DC_SI) + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: +#endif case CHIP_BONAIRE: case CHIP_KAVERI: case CHIP_KABINI: @@ -2831,7 +2871,7 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) */ bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) { - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) || adev->enable_virtual_display) return false; return amdgpu_device_asic_has_dc_support(adev->asic_type); @@ -2842,7 +2882,7 @@ static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) { struct amdgpu_device *adev = container_of(__work, struct amdgpu_device, xgmi_reset_work); - struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); + struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); /* It's a bug to not have a hive within this function */ if (WARN_ON(!hive)) @@ -2857,13 +2897,13 @@ static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { task_barrier_enter(&hive->tb); - adev->asic_reset_res = amdgpu_device_baco_enter(adev->ddev); + adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); if (adev->asic_reset_res) goto fail; task_barrier_exit(&hive->tb); - adev->asic_reset_res = amdgpu_device_baco_exit(adev->ddev); + adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); if (adev->asic_reset_res) goto fail; @@ -2879,7 +2919,8 @@ static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) fail: if (adev->asic_reset_res) DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", - adev->asic_reset_res, adev->ddev->unique); + adev->asic_reset_res, adev_to_drm(adev)->unique); + amdgpu_put_xgmi_hive(hive); } static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) @@ -2962,8 +3003,6 @@ static const struct attribute *amdgpu_dev_attributes[] = { * amdgpu_device_init - initialize the driver * * @adev: amdgpu_device pointer - * @ddev: drm dev pointer - * @pdev: pci dev pointer * @flags: driver flags * * Initializes the driver info and hw (all asics). @@ -2971,18 +3010,15 @@ static const struct attribute *amdgpu_dev_attributes[] = { * Called at driver startup. */ int amdgpu_device_init(struct amdgpu_device *adev, - struct drm_device *ddev, - struct pci_dev *pdev, uint32_t flags) { + struct drm_device *ddev = adev_to_drm(adev); + struct pci_dev *pdev = adev->pdev; int r, i; bool boco = false; u32 max_MBps; adev->shutdown = false; - adev->dev = &pdev->dev; - adev->ddev = ddev; - adev->pdev = pdev; adev->flags = flags; if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) @@ -3038,7 +3074,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->mn_lock); mutex_init(&adev->virt.vf_errors.lock); hash_init(adev->mn_hash); - mutex_init(&adev->lock_reset); + atomic_set(&adev->in_gpu_reset, 0); + init_rwsem(&adev->reset_sem); mutex_init(&adev->psp.mutex); mutex_init(&adev->notifier_lock); @@ -3188,7 +3225,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, goto failed; } DRM_INFO("GPU posting now...\n"); - r = amdgpu_atom_asic_init(adev->mode_info.atom_context); + r = amdgpu_device_asic_init(adev); if (r) { dev_err(adev->dev, "gpu post error!\n"); goto failed; @@ -3226,7 +3263,7 @@ fence_driver_init: } /* init the mode config */ - drm_mode_config_init(adev->ddev); + drm_mode_config_init(adev_to_drm(adev)); r = amdgpu_device_ip_init(adev); if (r) { @@ -3352,9 +3389,7 @@ failed: */ void amdgpu_device_fini(struct amdgpu_device *adev) { - int r; - - DRM_INFO("amdgpu: finishing device.\n"); + dev_info(adev->dev, "amdgpu: finishing device.\n"); flush_delayed_work(&adev->delayed_init_work); adev->shutdown = true; @@ -3368,15 +3403,15 @@ void amdgpu_device_fini(struct amdgpu_device *adev) amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized){ if (!amdgpu_device_has_dc_support(adev)) - drm_helper_force_disable_all(adev->ddev); + drm_helper_force_disable_all(adev_to_drm(adev)); else - drm_atomic_helper_shutdown(adev->ddev); + drm_atomic_helper_shutdown(adev_to_drm(adev)); } amdgpu_fence_driver_fini(adev); if (adev->pm_sysfs_en) amdgpu_pm_sysfs_fini(adev); amdgpu_fbdev_fini(adev); - r = amdgpu_device_ip_fini(adev); + amdgpu_device_ip_fini(adev); release_firmware(adev->firmware.gpu_info_fw); adev->firmware.gpu_info_fw = NULL; adev->accel_working = false; @@ -3394,7 +3429,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) amdgpu_has_atpx_dgpu_power_cntl()) && !pci_is_thunderbolt_attached(adev->pdev)) vga_switcheroo_unregister_client(adev->pdev); - if (amdgpu_device_supports_boco(adev->ddev)) + if (amdgpu_device_supports_boco(adev_to_drm(adev))) vga_switcheroo_fini_domain_pm_ops(adev->dev); vga_client_register(adev->pdev, NULL, NULL, NULL); if (adev->rio_mem) @@ -3410,7 +3445,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); if (IS_ENABLED(CONFIG_PERF_EVENTS)) amdgpu_pmu_fini(adev); - if (adev->discovery_bin) + if (adev->mman.discovery_bin) amdgpu_discovery_fini(adev); } @@ -3436,11 +3471,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) struct drm_connector_list_iter iter; int r; - if (dev == NULL || dev->dev_private == NULL) { - return -ENODEV; - } - - adev = dev->dev_private; + adev = drm_to_adev(dev); if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; @@ -3528,7 +3559,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) { struct drm_connector *connector; struct drm_connector_list_iter iter; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_crtc *crtc; int r = 0; @@ -3537,14 +3568,14 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) /* post card */ if (amdgpu_device_need_post(adev)) { - r = amdgpu_atom_asic_init(adev->mode_info.atom_context); + r = amdgpu_device_asic_init(adev); if (r) - DRM_ERROR("amdgpu asic init failed\n"); + dev_err(adev->dev, "amdgpu asic init failed\n"); } r = amdgpu_device_ip_resume(adev); if (r) { - DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r); + dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); return r; } amdgpu_fence_driver_resume(adev); @@ -3568,7 +3599,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) if (r == 0) { r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM); if (r != 0) - DRM_ERROR("Failed to pin cursor BO (%d)\n", r); + dev_err(adev->dev, "Failed to pin cursor BO (%d)\n", r); amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj); amdgpu_bo_unreserve(aobj); } @@ -3658,7 +3689,7 @@ static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) adev->ip_blocks[i].status.hang = adev->ip_blocks[i].version->funcs->check_soft_reset(adev); if (adev->ip_blocks[i].status.hang) { - DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); + dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); asic_hang = true; } } @@ -3719,7 +3750,7 @@ static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { if (adev->ip_blocks[i].status.hang) { - DRM_INFO("Some block need full reset!\n"); + dev_info(adev->dev, "Some block need full reset!\n"); return true; } } @@ -3807,7 +3838,7 @@ static int amdgpu_device_recover_vram(struct amdgpu_device *adev) else tmo = msecs_to_jiffies(100); - DRM_INFO("recover vram bo from shadow start\n"); + dev_info(adev->dev, "recover vram bo from shadow start\n"); mutex_lock(&adev->shadow_list_lock); list_for_each_entry(shadow, &adev->shadow_list, shadow_list) { @@ -3843,11 +3874,11 @@ static int amdgpu_device_recover_vram(struct amdgpu_device *adev) dma_fence_put(fence); if (r < 0 || tmo <= 0) { - DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); + dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); return -EIO; } - DRM_INFO("recover vram bo from shadow done\n"); + dev_info(adev->dev, "recover vram bo from shadow done\n"); return 0; } @@ -3908,6 +3939,34 @@ error: } /** + * amdgpu_device_has_job_running - check if there is any job in mirror list + * + * @adev: amdgpu device pointer + * + * check if there is any job in mirror list + */ +bool amdgpu_device_has_job_running(struct amdgpu_device *adev) +{ + int i; + struct drm_sched_job *job; + + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + + spin_lock(&ring->sched.job_list_lock); + job = list_first_entry_or_null(&ring->sched.ring_mirror_list, + struct drm_sched_job, node); + spin_unlock(&ring->sched.job_list_lock); + if (job) + return true; + } + return false; +} + +/** * amdgpu_device_should_recover_gpu - check if we should try GPU recovery * * @adev: amdgpu device pointer @@ -3918,7 +3977,7 @@ error: bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) { if (!amdgpu_device_ip_check_soft_reset(adev)) { - DRM_INFO("Timeout, but no hardware hang detected.\n"); + dev_info(adev->dev, "Timeout, but no hardware hang detected.\n"); return false; } @@ -3958,7 +4017,7 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) return true; disabled: - DRM_INFO("GPU recovery disabled.\n"); + dev_info(adev->dev, "GPU recovery disabled.\n"); return false; } @@ -3997,7 +4056,7 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, r = amdgpu_device_ip_soft_reset(adev); amdgpu_device_ip_post_soft_reset(adev); if (r || amdgpu_device_ip_check_soft_reset(adev)) { - DRM_INFO("soft reset failed, will fallback to full reset!\n"); + dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); need_full_reset = true; } } @@ -4033,8 +4092,8 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, r = amdgpu_asic_reset(tmp_adev); if (r) { - DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s", - r, tmp_adev->ddev->unique); + dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", + r, adev_to_drm(tmp_adev)->unique); break; } } @@ -4066,8 +4125,8 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { if (need_full_reset) { /* post card */ - if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context)) - DRM_WARN("asic atom init failed!"); + if (amdgpu_device_asic_init(tmp_adev)) + dev_warn(tmp_adev->dev, "asic atom init failed!"); if (!r) { dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); @@ -4108,8 +4167,23 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, amdgpu_fbdev_set_suspend(tmp_adev, 0); - /* must succeed. */ - amdgpu_ras_resume(tmp_adev); + /* + * The GPU enters bad state once faulty pages + * by ECC has reached the threshold, and ras + * recovery is scheduled next. So add one check + * here to break recovery if it indeed exceeds + * bad page threshold, and remind user to + * retire this GPU or setting one bigger + * bad_page_threshold value to fix this once + * probing driver again. + */ + if (!amdgpu_ras_check_err_threshold(tmp_adev)) { + /* must succeed. */ + amdgpu_ras_resume(tmp_adev); + } else { + r = -EINVAL; + goto out; + } /* Update PSP FW topology after reset */ if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1) @@ -4117,7 +4191,6 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, } } - out: if (!r) { amdgpu_irq_gpu_reset_resume_helper(tmp_adev); @@ -4142,16 +4215,19 @@ end: return r; } -static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock) +static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, + struct amdgpu_hive_info *hive) { - if (trylock) { - if (!mutex_trylock(&adev->lock_reset)) - return false; - } else - mutex_lock(&adev->lock_reset); + if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0) + return false; + + if (hive) { + down_write_nest_lock(&adev->reset_sem, &hive->hive_lock); + } else { + down_write(&adev->reset_sem); + } atomic_inc(&adev->gpu_reset_counter); - adev->in_gpu_reset = true; switch (amdgpu_asic_reset_method(adev)) { case AMD_RESET_METHOD_MODE1: adev->mp1_state = PP_MP1_STATE_SHUTDOWN; @@ -4171,8 +4247,8 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) { amdgpu_vf_error_trans_all(adev); adev->mp1_state = PP_MP1_STATE_NONE; - adev->in_gpu_reset = false; - mutex_unlock(&adev->lock_reset); + atomic_set(&adev->in_gpu_reset, 0); + up_write(&adev->reset_sem); } static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) @@ -4282,12 +4358,15 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, * We always reset all schedulers for device and all devices for XGMI * hive so that should take care of them too. */ - hive = amdgpu_get_xgmi_hive(adev, true); - if (hive && !mutex_trylock(&hive->reset_lock)) { - DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", - job ? job->base.id : -1, hive->hive_id); - mutex_unlock(&hive->hive_lock); - return 0; + hive = amdgpu_get_xgmi_hive(adev); + if (hive) { + if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) { + DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", + job ? job->base.id : -1, hive->hive_id); + amdgpu_put_xgmi_hive(hive); + return 0; + } + mutex_lock(&hive->hive_lock); } /* @@ -4309,11 +4388,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, /* block all schedulers and reset given job's ring */ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { - if (!amdgpu_device_lock_adev(tmp_adev, !hive)) { - DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress", + if (!amdgpu_device_lock_adev(tmp_adev, hive)) { + dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", job ? job->base.id : -1); - mutex_unlock(&hive->hive_lock); - return 0; + r = 0; + goto skip_recovery; } /* @@ -4385,8 +4464,8 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ &need_full_reset); /*TODO Should we stop ?*/ if (r) { - DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ", - r, tmp_adev->ddev->unique); + dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", + r, adev_to_drm(tmp_adev)->unique); tmp_adev->asic_reset_res = r; } } @@ -4422,7 +4501,7 @@ skip_hw_reset: } if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) { - drm_helper_resume_force_mode(tmp_adev->ddev); + drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); } tmp_adev->asic_reset_res = 0; @@ -4446,9 +4525,11 @@ skip_sched_resume: amdgpu_device_unlock_adev(tmp_adev); } +skip_recovery: if (hive) { - mutex_unlock(&hive->reset_lock); + atomic_set(&hive->in_reset, 0); mutex_unlock(&hive->hive_lock); + amdgpu_put_xgmi_hive(hive); } if (r) @@ -4594,10 +4675,10 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) int amdgpu_device_baco_enter(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); - if (!amdgpu_device_supports_baco(adev->ddev)) + if (!amdgpu_device_supports_baco(adev_to_drm(adev))) return -ENOTSUPP; if (ras && ras->supported) @@ -4608,11 +4689,11 @@ int amdgpu_device_baco_enter(struct drm_device *dev) int amdgpu_device_baco_exit(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); int ret = 0; - if (!amdgpu_device_supports_baco(adev->ddev)) + if (!amdgpu_device_supports_baco(adev_to_drm(adev))) return -ENOTSUPP; ret = amdgpu_dpm_baco_exit(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index a50ff2306504..bfb95143ba5e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -136,7 +136,7 @@ static int amdgpu_discovery_read_binary(struct amdgpu_device *adev, uint8_t *bin uint64_t pos = vram_size - DISCOVERY_TMR_OFFSET; amdgpu_device_vram_access(adev, pos, (uint32_t *)binary, - adev->discovery_tmr_size, false); + adev->mman.discovery_tmr_size, false); return 0; } @@ -168,18 +168,18 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) uint16_t checksum; int r; - adev->discovery_tmr_size = DISCOVERY_TMR_SIZE; - adev->discovery_bin = kzalloc(adev->discovery_tmr_size, GFP_KERNEL); - if (!adev->discovery_bin) + adev->mman.discovery_tmr_size = DISCOVERY_TMR_SIZE; + adev->mman.discovery_bin = kzalloc(adev->mman.discovery_tmr_size, GFP_KERNEL); + if (!adev->mman.discovery_bin) return -ENOMEM; - r = amdgpu_discovery_read_binary(adev, adev->discovery_bin); + r = amdgpu_discovery_read_binary(adev, adev->mman.discovery_bin); if (r) { DRM_ERROR("failed to read ip discovery binary\n"); goto out; } - bhdr = (struct binary_header *)adev->discovery_bin; + bhdr = (struct binary_header *)adev->mman.discovery_bin; if (le32_to_cpu(bhdr->binary_signature) != BINARY_SIGNATURE) { DRM_ERROR("invalid ip discovery binary signature\n"); @@ -192,7 +192,7 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) size = bhdr->binary_size - offset; checksum = bhdr->binary_checksum; - if (!amdgpu_discovery_verify_checksum(adev->discovery_bin + offset, + if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset, size, checksum)) { DRM_ERROR("invalid ip discovery binary checksum\n"); r = -EINVAL; @@ -202,7 +202,7 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) info = &bhdr->table_list[IP_DISCOVERY]; offset = le16_to_cpu(info->offset); checksum = le16_to_cpu(info->checksum); - ihdr = (struct ip_discovery_header *)(adev->discovery_bin + offset); + ihdr = (struct ip_discovery_header *)(adev->mman.discovery_bin + offset); if (le32_to_cpu(ihdr->signature) != DISCOVERY_TABLE_SIGNATURE) { DRM_ERROR("invalid ip discovery data table signature\n"); @@ -210,7 +210,7 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) goto out; } - if (!amdgpu_discovery_verify_checksum(adev->discovery_bin + offset, + if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset, ihdr->size, checksum)) { DRM_ERROR("invalid ip discovery data table checksum\n"); r = -EINVAL; @@ -220,9 +220,9 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) info = &bhdr->table_list[GC]; offset = le16_to_cpu(info->offset); checksum = le16_to_cpu(info->checksum); - ghdr = (struct gpu_info_header *)(adev->discovery_bin + offset); + ghdr = (struct gpu_info_header *)(adev->mman.discovery_bin + offset); - if (!amdgpu_discovery_verify_checksum(adev->discovery_bin + offset, + if (!amdgpu_discovery_verify_checksum(adev->mman.discovery_bin + offset, ghdr->size, checksum)) { DRM_ERROR("invalid gc data table checksum\n"); r = -EINVAL; @@ -232,16 +232,16 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) return 0; out: - kfree(adev->discovery_bin); - adev->discovery_bin = NULL; + kfree(adev->mman.discovery_bin); + adev->mman.discovery_bin = NULL; return r; } void amdgpu_discovery_fini(struct amdgpu_device *adev) { - kfree(adev->discovery_bin); - adev->discovery_bin = NULL; + kfree(adev->mman.discovery_bin); + adev->mman.discovery_bin = NULL; } int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) @@ -265,8 +265,8 @@ int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) return r; } - bhdr = (struct binary_header *)adev->discovery_bin; - ihdr = (struct ip_discovery_header *)(adev->discovery_bin + + bhdr = (struct binary_header *)adev->mman.discovery_bin; + ihdr = (struct ip_discovery_header *)(adev->mman.discovery_bin + le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset)); num_dies = le16_to_cpu(ihdr->num_dies); @@ -274,7 +274,7 @@ int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) for (i = 0; i < num_dies; i++) { die_offset = le16_to_cpu(ihdr->die_info[i].die_offset); - dhdr = (struct die_header *)(adev->discovery_bin + die_offset); + dhdr = (struct die_header *)(adev->mman.discovery_bin + die_offset); num_ips = le16_to_cpu(dhdr->num_ips); ip_offset = die_offset + sizeof(*dhdr); @@ -288,7 +288,7 @@ int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) le16_to_cpu(dhdr->die_id), num_ips); for (j = 0; j < num_ips; j++) { - ip = (struct ip *)(adev->discovery_bin + ip_offset); + ip = (struct ip *)(adev->mman.discovery_bin + ip_offset); num_base_address = ip->num_base_address; DRM_DEBUG("%s(%d) #%d v%d.%d.%d:\n", @@ -337,24 +337,24 @@ int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id, uint16_t num_ips; int i, j; - if (!adev->discovery_bin) { + if (!adev->mman.discovery_bin) { DRM_ERROR("ip discovery uninitialized\n"); return -EINVAL; } - bhdr = (struct binary_header *)adev->discovery_bin; - ihdr = (struct ip_discovery_header *)(adev->discovery_bin + + bhdr = (struct binary_header *)adev->mman.discovery_bin; + ihdr = (struct ip_discovery_header *)(adev->mman.discovery_bin + le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset)); num_dies = le16_to_cpu(ihdr->num_dies); for (i = 0; i < num_dies; i++) { die_offset = le16_to_cpu(ihdr->die_info[i].die_offset); - dhdr = (struct die_header *)(adev->discovery_bin + die_offset); + dhdr = (struct die_header *)(adev->mman.discovery_bin + die_offset); num_ips = le16_to_cpu(dhdr->num_ips); ip_offset = die_offset + sizeof(*dhdr); for (j = 0; j < num_ips; j++) { - ip = (struct ip *)(adev->discovery_bin + ip_offset); + ip = (struct ip *)(adev->mman.discovery_bin + ip_offset); if (le16_to_cpu(ip->hw_id) == hw_id) { if (major) @@ -377,13 +377,13 @@ int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev) struct binary_header *bhdr; struct gc_info_v1_0 *gc_info; - if (!adev->discovery_bin) { + if (!adev->mman.discovery_bin) { DRM_ERROR("ip discovery uninitialized\n"); return -EINVAL; } - bhdr = (struct binary_header *)adev->discovery_bin; - gc_info = (struct gc_info_v1_0 *)(adev->discovery_bin + + bhdr = (struct binary_header *)adev->mman.discovery_bin; + gc_info = (struct gc_info_v1_0 *)(adev->mman.discovery_bin + le16_to_cpu(bhdr->table_list[GC].offset)); adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->gc_num_se); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index d76172965199..c81206e6096f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -93,7 +93,7 @@ static void amdgpu_display_flip_work_func(struct work_struct *__work) * targeted by the flip */ if (amdgpu_crtc->enabled && - (amdgpu_display_get_crtc_scanoutpos(adev->ddev, work->crtc_id, 0, + (amdgpu_display_get_crtc_scanoutpos(adev_to_drm(adev), work->crtc_id, 0, &vpos, &hpos, NULL, NULL, &crtc->hwmode) & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) == @@ -152,7 +152,7 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc, struct drm_modeset_acquire_ctx *ctx) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_gem_object *obj; struct amdgpu_flip_work *work; @@ -292,7 +292,7 @@ int amdgpu_display_crtc_set_config(struct drm_mode_set *set, pm_runtime_mark_last_busy(dev->dev); - adev = dev->dev_private; + adev = drm_to_adev(dev); /* if we have active crtcs and we don't have a power ref, take the current one */ if (active && !adev->have_disp_power_ref) { @@ -619,51 +619,51 @@ int amdgpu_display_modeset_create_props(struct amdgpu_device *adev) int sz; adev->mode_info.coherent_mode_property = - drm_property_create_range(adev->ddev, 0 , "coherent", 0, 1); + drm_property_create_range(adev_to_drm(adev), 0, "coherent", 0, 1); if (!adev->mode_info.coherent_mode_property) return -ENOMEM; adev->mode_info.load_detect_property = - drm_property_create_range(adev->ddev, 0, "load detection", 0, 1); + drm_property_create_range(adev_to_drm(adev), 0, "load detection", 0, 1); if (!adev->mode_info.load_detect_property) return -ENOMEM; - drm_mode_create_scaling_mode_property(adev->ddev); + drm_mode_create_scaling_mode_property(adev_to_drm(adev)); sz = ARRAY_SIZE(amdgpu_underscan_enum_list); adev->mode_info.underscan_property = - drm_property_create_enum(adev->ddev, 0, - "underscan", - amdgpu_underscan_enum_list, sz); + drm_property_create_enum(adev_to_drm(adev), 0, + "underscan", + amdgpu_underscan_enum_list, sz); adev->mode_info.underscan_hborder_property = - drm_property_create_range(adev->ddev, 0, - "underscan hborder", 0, 128); + drm_property_create_range(adev_to_drm(adev), 0, + "underscan hborder", 0, 128); if (!adev->mode_info.underscan_hborder_property) return -ENOMEM; adev->mode_info.underscan_vborder_property = - drm_property_create_range(adev->ddev, 0, - "underscan vborder", 0, 128); + drm_property_create_range(adev_to_drm(adev), 0, + "underscan vborder", 0, 128); if (!adev->mode_info.underscan_vborder_property) return -ENOMEM; sz = ARRAY_SIZE(amdgpu_audio_enum_list); adev->mode_info.audio_property = - drm_property_create_enum(adev->ddev, 0, + drm_property_create_enum(adev_to_drm(adev), 0, "audio", amdgpu_audio_enum_list, sz); sz = ARRAY_SIZE(amdgpu_dither_enum_list); adev->mode_info.dither_property = - drm_property_create_enum(adev->ddev, 0, + drm_property_create_enum(adev_to_drm(adev), 0, "dither", amdgpu_dither_enum_list, sz); if (amdgpu_device_has_dc_support(adev)) { adev->mode_info.abm_level_property = - drm_property_create_range(adev->ddev, 0, - "abm level", 0, 4); + drm_property_create_range(adev_to_drm(adev), 0, + "abm level", 0, 4); if (!adev->mode_info.abm_level_property) return -ENOMEM; } @@ -813,7 +813,7 @@ int amdgpu_display_get_crtc_scanoutpos(struct drm_device *dev, int vbl_start, vbl_end, vtotal, ret = 0; bool in_vbl = true; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index d7050ab95946..957934926b24 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -35,6 +35,7 @@ #include "amdgpu_display.h" #include "amdgpu_gem.h" #include "amdgpu_dma_buf.h" +#include "amdgpu_xgmi.h" #include <drm/amdgpu_drm.h> #include <linux/dma-buf.h> #include <linux/dma-fence-array.h> @@ -455,7 +456,7 @@ static struct drm_gem_object * amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) { struct dma_resv *resv = dma_buf->resv; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_bo *bo; struct amdgpu_bo_param bp; int ret; @@ -596,3 +597,36 @@ struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, obj->import_attach = attach; return obj; } + +/** + * amdgpu_dmabuf_is_xgmi_accessible - Check if xgmi available for P2P transfer + * + * @adev: amdgpu_device pointer of the importer + * @bo: amdgpu buffer object + * + * Returns: + * True if dmabuf accessible over xgmi, false otherwise. + */ +bool amdgpu_dmabuf_is_xgmi_accessible(struct amdgpu_device *adev, + struct amdgpu_bo *bo) +{ + struct drm_gem_object *obj = &bo->tbo.base; + struct drm_gem_object *gobj; + + if (obj->import_attach) { + struct dma_buf *dma_buf = obj->import_attach->dmabuf; + + if (dma_buf->ops != &amdgpu_dmabuf_ops) + /* No XGMI with non AMD GPUs */ + return false; + + gobj = dma_buf->priv; + bo = gem_to_amdgpu_bo(gobj); + } + + if (amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) && + (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) + return true; + + return false; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h index ec447a7b6b28..2c5c84a06bb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h @@ -29,6 +29,8 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj, int flags); struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf); +bool amdgpu_dmabuf_is_xgmi_accessible(struct amdgpu_device *adev, + struct amdgpu_bo *bo); void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj); void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 81a79760ca61..6edde2b9e402 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -26,6 +26,7 @@ #include <drm/drm_drv.h> #include <drm/drm_gem.h> #include <drm/drm_vblank.h> +#include <drm/drm_managed.h> #include "amdgpu_drv.h" #include <drm/drm_pciids.h> @@ -88,9 +89,10 @@ * - 3.37.0 - L2 is invalidated before SDMA IBs, needed for correctness * - 3.38.0 - Add AMDGPU_IB_FLAG_EMIT_MEM_SYNC * - 3.39.0 - DMABUF implicit sync does a full pipeline sync + * - 3.40.0 - Add AMDGPU_IDS_FLAGS_TMZ */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 39 +#define KMS_DRIVER_MINOR 40 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit = 0; @@ -150,12 +152,14 @@ int amdgpu_noretry; int amdgpu_force_asic_type = -1; int amdgpu_tmz = 0; int amdgpu_reset_method = -1; /* auto */ +int amdgpu_num_kcq = -1; struct amdgpu_mgpu_info mgpu_info = { .mutex = __MUTEX_INITIALIZER(mgpu_info.mutex), }; int amdgpu_ras_enable = -1; uint amdgpu_ras_mask = 0xffffffff; +int amdgpu_bad_page_threshold = -1; /** * DOC: vramlimit (int) @@ -676,11 +680,14 @@ MODULE_PARM_DESC(debug_largebar, * Ignore CRAT table during KFD initialization. By default, KFD uses the ACPI CRAT * table to get information about AMD APUs. This option can serve as a workaround on * systems with a broken CRAT table. + * + * Default is auto (according to asic type, iommu_v2, and crat table, to decide + * whehter use CRAT) */ int ignore_crat; module_param(ignore_crat, int, 0444); MODULE_PARM_DESC(ignore_crat, - "Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)"); + "Ignore CRAT table during KFD initialization (0 = auto (default), 1 = ignore CRAT)"); /** * DOC: halt_if_hws_hang (int) @@ -715,6 +722,15 @@ MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1 bool debug_evictions; module_param(debug_evictions, bool, 0644); MODULE_PARM_DESC(debug_evictions, "enable eviction debug messages (false = default)"); + +/** + * DOC: no_system_mem_limit(bool) + * Disable system memory limit, to support multiple process shared memory + */ +bool no_system_mem_limit; +module_param(no_system_mem_limit, bool, 0644); +MODULE_PARM_DESC(no_system_mem_limit, "disable system memory limit (false = default)"); + #endif /** @@ -765,6 +781,19 @@ module_param_named(tmz, amdgpu_tmz, int, 0444); MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco)"); module_param_named(reset_method, amdgpu_reset_method, int, 0444); +/** + * DOC: bad_page_threshold (int) + * Bad page threshold is to specify the threshold value of faulty pages + * detected by RAS ECC, that may result in GPU entering bad status if total + * faulty pages by ECC exceed threshold value and leave it for user's further + * check. + */ +MODULE_PARM_DESC(bad_page_threshold, "Bad page threshold(-1 = auto(default value), 0 = disable bad page retirement)"); +module_param_named(bad_page_threshold, amdgpu_bad_page_threshold, int, 0444); + +MODULE_PARM_DESC(num_kcq, "number of kernel compute queue user want to setup (8 if set to greater than 8 or less than 0, only affect gfx 8+)"); +module_param_named(num_kcq, amdgpu_num_kcq, int, 0444); + static const struct pci_device_id pciidlist[] = { #ifdef CONFIG_DRM_AMDGPU_SI {0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI}, @@ -1057,7 +1086,7 @@ static struct drm_driver kms_driver; static int amdgpu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { - struct drm_device *dev; + struct drm_device *ddev; struct amdgpu_device *adev; unsigned long flags = ent->driver_data; int ret, retry = 0; @@ -1113,36 +1142,44 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, if (ret) return ret; - dev = drm_dev_alloc(&kms_driver, &pdev->dev); - if (IS_ERR(dev)) - return PTR_ERR(dev); + adev = kzalloc(sizeof(*adev), GFP_KERNEL); + if (!adev) + return -ENOMEM; + + adev->dev = &pdev->dev; + adev->pdev = pdev; + ddev = adev_to_drm(adev); + ret = drm_dev_init(ddev, &kms_driver, &pdev->dev); + if (ret) + goto err_free; + + drmm_add_final_kfree(ddev, adev); if (!supports_atomic) - dev->driver_features &= ~DRIVER_ATOMIC; + ddev->driver_features &= ~DRIVER_ATOMIC; ret = pci_enable_device(pdev); if (ret) goto err_free; - dev->pdev = pdev; - - pci_set_drvdata(pdev, dev); + ddev->pdev = pdev; + pci_set_drvdata(pdev, ddev); - ret = amdgpu_driver_load_kms(dev, ent->driver_data); + ret = amdgpu_driver_load_kms(adev, ent->driver_data); if (ret) goto err_pci; retry_init: - ret = drm_dev_register(dev, ent->driver_data); + ret = drm_dev_register(ddev, ent->driver_data); if (ret == -EAGAIN && ++retry <= 3) { DRM_INFO("retry init %d\n", retry); /* Don't request EX mode too frequently which is attacking */ msleep(5000); goto retry_init; - } else if (ret) + } else if (ret) { goto err_pci; + } - adev = dev->dev_private; ret = amdgpu_debugfs_init(adev); if (ret) DRM_ERROR("Creating debugfs files failed (%d).\n", ret); @@ -1152,7 +1189,7 @@ retry_init: err_pci: pci_disable_device(pdev); err_free: - drm_dev_put(dev); + drm_dev_put(ddev); return ret; } @@ -1176,7 +1213,7 @@ static void amdgpu_pci_shutdown(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (amdgpu_ras_intr_triggered()) return; @@ -1209,7 +1246,7 @@ static int amdgpu_pmops_resume(struct device *dev) static int amdgpu_pmops_freeze(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(drm_dev); int r; adev->in_hibernate = true; @@ -1245,7 +1282,7 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct drm_device *drm_dev = pci_get_drvdata(pdev); - struct amdgpu_device *adev = drm_dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(drm_dev); int ret, i; if (!adev->runpm) { @@ -1296,7 +1333,7 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct drm_device *drm_dev = pci_get_drvdata(pdev); - struct amdgpu_device *adev = drm_dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(drm_dev); int ret; if (!adev->runpm) @@ -1332,7 +1369,7 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) static int amdgpu_pmops_runtime_idle(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(drm_dev); /* we don't want the main rpm_idle to call suspend - we want to autosuspend */ int ret = 1; @@ -1500,8 +1537,6 @@ static struct pci_driver amdgpu_kms_pci_driver = { .driver.pm = &amdgpu_pm_ops, }; - - static int __init amdgpu_init(void) { int r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c index 61fcf247a638..af4ef84e27a7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c @@ -35,7 +35,7 @@ void amdgpu_link_encoder_connector(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_connector *connector; struct drm_connector_list_iter iter; struct amdgpu_connector *amdgpu_connector; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index db731f573f98..e2c2eb45a793 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -135,7 +135,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED; - info = drm_get_format_info(adev->ddev, mode_cmd); + info = drm_get_format_info(adev_to_drm(adev), mode_cmd); cpp = info->cpp[0]; /* need to align pitch with crtc limits */ @@ -231,7 +231,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, goto out; } - ret = amdgpu_display_framebuffer_init(adev->ddev, &rfbdev->rfb, + ret = amdgpu_display_framebuffer_init(adev_to_drm(adev), &rfbdev->rfb, &mode_cmd, gobj); if (ret) { DRM_ERROR("failed to initialize framebuffer %d\n", ret); @@ -254,7 +254,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, drm_fb_helper_fill_info(info, &rfbdev->helper, sizes); /* setup aperture base/size for vesafb takeover */ - info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base; + info->apertures->ranges[0].base = adev_to_drm(adev)->mode_config.fb_base; info->apertures->ranges[0].size = adev->gmc.aper_size; /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ @@ -270,7 +270,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, DRM_INFO("fb depth is %d\n", fb->format->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); - vga_switcheroo_client_fb_set(adev->ddev->pdev, info); + vga_switcheroo_client_fb_set(adev_to_drm(adev)->pdev, info); return 0; out: @@ -318,7 +318,7 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev) return 0; /* don't init fbdev if there are no connectors */ - if (list_empty(&adev->ddev->mode_config.connector_list)) + if (list_empty(&adev_to_drm(adev)->mode_config.connector_list)) return 0; /* select 8 bpp console on low vram cards */ @@ -332,10 +332,10 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev) rfbdev->adev = adev; adev->mode_info.rfbdev = rfbdev; - drm_fb_helper_prepare(adev->ddev, &rfbdev->helper, - &amdgpu_fb_helper_funcs); + drm_fb_helper_prepare(adev_to_drm(adev), &rfbdev->helper, + &amdgpu_fb_helper_funcs); - ret = drm_fb_helper_init(adev->ddev, &rfbdev->helper); + ret = drm_fb_helper_init(adev_to_drm(adev), &rfbdev->helper); if (ret) { kfree(rfbdev); return ret; @@ -343,7 +343,7 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev) /* disable all the possible outputs/crtcs before entering KMS mode */ if (!amdgpu_device_has_dc_support(adev)) - drm_helper_disable_unused_functions(adev->ddev); + drm_helper_disable_unused_functions(adev_to_drm(adev)); drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); return 0; @@ -354,7 +354,7 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev) if (!adev->mode_info.rfbdev) return; - amdgpu_fbdev_destroy(adev->ddev, adev->mode_info.rfbdev); + amdgpu_fbdev_destroy(adev_to_drm(adev), adev->mode_info.rfbdev); kfree(adev->mode_info.rfbdev); adev->mode_info.rfbdev = NULL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 58d4c219178a..fe2d495d08ab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -155,7 +155,7 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, seq); amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, seq, flags | AMDGPU_FENCE_FLAG_INT); - pm_runtime_get_noresume(adev->ddev->dev); + pm_runtime_get_noresume(adev_to_drm(adev)->dev); ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; if (unlikely(rcu_dereference_protected(*ptr, 1))) { struct dma_fence *old; @@ -284,8 +284,8 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring) BUG(); dma_fence_put(fence); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); } while (last_seq != seq); return true; @@ -700,7 +700,7 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int i; for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { @@ -749,7 +749,7 @@ static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int r; r = pm_runtime_get_sync(dev->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 6b1eb9d045ae..aa7f230c71bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -93,7 +93,7 @@ retry: void amdgpu_gem_force_release(struct amdgpu_device *adev) { - struct drm_device *ddev = adev->ddev; + struct drm_device *ddev = adev_to_drm(adev); struct drm_file *file; mutex_lock(&ddev->filelist_mutex); @@ -217,7 +217,7 @@ out_unlock: int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; union drm_amdgpu_gem_create *args = data; @@ -298,7 +298,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct ttm_operation_ctx ctx = { true, false }; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_amdgpu_gem_userptr *args = data; struct drm_gem_object *gobj; struct amdgpu_bo *bo; @@ -587,7 +587,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, struct drm_amdgpu_gem_va *args = data; struct drm_gem_object *gobj; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; struct amdgpu_bo *abo; struct amdgpu_bo_va *bo_va; @@ -711,7 +711,7 @@ error_unref: int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_amdgpu_gem_op *args = data; struct drm_gem_object *gobj; struct amdgpu_vm_bo_base *base; @@ -788,7 +788,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_gem_object *gobj; uint32_t handle; u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 78d37f92c7be..d6981425ec51 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -202,40 +202,29 @@ bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) { - int i, queue, pipe, mec; + int i, queue, pipe; bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); - - /* policy for amdgpu compute queue ownership */ - for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { - queue = i % adev->gfx.mec.num_queue_per_pipe; - pipe = (i / adev->gfx.mec.num_queue_per_pipe) - % adev->gfx.mec.num_pipe_per_mec; - mec = (i / adev->gfx.mec.num_queue_per_pipe) - / adev->gfx.mec.num_pipe_per_mec; - - /* we've run out of HW */ - if (mec >= adev->gfx.mec.num_mec) - break; - - if (multipipe_policy) { - /* policy: amdgpu owns the first two queues of the first MEC */ - if (mec == 0 && queue < 2) - set_bit(i, adev->gfx.mec.queue_bitmap); - } else { - /* policy: amdgpu owns all queues in the first pipe */ - if (mec == 0 && pipe == 0) - set_bit(i, adev->gfx.mec.queue_bitmap); + int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe, + adev->gfx.num_compute_rings); + + if (multipipe_policy) { + /* policy: make queues evenly cross all pipes on MEC1 only */ + for (i = 0; i < max_queues_per_mec; i++) { + pipe = i % adev->gfx.mec.num_pipe_per_mec; + queue = (i / adev->gfx.mec.num_pipe_per_mec) % + adev->gfx.mec.num_queue_per_pipe; + + set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue, + adev->gfx.mec.queue_bitmap); } + } else { + /* policy: amdgpu owns all queues in the given pipe */ + for (i = 0; i < max_queues_per_mec; ++i) + set_bit(i, adev->gfx.mec.queue_bitmap); } - /* update the number of active compute rings */ - adev->gfx.num_compute_rings = - bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); - - /* If you hit this case and edited the policy, you probably just - * need to increase AMDGPU_MAX_COMPUTE_RINGS */ - if (WARN_ON(adev->gfx.num_compute_rings > AMDGPU_MAX_COMPUTE_RINGS)) - adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); } void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) @@ -571,8 +560,14 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) if (enable && !adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); } else if (!enable && adev->gfx.gfx_off_state) { - if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) + if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) { adev->gfx.gfx_off_state = false; + + if (adev->gfx.funcs->init_spm_golden) { + dev_dbg(adev->dev, "GFXOFF is disabled, re-init SPM golden settings\n"); + amdgpu_gfx_init_spm_golden(adev); + } + } } mutex_unlock(&adev->gfx.gfx_off_mutex); @@ -724,7 +719,7 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) * * also don't wait anymore for IRQ context * */ - if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) goto failed_kiq_read; might_sleep(); @@ -748,7 +743,7 @@ failed_unlock: failed_kiq_read: if (reg_val_offs) amdgpu_device_wb_free(adev, reg_val_offs); - pr_err("failed to read reg:%x\n", reg); + dev_err(adev->dev, "failed to read reg:%x\n", reg); return ~0; } @@ -782,7 +777,7 @@ void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) * * also don't wait anymore for IRQ context * */ - if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) goto failed_kiq_write; might_sleep(); @@ -801,5 +796,5 @@ failed_undo: amdgpu_ring_undo(ring); spin_unlock_irqrestore(&kiq->ring_lock, flags); failed_kiq_write: - pr_err("failed to write reg:%x\n", reg); + dev_err(adev->dev, "failed to write reg:%x\n", reg); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index 1e7a2b0997c5..a611e78dd4ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -216,6 +216,7 @@ struct amdgpu_gfx_funcs { int (*ras_error_inject)(struct amdgpu_device *adev, void *inject_if); int (*query_ras_error_count) (struct amdgpu_device *adev, void *ras_error_status); void (*reset_ras_error_count) (struct amdgpu_device *adev); + void (*init_spm_golden)(struct amdgpu_device *adev); }; struct sq_work { @@ -324,6 +325,7 @@ struct amdgpu_gfx { #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev)) #define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance)) #define amdgpu_gfx_select_me_pipe_q(adev, me, pipe, q, vmid) (adev)->gfx.funcs->select_me_pipe_q((adev), (me), (pipe), (q), (vmid)) +#define amdgpu_gfx_init_spm_golden(adev) (adev)->gfx.funcs->init_spm_golden((adev)) /** * amdgpu_gfx_create_bitmask - create a bitmask diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 34cbd6f6a56b..213ef090bb0e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -27,6 +27,7 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include "amdgpu.h" +#include "amdgpu_gmc.h" #include "amdgpu_ras.h" #include "amdgpu_xgmi.h" @@ -411,3 +412,64 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) break; } } + +void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, + bool enable) +{ + struct amdgpu_vmhub *hub; + u32 tmp, reg, i; + + hub = &adev->vmhub[hub_type]; + for (i = 0; i < 16; i++) { + reg = hub->vm_context0_cntl + hub->ctx_distance * i; + + tmp = RREG32(reg); + if (enable) + tmp |= hub->vm_cntx_cntl_vm_fault; + else + tmp &= ~hub->vm_cntx_cntl_vm_fault; + + WREG32(reg, tmp); + } +} + +void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev) +{ + unsigned size; + + /* + * TODO: + * Currently there is a bug where some memory client outside + * of the driver writes to first 8M of VRAM on S3 resume, + * this overrides GART which by default gets placed in first 8M and + * causes VM_FAULTS once GTT is accessed. + * Keep the stolen memory reservation until the while this is not solved. + */ + switch (adev->asic_type) { + case CHIP_VEGA10: + case CHIP_RAVEN: + case CHIP_RENOIR: + adev->mman.keep_stolen_vga_memory = true; + break; + default: + adev->mman.keep_stolen_vga_memory = false; + break; + } + + if (!amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_DCE)) + size = 0; + else + size = amdgpu_gmc_get_vbios_fb_size(adev); + + /* set to 0 if the pre-OS buffer uses up most of vram */ + if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) + size = 0; + + if (size > AMDGPU_VBIOS_VGA_ALLOCATION) { + adev->mman.stolen_vga_size = AMDGPU_VBIOS_VGA_ALLOCATION; + adev->mman.stolen_extended_size = size - adev->mman.stolen_vga_size; + } else { + adev->mman.stolen_vga_size = size; + adev->mman.stolen_extended_size = 0; + } +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h index acdb61cfa24c..d61bbde4c7d2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h @@ -74,6 +74,12 @@ struct amdgpu_gmc_fault { /* * VMHUB structures, functions & helpers */ +struct amdgpu_vmhub_funcs { + void (*print_l2_protection_fault_status)(struct amdgpu_device *adev, + uint32_t status); + uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type); +}; + struct amdgpu_vmhub { uint32_t ctx0_ptb_addr_lo32; uint32_t ctx0_ptb_addr_hi32; @@ -92,6 +98,10 @@ struct amdgpu_vmhub { uint32_t ctx_addr_distance; /* include LO32/HI32 */ uint32_t eng_distance; uint32_t eng_addr_distance; /* include LO32/HI32 */ + + uint32_t vm_cntx_cntl_vm_fault; + + const struct amdgpu_vmhub_funcs *vmhub_funcs; }; /* @@ -121,6 +131,8 @@ struct amdgpu_gmc_funcs { void (*get_vm_pte)(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping, uint64_t *flags); + /* get the amount of memory used by the vbios for pre-OS console */ + unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev); }; struct amdgpu_xgmi { @@ -203,7 +215,6 @@ struct amdgpu_gmc { uint8_t vram_vendor; uint32_t srbm_soft_reset; bool prt_warning; - uint64_t stolen_size; uint32_t sdpif_register; /* apertures */ u64 shared_aperture_start; @@ -239,6 +250,7 @@ struct amdgpu_gmc { #define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags)) #define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags)) #define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags)) +#define amdgpu_gmc_get_vbios_fb_size(adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev)) /** * amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR @@ -289,4 +301,10 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev); extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev); +extern void +amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, + bool enable); + +void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index 697bc2c6fdb2..2b1b7c136343 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -46,8 +46,9 @@ static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); + return snprintf(buf, PAGE_SIZE, "%llu\n", man->size * PAGE_SIZE); } @@ -64,8 +65,9 @@ static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); + return snprintf(buf, PAGE_SIZE, "%llu\n", amdgpu_gtt_mgr_usage(man)); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c index 70dbe343f51d..47cad23a6b9e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c @@ -40,7 +40,7 @@ static int amdgpu_i2c_pre_xfer(struct i2c_adapter *i2c_adap) { struct amdgpu_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t temp; @@ -82,7 +82,7 @@ static int amdgpu_i2c_pre_xfer(struct i2c_adapter *i2c_adap) static void amdgpu_i2c_post_xfer(struct i2c_adapter *i2c_adap) { struct amdgpu_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t temp; @@ -101,7 +101,7 @@ static void amdgpu_i2c_post_xfer(struct i2c_adapter *i2c_adap) static int amdgpu_i2c_get_clock(void *i2c_priv) { struct amdgpu_i2c_chan *i2c = i2c_priv; - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t val; @@ -116,7 +116,7 @@ static int amdgpu_i2c_get_clock(void *i2c_priv) static int amdgpu_i2c_get_data(void *i2c_priv) { struct amdgpu_i2c_chan *i2c = i2c_priv; - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t val; @@ -130,7 +130,7 @@ static int amdgpu_i2c_get_data(void *i2c_priv) static void amdgpu_i2c_set_clock(void *i2c_priv, int clock) { struct amdgpu_i2c_chan *i2c = i2c_priv; - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t val; @@ -143,7 +143,7 @@ static void amdgpu_i2c_set_clock(void *i2c_priv, int clock) static void amdgpu_i2c_set_data(void *i2c_priv, int data) { struct amdgpu_i2c_chan *i2c = i2c_priv; - struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(i2c->dev); struct amdgpu_i2c_bus_rec *rec = &i2c->rec; uint32_t val; @@ -253,7 +253,7 @@ void amdgpu_i2c_add(struct amdgpu_device *adev, const struct amdgpu_i2c_bus_rec *rec, const char *name) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); int i; for (i = 0; i < AMDGPU_MAX_I2C_BUS; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index dcd492170598..2f53fa0ae9a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -445,7 +445,7 @@ static int amdgpu_debugfs_sa_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); seq_printf(m, "--------------------- DELAYED --------------------- \n"); amdgpu_sa_bo_dump_debug_info(&adev->ib_pools[AMDGPU_IB_POOL_DELAYED], diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 0cc4c67f95f7..300ac73b4738 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -85,7 +85,7 @@ static void amdgpu_hotplug_work_func(struct work_struct *work) { struct amdgpu_device *adev = container_of(work, struct amdgpu_device, hotplug_work); - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_mode_config *mode_config = &dev->mode_config; struct drm_connector *connector; struct drm_connector_list_iter iter; @@ -151,7 +151,7 @@ void amdgpu_irq_disable_all(struct amdgpu_device *adev) irqreturn_t amdgpu_irq_handler(int irq, void *arg) { struct drm_device *dev = (struct drm_device *) arg; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); irqreturn_t ret; ret = amdgpu_ih_process(adev, &adev->irq.ih); @@ -268,9 +268,9 @@ int amdgpu_irq_init(struct amdgpu_device *adev) if (!adev->enable_virtual_display) /* Disable vblank IRQs aggressively for power-saving */ /* XXX: can this be enabled for DC? */ - adev->ddev->vblank_disable_immediate = true; + adev_to_drm(adev)->vblank_disable_immediate = true; - r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc); + r = drm_vblank_init(adev_to_drm(adev), adev->mode_info.num_crtc); if (r) return r; @@ -284,14 +284,14 @@ int amdgpu_irq_init(struct amdgpu_device *adev) adev->irq.installed = true; /* Use vector 0 for MSI-X */ - r = drm_irq_install(adev->ddev, pci_irq_vector(adev->pdev, 0)); + r = drm_irq_install(adev_to_drm(adev), pci_irq_vector(adev->pdev, 0)); if (r) { adev->irq.installed = false; if (!amdgpu_device_has_dc_support(adev)) flush_work(&adev->hotplug_work); return r; } - adev->ddev->max_vblank_count = 0x00ffffff; + adev_to_drm(adev)->max_vblank_count = 0x00ffffff; DRM_DEBUG("amdgpu: irq initialized.\n"); return 0; @@ -311,7 +311,7 @@ void amdgpu_irq_fini(struct amdgpu_device *adev) unsigned i, j; if (adev->irq.installed) { - drm_irq_uninstall(adev->ddev); + drm_irq_uninstall(adev_to_drm(adev)); adev->irq.installed = false; if (adev->irq.msi_enabled) pci_free_irq_vectors(adev->pdev); @@ -522,7 +522,7 @@ void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev) int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type) { - if (!adev->ddev->irq_enabled) + if (!adev_to_drm(adev)->irq_enabled) return -ENOENT; if (type >= src->num_types) @@ -552,7 +552,7 @@ int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type) { - if (!adev->ddev->irq_enabled) + if (!adev_to_drm(adev)->irq_enabled) return -ENOENT; if (type >= src->num_types) @@ -583,7 +583,7 @@ int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src, bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type) { - if (!adev->ddev->irq_enabled) + if (!adev_to_drm(adev)->irq_enabled) return false; if (type >= src->num_types) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 937029ad5271..dcfe8a3b03ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -251,7 +251,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) int i; /* Signal all jobs not yet scheduled */ - for (i = DRM_SCHED_PRIORITY_MAX - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { struct drm_sched_rq *rq = &sched->sched_rq[i]; if (!rq) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 55ff071217a9..456a4a93b337 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -78,7 +78,7 @@ void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev) */ void amdgpu_driver_unload_kms(struct drm_device *dev) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (adev == NULL) return; @@ -86,7 +86,7 @@ void amdgpu_driver_unload_kms(struct drm_device *dev) amdgpu_unregister_gpu_instance(adev); if (adev->rmmio == NULL) - goto done_free; + return; if (adev->runpm) { pm_runtime_get_sync(dev->dev); @@ -94,12 +94,7 @@ void amdgpu_driver_unload_kms(struct drm_device *dev) } amdgpu_acpi_fini(adev); - amdgpu_device_fini(adev); - -done_free: - kfree(adev); - dev->dev_private = NULL; } void amdgpu_register_gpu_instance(struct amdgpu_device *adev) @@ -130,22 +125,18 @@ void amdgpu_register_gpu_instance(struct amdgpu_device *adev) /** * amdgpu_driver_load_kms - Main load function for KMS. * - * @dev: drm dev pointer + * @adev: pointer to struct amdgpu_device * @flags: device flags * * This is the main load function for KMS (all asics). * Returns 0 on success, error on failure. */ -int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) +int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) { - struct amdgpu_device *adev; + struct drm_device *dev; int r, acpi_status; - adev = kzalloc(sizeof(struct amdgpu_device), GFP_KERNEL); - if (adev == NULL) { - return -ENOMEM; - } - dev->dev_private = (void *)adev; + dev = adev_to_drm(adev); if (amdgpu_has_atpx() && (amdgpu_is_atpx_hybrid() || @@ -160,7 +151,7 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) * properly initialize the GPU MC controller and permit * VRAM allocation */ - r = amdgpu_device_init(adev, dev, dev->pdev, flags); + r = amdgpu_device_init(adev, flags); if (r) { dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); goto out; @@ -179,6 +170,7 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) case CHIP_VEGA20: case CHIP_ARCTURUS: case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: /* enable runpm if runpm=1 */ if (amdgpu_runtime_pm > 0) adev->runpm = true; @@ -479,7 +471,7 @@ static int amdgpu_hw_ip_info(struct amdgpu_device *adev, */ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_amdgpu_info *info = data; struct amdgpu_mode_info *minfo = &adev->mode_info; void __user *out = (void __user *)(uintptr_t)info->return_pointer; @@ -681,8 +673,12 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file * in the bitfields */ if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) se_num = 0xffffffff; + else if (se_num >= AMDGPU_GFX_MAX_SE) + return -EINVAL; if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) sh_num = 0xffffffff; + else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) + return -EINVAL; if (info->read_mmr_reg.count > 128) return -EINVAL; @@ -740,6 +736,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION; if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION; + if (amdgpu_is_tmz(adev)) + dev_info.ids_flags |= AMDGPU_IDS_FLAGS_TMZ; vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_SIZE; @@ -993,7 +991,7 @@ void amdgpu_driver_lastclose_kms(struct drm_device *dev) */ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv; int r, pasid; @@ -1078,7 +1076,7 @@ pm_put: void amdgpu_driver_postclose_kms(struct drm_device *dev, struct drm_file *file_priv) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = file_priv->driver_priv; struct amdgpu_bo_list *list; struct amdgpu_bo *pd; @@ -1143,7 +1141,7 @@ u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; unsigned int pipe = crtc->index; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int vpos, hpos, stat; u32 count; @@ -1211,7 +1209,7 @@ int amdgpu_enable_vblank_kms(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; unsigned int pipe = crtc->index; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe); return amdgpu_irq_get(adev, &adev->crtc_irq, idx); @@ -1228,7 +1226,7 @@ void amdgpu_disable_vblank_kms(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; unsigned int pipe = crtc->index; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe); amdgpu_irq_put(adev, &adev->crtc_irq, idx); @@ -1264,7 +1262,7 @@ static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_amdgpu_info_firmware fw_info; struct drm_amdgpu_query_fw query_fw; struct atom_context *ctx = adev->mode_info.atom_context; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h index e89fb35fec71..0c43d7fe893c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h @@ -27,6 +27,19 @@ struct amdgpu_mmhub_funcs { void (*query_ras_error_count)(struct amdgpu_device *adev, void *ras_error_status); void (*reset_ras_error_count)(struct amdgpu_device *adev); + u64 (*get_fb_location)(struct amdgpu_device *adev); + void (*init)(struct amdgpu_device *adev); + int (*gart_enable)(struct amdgpu_device *adev); + void (*set_fault_enable_default)(struct amdgpu_device *adev, + bool value); + void (*gart_disable)(struct amdgpu_device *adev); + int (*set_clockgating)(struct amdgpu_device *adev, + enum amd_clockgating_state state); + void (*get_clockgating)(struct amdgpu_device *adev, u32 *flags); + void (*setup_vm_pt_regs)(struct amdgpu_device *adev, uint32_t vmid, + uint64_t page_table_base); + void (*update_power_gating)(struct amdgpu_device *adev, + bool enable); }; struct amdgpu_mmhub { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index eaff4c4506eb..ac043baac05d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -380,6 +380,9 @@ int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, if (r) return r; + if ((*bo_ptr) == NULL) + return 0; + /* * Remove the original mem node and create a new one at the request * position. @@ -558,7 +561,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); if (bo == NULL) return -ENOMEM; - drm_gem_private_object_init(adev->ddev, &bo->tbo.base, size); + drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); INIT_LIST_HEAD(&bo->shadow_list); bo->vm_bo = NULL; bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : @@ -1305,7 +1308,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, } /** - * amdgpu_bo_move_notify - notification about a BO being released + * amdgpu_bo_release_notify - notification about a BO being released * @bo: pointer to a buffer object * * Wipes VRAM buffers whose contents should not be leaked before the diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c index 1311d6aec5d4..69af462db34d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c @@ -226,7 +226,7 @@ static int init_pmu_by_type(struct amdgpu_device *adev, pmu_entry->pmu.attr_groups = attr_groups; pmu_entry->pmu_perf_type = pmu_perf_type; snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d", - pmu_file_prefix, adev->ddev->primary->index); + pmu_file_prefix, adev_to_drm(adev)->primary->index); ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 7fe564275457..d6c38e24f130 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -522,8 +522,7 @@ static int psp_asd_load(struct psp_context *psp) * add workaround to bypass it for sriov now. * TODO: add version check to make it common */ - if (amdgpu_sriov_vf(psp->adev) || - (psp->adev->asic_type == CHIP_NAVY_FLOUNDER)) + if (amdgpu_sriov_vf(psp->adev) || !psp->asd_fw) return 0; cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); @@ -1430,6 +1429,168 @@ static int psp_dtm_terminate(struct psp_context *psp) } // DTM end +// RAP start +static int psp_rap_init_shared_buf(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for rap ta <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAP_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->rap_context.rap_shared_bo, + &psp->rap_context.rap_shared_mc_addr, + &psp->rap_context.rap_shared_buf); + + return ret; +} + +static int psp_rap_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->ta_rap_start_addr, psp->ta_rap_ucode_size); + + psp_prep_ta_load_cmd_buf(cmd, + psp->fw_pri_mc_addr, + psp->ta_rap_ucode_size, + psp->rap_context.rap_shared_mc_addr, + PSP_RAP_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + if (!ret) { + psp->rap_context.rap_initialized = true; + psp->rap_context.session_id = cmd->resp.session_id; + mutex_init(&psp->rap_context.mutex); + } + + kfree(cmd); + + return ret; +} + +static int psp_rap_unload(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_ta_unload_cmd_buf(cmd, psp->rap_context.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static int psp_rap_initialize(struct psp_context *psp) +{ + int ret; + + /* + * TODO: bypass the initialize in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + if (!psp->adev->psp.ta_rap_ucode_size || + !psp->adev->psp.ta_rap_start_addr) { + dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n"); + return 0; + } + + if (!psp->rap_context.rap_initialized) { + ret = psp_rap_init_shared_buf(psp); + if (ret) + return ret; + } + + ret = psp_rap_load(psp); + if (ret) + return ret; + + ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE); + if (ret != TA_RAP_STATUS__SUCCESS) { + psp_rap_unload(psp); + + amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo, + &psp->rap_context.rap_shared_mc_addr, + &psp->rap_context.rap_shared_buf); + + psp->rap_context.rap_initialized = false; + + dev_warn(psp->adev->dev, "RAP TA initialize fail.\n"); + return -EINVAL; + } + + return 0; +} + +static int psp_rap_terminate(struct psp_context *psp) +{ + int ret; + + if (!psp->rap_context.rap_initialized) + return 0; + + ret = psp_rap_unload(psp); + + psp->rap_context.rap_initialized = false; + + /* free rap shared memory */ + amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo, + &psp->rap_context.rap_shared_mc_addr, + &psp->rap_context.rap_shared_buf); + + return ret; +} + +int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id) +{ + struct ta_rap_shared_memory *rap_cmd; + int ret; + + if (!psp->rap_context.rap_initialized) + return -EINVAL; + + if (ta_cmd_id != TA_CMD_RAP__INITIALIZE && + ta_cmd_id != TA_CMD_RAP__VALIDATE_L0) + return -EINVAL; + + mutex_lock(&psp->rap_context.mutex); + + rap_cmd = (struct ta_rap_shared_memory *) + psp->rap_context.rap_shared_buf; + memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory)); + + rap_cmd->cmd_id = ta_cmd_id; + rap_cmd->validation_method_id = METHOD_A; + + ret = psp_ta_invoke(psp, rap_cmd->cmd_id, psp->rap_context.session_id); + if (ret) { + mutex_unlock(&psp->rap_context.mutex); + return ret; + } + + mutex_unlock(&psp->rap_context.mutex); + + return rap_cmd->rap_status; +} +// RAP end + static int psp_hw_start(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -1707,7 +1868,7 @@ static int psp_load_smu_fw(struct psp_context *psp) return 0; - if (adev->in_gpu_reset && ras && ras->supported) { + if (amdgpu_in_reset(adev) && ras && ras->supported) { ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD); if (ret) { DRM_WARN("Failed to set MP1 state prepare for reload\n"); @@ -1822,7 +1983,7 @@ static int psp_load_fw(struct amdgpu_device *adev) int ret; struct psp_context *psp = &adev->psp; - if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) { + if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) { psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */ goto skip_memalloc; } @@ -1892,6 +2053,11 @@ skip_memalloc: if (ret) dev_err(psp->adev->dev, "DTM: Failed to initialize DTM\n"); + + ret = psp_rap_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "RAP: Failed to initialize RAP\n"); } return 0; @@ -1942,6 +2108,7 @@ static int psp_hw_fini(void *handle) if (psp->adev->psp.ta_fw) { psp_ras_terminate(psp); + psp_rap_terminate(psp); psp_dtm_terminate(psp); psp_hdcp_terminate(psp); } @@ -2000,6 +2167,11 @@ static int psp_suspend(void *handle) DRM_ERROR("Failed to terminate dtm ta\n"); return ret; } + ret = psp_rap_terminate(psp); + if (ret) { + DRM_ERROR("Failed to terminate rap ta\n"); + return ret; + } } ret = psp_asd_unload(psp); @@ -2078,6 +2250,11 @@ static int psp_resume(void *handle) if (ret) dev_err(psp->adev->dev, "DTM: Failed to initialize DTM\n"); + + ret = psp_rap_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "RAP: Failed to initialize RAP\n"); } mutex_unlock(&adev->firmware.mutex); @@ -2343,6 +2520,11 @@ int parse_ta_bin_descriptor(struct psp_context *psp, psp->ta_dtm_ucode_size = le32_to_cpu(desc->size_bytes); psp->ta_dtm_start_addr = ucode_start_addr; break; + case TA_FW_TYPE_PSP_RAP: + psp->ta_rap_ucode_version = le32_to_cpu(desc->fw_version); + psp->ta_rap_ucode_size = le32_to_cpu(desc->size_bytes); + psp->ta_rap_start_addr = ucode_start_addr; + break; default: dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type); break; @@ -2421,7 +2603,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t fw_ver; int ret; @@ -2448,7 +2630,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); void *cpu_addr; dma_addr_t dma_addr; int ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 623888bf30cb..919d2fb7427b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -29,6 +29,7 @@ #include "psp_gfx_if.h" #include "ta_xgmi_if.h" #include "ta_ras_if.h" +#include "ta_rap_if.h" #define PSP_FENCE_BUFFER_SIZE 0x1000 #define PSP_CMD_BUFFER_SIZE 0x1000 @@ -38,6 +39,7 @@ #define PSP_TMR_SIZE 0x400000 #define PSP_HDCP_SHARED_MEM_SIZE 0x4000 #define PSP_DTM_SHARED_MEM_SIZE 0x4000 +#define PSP_RAP_SHARED_MEM_SIZE 0x4000 #define PSP_SHARED_MEM_SIZE 0x4000 struct psp_context; @@ -159,6 +161,15 @@ struct psp_dtm_context { struct mutex mutex; }; +struct psp_rap_context { + bool rap_initialized; + uint32_t session_id; + struct amdgpu_bo *rap_shared_bo; + uint64_t rap_shared_mc_addr; + void *rap_shared_buf; + struct mutex mutex; +}; + #define MEM_TRAIN_SYSTEM_SIGNATURE 0x54534942 #define GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES 0x1000 #define GDDR6_MEM_TRAINING_OFFSET 0x8000 @@ -277,11 +288,16 @@ struct psp_context uint32_t ta_dtm_ucode_size; uint8_t *ta_dtm_start_addr; + uint32_t ta_rap_ucode_version; + uint32_t ta_rap_ucode_size; + uint8_t *ta_rap_start_addr; + struct psp_asd_context asd_context; struct psp_xgmi_context xgmi_context; struct psp_ras_context ras; struct psp_hdcp_context hdcp_context; struct psp_dtm_context dtm_context; + struct psp_rap_context rap_context; struct mutex mutex; struct psp_memory_training_context mem_train_ctx; }; @@ -357,6 +373,7 @@ int psp_ras_trigger_error(struct psp_context *psp, int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id); +int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_rlc_autoload_start(struct psp_context *psp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c new file mode 100644 index 000000000000..8da5356c36f1 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include <linux/debugfs.h> +#include <linux/pm_runtime.h> + +#include "amdgpu.h" +#include "amdgpu_rap.h" + +/** + * DOC: AMDGPU RAP debugfs test interface + * + * how to use? + * echo opcode > <debugfs_dir>/dri/xxx/rap_test + * + * opcode: + * currently, only 2 is supported by Linux host driver, + * opcode 2 stands for TA_CMD_RAP__VALIDATE_L0, used to + * trigger L0 policy validation, you can refer more detail + * from header file ta_rap_if.h + * + */ +static ssize_t amdgpu_rap_debugfs_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; + struct ta_rap_shared_memory *rap_shared_mem; + struct ta_rap_cmd_output_data *rap_cmd_output; + struct drm_device *dev = adev_to_drm(adev); + uint32_t op; + int ret; + + if (*pos || size != 2) + return -EINVAL; + + ret = kstrtouint_from_user(buf, size, *pos, &op); + if (ret) + return ret; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(dev->dev); + return ret; + } + + /* make sure gfx core is on, RAP TA cann't handle + * GFX OFF case currently. + */ + amdgpu_gfx_off_ctrl(adev, false); + + switch (op) { + case 2: + ret = psp_rap_invoke(&adev->psp, op); + + if (ret == TA_RAP_STATUS__SUCCESS) { + dev_info(adev->dev, "RAP L0 validate test success.\n"); + } else { + rap_shared_mem = (struct ta_rap_shared_memory *) + adev->psp.rap_context.rap_shared_buf; + rap_cmd_output = &(rap_shared_mem->rap_out_message.output); + + dev_info(adev->dev, "RAP test failed, the output is:\n"); + dev_info(adev->dev, "\tlast_subsection: 0x%08x.\n", + rap_cmd_output->last_subsection); + dev_info(adev->dev, "\tnum_total_validate: 0x%08x.\n", + rap_cmd_output->num_total_validate); + dev_info(adev->dev, "\tnum_valid: 0x%08x.\n", + rap_cmd_output->num_valid); + dev_info(adev->dev, "\tlast_validate_addr: 0x%08x.\n", + rap_cmd_output->last_validate_addr); + dev_info(adev->dev, "\tlast_validate_val: 0x%08x.\n", + rap_cmd_output->last_validate_val); + dev_info(adev->dev, "\tlast_validate_val_exptd: 0x%08x.\n", + rap_cmd_output->last_validate_val_exptd); + } + break; + default: + dev_info(adev->dev, "Unsupported op id: %d, ", op); + dev_info(adev->dev, "Only support op 2(L0 validate test).\n"); + } + + amdgpu_gfx_off_ctrl(adev, true); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + return size; +} + +static const struct file_operations amdgpu_rap_debugfs_ops = { + .owner = THIS_MODULE, + .read = NULL, + .write = amdgpu_rap_debugfs_write, + .llseek = default_llseek +}; + +void amdgpu_rap_debugfs_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + struct drm_minor *minor = adev_to_drm(adev)->primary; + + if (!adev->psp.rap_context.rap_initialized) + return; + + debugfs_create_file("rap_test", S_IWUSR, minor->debugfs_root, + adev, &amdgpu_rap_debugfs_ops); +#endif +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.h new file mode 100644 index 000000000000..ec6d7632d3a0 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.h @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _AMDGPU_RAP_H +#define _AMDGPU_RAP_H + +#include "amdgpu.h" + +void amdgpu_rap_debugfs_init(struct amdgpu_device *adev); +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index bcce4c0be462..e5ea14774c0c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -34,6 +34,8 @@ #include "amdgpu_xgmi.h" #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" +static const char *RAS_FS_NAME = "ras"; + const char *ras_error_string[] = { "none", "parity", @@ -62,13 +64,14 @@ const char *ras_block_string[] = { #define ras_err_str(i) (ras_error_string[ffs(i)]) #define ras_block_str(i) (ras_block_string[i]) -#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS 1 -#define AMDGPU_RAS_FLAG_INIT_NEED_RESET 2 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) /* inject address is 52 bits */ #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) +/* typical ECC bad page rate(1 bad page per 100MB VRAM) */ +#define RAS_BAD_PAGE_RATE (100 * 1024 * 1024ULL) + enum amdgpu_ras_retire_page_reservation { AMDGPU_RAS_RETIRE_PAGE_RESERVED, AMDGPU_RAS_RETIRE_PAGE_PENDING, @@ -367,12 +370,19 @@ static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user * static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; + struct amdgpu_device *adev = + (struct amdgpu_device *)file_inode(f)->i_private; int ret; - ret = amdgpu_ras_eeprom_reset_table(&adev->psp.ras.ras->eeprom_control); + ret = amdgpu_ras_eeprom_reset_table( + &(amdgpu_ras_get_context(adev)->eeprom_control)); - return ret == 1 ? size : -EIO; + if (ret == 1) { + amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; + return size; + } else { + return -EIO; + } } static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { @@ -1017,6 +1027,33 @@ static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, return scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features); } +static void amdgpu_ras_sysfs_add_bad_page_node(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct attribute_group group; + struct bin_attribute *bin_attrs[] = { + &con->badpages_attr, + NULL, + }; + + con->badpages_attr = (struct bin_attribute) { + .attr = { + .name = "gpu_vram_bad_pages", + .mode = S_IRUGO, + }, + .size = 0, + .private = NULL, + .read = amdgpu_ras_sysfs_badpages_read, + }; + + group.name = RAS_FS_NAME; + group.bin_attrs = bin_attrs; + + sysfs_bin_attr_init(bin_attrs[0]); + + sysfs_update_group(&adev->dev->kobj, &group); +} + static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); @@ -1024,14 +1061,9 @@ static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev) &con->features_attr.attr, NULL }; - struct bin_attribute *bin_attrs[] = { - &con->badpages_attr, - NULL - }; struct attribute_group group = { - .name = "ras", + .name = RAS_FS_NAME, .attrs = attrs, - .bin_attrs = bin_attrs, }; con->features_attr = (struct device_attribute) { @@ -1042,22 +1074,20 @@ static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev) .show = amdgpu_ras_sysfs_features_read, }; - con->badpages_attr = (struct bin_attribute) { - .attr = { - .name = "gpu_vram_bad_pages", - .mode = S_IRUGO, - }, - .size = 0, - .private = NULL, - .read = amdgpu_ras_sysfs_badpages_read, - }; - sysfs_attr_init(attrs[0]); - sysfs_bin_attr_init(bin_attrs[0]); return sysfs_create_group(&adev->dev->kobj, &group); } +static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + sysfs_remove_file_from_group(&adev->dev->kobj, + &con->badpages_attr.attr, + RAS_FS_NAME); +} + static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); @@ -1065,14 +1095,9 @@ static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev) &con->features_attr.attr, NULL }; - struct bin_attribute *bin_attrs[] = { - &con->badpages_attr, - NULL - }; struct attribute_group group = { - .name = "ras", + .name = RAS_FS_NAME, .attrs = attrs, - .bin_attrs = bin_attrs, }; sysfs_remove_group(&adev->dev->kobj, &group); @@ -1105,7 +1130,7 @@ int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, if (sysfs_add_file_to_group(&adev->dev->kobj, &obj->sysfs_attr.attr, - "ras")) { + RAS_FS_NAME)) { put_obj(obj); return -EINVAL; } @@ -1125,7 +1150,7 @@ int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, sysfs_remove_file_from_group(&adev->dev->kobj, &obj->sysfs_attr.attr, - "ras"); + RAS_FS_NAME); obj->attr_inuse = 0; put_obj(obj); @@ -1141,6 +1166,9 @@ static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) amdgpu_ras_sysfs_remove(adev, &obj->head); } + if (amdgpu_bad_page_threshold != 0) + amdgpu_ras_sysfs_remove_bad_page_node(adev); + amdgpu_ras_sysfs_remove_feature_node(adev); return 0; @@ -1169,9 +1197,9 @@ static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) static void amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); - struct drm_minor *minor = adev->ddev->primary; + struct drm_minor *minor = adev_to_drm(adev)->primary; - con->dir = debugfs_create_dir("ras", minor->debugfs_root); + con->dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, con->dir, adev, &amdgpu_ras_debugfs_ctrl_ops); debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, con->dir, @@ -1187,6 +1215,13 @@ static void amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) */ debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, con->dir, &con->reboot); + + /* + * User could set this not to clean up hardware's error count register + * of RAS IPs during ras recovery. + */ + debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, + con->dir, &con->disable_ras_err_cnt_harvest); } void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, @@ -1211,6 +1246,7 @@ void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) { +#if defined(CONFIG_DEBUG_FS) struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_manager *obj; struct ras_fs_if fs_info; @@ -1233,6 +1269,7 @@ void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) amdgpu_ras_debugfs_create(adev, &fs_info); } } +#endif } void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev, @@ -1243,13 +1280,13 @@ void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev, if (!obj || !obj->ent) return; - debugfs_remove(obj->ent); obj->ent = NULL; put_obj(obj); } static void amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev) { +#if defined(CONFIG_DEBUG_FS) struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_manager *obj, *tmp; @@ -1257,8 +1294,8 @@ static void amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev) amdgpu_ras_debugfs_remove(adev, &obj->head); } - debugfs_remove_recursive(con->dir); con->dir = NULL; +#endif } /* debugfs end */ @@ -1268,6 +1305,9 @@ static int amdgpu_ras_fs_init(struct amdgpu_device *adev) { amdgpu_ras_sysfs_create_feature_node(adev); + if (amdgpu_bad_page_threshold != 0) + amdgpu_ras_sysfs_add_bad_page_node(adev); + return 0; } @@ -1514,23 +1554,28 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) struct amdgpu_device *remote_adev = NULL; struct amdgpu_device *adev = ras->adev; struct list_head device_list, *device_list_handle = NULL; - struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, false); - - /* Build list of devices to query RAS related errors */ - if (hive && adev->gmc.xgmi.num_physical_nodes > 1) - device_list_handle = &hive->device_list; - else { - INIT_LIST_HEAD(&device_list); - list_add_tail(&adev->gmc.xgmi.head, &device_list); - device_list_handle = &device_list; - } - list_for_each_entry(remote_adev, device_list_handle, gmc.xgmi.head) { - amdgpu_ras_log_on_err_counter(remote_adev); + if (!ras->disable_ras_err_cnt_harvest) { + struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); + + /* Build list of devices to query RAS related errors */ + if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { + device_list_handle = &hive->device_list; + } else { + INIT_LIST_HEAD(&device_list); + list_add_tail(&adev->gmc.xgmi.head, &device_list); + device_list_handle = &device_list; + } + + list_for_each_entry(remote_adev, + device_list_handle, gmc.xgmi.head) + amdgpu_ras_log_on_err_counter(remote_adev); + + amdgpu_put_xgmi_hive(hive); } if (amdgpu_device_should_recover_gpu(ras->adev)) - amdgpu_device_gpu_recover(ras->adev, 0); + amdgpu_device_gpu_recover(ras->adev, NULL); atomic_set(&ras->in_recovery, 0); } @@ -1645,7 +1690,7 @@ static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) int ret = 0; /* no bad page record, skip eeprom access */ - if (!control->num_recs) + if (!control->num_recs || (amdgpu_bad_page_threshold == 0)) return ret; bps = kcalloc(control->num_recs, sizeof(*bps), GFP_KERNEL); @@ -1699,6 +1744,47 @@ out: return ret; } +static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, + uint32_t max_length) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + int tmp_threshold = amdgpu_bad_page_threshold; + u64 val; + + /* + * Justification of value bad_page_cnt_threshold in ras structure + * + * Generally, -1 <= amdgpu_bad_page_threshold <= max record length + * in eeprom, and introduce two scenarios accordingly. + * + * Bad page retirement enablement: + * - If amdgpu_bad_page_threshold = -1, + * bad_page_cnt_threshold = typical value by formula. + * + * - When the value from user is 0 < amdgpu_bad_page_threshold < + * max record length in eeprom, use it directly. + * + * Bad page retirement disablement: + * - If amdgpu_bad_page_threshold = 0, bad page retirement + * functionality is disabled, and bad_page_cnt_threshold will + * take no effect. + */ + + if (tmp_threshold < -1) + tmp_threshold = -1; + else if (tmp_threshold > max_length) + tmp_threshold = max_length; + + if (tmp_threshold == -1) { + val = adev->gmc.mc_vram_size; + do_div(val, RAS_BAD_PAGE_RATE); + con->bad_page_cnt_threshold = min(lower_32_bits(val), + max_length); + } else { + con->bad_page_cnt_threshold = tmp_threshold; + } +} + /* called in gpu recovery/init */ int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev) { @@ -1708,7 +1794,8 @@ int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev) struct amdgpu_bo *bo = NULL; int i, ret = 0; - if (!con || !con->eh_data) + /* Not reserve bad page when amdgpu_bad_page_threshold == 0. */ + if (!con || !con->eh_data || (amdgpu_bad_page_threshold == 0)) return 0; mutex_lock(&con->recovery_lock); @@ -1776,6 +1863,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_err_handler_data **data; + uint32_t max_eeprom_records_len = 0; + bool exc_err_limit = false; int ret; if (con) @@ -1794,8 +1883,15 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev) atomic_set(&con->in_recovery, 0); con->adev = adev; - ret = amdgpu_ras_eeprom_init(&con->eeprom_control); - if (ret) + max_eeprom_records_len = amdgpu_ras_eeprom_get_record_max_length(); + amdgpu_ras_validate_threshold(adev, max_eeprom_records_len); + + ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit); + /* + * This calling fails when exc_err_limit is true or + * ret != 0. + */ + if (exc_err_limit || ret) goto free; if (con->eeprom_control.num_recs) { @@ -1819,6 +1915,15 @@ free: out: dev_warn(adev->dev, "Failed to initialize ras recovery!\n"); + /* + * Except error threshold exceeding case, other failure cases in this + * function would not fail amdgpu driver init. + */ + if (!exc_err_limit) + ret = 0; + else + ret = -EINVAL; + return ret; } @@ -1858,6 +1963,17 @@ int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev, return 0; } +static int amdgpu_ras_check_asic_type(struct amdgpu_device *adev) +{ + if (adev->asic_type != CHIP_VEGA10 && + adev->asic_type != CHIP_VEGA20 && + adev->asic_type != CHIP_ARCTURUS && + adev->asic_type != CHIP_SIENNA_CICHLID) + return 1; + else + return 0; +} + /* * check hardware's ras ability which will be saved in hw_supported. * if hardware does not support ras, we can skip some ras initializtion and @@ -1874,8 +1990,7 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev, *supported = 0; if (amdgpu_sriov_vf(adev) || !adev->is_atom_fw || - (adev->asic_type != CHIP_VEGA20 && - adev->asic_type != CHIP_ARCTURUS)) + amdgpu_ras_check_asic_type(adev)) return; if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { @@ -1897,6 +2012,7 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev, *supported = amdgpu_ras_enable == 0 ? 0 : *hw_supported & amdgpu_ras_mask; + adev->ras_features = *supported; } int amdgpu_ras_init(struct amdgpu_device *adev) @@ -1919,9 +2035,9 @@ int amdgpu_ras_init(struct amdgpu_device *adev) amdgpu_ras_check_supported(adev, &con->hw_supported, &con->supported); - if (!con->hw_supported) { + if (!con->hw_supported || (adev->asic_type == CHIP_VEGA10)) { r = 0; - goto err_out; + goto release_con; } con->features = 0; @@ -1932,25 +2048,25 @@ int amdgpu_ras_init(struct amdgpu_device *adev) if (adev->nbio.funcs->init_ras_controller_interrupt) { r = adev->nbio.funcs->init_ras_controller_interrupt(adev); if (r) - goto err_out; + goto release_con; } if (adev->nbio.funcs->init_ras_err_event_athub_interrupt) { r = adev->nbio.funcs->init_ras_err_event_athub_interrupt(adev); if (r) - goto err_out; + goto release_con; } if (amdgpu_ras_fs_init(adev)) { r = -EINVAL; - goto err_out; + goto release_con; } dev_info(adev->dev, "RAS INFO: ras initialized successfully, " "hardware ability[%x] ras_mask[%x]\n", con->hw_supported, con->supported); return 0; -err_out: +release_con: amdgpu_ras_set_context(adev, NULL); kfree(con); @@ -1978,7 +2094,7 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev, amdgpu_ras_request_reset_on_boot(adev, ras_block->block); return 0; - } else if (adev->in_suspend || adev->in_gpu_reset) { + } else if (adev->in_suspend || amdgpu_in_reset(adev)) { /* in resume phase, if fail to enable ras, * clean up all ras fs nodes, and disable ras */ goto cleanup; @@ -1987,7 +2103,7 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev, } /* in resume phase, no need to create ras fs node */ - if (adev->in_suspend || adev->in_gpu_reset) + if (adev->in_suspend || amdgpu_in_reset(adev)) return 0; if (ih_info->cb) { @@ -2145,3 +2261,19 @@ bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) return false; } + +bool amdgpu_ras_check_err_threshold(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + bool exc_err_limit = false; + + if (con && (amdgpu_bad_page_threshold != 0)) + amdgpu_ras_eeprom_check_err_threshold(&con->eeprom_control, + &exc_err_limit); + + /* + * We are only interested in variable exc_err_limit, + * as it says if GPU is in bad state or not. + */ + return exc_err_limit; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h index b2667342cf67..6b8d7bb83bb3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h @@ -31,6 +31,10 @@ #include "ta_ras_if.h" #include "amdgpu_ras_eeprom.h" +#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS (0x1 << 0) +#define AMDGPU_RAS_FLAG_INIT_NEED_RESET (0x1 << 1) +#define AMDGPU_RAS_FLAG_SKIP_BAD_PAGE_RESV (0x1 << 2) + enum amdgpu_ras_block { AMDGPU_RAS_BLOCK__UMC = 0, AMDGPU_RAS_BLOCK__SDMA, @@ -336,6 +340,12 @@ struct amdgpu_ras { struct amdgpu_ras_eeprom_control eeprom_control; bool error_query_ready; + + /* bad page count threshold */ + uint32_t bad_page_cnt_threshold; + + /* disable ras error count harvest in recovery */ + bool disable_ras_err_cnt_harvest; }; struct ras_fs_data { @@ -490,6 +500,8 @@ void amdgpu_ras_suspend(struct amdgpu_device *adev); unsigned long amdgpu_ras_query_error_count(struct amdgpu_device *adev, bool is_ce); +bool amdgpu_ras_check_err_threshold(struct amdgpu_device *adev); + /* error handling functions */ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, struct eeprom_table_record *bps, int pages); @@ -500,10 +512,14 @@ static inline int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) { struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); - /* save bad page to eeprom before gpu reset, - * i2c may be unstable in gpu reset + /* + * Save bad page to eeprom before gpu reset, i2c may be unstable + * in gpu reset. + * + * Also, exclude the case when ras recovery issuer is + * eeprom page write itself. */ - if (in_task()) + if (!(ras->flags & AMDGPU_RAS_FLAG_SKIP_BAD_PAGE_RESV) && in_task()) amdgpu_ras_reserve_bad_pages(adev); if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index c0096097bbcf..0e64c39a2372 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -46,6 +46,9 @@ #define EEPROM_TABLE_HDR_VAL 0x414d4452 #define EEPROM_TABLE_VER 0x00010000 +/* Bad GPU tag ‘BADG’ */ +#define EEPROM_TABLE_HDR_BAD 0x42414447 + /* Assume 2 Mbit size */ #define EEPROM_SIZE_BYTES 256000 #define EEPROM_PAGE__SIZE_BYTES 256 @@ -56,6 +59,15 @@ #define to_amdgpu_device(x) (container_of(x, struct amdgpu_ras, eeprom_control))->adev +static bool __is_ras_eeprom_supported(struct amdgpu_device *adev) +{ + if ((adev->asic_type == CHIP_VEGA20) || + (adev->asic_type == CHIP_ARCTURUS)) + return true; + + return false; +} + static bool __get_eeprom_i2c_addr_arct(struct amdgpu_device *adev, uint16_t *i2c_addr) { @@ -213,6 +225,24 @@ static bool __validate_tbl_checksum(struct amdgpu_ras_eeprom_control *control, return true; } +static int amdgpu_ras_eeprom_correct_header_tag( + struct amdgpu_ras_eeprom_control *control, + uint32_t header) +{ + unsigned char buff[EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE]; + struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr; + int ret = 0; + + memset(buff, 0, EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE); + + mutex_lock(&control->tbl_mutex); + hdr->header = header; + ret = __update_table_header(control, buff); + mutex_unlock(&control->tbl_mutex); + + return ret; +} + int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control) { unsigned char buff[EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE] = { 0 }; @@ -238,12 +268,14 @@ int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control) } -int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) +int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control, + bool *exceed_err_limit) { int ret = 0; struct amdgpu_device *adev = to_amdgpu_device(control); unsigned char buff[EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE] = { 0 }; struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr; + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); struct i2c_msg msg = { .addr = 0, .flags = I2C_M_RD, @@ -251,6 +283,11 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) .buf = buff, }; + *exceed_err_limit = false; + + if (!__is_ras_eeprom_supported(adev)) + return 0; + /* Verify i2c adapter is initialized */ if (!adev->pm.smu_i2c.algo) return -ENOENT; @@ -279,6 +316,18 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) DRM_DEBUG_DRIVER("Found existing EEPROM table with %d records", control->num_recs); + } else if ((hdr->header == EEPROM_TABLE_HDR_BAD) && + (amdgpu_bad_page_threshold != 0)) { + if (ras->bad_page_cnt_threshold > control->num_recs) { + dev_info(adev->dev, "Using one valid bigger bad page " + "threshold and correcting eeprom header tag.\n"); + ret = amdgpu_ras_eeprom_correct_header_tag(control, + EEPROM_TABLE_HDR_VAL); + } else { + *exceed_err_limit = true; + dev_err(adev->dev, "Exceeding the bad_page_threshold parameter, " + "disabling the GPU.\n"); + } } else { DRM_INFO("Creating new EEPROM table"); @@ -375,6 +424,49 @@ static uint32_t __correct_eeprom_dest_address(uint32_t curr_address) return curr_address; } +int amdgpu_ras_eeprom_check_err_threshold( + struct amdgpu_ras_eeprom_control *control, + bool *exceed_err_limit) +{ + struct amdgpu_device *adev = to_amdgpu_device(control); + unsigned char buff[EEPROM_ADDRESS_SIZE + + EEPROM_TABLE_HEADER_SIZE] = { 0 }; + struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr; + struct i2c_msg msg = { + .addr = control->i2c_address, + .flags = I2C_M_RD, + .len = EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE, + .buf = buff, + }; + int ret; + + *exceed_err_limit = false; + + if (!__is_ras_eeprom_supported(adev)) + return 0; + + /* read EEPROM table header */ + mutex_lock(&control->tbl_mutex); + ret = i2c_transfer(&adev->pm.smu_i2c, &msg, 1); + if (ret < 1) { + dev_err(adev->dev, "Failed to read EEPROM table header.\n"); + goto err; + } + + __decode_table_header_from_buff(hdr, &buff[2]); + + if (hdr->header == EEPROM_TABLE_HDR_BAD) { + dev_warn(adev->dev, "This GPU is in BAD status."); + dev_warn(adev->dev, "Please retire it or setting one bigger " + "threshold value when reloading driver.\n"); + *exceed_err_limit = true; + } + +err: + mutex_unlock(&control->tbl_mutex); + return 0; +} + int amdgpu_ras_eeprom_process_recods(struct amdgpu_ras_eeprom_control *control, struct eeprom_table_record *records, bool write, @@ -383,10 +475,12 @@ int amdgpu_ras_eeprom_process_recods(struct amdgpu_ras_eeprom_control *control, int i, ret = 0; struct i2c_msg *msgs, *msg; unsigned char *buffs, *buff; + bool sched_ras_recovery = false; struct eeprom_table_record *record; struct amdgpu_device *adev = to_amdgpu_device(control); + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); - if (adev->asic_type != CHIP_VEGA20 && adev->asic_type != CHIP_ARCTURUS) + if (!__is_ras_eeprom_supported(adev)) return 0; buffs = kcalloc(num, EEPROM_ADDRESS_SIZE + EEPROM_TABLE_RECORD_SIZE, @@ -402,11 +496,30 @@ int amdgpu_ras_eeprom_process_recods(struct amdgpu_ras_eeprom_control *control, goto free_buff; } + /* + * If saved bad pages number exceeds the bad page threshold for + * the whole VRAM, update table header to mark the BAD GPU tag + * and schedule one ras recovery after eeprom write is done, + * this can avoid the missing for latest records. + * + * This new header will be picked up and checked in the bootup + * by ras recovery, which may break bootup process to notify + * user this GPU is in bad state and to retire such GPU for + * further check. + */ + if (write && (amdgpu_bad_page_threshold != 0) && + ((control->num_recs + num) >= ras->bad_page_cnt_threshold)) { + dev_warn(adev->dev, + "Saved bad pages(%d) reaches threshold value(%d).\n", + control->num_recs + num, ras->bad_page_cnt_threshold); + control->tbl_hdr.header = EEPROM_TABLE_HDR_BAD; + sched_ras_recovery = true; + } + /* In case of overflow just start from beginning to not lose newest records */ if (write && (control->next_addr + EEPROM_TABLE_RECORD_SIZE * num > EEPROM_SIZE_BYTES)) control->next_addr = EEPROM_RECORD_START; - /* * TODO Currently makes EEPROM writes for each record, this creates * internal fragmentation. Optimized the code to do full page write of @@ -482,6 +595,20 @@ int amdgpu_ras_eeprom_process_recods(struct amdgpu_ras_eeprom_control *control, __update_tbl_checksum(control, records, num, old_hdr_byte_sum); __update_table_header(control, buffs); + + if (sched_ras_recovery) { + /* + * Before scheduling ras recovery, assert the related + * flag first, which shall bypass common bad page + * reservation execution in amdgpu_ras_reset_gpu. + */ + amdgpu_ras_get_context(adev)->flags |= + AMDGPU_RAS_FLAG_SKIP_BAD_PAGE_RESV; + + dev_warn(adev->dev, "Conduct ras recovery due to bad " + "page threshold reached.\n"); + amdgpu_ras_reset_gpu(adev); + } } else if (!__validate_tbl_checksum(control, records, num)) { DRM_WARN("EEPROM Table checksum mismatch!"); /* TODO Uncomment when EEPROM read/write is relliable */ @@ -499,6 +626,11 @@ free_buff: return ret == num ? 0 : -EIO; } +inline uint32_t amdgpu_ras_eeprom_get_record_max_length(void) +{ + return EEPROM_MAX_RECORD_NUM; +} + /* Used for testing if bugs encountered */ #if 0 void amdgpu_ras_eeprom_test(struct amdgpu_ras_eeprom_control *control) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h index 9e7d640920fb..c7a5e5c7c61e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h @@ -76,14 +76,21 @@ struct eeprom_table_record { unsigned char mcumc_id; }__attribute__((__packed__)); -int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control); +int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control, + bool *exceed_err_limit); int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control); +int amdgpu_ras_eeprom_check_err_threshold( + struct amdgpu_ras_eeprom_control *control, + bool *exceed_err_limit); + int amdgpu_ras_eeprom_process_recods(struct amdgpu_ras_eeprom_control *control, struct eeprom_table_record *records, bool write, int num); +inline uint32_t amdgpu_ras_eeprom_get_record_max_length(void); + void amdgpu_ras_eeprom_test(struct amdgpu_ras_eeprom_control *control); #endif // _AMDGPU_RAS_EEPROM_H diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 13ea8ebc421c..15ee13c3bd9e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -267,7 +267,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, &ring->sched; } - for (i = 0; i < DRM_SCHED_PRIORITY_MAX; ++i) + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; ++i) atomic_set(&ring->num_jobs[i], 0); return 0; @@ -420,7 +420,7 @@ int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring) { #if defined(CONFIG_DEBUG_FS) - struct drm_minor *minor = adev->ddev->primary; + struct drm_minor *minor = adev_to_drm(adev)->primary; struct dentry *ent, *root = minor->debugfs_root; char name[32]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index da871d84b742..7112137689db 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -243,7 +243,7 @@ struct amdgpu_ring { bool has_compute_vm_bug; bool no_scheduler; - atomic_t num_jobs[DRM_SCHED_PRIORITY_MAX]; + atomic_t num_jobs[DRM_SCHED_PRIORITY_COUNT]; struct mutex priority_mutex; /* protected by priority_mutex */ int priority; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index c799691dfa84..0da0a0d98672 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -32,24 +32,32 @@ #include "amdgpu_vm.h" -enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority) +int amdgpu_to_sched_priority(int amdgpu_priority, + enum drm_sched_priority *prio) { switch (amdgpu_priority) { case AMDGPU_CTX_PRIORITY_VERY_HIGH: - return DRM_SCHED_PRIORITY_HIGH_HW; + *prio = DRM_SCHED_PRIORITY_HIGH; + break; case AMDGPU_CTX_PRIORITY_HIGH: - return DRM_SCHED_PRIORITY_HIGH_SW; + *prio = DRM_SCHED_PRIORITY_HIGH; + break; case AMDGPU_CTX_PRIORITY_NORMAL: - return DRM_SCHED_PRIORITY_NORMAL; + *prio = DRM_SCHED_PRIORITY_NORMAL; + break; case AMDGPU_CTX_PRIORITY_LOW: case AMDGPU_CTX_PRIORITY_VERY_LOW: - return DRM_SCHED_PRIORITY_LOW; + *prio = DRM_SCHED_PRIORITY_MIN; + break; case AMDGPU_CTX_PRIORITY_UNSET: - return DRM_SCHED_PRIORITY_UNSET; + *prio = DRM_SCHED_PRIORITY_UNSET; + break; default: WARN(1, "Invalid context priority %d\n", amdgpu_priority); - return DRM_SCHED_PRIORITY_INVALID; + return -EINVAL; } + + return 0; } static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, @@ -115,13 +123,24 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { union drm_amdgpu_sched *args = data; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); enum drm_sched_priority priority; int r; - priority = amdgpu_to_sched_priority(args->in.priority); - if (priority == DRM_SCHED_PRIORITY_INVALID) + /* First check the op, then the op's argument. + */ + switch (args->in.op) { + case AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE: + case AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE: + break; + default: + DRM_ERROR("Invalid sched op specified: %d\n", args->in.op); return -EINVAL; + } + + r = amdgpu_to_sched_priority(args->in.priority, &priority); + if (r) + return r; switch (args->in.op) { case AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE: @@ -136,7 +155,8 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data, priority); break; default: - DRM_ERROR("Invalid sched op specified: %d\n", args->in.op); + /* Impossible. + */ r = -EINVAL; break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h index 12299fd95691..67e5b2472f6a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h @@ -30,7 +30,8 @@ enum drm_sched_priority; struct drm_device; struct drm_file; -enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority); +int amdgpu_to_sched_priority(int amdgpu_priority, + enum drm_sched_priority *prio); int amdgpu_sched_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 651365183e75..6fc3af082f6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1704,8 +1704,8 @@ static struct ttm_bo_driver amdgpu_bo_driver = { */ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) { - amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo, - NULL, &adev->fw_vram_usage.va); + amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, + NULL, &adev->mman.fw_vram_usage_va); } /** @@ -1719,19 +1719,19 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) { uint64_t vram_size = adev->gmc.visible_vram_size; - adev->fw_vram_usage.va = NULL; - adev->fw_vram_usage.reserved_bo = NULL; + adev->mman.fw_vram_usage_va = NULL; + adev->mman.fw_vram_usage_reserved_bo = NULL; - if (adev->fw_vram_usage.size == 0 || - adev->fw_vram_usage.size > vram_size) + if (adev->mman.fw_vram_usage_size == 0 || + adev->mman.fw_vram_usage_size > vram_size) return 0; return amdgpu_bo_create_kernel_at(adev, - adev->fw_vram_usage.start_offset, - adev->fw_vram_usage.size, + adev->mman.fw_vram_usage_start_offset, + adev->mman.fw_vram_usage_size, AMDGPU_GEM_DOMAIN_VRAM, - &adev->fw_vram_usage.reserved_bo, - &adev->fw_vram_usage.va); + &adev->mman.fw_vram_usage_reserved_bo, + &adev->mman.fw_vram_usage_va); } /* @@ -1763,7 +1763,7 @@ static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev) memset(ctx, 0, sizeof(*ctx)); ctx->c2p_train_data_offset = - ALIGN((adev->gmc.mc_vram_size - adev->discovery_tmr_size - SZ_1M), SZ_1M); + ALIGN((adev->gmc.mc_vram_size - adev->mman.discovery_tmr_size - SZ_1M), SZ_1M); ctx->p2c_train_data_offset = (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); ctx->train_data_size = @@ -1802,10 +1802,10 @@ static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) * Otherwise, fallback to legacy approach to check and reserve tmr block for ip * discovery data and G6 memory training data respectively */ - adev->discovery_tmr_size = + adev->mman.discovery_tmr_size = amdgpu_atomfirmware_get_fw_reserved_fb_size(adev); - if (!adev->discovery_tmr_size) - adev->discovery_tmr_size = DISCOVERY_TMR_OFFSET; + if (!adev->mman.discovery_tmr_size) + adev->mman.discovery_tmr_size = DISCOVERY_TMR_OFFSET; if (mem_train_support) { /* reserve vram for mem train according to TMR location */ @@ -1825,14 +1825,14 @@ static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) } ret = amdgpu_bo_create_kernel_at(adev, - adev->gmc.real_vram_size - adev->discovery_tmr_size, - adev->discovery_tmr_size, + adev->gmc.real_vram_size - adev->mman.discovery_tmr_size, + adev->mman.discovery_tmr_size, AMDGPU_GEM_DOMAIN_VRAM, - &adev->discovery_memory, + &adev->mman.discovery_memory, NULL); if (ret) { DRM_ERROR("alloc tmr failed(%d)!\n", ret); - amdgpu_bo_free_kernel(&adev->discovery_memory, NULL, NULL); + amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); return ret; } @@ -1853,15 +1853,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) uint64_t gtt_size; int r; u64 vis_vram_limit; - void *stolen_vga_buf; mutex_init(&adev->mman.gtt_window_lock); /* No others user of address space so set it to 0 */ r = ttm_bo_device_init(&adev->mman.bdev, &amdgpu_bo_driver, - adev->ddev->anon_inode->i_mapping, - adev->ddev->vma_offset_manager, + adev_to_drm(adev)->anon_inode->i_mapping, + adev_to_drm(adev)->vma_offset_manager, dma_addressing_limited(adev->dev)); if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); @@ -1906,7 +1905,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) * If IP discovery enabled, a block of memory should be * reserved for IP discovey. */ - if (adev->discovery_bin) { + if (adev->mman.discovery_bin) { r = amdgpu_ttm_reserve_tmr(adev); if (r) return r; @@ -1916,10 +1915,17 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) * This is used for VGA emulation and pre-OS scanout buffers to * avoid display artifacts while transitioning between pre-OS * and driver. */ - r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, - &adev->stolen_vga_memory, - NULL, &stolen_vga_buf); + r = amdgpu_bo_create_kernel_at(adev, 0, adev->mman.stolen_vga_size, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->mman.stolen_vga_memory, + NULL); + if (r) + return r; + r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, + adev->mman.stolen_extended_size, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->mman.stolen_extended_memory, + NULL); if (r) return r; @@ -1975,9 +1981,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) */ void amdgpu_ttm_late_init(struct amdgpu_device *adev) { - void *stolen_vga_buf; /* return the VGA stolen memory (if any) back to VRAM */ - amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf); + if (!adev->mman.keep_stolen_vga_memory) + amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); + amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); } /** @@ -1989,8 +1996,11 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) return; amdgpu_ttm_training_reserve_vram_fini(adev); + /* return the stolen vga memory back to VRAM */ + if (adev->mman.keep_stolen_vga_memory) + amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); /* return the IP Discovery TMR memory back to VRAM */ - amdgpu_bo_free_kernel(&adev->discovery_memory, NULL, NULL); + amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); amdgpu_ttm_fw_reserve_vram_fini(adev); if (adev->mman.aper_base_kaddr) @@ -2022,7 +2032,7 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) uint64_t size; int r; - if (!adev->mman.initialized || adev->in_gpu_reset || + if (!adev->mman.initialized || amdgpu_in_reset(adev) || adev->mman.buffer_funcs_enabled == enable) return; @@ -2033,7 +2043,7 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) ring = adev->mman.buffer_funcs_ring; sched = &ring->sched; r = drm_sched_entity_init(&adev->mman.entity, - DRM_SCHED_PRIORITY_KERNEL, &sched, + DRM_SCHED_PRIORITY_KERNEL, &sched, 1, NULL); if (r) { DRM_ERROR("Failed setting up TTM BO move entity (%d)\n", @@ -2058,7 +2068,7 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma) { struct drm_file *file_priv = filp->private_data; - struct amdgpu_device *adev = file_priv->minor->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(file_priv->minor->dev); if (adev == NULL) return -EINVAL; @@ -2239,7 +2249,7 @@ static int amdgpu_mm_dump_table(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *)m->private; unsigned ttm_pl = (uintptr_t)node->info_ent->data; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, ttm_pl); struct drm_printer p = drm_seq_file_printer(m); @@ -2530,7 +2540,7 @@ int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) #if defined(CONFIG_DEBUG_FS) unsigned count; - struct drm_minor *minor = adev->ddev->primary; + struct drm_minor *minor = adev_to_drm(adev)->primary; struct dentry *ent, *root = minor->debugfs_root; for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 4f9426ecf039..a87951b2f06d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -73,6 +73,23 @@ struct amdgpu_mman { struct amdgpu_vram_mgr vram_mgr; struct amdgpu_gtt_mgr gtt_mgr; + + uint64_t stolen_vga_size; + struct amdgpu_bo *stolen_vga_memory; + uint64_t stolen_extended_size; + struct amdgpu_bo *stolen_extended_memory; + bool keep_stolen_vga_memory; + + /* discovery */ + uint8_t *discovery_bin; + uint32_t discovery_tmr_size; + struct amdgpu_bo *discovery_memory; + + /* firmware VRAM reservation */ + u64 fw_vram_usage_start_offset; + u64 fw_vram_usage_size; + struct amdgpu_bo *fw_vram_usage_reserved_bo; + void *fw_vram_usage_va; }; struct amdgpu_copy_mem { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index 183743c5fb7b..55fe19a2f332 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -408,7 +408,7 @@ static ssize_t show_##name(struct device *dev, \ char *buf) \ { \ struct drm_device *ddev = dev_get_drvdata(dev); \ - struct amdgpu_device *adev = ddev->dev_private; \ + struct amdgpu_device *adev = drm_to_adev(ddev); \ \ return snprintf(buf, PAGE_SIZE, "0x%08x\n", adev->field); \ } \ @@ -628,7 +628,7 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev) struct amdgpu_firmware_info *ucode = NULL; /* for baremetal, the ucode is allocated in gtt, so don't need to fill the bo when reset/suspend */ - if (!amdgpu_sriov_vf(adev) && (adev->in_gpu_reset || adev->in_suspend)) + if (!amdgpu_sriov_vf(adev) && (amdgpu_in_reset(adev) || adev->in_suspend)) return 0; /* * if SMU loaded firmware, it needn't add SMC, UVD, and VCE diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 12a8bc8fca0b..3c23c6293ff9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -131,6 +131,7 @@ enum ta_fw_type { TA_FW_TYPE_PSP_RAS, TA_FW_TYPE_PSP_HDCP, TA_FW_TYPE_PSP_DTM, + TA_FW_TYPE_PSP_RAP, }; struct ta_fw_bin_desc { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index af1b1ccf613c..262baf0f61ea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -125,8 +125,9 @@ int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev, "detected in UMC block\n", err_data->ue_count); - if (err_data->err_addr_cnt && - amdgpu_ras_add_bad_pages(adev, err_data->err_addr, + if ((amdgpu_bad_page_threshold != 0) && + err_data->err_addr_cnt && + amdgpu_ras_add_bad_pages(adev, err_data->err_addr, err_data->err_addr_cnt)) dev_warn(adev->dev, "Failed to add ras bad page!\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h index a615a1eb750b..183814493658 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h @@ -21,6 +21,20 @@ #ifndef __AMDGPU_UMC_H__ #define __AMDGPU_UMC_H__ +/* + * (addr / 256) * 8192, the higher 26 bits in ErrorAddr + * is the index of 8KB block + */ +#define ADDR_OF_8KB_BLOCK(addr) (((addr) & ~0xffULL) << 5) +/* channel index is the index of 256B block */ +#define ADDR_OF_256B_BLOCK(channel_index) ((channel_index) << 8) +/* offset in 256B block */ +#define OFFSET_IN_256B_BLOCK(addr) ((addr) & 0xffULL) + +#define LOOP_UMC_INST(umc_inst) for ((umc_inst) = 0; (umc_inst) < adev->umc.umc_inst_num; (umc_inst)++) +#define LOOP_UMC_CH_INST(ch_inst) for ((ch_inst) = 0; (ch_inst) < adev->umc.channel_inst_num; (ch_inst)++) +#define LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) LOOP_UMC_INST((umc_inst)) LOOP_UMC_CH_INST((ch_inst)) + struct amdgpu_umc_funcs { void (*err_cnt_init)(struct amdgpu_device *adev); int (*ras_late_init)(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 1203c20491e6..f76961d17246 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -45,7 +45,7 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev) if (adev->mode_info.num_crtc == 0) adev->mode_info.num_crtc = 1; adev->enable_virtual_display = true; - adev->ddev->driver->driver_features &= ~DRIVER_ATOMIC; + adev_to_drm(adev)->driver->driver_features &= ~DRIVER_ATOMIC; adev->cg_flags = 0; adev->pg_flags = 0; } @@ -93,7 +93,7 @@ failed_undo: amdgpu_ring_undo(ring); spin_unlock_irqrestore(&kiq->ring_lock, flags); failed_kiq: - pr_err("failed to write reg %x wait reg %x\n", reg0, reg1); + dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); } /** @@ -401,7 +401,7 @@ static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, if (bp_block_size) { bp_cnt = bp_block_size / sizeof(uint64_t); for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { - retired_page = *(uint64_t *)(adev->fw_vram_usage.va + + retired_page = *(uint64_t *)(adev->mman.fw_vram_usage_va + bp_block_offset + bp_idx * sizeof(uint64_t)); bp.retired_page = retired_page; @@ -428,10 +428,10 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) adev->virt.fw_reserve.p_pf2vf = NULL; adev->virt.fw_reserve.p_vf2pf = NULL; - if (adev->fw_vram_usage.va != NULL) { + if (adev->mman.fw_vram_usage_va != NULL) { adev->virt.fw_reserve.p_pf2vf = (struct amd_sriov_msg_pf2vf_info_header *)( - adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET); + adev->mman.fw_vram_usage_va + AMDGIM_DATAEXCHANGE_OFFSET); AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size); AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum); AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index f826945989c7..b2046c3a404d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -325,9 +325,9 @@ static inline bool is_virtual_machine(void) #define amdgpu_sriov_is_pp_one_vf(adev) \ ((adev)->virt.gim_feature & AMDGIM_FEATURE_PP_ONE_VF) #define amdgpu_sriov_is_debug(adev) \ - ((!adev->in_gpu_reset) && adev->virt.tdr_debug) + ((!amdgpu_in_reset(adev)) && adev->virt.tdr_debug) #define amdgpu_sriov_is_normal(adev) \ - ((!adev->in_gpu_reset) && (!adev->virt.tdr_debug)) + ((!amdgpu_in_reset(adev)) && (!adev->virt.tdr_debug)) bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev); void amdgpu_virt_init_setting(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 8bc2253939be..420931d36732 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -28,6 +28,7 @@ #include <linux/dma-fence-array.h> #include <linux/interval_tree_generic.h> #include <linux/idr.h> +#include <linux/dma-buf.h> #include <drm/amdgpu_drm.h> #include "amdgpu.h" @@ -35,6 +36,7 @@ #include "amdgpu_amdkfd.h" #include "amdgpu_gmc.h" #include "amdgpu_xgmi.h" +#include "amdgpu_dma_buf.h" /** * DOC: GPUVM @@ -1691,13 +1693,13 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, uint64_t max_entries; uint64_t addr, last; + max_entries = mapping->last - start + 1; if (nodes) { addr = nodes->start << PAGE_SHIFT; - max_entries = (nodes->size - pfn) * - AMDGPU_GPU_PAGES_IN_CPU_PAGE; + max_entries = min((nodes->size - pfn) * + AMDGPU_GPU_PAGES_IN_CPU_PAGE, max_entries); } else { addr = 0; - max_entries = S64_MAX; } if (pages_addr) { @@ -1727,7 +1729,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, addr += pfn << PAGE_SHIFT; } - last = min((uint64_t)mapping->last, start + max_entries - 1); + last = start + max_entries - 1; r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv, start, last, flags, addr, dma_addr, fence); @@ -1778,15 +1780,24 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, nodes = NULL; resv = vm->root.base.bo->tbo.base.resv; } else { + struct drm_gem_object *obj = &bo->tbo.base; struct ttm_dma_tt *ttm; + resv = bo->tbo.base.resv; + if (obj->import_attach && bo_va->is_xgmi) { + struct dma_buf *dma_buf = obj->import_attach->dmabuf; + struct drm_gem_object *gobj = dma_buf->priv; + struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); + + if (abo->tbo.mem.mem_type == TTM_PL_VRAM) + bo = gem_to_amdgpu_bo(gobj); + } mem = &bo->tbo.mem; nodes = mem->mm_node; if (mem->mem_type == TTM_PL_TT) { ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); pages_addr = ttm->dma_address; } - resv = bo->tbo.base.resv; } if (bo) { @@ -2132,8 +2143,10 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, INIT_LIST_HEAD(&bo_va->valids); INIT_LIST_HEAD(&bo_va->invalids); - if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) && - (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { + if (!bo) + return bo_va; + + if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) { bo_va->is_xgmi = true; /* Power up XGMI if it can be potentially used */ amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20); @@ -3209,7 +3222,7 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { union drm_amdgpu_vm *args = data; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_fpriv *fpriv = filp->driver_priv; long timeout = msecs_to_jiffies(2000); int r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 7574be6cd7a0..b2adc2abc581 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -50,7 +50,7 @@ static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.real_vram_size); } @@ -67,7 +67,7 @@ static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.visible_vram_size); } @@ -84,8 +84,9 @@ static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); + return snprintf(buf, PAGE_SIZE, "%llu\n", amdgpu_vram_mgr_usage(man)); } @@ -102,8 +103,9 @@ static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); + return snprintf(buf, PAGE_SIZE, "%llu\n", amdgpu_vram_mgr_vis_usage(man)); } @@ -113,7 +115,7 @@ static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); switch (adev->gmc.vram_vendor) { case SAMSUNG: @@ -478,7 +480,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, unsigned int pages; int i, r; - *sgt = kmalloc(sizeof(*sg), GFP_KERNEL); + *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); if (!*sgt) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index e3a3755cb999..1162913c8bf4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -35,11 +35,9 @@ static DEFINE_MUTEX(xgmi_mutex); -#define AMDGPU_MAX_XGMI_HIVE 8 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4 -static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE]; -static unsigned hive_count = 0; +static LIST_HEAD(xgmi_hive_list); static const int xgmi_pcs_err_status_reg_vg20[] = { smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS, @@ -171,65 +169,53 @@ static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = { * */ +static struct attribute amdgpu_xgmi_hive_id = { + .name = "xgmi_hive_id", + .mode = S_IRUGO +}; -static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct amdgpu_hive_info *hive = - container_of(attr, struct amdgpu_hive_info, dev_attr); - - return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id); -} +static struct attribute *amdgpu_xgmi_hive_attrs[] = { + &amdgpu_xgmi_hive_id, + NULL +}; -static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev, - struct amdgpu_hive_info *hive) +static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj, + struct attribute *attr, char *buf) { - int ret = 0; + struct amdgpu_hive_info *hive = container_of( + kobj, struct amdgpu_hive_info, kobj); - if (WARN_ON(hive->kobj)) - return -EINVAL; - - hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj); - if (!hive->kobj) { - dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n"); - return -EINVAL; - } - - hive->dev_attr = (struct device_attribute) { - .attr = { - .name = "xgmi_hive_id", - .mode = S_IRUGO, - - }, - .show = amdgpu_xgmi_show_hive_id, - }; + if (attr == &amdgpu_xgmi_hive_id) + return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id); - ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr); - if (ret) { - dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n"); - kobject_del(hive->kobj); - kobject_put(hive->kobj); - hive->kobj = NULL; - } - - return ret; + return 0; } -static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev, - struct amdgpu_hive_info *hive) +static void amdgpu_xgmi_hive_release(struct kobject *kobj) { - sysfs_remove_file(hive->kobj, &hive->dev_attr.attr); - kobject_del(hive->kobj); - kobject_put(hive->kobj); - hive->kobj = NULL; + struct amdgpu_hive_info *hive = container_of( + kobj, struct amdgpu_hive_info, kobj); + + mutex_destroy(&hive->hive_lock); + kfree(hive); } +static const struct sysfs_ops amdgpu_xgmi_hive_ops = { + .show = amdgpu_xgmi_show_attrs, +}; + +struct kobj_type amdgpu_xgmi_hive_type = { + .release = amdgpu_xgmi_hive_release, + .sysfs_ops = &amdgpu_xgmi_hive_ops, + .default_attrs = amdgpu_xgmi_hive_attrs, +}; + static ssize_t amdgpu_xgmi_show_device_id(struct device *dev, struct device_attribute *attr, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id); @@ -241,7 +227,7 @@ static ssize_t amdgpu_xgmi_show_error(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in; uint64_t fica_out; unsigned int error_count = 0; @@ -287,8 +273,8 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, /* Create sysfs link to hive info folder on the first device */ - if (adev != hive->adev) { - ret = sysfs_create_link(&adev->dev->kobj, hive->kobj, + if (hive->kobj.parent != (&adev->dev->kobj)) { + ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj, "xgmi_hive_info"); if (ret) { dev_err(adev->dev, "XGMI: Failed to create link to hive info"); @@ -296,9 +282,9 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, } } - sprintf(node, "node%d", hive->number_devices); + sprintf(node, "node%d", atomic_read(&hive->number_devices)); /* Create sysfs link form the hive folder to yourself */ - ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node); + ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node); if (ret) { dev_err(adev->dev, "XGMI: Failed to create link from hive info"); goto remove_link; @@ -308,7 +294,7 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, remove_link: - sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); + sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique); remove_file: device_remove_file(adev->dev, &dev_attr_xgmi_device_id); @@ -326,78 +312,96 @@ static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, device_remove_file(adev->dev, &dev_attr_xgmi_device_id); device_remove_file(adev->dev, &dev_attr_xgmi_error); - if (adev != hive->adev) + if (hive->kobj.parent != (&adev->dev->kobj)) sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info"); - sprintf(node, "node%d", hive->number_devices); - sysfs_remove_link(hive->kobj, node); + sprintf(node, "node%d", atomic_read(&hive->number_devices)); + sysfs_remove_link(&hive->kobj, node); } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) { - int i; - struct amdgpu_hive_info *tmp; + struct amdgpu_hive_info *hive = NULL, *tmp = NULL; + int ret; if (!adev->gmc.xgmi.hive_id) return NULL; + if (adev->hive) { + kobject_get(&adev->hive->kobj); + return adev->hive; + } + mutex_lock(&xgmi_mutex); - for (i = 0 ; i < hive_count; ++i) { - tmp = &xgmi_hives[i]; - if (tmp->hive_id == adev->gmc.xgmi.hive_id) { - if (lock) - mutex_lock(&tmp->hive_lock); - mutex_unlock(&xgmi_mutex); - return tmp; + if (!list_empty(&xgmi_hive_list)) { + list_for_each_entry_safe(hive, tmp, &xgmi_hive_list, node) { + if (hive->hive_id == adev->gmc.xgmi.hive_id) + goto pro_end; } } - if (i >= AMDGPU_MAX_XGMI_HIVE) { - mutex_unlock(&xgmi_mutex); - return NULL; + + hive = kzalloc(sizeof(*hive), GFP_KERNEL); + if (!hive) { + dev_err(adev->dev, "XGMI: allocation failed\n"); + hive = NULL; + goto pro_end; } /* initialize new hive if not exist */ - tmp = &xgmi_hives[hive_count++]; - - if (amdgpu_xgmi_sysfs_create(adev, tmp)) { - mutex_unlock(&xgmi_mutex); - return NULL; + ret = kobject_init_and_add(&hive->kobj, + &amdgpu_xgmi_hive_type, + &adev->dev->kobj, + "%s", "xgmi_hive_info"); + if (ret) { + dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n"); + kfree(hive); + hive = NULL; + goto pro_end; } - tmp->adev = adev; - tmp->hive_id = adev->gmc.xgmi.hive_id; - INIT_LIST_HEAD(&tmp->device_list); - mutex_init(&tmp->hive_lock); - mutex_init(&tmp->reset_lock); - task_barrier_init(&tmp->tb); - - if (lock) - mutex_lock(&tmp->hive_lock); - tmp->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN; - tmp->hi_req_gpu = NULL; + hive->hive_id = adev->gmc.xgmi.hive_id; + INIT_LIST_HEAD(&hive->device_list); + INIT_LIST_HEAD(&hive->node); + mutex_init(&hive->hive_lock); + atomic_set(&hive->in_reset, 0); + atomic_set(&hive->number_devices, 0); + task_barrier_init(&hive->tb); + hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN; + hive->hi_req_gpu = NULL; /* * hive pstate on boot is high in vega20 so we have to go to low * pstate on after boot. */ - tmp->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE; + hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE; + list_add_tail(&hive->node, &xgmi_hive_list); + +pro_end: + if (hive) + kobject_get(&hive->kobj); mutex_unlock(&xgmi_mutex); + return hive; +} - return tmp; +void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive) +{ + if (hive) + kobject_put(&hive->kobj); } int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) { int ret = 0; - struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); + struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); struct amdgpu_device *request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev; bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20; bool init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN; + amdgpu_put_xgmi_hive(hive); /* fw bug so temporarily disable pstate switching */ return 0; @@ -449,7 +453,7 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev /* Each psp need to set the latest topology */ ret = psp_xgmi_set_topology_info(&adev->psp, - hive->number_devices, + atomic_read(&hive->number_devices), &adev->psp.xgmi_context.top_info); if (ret) dev_err(adev->dev, @@ -511,7 +515,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16; } - hive = amdgpu_get_xgmi_hive(adev, 1); + hive = amdgpu_get_xgmi_hive(adev); if (!hive) { ret = -EINVAL; dev_err(adev->dev, @@ -519,6 +523,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id); goto exit; } + mutex_lock(&hive->hive_lock); top_info = &adev->psp.xgmi_context.top_info; @@ -526,7 +531,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) list_for_each_entry(entry, &hive->device_list, head) top_info->nodes[count++].node_id = entry->node_id; top_info->num_nodes = count; - hive->number_devices = count; + atomic_set(&hive->number_devices, count); task_barrier_add_task(&hive->tb); @@ -541,7 +546,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) } ret = amdgpu_xgmi_update_topology(hive, tmp_adev); if (ret) - goto exit; + goto exit_unlock; } /* get latest topology info for each device from psp */ @@ -554,7 +559,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) tmp_adev->gmc.xgmi.node_id, tmp_adev->gmc.xgmi.hive_id, ret); /* To do : continue with some node failed or disable the whole hive */ - goto exit; + goto exit_unlock; } } } @@ -562,39 +567,51 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) if (!ret) ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive); - +exit_unlock: mutex_unlock(&hive->hive_lock); exit: - if (!ret) + if (!ret) { + adev->hive = hive; dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n", adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id); - else + } else { + amdgpu_put_xgmi_hive(hive); dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n", adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id, ret); + } return ret; } int amdgpu_xgmi_remove_device(struct amdgpu_device *adev) { - struct amdgpu_hive_info *hive; + struct amdgpu_hive_info *hive = adev->hive; if (!adev->gmc.xgmi.supported) return -EINVAL; - hive = amdgpu_get_xgmi_hive(adev, 1); if (!hive) return -EINVAL; + mutex_lock(&hive->hive_lock); task_barrier_rem_task(&hive->tb); amdgpu_xgmi_sysfs_rem_dev_info(adev, hive); + if (hive->hi_req_gpu == adev) + hive->hi_req_gpu = NULL; + list_del(&adev->gmc.xgmi.head); mutex_unlock(&hive->hive_lock); - if(!(--hive->number_devices)){ - amdgpu_xgmi_sysfs_destroy(adev, hive); - mutex_destroy(&hive->hive_lock); - mutex_destroy(&hive->reset_lock); + amdgpu_put_xgmi_hive(hive); + adev->hive = NULL; + + if (atomic_dec_return(&hive->number_devices) == 0) { + /* Remove the hive from global hive list */ + mutex_lock(&xgmi_mutex); + list_del(&hive->node); + mutex_unlock(&xgmi_mutex); + + amdgpu_put_xgmi_hive(hive); } return psp_xgmi_terminate(&adev->psp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h index 6999eab16a72..148560d63554 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h @@ -27,13 +27,13 @@ struct amdgpu_hive_info { - uint64_t hive_id; - struct list_head device_list; - int number_devices; - struct mutex hive_lock, reset_lock; - struct kobject *kobj; - struct device_attribute dev_attr; - struct amdgpu_device *adev; + struct kobject kobj; + uint64_t hive_id; + struct list_head device_list; + struct list_head node; + atomic_t number_devices; + struct mutex hive_lock; + atomic_t in_reset; int hi_req_count; struct amdgpu_device *hi_req_gpu; struct task_barrier tb; @@ -50,7 +50,8 @@ struct amdgpu_pcs_ras_field { uint32_t pcs_err_shift; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev); +void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive); int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev); int amdgpu_xgmi_add_device(struct amdgpu_device *adev); int amdgpu_xgmi_remove_device(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c index 847ca9b3ce4e..3ea557864320 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c @@ -73,6 +73,7 @@ int athub_v1_0_set_clockgating(struct amdgpu_device *adev, case CHIP_VEGA12: case CHIP_VEGA20: case CHIP_RAVEN: + case CHIP_RENOIR: athub_update_medium_grain_clock_gating(adev, state == AMD_CG_STATE_GATE); athub_update_medium_grain_light_sleep(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c index 213e62a28ba0..159a2a4385a1 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c @@ -41,7 +41,7 @@ void amdgpu_atombios_crtc_overscan_setup(struct drm_crtc *crtc, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); SET_CRTC_OVERSCAN_PS_ALLOCATION args; int index = GetIndexIntoMasterTable(COMMAND, SetCRTC_OverScan); @@ -84,7 +84,7 @@ void amdgpu_atombios_crtc_overscan_setup(struct drm_crtc *crtc, void amdgpu_atombios_crtc_scaler_setup(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); ENABLE_SCALER_PS_ALLOCATION args; int index = GetIndexIntoMasterTable(COMMAND, EnableScaler); @@ -114,7 +114,7 @@ void amdgpu_atombios_crtc_lock(struct drm_crtc *crtc, int lock) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int index = GetIndexIntoMasterTable(COMMAND, UpdateCRTC_DoubleBufferRegisters); ENABLE_CRTC_PS_ALLOCATION args; @@ -131,7 +131,7 @@ void amdgpu_atombios_crtc_enable(struct drm_crtc *crtc, int state) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int index = GetIndexIntoMasterTable(COMMAND, EnableCRTC); ENABLE_CRTC_PS_ALLOCATION args; @@ -147,7 +147,7 @@ void amdgpu_atombios_crtc_blank(struct drm_crtc *crtc, int state) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int index = GetIndexIntoMasterTable(COMMAND, BlankCRTC); BLANK_CRTC_PS_ALLOCATION args; @@ -163,7 +163,7 @@ void amdgpu_atombios_crtc_powergate(struct drm_crtc *crtc, int state) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int index = GetIndexIntoMasterTable(COMMAND, EnableDispPowerGating); ENABLE_DISP_POWER_GATING_PS_ALLOCATION args; @@ -192,7 +192,7 @@ void amdgpu_atombios_crtc_set_dtd_timing(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); SET_CRTC_USING_DTD_TIMING_PARAMETERS args; int index = GetIndexIntoMasterTable(COMMAND, SetCRTC_UsingDTDTiming); u16 misc = 0; @@ -307,7 +307,7 @@ static u32 amdgpu_atombios_crtc_adjust_pll(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_encoder *encoder = amdgpu_crtc->encoder; struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -588,7 +588,7 @@ void amdgpu_atombios_crtc_program_pll(struct drm_crtc *crtc, struct amdgpu_atom_ss *ss) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u8 frev, crev; int index = GetIndexIntoMasterTable(COMMAND, SetPixelClock); union set_pixel_clock args; @@ -749,7 +749,7 @@ int amdgpu_atombios_crtc_prepare_pll(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(amdgpu_crtc->encoder); int encoder_mode = amdgpu_atombios_encoder_get_encoder_mode(amdgpu_crtc->encoder); @@ -818,7 +818,7 @@ void amdgpu_atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(amdgpu_crtc->encoder); u32 pll_clock = mode->clock; diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c index 900b2727f432..a3ba9ca11e98 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c @@ -60,7 +60,7 @@ static int amdgpu_atombios_dp_process_aux_ch(struct amdgpu_i2c_chan *chan, u8 delay, u8 *ack) { struct drm_device *dev = chan->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); union aux_channel_transaction args; int index = GetIndexIntoMasterTable(COMMAND, ProcessAuxChannelTransaction); unsigned char *base; @@ -305,7 +305,7 @@ static u8 amdgpu_atombios_dp_encoder_service(struct amdgpu_device *adev, u8 amdgpu_atombios_dp_get_sinktype(struct amdgpu_connector *amdgpu_connector) { struct drm_device *dev = amdgpu_connector->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); return amdgpu_atombios_dp_encoder_service(adev, ATOM_DP_ACTION_GET_SINK_TYPE, 0, amdgpu_connector->ddc_bus->rec.i2c_id, 0); @@ -718,7 +718,7 @@ void amdgpu_atombios_dp_link_train(struct drm_encoder *encoder, struct drm_connector *connector) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_connector *amdgpu_connector; struct amdgpu_connector_atom_dig *dig_connector; diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c index 1e94a9b652f7..8339c8c3a328 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c @@ -70,7 +70,7 @@ u8 amdgpu_atombios_encoder_get_backlight_level(struct amdgpu_encoder *amdgpu_encoder) { struct drm_device *dev = amdgpu_encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) return 0; @@ -84,7 +84,7 @@ amdgpu_atombios_encoder_set_backlight_level(struct amdgpu_encoder *amdgpu_encode { struct drm_encoder *encoder = &amdgpu_encoder->base; struct drm_device *dev = amdgpu_encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder_atom_dig *dig; if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) @@ -152,7 +152,7 @@ amdgpu_atombios_encoder_get_backlight_brightness(struct backlight_device *bd) struct amdgpu_backlight_privdata *pdata = bl_get_data(bd); struct amdgpu_encoder *amdgpu_encoder = pdata->encoder; struct drm_device *dev = amdgpu_encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); return amdgpu_atombios_encoder_get_backlight_level_from_reg(adev); } @@ -166,7 +166,7 @@ void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encode struct drm_connector *drm_connector) { struct drm_device *dev = amdgpu_encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct backlight_device *bd; struct backlight_properties props; struct amdgpu_backlight_privdata *pdata; @@ -229,7 +229,7 @@ void amdgpu_atombios_encoder_fini_backlight(struct amdgpu_encoder *amdgpu_encoder) { struct drm_device *dev = amdgpu_encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct backlight_device *bd = NULL; struct amdgpu_encoder_atom_dig *dig; @@ -319,7 +319,7 @@ static void amdgpu_atombios_encoder_setup_dac(struct drm_encoder *encoder, int action) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); DAC_ENCODER_CONTROL_PS_ALLOCATION args; int index = 0; @@ -382,7 +382,7 @@ static void amdgpu_atombios_encoder_setup_dvo(struct drm_encoder *encoder, int action) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); union dvo_encoder_control args; int index = GetIndexIntoMasterTable(COMMAND, DVOEncoderControl); @@ -573,7 +573,7 @@ amdgpu_atombios_encoder_setup_dig_encoder(struct drm_encoder *encoder, int action, int panel_mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -762,7 +762,7 @@ amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int a uint8_t lane_num, uint8_t lane_set) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1178,7 +1178,7 @@ amdgpu_atombios_encoder_set_edp_panel_power(struct drm_connector *connector, { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); struct drm_device *dev = amdgpu_connector->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); union dig_transmitter_control args; int index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl); uint8_t frev, crev; @@ -1225,7 +1225,7 @@ amdgpu_atombios_encoder_setup_external_encoder(struct drm_encoder *encoder, int action) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder *ext_amdgpu_encoder = to_amdgpu_encoder(ext_encoder); union external_encoder_control args; @@ -1466,7 +1466,7 @@ void amdgpu_atombios_encoder_set_crtc_source(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); union crtc_source_param args; @@ -1673,7 +1673,7 @@ amdgpu_atombios_encoder_set_crtc_source(struct drm_encoder *encoder) void amdgpu_atombios_encoder_init_dig(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_encoder *encoder; list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { @@ -1701,7 +1701,7 @@ amdgpu_atombios_encoder_dac_load_detect(struct drm_encoder *encoder, struct drm_connector *connector) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); @@ -1751,7 +1751,7 @@ amdgpu_atombios_encoder_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); uint32_t bios_0_scratch; @@ -1790,7 +1790,7 @@ amdgpu_atombios_encoder_dig_detect(struct drm_encoder *encoder, struct drm_connector *connector) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); struct drm_encoder *ext_encoder = amdgpu_get_external_encoder(encoder); @@ -1848,7 +1848,7 @@ amdgpu_atombios_encoder_set_bios_scratch_regs(struct drm_connector *connector, bool connected) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); @@ -1999,7 +1999,7 @@ struct amdgpu_encoder_atom_dig * amdgpu_atombios_encoder_get_lcd_info(struct amdgpu_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_mode_info *mode_info = &adev->mode_info; int index = GetIndexIntoMasterTable(DATA, LVDS_Info); uint16_t data_offset, misc; diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c index b4cc7c55fa16..09a538465ffd 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c @@ -40,7 +40,7 @@ static int amdgpu_atombios_i2c_process_i2c_ch(struct amdgpu_i2c_chan *chan, u8 *buf, u8 num) { struct drm_device *dev = chan->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); PROCESS_I2C_CHANNEL_TRANSACTION_PS_ALLOCATION args; int index = GetIndexIntoMasterTable(COMMAND, ProcessI2cChannelTransaction); unsigned char *base; diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index c2c67ab68a43..03ff8bd1fee8 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -1366,8 +1366,10 @@ static int cik_asic_reset(struct amdgpu_device *adev) int r; if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { + dev_info(adev->dev, "BACO reset\n"); r = amdgpu_dpm_baco_reset(adev); } else { + dev_info(adev->dev, "PCI CONFIG reset\n"); r = cik_asic_pci_config_reset(adev); } @@ -1919,6 +1921,10 @@ static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev) return (nak_r + nak_g); } +static void cik_pre_asic_init(struct amdgpu_device *adev) +{ +} + static const struct amdgpu_asic_funcs cik_asic_funcs = { .read_disabled_bios = &cik_read_disabled_bios, @@ -1939,6 +1945,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs = .need_reset_on_init = &cik_need_reset_on_init, .get_pcie_replay_count = &cik_get_pcie_replay_count, .supports_baco = &cik_asic_supports_baco, + .pre_asic_init = &cik_pre_asic_init, }; static int cik_common_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 84b45a019a36..5963cbe0d455 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -328,7 +328,7 @@ static void dce_v10_0_hpd_set_polarity(struct amdgpu_device *adev, */ static void dce_v10_0_hpd_init(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -383,7 +383,7 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev) */ static void dce_v10_0_hpd_fini(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -504,7 +504,7 @@ void dce_v10_0_disable_dce(struct amdgpu_device *adev) static void dce_v10_0_program_fmt(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1209,7 +1209,7 @@ static struct amdgpu_audio_pin *dce_v10_0_audio_get_pin(struct amdgpu_device *ad static void dce_v10_0_afmt_audio_select_pin(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1226,7 +1226,7 @@ static void dce_v10_0_audio_write_latency_fields(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1272,7 +1272,7 @@ static void dce_v10_0_audio_write_latency_fields(struct drm_encoder *encoder, static void dce_v10_0_audio_write_speaker_allocation(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1328,7 +1328,7 @@ static void dce_v10_0_audio_write_speaker_allocation(struct drm_encoder *encoder static void dce_v10_0_audio_write_sad_regs(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1483,7 +1483,7 @@ static void dce_v10_0_audio_fini(struct amdgpu_device *adev) static void dce_v10_0_afmt_update_ACR(struct drm_encoder *encoder, uint32_t clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_afmt_acr acr = amdgpu_afmt_acr(clock); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1519,7 +1519,7 @@ static void dce_v10_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, void *buffer, size_t size) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; uint8_t *frame = buffer + 3; @@ -1538,7 +1538,7 @@ static void dce_v10_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, static void dce_v10_0_audio_set_dto(struct drm_encoder *encoder, u32 clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); @@ -1569,7 +1569,7 @@ static void dce_v10_0_afmt_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1749,7 +1749,7 @@ static void dce_v10_0_afmt_setmode(struct drm_encoder *encoder, static void dce_v10_0_afmt_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1822,7 +1822,7 @@ static void dce_v10_0_vga_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 vga_control; vga_control = RREG32(vga_control_regs[amdgpu_crtc->crtc_id]) & ~1; @@ -1836,7 +1836,7 @@ static void dce_v10_0_grph_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (enable) WREG32(mmGRPH_ENABLE + amdgpu_crtc->crtc_offset, 1); @@ -1850,7 +1850,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_framebuffer *target_fb; struct drm_gem_object *obj; struct amdgpu_bo *abo; @@ -2095,7 +2095,7 @@ static void dce_v10_0_set_interleave(struct drm_crtc *crtc, struct drm_display_mode *mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); u32 tmp; @@ -2111,7 +2111,7 @@ static void dce_v10_0_crtc_load_lut(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u16 *r, *g, *b; int i; u32 tmp; @@ -2250,7 +2250,7 @@ static u32 dce_v10_0_pick_pll(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 pll_in_use; int pll; @@ -2285,7 +2285,7 @@ static u32 dce_v10_0_pick_pll(struct drm_crtc *crtc) static void dce_v10_0_lock_cursor(struct drm_crtc *crtc, bool lock) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); uint32_t cur_lock; @@ -2300,7 +2300,7 @@ static void dce_v10_0_lock_cursor(struct drm_crtc *crtc, bool lock) static void dce_v10_0_hide_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); u32 tmp; tmp = RREG32(mmCUR_CONTROL + amdgpu_crtc->crtc_offset); @@ -2311,7 +2311,7 @@ static void dce_v10_0_hide_cursor(struct drm_crtc *crtc) static void dce_v10_0_show_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); u32 tmp; WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset, @@ -2329,7 +2329,7 @@ static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); int xorigin = 0, yorigin = 0; amdgpu_crtc->cursor_x = x; @@ -2503,7 +2503,7 @@ static const struct drm_crtc_funcs dce_v10_0_crtc_funcs = { static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); unsigned type; @@ -2557,7 +2557,7 @@ static void dce_v10_0_crtc_disable(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_atom_ss ss; int i; @@ -2701,7 +2701,7 @@ static int dce_v10_0_crtc_init(struct amdgpu_device *adev, int index) if (amdgpu_crtc == NULL) return -ENOMEM; - drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_v10_0_crtc_funcs); + drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_v10_0_crtc_funcs); drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); amdgpu_crtc->crtc_id = index; @@ -2709,8 +2709,8 @@ static int dce_v10_0_crtc_init(struct amdgpu_device *adev, int index) amdgpu_crtc->max_cursor_width = 128; amdgpu_crtc->max_cursor_height = 128; - adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; - adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; + adev_to_drm(adev)->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; + adev_to_drm(adev)->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; switch (amdgpu_crtc->crtc_id) { case 0: @@ -2792,24 +2792,24 @@ static int dce_v10_0_sw_init(void *handle) if (r) return r; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; + adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; - adev->ddev->mode_config.async_page_flip = true; + adev_to_drm(adev)->mode_config.async_page_flip = true; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; r = amdgpu_display_modeset_create_props(adev); if (r) return r; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; /* allocate crtcs */ for (i = 0; i < adev->mode_info.num_crtc; i++) { @@ -2819,7 +2819,7 @@ static int dce_v10_0_sw_init(void *handle) } if (amdgpu_atombios_get_connector_info_from_object_table(adev)) - amdgpu_display_print_display_setup(adev->ddev); + amdgpu_display_print_display_setup(adev_to_drm(adev)); else return -EINVAL; @@ -2832,7 +2832,7 @@ static int dce_v10_0_sw_init(void *handle) if (r) return r; - drm_kms_helper_poll_init(adev->ddev); + drm_kms_helper_poll_init(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = true; return 0; @@ -2844,13 +2844,13 @@ static int dce_v10_0_sw_fini(void *handle) kfree(adev->mode_info.bios_hardcoded_edid); - drm_kms_helper_poll_fini(adev->ddev); + drm_kms_helper_poll_fini(adev_to_drm(adev)); dce_v10_0_audio_fini(adev); dce_v10_0_afmt_fini(adev); - drm_mode_config_cleanup(adev->ddev); + drm_mode_config_cleanup(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = false; return 0; @@ -3157,14 +3157,14 @@ static int dce_v10_0_pageflip_irq(struct amdgpu_device *adev, if (amdgpu_crtc == NULL) return 0; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); works = amdgpu_crtc->pflip_works; if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " "AMDGPU_FLIP_SUBMITTED(%d)\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return 0; } @@ -3176,7 +3176,7 @@ static int dce_v10_0_pageflip_irq(struct amdgpu_device *adev, if (works->event) drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); @@ -3245,7 +3245,7 @@ static int dce_v10_0_crtc_irq(struct amdgpu_device *adev, DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); if (amdgpu_irq_enabled(adev, source, irq_type)) { - drm_handle_vblank(adev->ddev, crtc); + drm_handle_vblank(adev_to_drm(adev), crtc); } DRM_DEBUG("IH: D%d vblank\n", crtc + 1); @@ -3345,7 +3345,7 @@ dce_v10_0_encoder_mode_set(struct drm_encoder *encoder, static void dce_v10_0_encoder_prepare(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -3385,7 +3385,7 @@ static void dce_v10_0_encoder_prepare(struct drm_encoder *encoder) static void dce_v10_0_encoder_commit(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* need to call this here as we need the crtc set up */ amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_ON); @@ -3485,7 +3485,7 @@ static void dce_v10_0_encoder_add(struct amdgpu_device *adev, uint32_t supported_device, u16 caps) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_encoder *encoder; struct amdgpu_encoder *amdgpu_encoder; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 01ce52266966..1954472c8e8f 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -346,7 +346,7 @@ static void dce_v11_0_hpd_set_polarity(struct amdgpu_device *adev, */ static void dce_v11_0_hpd_init(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -400,7 +400,7 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev) */ static void dce_v11_0_hpd_fini(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -530,7 +530,7 @@ void dce_v11_0_disable_dce(struct amdgpu_device *adev) static void dce_v11_0_program_fmt(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1235,7 +1235,7 @@ static struct amdgpu_audio_pin *dce_v11_0_audio_get_pin(struct amdgpu_device *ad static void dce_v11_0_afmt_audio_select_pin(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1252,7 +1252,7 @@ static void dce_v11_0_audio_write_latency_fields(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1298,7 +1298,7 @@ static void dce_v11_0_audio_write_latency_fields(struct drm_encoder *encoder, static void dce_v11_0_audio_write_speaker_allocation(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1354,7 +1354,7 @@ static void dce_v11_0_audio_write_speaker_allocation(struct drm_encoder *encoder static void dce_v11_0_audio_write_sad_regs(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1525,7 +1525,7 @@ static void dce_v11_0_audio_fini(struct amdgpu_device *adev) static void dce_v11_0_afmt_update_ACR(struct drm_encoder *encoder, uint32_t clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_afmt_acr acr = amdgpu_afmt_acr(clock); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1561,7 +1561,7 @@ static void dce_v11_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, void *buffer, size_t size) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; uint8_t *frame = buffer + 3; @@ -1580,7 +1580,7 @@ static void dce_v11_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, static void dce_v11_0_audio_set_dto(struct drm_encoder *encoder, u32 clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); @@ -1611,7 +1611,7 @@ static void dce_v11_0_afmt_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1791,7 +1791,7 @@ static void dce_v11_0_afmt_setmode(struct drm_encoder *encoder, static void dce_v11_0_afmt_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1864,7 +1864,7 @@ static void dce_v11_0_vga_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 vga_control; vga_control = RREG32(vga_control_regs[amdgpu_crtc->crtc_id]) & ~1; @@ -1878,7 +1878,7 @@ static void dce_v11_0_grph_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (enable) WREG32(mmGRPH_ENABLE + amdgpu_crtc->crtc_offset, 1); @@ -1892,7 +1892,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_framebuffer *target_fb; struct drm_gem_object *obj; struct amdgpu_bo *abo; @@ -2137,7 +2137,7 @@ static void dce_v11_0_set_interleave(struct drm_crtc *crtc, struct drm_display_mode *mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); u32 tmp; @@ -2153,7 +2153,7 @@ static void dce_v11_0_crtc_load_lut(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u16 *r, *g, *b; int i; u32 tmp; @@ -2283,7 +2283,7 @@ static u32 dce_v11_0_pick_pll(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 pll_in_use; int pll; @@ -2364,7 +2364,7 @@ static u32 dce_v11_0_pick_pll(struct drm_crtc *crtc) static void dce_v11_0_lock_cursor(struct drm_crtc *crtc, bool lock) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); uint32_t cur_lock; @@ -2379,7 +2379,7 @@ static void dce_v11_0_lock_cursor(struct drm_crtc *crtc, bool lock) static void dce_v11_0_hide_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); u32 tmp; tmp = RREG32(mmCUR_CONTROL + amdgpu_crtc->crtc_offset); @@ -2390,7 +2390,7 @@ static void dce_v11_0_hide_cursor(struct drm_crtc *crtc) static void dce_v11_0_show_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); u32 tmp; WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset, @@ -2408,7 +2408,7 @@ static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); int xorigin = 0, yorigin = 0; amdgpu_crtc->cursor_x = x; @@ -2582,7 +2582,7 @@ static const struct drm_crtc_funcs dce_v11_0_crtc_funcs = { static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); unsigned type; @@ -2636,7 +2636,7 @@ static void dce_v11_0_crtc_disable(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_atom_ss ss; int i; @@ -2706,7 +2706,7 @@ static int dce_v11_0_crtc_mode_set(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (!amdgpu_crtc->adjusted_clock) return -EINVAL; @@ -2809,7 +2809,7 @@ static int dce_v11_0_crtc_init(struct amdgpu_device *adev, int index) if (amdgpu_crtc == NULL) return -ENOMEM; - drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_v11_0_crtc_funcs); + drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_v11_0_crtc_funcs); drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); amdgpu_crtc->crtc_id = index; @@ -2817,8 +2817,8 @@ static int dce_v11_0_crtc_init(struct amdgpu_device *adev, int index) amdgpu_crtc->max_cursor_width = 128; amdgpu_crtc->max_cursor_height = 128; - adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; - adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; + adev_to_drm(adev)->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; + adev_to_drm(adev)->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; switch (amdgpu_crtc->crtc_id) { case 0: @@ -2913,24 +2913,24 @@ static int dce_v11_0_sw_init(void *handle) if (r) return r; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; + adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; - adev->ddev->mode_config.async_page_flip = true; + adev_to_drm(adev)->mode_config.async_page_flip = true; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; r = amdgpu_display_modeset_create_props(adev); if (r) return r; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; /* allocate crtcs */ @@ -2941,7 +2941,7 @@ static int dce_v11_0_sw_init(void *handle) } if (amdgpu_atombios_get_connector_info_from_object_table(adev)) - amdgpu_display_print_display_setup(adev->ddev); + amdgpu_display_print_display_setup(adev_to_drm(adev)); else return -EINVAL; @@ -2954,7 +2954,7 @@ static int dce_v11_0_sw_init(void *handle) if (r) return r; - drm_kms_helper_poll_init(adev->ddev); + drm_kms_helper_poll_init(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = true; return 0; @@ -2966,13 +2966,13 @@ static int dce_v11_0_sw_fini(void *handle) kfree(adev->mode_info.bios_hardcoded_edid); - drm_kms_helper_poll_fini(adev->ddev); + drm_kms_helper_poll_fini(adev_to_drm(adev)); dce_v11_0_audio_fini(adev); dce_v11_0_afmt_fini(adev); - drm_mode_config_cleanup(adev->ddev); + drm_mode_config_cleanup(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = false; return 0; @@ -3283,14 +3283,14 @@ static int dce_v11_0_pageflip_irq(struct amdgpu_device *adev, if(amdgpu_crtc == NULL) return 0; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); works = amdgpu_crtc->pflip_works; if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED){ DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " "AMDGPU_FLIP_SUBMITTED(%d)\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return 0; } @@ -3302,7 +3302,7 @@ static int dce_v11_0_pageflip_irq(struct amdgpu_device *adev, if(works->event) drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); @@ -3372,7 +3372,7 @@ static int dce_v11_0_crtc_irq(struct amdgpu_device *adev, DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); if (amdgpu_irq_enabled(adev, source, irq_type)) { - drm_handle_vblank(adev->ddev, crtc); + drm_handle_vblank(adev_to_drm(adev), crtc); } DRM_DEBUG("IH: D%d vblank\n", crtc + 1); @@ -3471,7 +3471,7 @@ dce_v11_0_encoder_mode_set(struct drm_encoder *encoder, static void dce_v11_0_encoder_prepare(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -3511,7 +3511,7 @@ static void dce_v11_0_encoder_prepare(struct drm_encoder *encoder) static void dce_v11_0_encoder_commit(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* need to call this here as we need the crtc set up */ amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_ON); @@ -3611,7 +3611,7 @@ static void dce_v11_0_encoder_add(struct amdgpu_device *adev, uint32_t supported_device, u16 caps) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_encoder *encoder; struct amdgpu_encoder *amdgpu_encoder; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index cbddead3dafb..3a44753a80d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -279,7 +279,7 @@ static void dce_v6_0_hpd_set_polarity(struct amdgpu_device *adev, */ static void dce_v6_0_hpd_init(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -324,7 +324,7 @@ static void dce_v6_0_hpd_init(struct amdgpu_device *adev) */ static void dce_v6_0_hpd_fini(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -401,7 +401,7 @@ static void dce_v6_0_program_fmt(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); @@ -1114,7 +1114,7 @@ static struct amdgpu_audio_pin *dce_v6_0_audio_get_pin(struct amdgpu_device *ade static void dce_v6_0_audio_select_pin(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1130,7 +1130,7 @@ static void dce_v6_0_audio_write_latency_fields(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1174,7 +1174,7 @@ static void dce_v6_0_audio_write_latency_fields(struct drm_encoder *encoder, static void dce_v6_0_audio_write_speaker_allocation(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1235,7 +1235,7 @@ static void dce_v6_0_audio_write_speaker_allocation(struct drm_encoder *encoder) static void dce_v6_0_audio_write_sad_regs(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1392,7 +1392,7 @@ static void dce_v6_0_audio_fini(struct amdgpu_device *adev) static void dce_v6_0_audio_set_vbi_packet(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1408,7 +1408,7 @@ static void dce_v6_0_audio_set_acr(struct drm_encoder *encoder, uint32_t clock, int bpc) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_afmt_acr acr = amdgpu_afmt_acr(clock); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1446,7 +1446,7 @@ static void dce_v6_0_audio_set_avi_infoframe(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1488,7 +1488,7 @@ static void dce_v6_0_audio_set_avi_infoframe(struct drm_encoder *encoder, static void dce_v6_0_audio_set_dto(struct drm_encoder *encoder, u32 clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); int em = amdgpu_atombios_encoder_get_encoder_mode(encoder); u32 tmp; @@ -1522,7 +1522,7 @@ static void dce_v6_0_audio_set_dto(struct drm_encoder *encoder, u32 clock) static void dce_v6_0_audio_set_packet(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1566,7 +1566,7 @@ static void dce_v6_0_audio_set_packet(struct drm_encoder *encoder) static void dce_v6_0_audio_set_mute(struct drm_encoder *encoder, bool mute) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1579,7 +1579,7 @@ static void dce_v6_0_audio_set_mute(struct drm_encoder *encoder, bool mute) static void dce_v6_0_audio_hdmi_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1616,7 +1616,7 @@ static void dce_v6_0_audio_hdmi_enable(struct drm_encoder *encoder, bool enable) static void dce_v6_0_audio_dp_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 tmp; @@ -1645,7 +1645,7 @@ static void dce_v6_0_afmt_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1714,7 +1714,7 @@ static void dce_v6_0_afmt_setmode(struct drm_encoder *encoder, static void dce_v6_0_afmt_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1788,7 +1788,7 @@ static void dce_v6_0_vga_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 vga_control; vga_control = RREG32(vga_control_regs[amdgpu_crtc->crtc_id]) & ~1; @@ -1799,7 +1799,7 @@ static void dce_v6_0_grph_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); WREG32(mmGRPH_ENABLE + amdgpu_crtc->crtc_offset, enable ? 1 : 0); } @@ -1810,7 +1810,7 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_framebuffer *target_fb; struct drm_gem_object *obj; struct amdgpu_bo *abo; @@ -2033,7 +2033,7 @@ static void dce_v6_0_set_interleave(struct drm_crtc *crtc, struct drm_display_mode *mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); if (mode->flags & DRM_MODE_FLAG_INTERLACE) @@ -2048,7 +2048,7 @@ static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc) struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u16 *r, *g, *b; int i; @@ -2148,7 +2148,7 @@ static u32 dce_v6_0_pick_pll(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 pll_in_use; int pll; @@ -2177,7 +2177,7 @@ static u32 dce_v6_0_pick_pll(struct drm_crtc *crtc) static void dce_v6_0_lock_cursor(struct drm_crtc *crtc, bool lock) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); uint32_t cur_lock; @@ -2192,7 +2192,7 @@ static void dce_v6_0_lock_cursor(struct drm_crtc *crtc, bool lock) static void dce_v6_0_hide_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); WREG32(mmCUR_CONTROL + amdgpu_crtc->crtc_offset, (CURSOR_24_8_PRE_MULT << CUR_CONTROL__CURSOR_MODE__SHIFT) | @@ -2204,7 +2204,7 @@ static void dce_v6_0_hide_cursor(struct drm_crtc *crtc) static void dce_v6_0_show_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset, upper_32_bits(amdgpu_crtc->cursor_addr)); @@ -2222,7 +2222,7 @@ static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); int xorigin = 0, yorigin = 0; int w = amdgpu_crtc->cursor_width; @@ -2397,7 +2397,7 @@ static const struct drm_crtc_funcs dce_v6_0_crtc_funcs = { static void dce_v6_0_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); unsigned type; @@ -2447,7 +2447,7 @@ static void dce_v6_0_crtc_disable(struct drm_crtc *crtc) struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_atom_ss ss; int i; @@ -2591,7 +2591,7 @@ static int dce_v6_0_crtc_init(struct amdgpu_device *adev, int index) if (amdgpu_crtc == NULL) return -ENOMEM; - drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_v6_0_crtc_funcs); + drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_v6_0_crtc_funcs); drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); amdgpu_crtc->crtc_id = index; @@ -2599,8 +2599,8 @@ static int dce_v6_0_crtc_init(struct amdgpu_device *adev, int index) amdgpu_crtc->max_cursor_width = CURSOR_WIDTH; amdgpu_crtc->max_cursor_height = CURSOR_HEIGHT; - adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; - adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; + adev_to_drm(adev)->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; + adev_to_drm(adev)->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; amdgpu_crtc->crtc_offset = crtc_offsets[amdgpu_crtc->crtc_id]; @@ -2669,20 +2669,20 @@ static int dce_v6_0_sw_init(void *handle) adev->mode_info.mode_config_initialized = true; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; - adev->ddev->mode_config.async_page_flip = true; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; + adev_to_drm(adev)->mode_config.async_page_flip = true; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; r = amdgpu_display_modeset_create_props(adev); if (r) return r; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; /* allocate crtcs */ for (i = 0; i < adev->mode_info.num_crtc; i++) { @@ -2693,7 +2693,7 @@ static int dce_v6_0_sw_init(void *handle) ret = amdgpu_atombios_get_connector_info_from_object_table(adev); if (ret) - amdgpu_display_print_display_setup(adev->ddev); + amdgpu_display_print_display_setup(adev_to_drm(adev)); else return -EINVAL; @@ -2706,7 +2706,7 @@ static int dce_v6_0_sw_init(void *handle) if (r) return r; - drm_kms_helper_poll_init(adev->ddev); + drm_kms_helper_poll_init(adev_to_drm(adev)); return r; } @@ -2717,12 +2717,12 @@ static int dce_v6_0_sw_fini(void *handle) kfree(adev->mode_info.bios_hardcoded_edid); - drm_kms_helper_poll_fini(adev->ddev); + drm_kms_helper_poll_fini(adev_to_drm(adev)); dce_v6_0_audio_fini(adev); dce_v6_0_afmt_fini(adev); - drm_mode_config_cleanup(adev->ddev); + drm_mode_config_cleanup(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = false; return 0; @@ -2967,7 +2967,7 @@ static int dce_v6_0_crtc_irq(struct amdgpu_device *adev, DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); if (amdgpu_irq_enabled(adev, source, irq_type)) { - drm_handle_vblank(adev->ddev, crtc); + drm_handle_vblank(adev_to_drm(adev), crtc); } DRM_DEBUG("IH: D%d vblank\n", crtc + 1); break; @@ -3036,14 +3036,14 @@ static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev, if (amdgpu_crtc == NULL) return 0; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); works = amdgpu_crtc->pflip_works; if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED){ DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " "AMDGPU_FLIP_SUBMITTED(%d)\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return 0; } @@ -3055,7 +3055,7 @@ static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev, if (works->event) drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); @@ -3146,7 +3146,7 @@ dce_v6_0_encoder_mode_set(struct drm_encoder *encoder, static void dce_v6_0_encoder_prepare(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -3187,7 +3187,7 @@ static void dce_v6_0_encoder_commit(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* need to call this here as we need the crtc set up */ amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_ON); @@ -3297,7 +3297,7 @@ static void dce_v6_0_encoder_add(struct amdgpu_device *adev, uint32_t supported_device, u16 caps) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_encoder *encoder; struct amdgpu_encoder *amdgpu_encoder; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index fa0ad50b628c..3603e5f13077 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -273,7 +273,7 @@ static void dce_v8_0_hpd_set_polarity(struct amdgpu_device *adev, */ static void dce_v8_0_hpd_init(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -318,7 +318,7 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev) */ static void dce_v8_0_hpd_fini(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; u32 tmp; @@ -444,7 +444,7 @@ void dce_v8_0_disable_dce(struct amdgpu_device *adev) static void dce_v8_0_program_fmt(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1146,7 +1146,7 @@ static struct amdgpu_audio_pin *dce_v8_0_audio_get_pin(struct amdgpu_device *ade static void dce_v8_0_afmt_audio_select_pin(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 offset; @@ -1164,7 +1164,7 @@ static void dce_v8_0_audio_write_latency_fields(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1225,7 +1225,7 @@ static void dce_v8_0_audio_write_latency_fields(struct drm_encoder *encoder, static void dce_v8_0_audio_write_speaker_allocation(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector; @@ -1278,7 +1278,7 @@ static void dce_v8_0_audio_write_speaker_allocation(struct drm_encoder *encoder) static void dce_v8_0_audio_write_sad_regs(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; u32 offset; @@ -1446,7 +1446,7 @@ static void dce_v8_0_audio_fini(struct amdgpu_device *adev) static void dce_v8_0_afmt_update_ACR(struct drm_encoder *encoder, uint32_t clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_afmt_acr acr = amdgpu_afmt_acr(clock); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1469,7 +1469,7 @@ static void dce_v8_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, void *buffer, size_t size) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; uint32_t offset = dig->afmt->offset; @@ -1489,7 +1489,7 @@ static void dce_v8_0_afmt_update_avi_infoframe(struct drm_encoder *encoder, static void dce_v8_0_audio_set_dto(struct drm_encoder *encoder, u32 clock) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); @@ -1516,7 +1516,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -1678,7 +1678,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder, static void dce_v8_0_afmt_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; @@ -1751,7 +1751,7 @@ static void dce_v8_0_vga_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 vga_control; vga_control = RREG32(vga_control_regs[amdgpu_crtc->crtc_id]) & ~1; @@ -1765,7 +1765,7 @@ static void dce_v8_0_grph_enable(struct drm_crtc *crtc, bool enable) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); if (enable) WREG32(mmGRPH_ENABLE + amdgpu_crtc->crtc_offset, 1); @@ -1779,7 +1779,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc, { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_framebuffer *target_fb; struct drm_gem_object *obj; struct amdgpu_bo *abo; @@ -2004,7 +2004,7 @@ static void dce_v8_0_set_interleave(struct drm_crtc *crtc, struct drm_display_mode *mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); if (mode->flags & DRM_MODE_FLAG_INTERLACE) @@ -2018,7 +2018,7 @@ static void dce_v8_0_crtc_load_lut(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u16 *r, *g, *b; int i; @@ -2140,7 +2140,7 @@ static u32 dce_v8_0_pick_pll(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 pll_in_use; int pll; @@ -2188,7 +2188,7 @@ static u32 dce_v8_0_pick_pll(struct drm_crtc *crtc) static void dce_v8_0_lock_cursor(struct drm_crtc *crtc, bool lock) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); uint32_t cur_lock; @@ -2203,7 +2203,7 @@ static void dce_v8_0_lock_cursor(struct drm_crtc *crtc, bool lock) static void dce_v8_0_hide_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); WREG32(mmCUR_CONTROL + amdgpu_crtc->crtc_offset, (CURSOR_24_8_PRE_MULT << CUR_CONTROL__CURSOR_MODE__SHIFT) | @@ -2213,7 +2213,7 @@ static void dce_v8_0_hide_cursor(struct drm_crtc *crtc) static void dce_v8_0_show_cursor(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset, upper_32_bits(amdgpu_crtc->cursor_addr)); @@ -2230,7 +2230,7 @@ static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); int xorigin = 0, yorigin = 0; amdgpu_crtc->cursor_x = x; @@ -2404,7 +2404,7 @@ static const struct drm_crtc_funcs dce_v8_0_crtc_funcs = { static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); unsigned type; @@ -2458,7 +2458,7 @@ static void dce_v8_0_crtc_disable(struct drm_crtc *crtc) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_atom_ss ss; int i; @@ -2609,7 +2609,7 @@ static int dce_v8_0_crtc_init(struct amdgpu_device *adev, int index) if (amdgpu_crtc == NULL) return -ENOMEM; - drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_v8_0_crtc_funcs); + drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_v8_0_crtc_funcs); drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); amdgpu_crtc->crtc_id = index; @@ -2617,8 +2617,8 @@ static int dce_v8_0_crtc_init(struct amdgpu_device *adev, int index) amdgpu_crtc->max_cursor_width = CIK_CURSOR_WIDTH; amdgpu_crtc->max_cursor_height = CIK_CURSOR_HEIGHT; - adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; - adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; + adev_to_drm(adev)->mode_config.cursor_width = amdgpu_crtc->max_cursor_width; + adev_to_drm(adev)->mode_config.cursor_height = amdgpu_crtc->max_cursor_height; amdgpu_crtc->crtc_offset = crtc_offsets[amdgpu_crtc->crtc_id]; @@ -2689,24 +2689,24 @@ static int dce_v8_0_sw_init(void *handle) if (r) return r; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; + adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; - adev->ddev->mode_config.async_page_flip = true; + adev_to_drm(adev)->mode_config.async_page_flip = true; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; r = amdgpu_display_modeset_create_props(adev); if (r) return r; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; /* allocate crtcs */ for (i = 0; i < adev->mode_info.num_crtc; i++) { @@ -2716,7 +2716,7 @@ static int dce_v8_0_sw_init(void *handle) } if (amdgpu_atombios_get_connector_info_from_object_table(adev)) - amdgpu_display_print_display_setup(adev->ddev); + amdgpu_display_print_display_setup(adev_to_drm(adev)); else return -EINVAL; @@ -2729,7 +2729,7 @@ static int dce_v8_0_sw_init(void *handle) if (r) return r; - drm_kms_helper_poll_init(adev->ddev); + drm_kms_helper_poll_init(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = true; return 0; @@ -2741,13 +2741,13 @@ static int dce_v8_0_sw_fini(void *handle) kfree(adev->mode_info.bios_hardcoded_edid); - drm_kms_helper_poll_fini(adev->ddev); + drm_kms_helper_poll_fini(adev_to_drm(adev)); dce_v8_0_audio_fini(adev); dce_v8_0_afmt_fini(adev); - drm_mode_config_cleanup(adev->ddev); + drm_mode_config_cleanup(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = false; return 0; @@ -3057,7 +3057,7 @@ static int dce_v8_0_crtc_irq(struct amdgpu_device *adev, DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); if (amdgpu_irq_enabled(adev, source, irq_type)) { - drm_handle_vblank(adev->ddev, crtc); + drm_handle_vblank(adev_to_drm(adev), crtc); } DRM_DEBUG("IH: D%d vblank\n", crtc + 1); break; @@ -3126,14 +3126,14 @@ static int dce_v8_0_pageflip_irq(struct amdgpu_device *adev, if (amdgpu_crtc == NULL) return 0; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); works = amdgpu_crtc->pflip_works; if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED){ DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " "AMDGPU_FLIP_SUBMITTED(%d)\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return 0; } @@ -3145,7 +3145,7 @@ static int dce_v8_0_pageflip_irq(struct amdgpu_device *adev, if (works->event) drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); @@ -3233,7 +3233,7 @@ dce_v8_0_encoder_mode_set(struct drm_encoder *encoder, static void dce_v8_0_encoder_prepare(struct drm_encoder *encoder) { - struct amdgpu_device *adev = encoder->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(encoder->dev); struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); @@ -3273,7 +3273,7 @@ static void dce_v8_0_encoder_prepare(struct drm_encoder *encoder) static void dce_v8_0_encoder_commit(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); /* need to call this here as we need the crtc set up */ amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_ON); @@ -3373,7 +3373,7 @@ static void dce_v8_0_encoder_add(struct amdgpu_device *adev, uint32_t supported_device, u16 caps) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_encoder *encoder; struct amdgpu_encoder *amdgpu_encoder; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c index d5ff7b6331ff..cc93577dee03 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c @@ -132,7 +132,7 @@ static const struct drm_crtc_funcs dce_virtual_crtc_funcs = { static void dce_virtual_crtc_dpms(struct drm_crtc *crtc, int mode) { struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); unsigned type; @@ -235,7 +235,7 @@ static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index) if (amdgpu_crtc == NULL) return -ENOMEM; - drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_virtual_crtc_funcs); + drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_virtual_crtc_funcs); drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); amdgpu_crtc->crtc_id = index; @@ -374,24 +374,24 @@ static int dce_virtual_sw_init(void *handle) if (r) return r; - adev->ddev->max_vblank_count = 0; + adev_to_drm(adev)->max_vblank_count = 0; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; + adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; r = amdgpu_display_modeset_create_props(adev); if (r) return r; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; /* allocate crtcs, encoders, connectors */ for (i = 0; i < adev->mode_info.num_crtc; i++) { @@ -403,7 +403,7 @@ static int dce_virtual_sw_init(void *handle) return r; } - drm_kms_helper_poll_init(adev->ddev); + drm_kms_helper_poll_init(adev_to_drm(adev)); adev->mode_info.mode_config_initialized = true; return 0; @@ -415,9 +415,9 @@ static int dce_virtual_sw_fini(void *handle) kfree(adev->mode_info.bios_hardcoded_edid); - drm_kms_helper_poll_fini(adev->ddev); + drm_kms_helper_poll_fini(adev_to_drm(adev)); - drm_mode_config_cleanup(adev->ddev); + drm_mode_config_cleanup(adev_to_drm(adev)); /* clear crtcs pointer to avoid dce irq finish routine access freed data */ memset(adev->mode_info.crtcs, 0, sizeof(adev->mode_info.crtcs[0]) * AMDGPU_MAX_CRTCS); adev->mode_info.mode_config_initialized = false; @@ -602,7 +602,7 @@ static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev, if (!encoder) return -ENOMEM; encoder->possible_crtcs = 1 << index; - drm_encoder_init(adev->ddev, encoder, &dce_virtual_encoder_funcs, + drm_encoder_init(adev_to_drm(adev), encoder, &dce_virtual_encoder_funcs, DRM_MODE_ENCODER_VIRTUAL, NULL); drm_encoder_helper_add(encoder, &dce_virtual_encoder_helper_funcs); @@ -613,7 +613,7 @@ static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev, } /* add a new connector */ - drm_connector_init(adev->ddev, connector, &dce_virtual_connector_funcs, + drm_connector_init(adev_to_drm(adev), connector, &dce_virtual_connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL); drm_connector_helper_add(connector, &dce_virtual_connector_helper_funcs); connector->display_info.subpixel_order = SubPixelHorizontalRGB; @@ -663,14 +663,14 @@ static int dce_virtual_pageflip(struct amdgpu_device *adev, if (amdgpu_crtc == NULL) return 0; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); works = amdgpu_crtc->pflip_works; if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " "AMDGPU_FLIP_SUBMITTED(%d)\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return 0; } @@ -682,7 +682,7 @@ static int dce_virtual_pageflip(struct amdgpu_device *adev, if (works->event) drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_crtc_vblank_put(&amdgpu_crtc->base); amdgpu_bo_unref(&works->old_abo); @@ -697,7 +697,7 @@ static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vbla struct amdgpu_crtc *amdgpu_crtc = container_of(vblank_timer, struct amdgpu_crtc, vblank_timer); struct drm_device *ddev = amdgpu_crtc->base.dev; - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); drm_handle_vblank(ddev, amdgpu_crtc->crtc_id); dce_virtual_pageflip(adev, amdgpu_crtc->crtc_id); diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c index 1ab261836983..2eab808fffeb 100644 --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c @@ -251,7 +251,7 @@ static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev, int i, count; ddev = dev_get_drvdata(dev); - adev = ddev->dev_private; + adev = drm_to_adev(ddev); count = 0; for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) { @@ -646,7 +646,7 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev, uint64_t config, uint64_t *count) { - uint32_t lo_base_addr, hi_base_addr, lo_val = 0, hi_val = 0; + uint32_t lo_base_addr = 0, hi_base_addr = 0, lo_val = 0, hi_val = 0; *count = 0; switch (adev->asic_type) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 65997ffaed45..d502e30f67d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -3307,6 +3307,29 @@ static void gfx_v10_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) adev->gfx.kiq.pmf = &gfx_v10_0_kiq_pm4_funcs; } +static void gfx_v10_0_init_spm_golden_registers(struct amdgpu_device *adev) +{ + switch (adev->asic_type) { + case CHIP_NAVI10: + soc15_program_register_sequence(adev, + golden_settings_gc_rlc_spm_10_0_nv10, + (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_0_nv10)); + break; + case CHIP_NAVI14: + soc15_program_register_sequence(adev, + golden_settings_gc_rlc_spm_10_1_nv14, + (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_1_nv14)); + break; + case CHIP_NAVI12: + soc15_program_register_sequence(adev, + golden_settings_gc_rlc_spm_10_1_2_nv12, + (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_1_2_nv12)); + break; + default: + break; + } +} + static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev) { switch (adev->asic_type) { @@ -3317,9 +3340,6 @@ static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev) soc15_program_register_sequence(adev, golden_settings_gc_10_0_nv10, (const u32)ARRAY_SIZE(golden_settings_gc_10_0_nv10)); - soc15_program_register_sequence(adev, - golden_settings_gc_rlc_spm_10_0_nv10, - (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_0_nv10)); break; case CHIP_NAVI14: soc15_program_register_sequence(adev, @@ -3328,9 +3348,6 @@ static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev) soc15_program_register_sequence(adev, golden_settings_gc_10_1_nv14, (const u32)ARRAY_SIZE(golden_settings_gc_10_1_nv14)); - soc15_program_register_sequence(adev, - golden_settings_gc_rlc_spm_10_1_nv14, - (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_1_nv14)); break; case CHIP_NAVI12: soc15_program_register_sequence(adev, @@ -3339,9 +3356,6 @@ static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev) soc15_program_register_sequence(adev, golden_settings_gc_10_1_2_nv12, (const u32)ARRAY_SIZE(golden_settings_gc_10_1_2_nv12)); - soc15_program_register_sequence(adev, - golden_settings_gc_rlc_spm_10_1_2_nv12, - (const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_10_1_2_nv12)); break; case CHIP_SIENNA_CICHLID: soc15_program_register_sequence(adev, @@ -3360,6 +3374,7 @@ static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev) default: break; } + gfx_v10_0_init_spm_golden_registers(adev); } static void gfx_v10_0_scratch_init(struct amdgpu_device *adev) @@ -4022,21 +4037,23 @@ static int gfx_v10_0_mec_init(struct amdgpu_device *adev) amdgpu_gfx_compute_queue_acquire(adev); mec_hpd_size = adev->gfx.num_compute_rings * GFX10_MEC_HPD_SIZE; - r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_GTT, - &adev->gfx.mec.hpd_eop_obj, - &adev->gfx.mec.hpd_eop_gpu_addr, - (void **)&hpd); - if (r) { - dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); - gfx_v10_0_mec_fini(adev); - return r; - } + if (mec_hpd_size) { + r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, + &adev->gfx.mec.hpd_eop_obj, + &adev->gfx.mec.hpd_eop_gpu_addr, + (void **)&hpd); + if (r) { + dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); + gfx_v10_0_mec_fini(adev); + return r; + } - memset(hpd, 0, mec_hpd_size); + memset(hpd, 0, mec_hpd_size); - amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); - amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + } if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; @@ -4147,6 +4164,7 @@ static const struct amdgpu_gfx_funcs gfx_v10_0_gfx_funcs = { .read_wave_sgprs = &gfx_v10_0_read_wave_sgprs, .read_wave_vgprs = &gfx_v10_0_read_wave_vgprs, .select_me_pipe_q = &gfx_v10_0_select_me_pipe_q, + .init_spm_golden = &gfx_v10_0_init_spm_golden_registers, }; static void gfx_v10_0_gpu_early_init(struct amdgpu_device *adev) @@ -6180,7 +6198,7 @@ static int gfx_v10_0_gfx_init_queue(struct amdgpu_ring *ring) struct v10_gfx_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.gfx_ring[0]; - if (!adev->in_gpu_reset && !adev->in_suspend) { + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { memset((void *)mqd, 0, sizeof(*mqd)); mutex_lock(&adev->srbm_mutex); nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); @@ -6192,7 +6210,7 @@ static int gfx_v10_0_gfx_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); if (adev->gfx.me.mqd_backup[mqd_idx]) memcpy(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); - } else if (adev->in_gpu_reset) { + } else if (amdgpu_in_reset(adev)) { /* reset mqd with the backup copy */ if (adev->gfx.me.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); @@ -6433,6 +6451,10 @@ static int gfx_v10_0_kiq_init_register(struct amdgpu_ring *ring) struct v10_compute_mqd *mqd = ring->mqd_ptr; int j; + /* inactivate the queue */ + if (amdgpu_sriov_vf(adev)) + WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0); + /* disable wptr polling */ WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); @@ -6541,7 +6563,7 @@ static int gfx_v10_0_kiq_init_queue(struct amdgpu_ring *ring) gfx_v10_0_kiq_setting(ring); - if (adev->in_gpu_reset) { /* for GPU_RESET case */ + if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); @@ -6577,7 +6599,7 @@ static int gfx_v10_0_kcq_init_queue(struct amdgpu_ring *ring) struct v10_compute_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.compute_ring[0]; - if (!adev->in_gpu_reset && !adev->in_suspend) { + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { memset((void *)mqd, 0, sizeof(*mqd)); mutex_lock(&adev->srbm_mutex); nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); @@ -6587,7 +6609,7 @@ static int gfx_v10_0_kcq_init_queue(struct amdgpu_ring *ring) if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); - } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ + } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); @@ -7033,8 +7055,7 @@ static int gfx_v10_0_soft_reset(void *handle) GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | GRBM_STATUS__GDS_BUSY_MASK | - GRBM_STATUS__SPI_BUSY_MASK | GRBM_STATUS__GE_BUSY_NO_DMA_MASK - | GRBM_STATUS__BCI_BUSY_MASK)) { + GRBM_STATUS__SPI_BUSY_MASK | GRBM_STATUS__GE_BUSY_NO_DMA_MASK)) { grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP, 1); @@ -7159,7 +7180,7 @@ static int gfx_v10_0_early_init(void *handle) break; } - adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + adev->gfx.num_compute_rings = amdgpu_num_kcq; gfx_v10_0_set_kiq_pm4_funcs(adev); gfx_v10_0_set_ring_funcs(adev); @@ -7263,10 +7284,8 @@ static void gfx_v10_0_update_medium_grain_clock_gating(struct amdgpu_device *ade def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | - RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); - - /* only for Vega10 & Raven1 */ - data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; + RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK | + RLC_CGTT_MGCG_OVERRIDE__ENABLE_CGTS_LEGACY_MASK); if (def != data) WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); @@ -7429,7 +7448,6 @@ static int gfx_v10_0_update_gfx_clock_gating(struct amdgpu_device *adev, (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_CGLS | AMD_CG_SUPPORT_GFX_CGCG | - AMD_CG_SUPPORT_GFX_CGLS | AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) gfx_v10_0_enable_gui_idle_interrupt(adev, enable); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 33f1c4a46ebe..94b7e0531d09 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -1343,21 +1343,22 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev) amdgpu_gfx_compute_queue_acquire(adev); mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; + if (mec_hpd_size) { + r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.mec.hpd_eop_obj, + &adev->gfx.mec.hpd_eop_gpu_addr, + (void **)&hpd); + if (r) { + dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); + return r; + } - r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, - &adev->gfx.mec.hpd_eop_obj, - &adev->gfx.mec.hpd_eop_gpu_addr, - (void **)&hpd); - if (r) { - dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); - return r; - } - - memset(hpd, 0, mec_hpd_size); + memset(hpd, 0, mec_hpd_size); - amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); - amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + } return 0; } @@ -3250,7 +3251,7 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) dev_warn(adev->dev, "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n", adev->asic_type); - /* fall through */ + fallthrough; case CHIP_CARRIZO: modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | @@ -4632,7 +4633,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) gfx_v8_0_kiq_setting(ring); - if (adev->in_gpu_reset) { /* for GPU_RESET case */ + if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); @@ -4669,7 +4670,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) struct vi_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.compute_ring[0]; - if (!adev->in_gpu_reset && !adev->in_suspend) { + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; @@ -4681,7 +4682,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); - } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ + } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); @@ -5294,7 +5295,7 @@ static int gfx_v8_0_early_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; - adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + adev->gfx.num_compute_rings = amdgpu_num_kcq; adev->gfx.funcs = &gfx_v8_0_gfx_funcs; gfx_v8_0_set_ring_funcs(adev); gfx_v8_0_set_irq_funcs(adev); @@ -5342,10 +5343,9 @@ static int gfx_v8_0_late_init(void *handle) static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, bool enable) { - if (((adev->asic_type == CHIP_POLARIS11) || + if ((adev->asic_type == CHIP_POLARIS11) || (adev->asic_type == CHIP_POLARIS12) || - (adev->asic_type == CHIP_VEGAM)) && - adev->powerplay.pp_funcs->set_powergating_by_smu) + (adev->asic_type == CHIP_VEGAM)) /* Send msg to SMU via Powerplay */ amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable); @@ -5879,8 +5879,7 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_CG, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { @@ -5901,8 +5900,7 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_MG, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } return 0; @@ -5931,8 +5929,7 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_CG, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { @@ -5951,8 +5948,7 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_3D, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { @@ -5973,8 +5969,7 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_MG, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { @@ -5989,8 +5984,7 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_RLC, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { @@ -6004,8 +5998,7 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, PP_BLOCK_GFX_CP, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index cb9d60a4e05e..93c63ff3b35e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -691,6 +691,7 @@ static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), + SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) }; static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { @@ -1938,22 +1939,23 @@ static int gfx_v9_0_mec_init(struct amdgpu_device *adev) /* take ownership of the relevant compute queues */ amdgpu_gfx_compute_queue_acquire(adev); mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; + if (mec_hpd_size) { + r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.mec.hpd_eop_obj, + &adev->gfx.mec.hpd_eop_gpu_addr, + (void **)&hpd); + if (r) { + dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); + gfx_v9_0_mec_fini(adev); + return r; + } - r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, - &adev->gfx.mec.hpd_eop_obj, - &adev->gfx.mec.hpd_eop_gpu_addr, - (void **)&hpd); - if (r) { - dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); - gfx_v9_0_mec_fini(adev); - return r; - } - - memset(hpd, 0, mec_hpd_size); + memset(hpd, 0, mec_hpd_size); - amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); - amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); + amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); + } mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; @@ -3684,7 +3686,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) gfx_v9_0_kiq_setting(ring); - if (adev->in_gpu_reset) { /* for GPU_RESET case */ + if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); @@ -3722,7 +3724,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) struct v9_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.compute_ring[0]; - if (!adev->in_gpu_reset && !adev->in_suspend) { + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; @@ -3734,7 +3736,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); - } else if (adev->in_gpu_reset) { /* for GPU_RESET case */ + } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); @@ -3928,7 +3930,7 @@ static int gfx_v9_0_hw_fini(void *handle) /* Use deinitialize sequence from CAIL when unbinding device from driver, * otherwise KIQ is hanging when binding back */ - if (!adev->in_gpu_reset && !adev->in_suspend) { + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { mutex_lock(&adev->srbm_mutex); soc15_grbm_select(adev, adev->gfx.kiq.ring.me, adev->gfx.kiq.ring.pipe, @@ -4086,7 +4088,7 @@ static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) * * also don't wait anymore for IRQ context * */ - if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) goto failed_kiq_read; might_sleep(); @@ -4625,7 +4627,7 @@ static int gfx_v9_0_early_init(void *handle) adev->gfx.num_gfx_rings = 0; else adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; - adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + adev->gfx.num_compute_rings = amdgpu_num_kcq; gfx_v9_0_set_kiq_pm4_funcs(adev); gfx_v9_0_set_ring_funcs(adev); gfx_v9_0_set_irq_funcs(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c index 46351db36922..bd85aed3523a 100755..100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c @@ -57,10 +57,10 @@ static const struct soc15_reg_entry gfx_v9_4_edc_counter_regs[] = { /* SPI */ { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1 }, /* SQ */ - { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16 }, - { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16 }, - { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16 }, - { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 8, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 8, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 8, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 8, 16 }, /* SQC */ { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6 }, { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6 }, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 394e6f56948a..b882ac59879a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -31,6 +31,77 @@ #include "soc15_common.h" +static const char *gfxhub_client_ids[] = { + "CB/DB", + "Reserved", + "GE1", + "GE2", + "CPF", + "CPC", + "CPG", + "RLC", + "TCP", + "SQC (inst)", + "SQC (data)", + "SQG", + "Reserved", + "SDMA0", + "SDMA1", + "GCR", + "SDMA2", + "SDMA3", +}; + +static uint32_t gfxhub_v2_0_get_invalidate_req(unsigned int vmid, + uint32_t flush_type) +{ + u32 req = 0; + + /* invalidate using legacy mode on vmid*/ + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, + PER_VMID_INVALIDATE_REQ, 1 << vmid); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, + CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); + + return req; +} + +static void +gfxhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev, + uint32_t status) +{ + u32 cid = REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, CID); + + dev_err(adev->dev, + "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", + status); + dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", + cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], + cid); + dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); + dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); + dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); + dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); + dev_err(adev->dev, "\t RW: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, RW)); +} + u64 gfxhub_v2_0_get_fb_location(struct amdgpu_device *adev) { u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE); @@ -360,6 +431,11 @@ void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp); } +static const struct amdgpu_vmhub_funcs gfxhub_v2_0_vmhub_funcs = { + .print_l2_protection_fault_status = gfxhub_v2_0_print_l2_protection_fault_status, + .get_invalidate_req = gfxhub_v2_0_get_invalidate_req, +}; + void gfxhub_v2_0_init(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; @@ -390,4 +466,14 @@ void gfxhub_v2_0_init(struct amdgpu_device *adev) mmGCVM_INVALIDATE_ENG0_REQ; hub->eng_addr_distance = mmGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; + + hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; + + hub->vmhub_funcs = &gfxhub_v2_0_vmhub_funcs; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c index fa0bca3e1f73..237a9ff5afa0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c @@ -31,6 +31,77 @@ #include "soc15_common.h" +static const char *gfxhub_client_ids[] = { + "CB/DB", + "Reserved", + "GE1", + "GE2", + "CPF", + "CPC", + "CPG", + "RLC", + "TCP", + "SQC (inst)", + "SQC (data)", + "SQG", + "Reserved", + "SDMA0", + "SDMA1", + "GCR", + "SDMA2", + "SDMA3", +}; + +static uint32_t gfxhub_v2_1_get_invalidate_req(unsigned int vmid, + uint32_t flush_type) +{ + u32 req = 0; + + /* invalidate using legacy mode on vmid*/ + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, + PER_VMID_INVALIDATE_REQ, 1 << vmid); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); + req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, + CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); + + return req; +} + +static void +gfxhub_v2_1_print_l2_protection_fault_status(struct amdgpu_device *adev, + uint32_t status) +{ + u32 cid = REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, CID); + + dev_err(adev->dev, + "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", + status); + dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", + cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], + cid); + dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); + dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); + dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); + dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); + dev_err(adev->dev, "\t RW: 0x%lx\n", + REG_GET_FIELD(status, + GCVM_L2_PROTECTION_FAULT_STATUS, RW)); +} + u64 gfxhub_v2_1_get_fb_location(struct amdgpu_device *adev) { u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE); @@ -135,6 +206,12 @@ static void gfxhub_v2_1_init_cache_regs(struct amdgpu_device *adev) { uint32_t tmp; + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + /* Setup L2 cache */ tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL); tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1); @@ -190,6 +267,12 @@ static void gfxhub_v2_1_enable_system_domain(struct amdgpu_device *adev) static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev) { + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 0xFFFFFFFF); WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, @@ -326,6 +409,13 @@ void gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device *adev, bool value) { u32 tmp; + + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL); tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); @@ -359,6 +449,11 @@ void gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device *adev, WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp); } +static const struct amdgpu_vmhub_funcs gfxhub_v2_1_vmhub_funcs = { + .print_l2_protection_fault_status = gfxhub_v2_1_print_l2_protection_fault_status, + .get_invalidate_req = gfxhub_v2_1_get_invalidate_req, +}; + void gfxhub_v2_1_init(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; @@ -389,6 +484,16 @@ void gfxhub_v2_1_init(struct amdgpu_device *adev) mmGCVM_INVALIDATE_ENG0_REQ; hub->eng_addr_distance = mmGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; + + hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; + + hub->vmhub_funcs = &gfxhub_v2_1_vmhub_funcs; } int gfxhub_v2_1_get_xgmi_info(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index ec90c62078d9..31359e519d69 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -25,11 +25,10 @@ #include "amdgpu.h" #include "amdgpu_atomfirmware.h" #include "gmc_v10_0.h" +#include "umc_v8_7.h" #include "hdp/hdp_5_0_0_offset.h" #include "hdp/hdp_5_0_0_sh_mask.h" -#include "gc/gc_10_1_0_sh_mask.h" -#include "mmhub/mmhub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_offset.h" #include "dcn/dcn_2_0_0_offset.h" @@ -57,68 +56,31 @@ static const struct soc15_reg_golden golden_settings_navi10_hdp[] = }; #endif +static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned type, + enum amdgpu_interrupt_state state) +{ + return 0; +} + static int gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type, enum amdgpu_interrupt_state state) { - struct amdgpu_vmhub *hub; - u32 tmp, reg, bits[AMDGPU_MAX_VMHUBS], i; - - bits[AMDGPU_GFXHUB_0] = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; - - bits[AMDGPU_MMHUB_0] = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | - MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; - switch (state) { case AMDGPU_IRQ_STATE_DISABLE: /* MM HUB */ - hub = &adev->vmhub[AMDGPU_MMHUB_0]; - for (i = 0; i < 16; i++) { - reg = hub->vm_context0_cntl + hub->ctx_distance * i; - tmp = RREG32(reg); - tmp &= ~bits[AMDGPU_MMHUB_0]; - WREG32(reg, tmp); - } - + amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_0, false); /* GFX HUB */ - hub = &adev->vmhub[AMDGPU_GFXHUB_0]; - for (i = 0; i < 16; i++) { - reg = hub->vm_context0_cntl + hub->ctx_distance * i; - tmp = RREG32(reg); - tmp &= ~bits[AMDGPU_GFXHUB_0]; - WREG32(reg, tmp); - } + amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_0, false); break; case AMDGPU_IRQ_STATE_ENABLE: /* MM HUB */ - hub = &adev->vmhub[AMDGPU_MMHUB_0]; - for (i = 0; i < 16; i++) { - reg = hub->vm_context0_cntl + hub->ctx_distance * i; - tmp = RREG32(reg); - tmp |= bits[AMDGPU_MMHUB_0]; - WREG32(reg, tmp); - } - + amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_0, true); /* GFX HUB */ - hub = &adev->vmhub[AMDGPU_GFXHUB_0]; - for (i = 0; i < 16; i++) { - reg = hub->vm_context0_cntl + hub->ctx_distance * i; - tmp = RREG32(reg); - tmp |= bits[AMDGPU_GFXHUB_0]; - WREG32(reg, tmp); - } + amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_0, true); break; default: break; @@ -166,29 +128,8 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, task_info.task_name, task_info.pid); dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", addr, entry->client_id); - if (!amdgpu_sriov_vf(adev)) { - dev_err(adev->dev, - "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", - status); - dev_err(adev->dev, "\t Faulty UTCL2 client ID: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, CID)); - dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); - dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); - dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); - dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); - dev_err(adev->dev, "\t RW: 0x%lx\n", - REG_GET_FIELD(status, - GCVM_L2_PROTECTION_FAULT_STATUS, RW)); - } + if (!amdgpu_sriov_vf(adev)) + hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); } return 0; @@ -199,30 +140,20 @@ static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = { .process = gmc_v10_0_process_interrupt, }; -static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev) +static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = { + .set = gmc_v10_0_ecc_interrupt_state, + .process = amdgpu_umc_process_ecc_irq, +}; + + static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev) { adev->gmc.vm_fault.num_types = 1; adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs; -} -static uint32_t gmc_v10_0_get_invalidate_req(unsigned int vmid, - uint32_t flush_type) -{ - u32 req = 0; - - /* invalidate using legacy mode on vmid*/ - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, - PER_VMID_INVALIDATE_REQ, 1 << vmid); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); - req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, - CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); - - return req; + if (!amdgpu_sriov_vf(adev)) { + adev->gmc.ecc_irq.num_types = 1; + adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs; + } } /** @@ -265,7 +196,7 @@ static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, { bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub); struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; - u32 inv_req = gmc_v10_0_get_invalidate_req(vmid, flush_type); + u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); u32 tmp; /* Use register 17 for GART */ const unsigned eng = 17; @@ -356,16 +287,17 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, */ if (adev->gfx.kiq.ring.sched.ready && (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && - !adev->in_gpu_reset) { - + down_read_trylock(&adev->reset_sem)) { struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; const unsigned eng = 17; - u32 inv_req = gmc_v10_0_get_invalidate_req(vmid, flush_type); + u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 1 << vmid); + + up_read(&adev->reset_sem); return; } @@ -381,7 +313,7 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, if (!adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || - adev->in_gpu_reset || + amdgpu_in_reset(adev) || ring->sched.ready == false) { gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB_0, 0); mutex_unlock(&adev->mman.gtt_window_lock); @@ -459,7 +391,7 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, spin_unlock(&adev->gfx.kiq.ring_lock); r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); if (r < 1) { - DRM_ERROR("wait for kiq fence error: %ld.\n", r); + dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); return -ETIME; } @@ -491,7 +423,7 @@ static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, { bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub]; - uint32_t req = gmc_v10_0_get_invalidate_req(vmid, 0); + uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); unsigned eng = ring->vm_inv_eng; /* @@ -641,6 +573,28 @@ static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, } } +static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) +{ + u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); + unsigned size; + + if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { + size = AMDGPU_VBIOS_VGA_ALLOCATION; + } else { + u32 viewport; + u32 pitch; + + viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); + pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH); + size = (REG_GET_FIELD(viewport, + HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * + REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) * + 4); + } + + return size; +} + static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = { .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb, .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid, @@ -648,7 +602,8 @@ static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = { .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping, .map_mtype = gmc_v10_0_map_mtype, .get_vm_pde = gmc_v10_0_get_vm_pde, - .get_vm_pte = gmc_v10_0_get_vm_pte + .get_vm_pte = gmc_v10_0_get_vm_pte, + .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size, }; static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev) @@ -657,12 +612,36 @@ static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev) adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs; } +static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev) +{ + switch (adev->asic_type) { + case CHIP_SIENNA_CICHLID: + adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM; + adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM; + adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM; + adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA; + adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0]; + adev->umc.funcs = &umc_v8_7_funcs; + break; + default: + break; + } +} + + +static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev) +{ + adev->mmhub.funcs = &mmhub_v2_0_funcs; +} + static int gmc_v10_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + gmc_v10_0_set_mmhub_funcs(adev); gmc_v10_0_set_gmc_funcs(adev); gmc_v10_0_set_irq_funcs(adev); + gmc_v10_0_set_umc_funcs(adev); adev->gmc.shared_aperture_start = 0x2000000000000000ULL; adev->gmc.shared_aperture_end = @@ -685,6 +664,10 @@ static int gmc_v10_0_late_init(void *handle) if (r) return r; + r = amdgpu_gmc_ras_late_init(adev); + if (r) + return r; + return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); } @@ -789,36 +772,6 @@ static int gmc_v10_0_gart_init(struct amdgpu_device *adev) return amdgpu_gart_table_vram_alloc(adev); } -static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) -{ - u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); - unsigned size; - - if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ - } else { - u32 viewport; - u32 pitch; - - viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); - pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH); - size = (REG_GET_FIELD(viewport, - HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * - REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) * - 4); - } - /* return 0 if the pre-OS buffer uses up most of vram */ - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) { - DRM_ERROR("Warning: pre-OS buffer uses most of vram, \ - be aware of gart table overwrite\n"); - return 0; - } - - return size; -} - - - static int gmc_v10_0_sw_init(void *handle) { int r, vram_width = 0, vram_type = 0, vram_vendor = 0; @@ -830,7 +783,7 @@ static int gmc_v10_0_sw_init(void *handle) else gfxhub_v2_0_init(adev); - mmhub_v2_0_init(adev); + adev->mmhub.funcs->init(adev); spin_lock_init(&adev->gmc.invalidate_lock); @@ -878,6 +831,14 @@ static int gmc_v10_0_sw_init(void *handle) if (r) return r; + if (!amdgpu_sriov_vf(adev)) { + /* interrupt sent to DF. */ + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, + &adev->gmc.ecc_irq); + if (r) + return r; + } + /* * Set the internal MC address mask This is the max address of the GPU's * internal address space. @@ -900,7 +861,7 @@ static int gmc_v10_0_sw_init(void *handle) if (r) return r; - adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev); + amdgpu_gmc_get_vbios_allocations(adev); /* Memory manager */ r = amdgpu_bo_init(adev); @@ -991,7 +952,7 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) if (r) return r; - r = mmhub_v2_0_gart_enable(adev); + r = adev->mmhub.funcs->gart_enable(adev); if (r) return r; @@ -1013,7 +974,7 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) gfxhub_v2_1_set_fault_enable_default(adev, value); else gfxhub_v2_0_set_fault_enable_default(adev, value); - mmhub_v2_0_set_fault_enable_default(adev, value); + adev->mmhub.funcs->set_fault_enable_default(adev, value); gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB_0, 0); gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_0, 0); @@ -1038,6 +999,9 @@ static int gmc_v10_0_hw_init(void *handle) if (r) return r; + if (adev->umc.funcs && adev->umc.funcs->init_registers) + adev->umc.funcs->init_registers(adev); + return 0; } @@ -1055,7 +1019,7 @@ static void gmc_v10_0_gart_disable(struct amdgpu_device *adev) gfxhub_v2_1_gart_disable(adev); else gfxhub_v2_0_gart_disable(adev); - mmhub_v2_0_gart_disable(adev); + adev->mmhub.funcs->gart_disable(adev); amdgpu_gart_table_vram_unpin(adev); } @@ -1069,6 +1033,7 @@ static int gmc_v10_0_hw_fini(void *handle) return 0; } + amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); gmc_v10_0_gart_disable(adev); @@ -1121,7 +1086,7 @@ static int gmc_v10_0_set_clockgating_state(void *handle, int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - r = mmhub_v2_0_set_clockgating(adev, state); + r = adev->mmhub.funcs->set_clockgating(adev, state); if (r) return r; @@ -1136,7 +1101,7 @@ static void gmc_v10_0_get_clockgating_state(void *handle, u32 *flags) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - mmhub_v2_0_get_clockgating(adev, flags); + adev->mmhub.funcs->get_clockgating(adev, flags); if (adev->asic_type == CHIP_SIENNA_CICHLID || adev->asic_type == CHIP_NAVY_FLOUNDER) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index 538e7ee35cdf..95a9117e9564 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -805,16 +805,13 @@ static unsigned gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev) unsigned size; if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ + size = AMDGPU_VBIOS_VGA_ALLOCATION; } else { u32 viewport = RREG32(mmVIEWPORT_SIZE); size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * 4); } - /* return 0 if the pre-OS buffer uses up most of vram */ - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) - return 0; return size; } @@ -862,7 +859,7 @@ static int gmc_v6_0_sw_init(void *handle) if (r) return r; - adev->gmc.stolen_size = gmc_v6_0_get_vbios_fb_size(adev); + amdgpu_gmc_get_vbios_allocations(adev); r = amdgpu_bo_init(adev); if (r) @@ -1136,6 +1133,7 @@ static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = { .set_prt = gmc_v6_0_set_prt, .get_vm_pde = gmc_v6_0_get_vm_pde, .get_vm_pte = gmc_v6_0_get_vm_pte, + .get_vbios_fb_size = gmc_v6_0_get_vbios_fb_size, }; static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index e18296dc1386..80c146df338a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -434,7 +434,7 @@ static int gmc_v7_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, int vmid; unsigned int tmp; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; for (vmid = 1; vmid < 16; vmid++) { @@ -970,16 +970,14 @@ static unsigned gmc_v7_0_get_vbios_fb_size(struct amdgpu_device *adev) unsigned size; if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ + size = AMDGPU_VBIOS_VGA_ALLOCATION; } else { u32 viewport = RREG32(mmVIEWPORT_SIZE); size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * 4); } - /* return 0 if the pre-OS buffer uses up most of vram */ - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) - return 0; + return size; } @@ -1035,7 +1033,7 @@ static int gmc_v7_0_sw_init(void *handle) if (r) return r; - adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev); + amdgpu_gmc_get_vbios_allocations(adev); /* Memory manager */ r = amdgpu_bo_init(adev); @@ -1372,7 +1370,8 @@ static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = { .emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping, .set_prt = gmc_v7_0_set_prt, .get_vm_pde = gmc_v7_0_get_vm_pde, - .get_vm_pte = gmc_v7_0_get_vm_pte + .get_vm_pte = gmc_v7_0_get_vm_pte, + .get_vbios_fb_size = gmc_v7_0_get_vbios_fb_size, }; static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index a9e722b8a458..9ab65ca7df77 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -635,7 +635,7 @@ static int gmc_v8_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, int vmid; unsigned int tmp; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; for (vmid = 1; vmid < 16; vmid++) { @@ -1087,16 +1087,14 @@ static unsigned gmc_v8_0_get_vbios_fb_size(struct amdgpu_device *adev) unsigned size; if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ + size = AMDGPU_VBIOS_VGA_ALLOCATION; } else { u32 viewport = RREG32(mmVIEWPORT_SIZE); size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * 4); } - /* return 0 if the pre-OS buffer uses up most of vram */ - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) - return 0; + return size; } @@ -1160,7 +1158,7 @@ static int gmc_v8_0_sw_init(void *handle) if (r) return r; - adev->gmc.stolen_size = gmc_v8_0_get_vbios_fb_size(adev); + amdgpu_gmc_get_vbios_allocations(adev); /* Memory manager */ r = amdgpu_bo_init(adev); @@ -1739,7 +1737,8 @@ static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = { .emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping, .set_prt = gmc_v8_0_set_prt, .get_vm_pde = gmc_v8_0_get_vm_pde, - .get_vm_pte = gmc_v8_0_get_vm_pte + .get_vm_pte = gmc_v8_0_get_vm_pte, + .get_vbios_fb_size = gmc_v8_0_get_vbios_fb_size, }; static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 6e4f3ff4810f..91629c2b1d5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -67,6 +67,222 @@ #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L +#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d +#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 + + +static const char *gfxhub_client_ids[] = { + "CB", + "DB", + "IA", + "WD", + "CPF", + "CPC", + "CPG", + "RLC", + "TCP", + "SQC (inst)", + "SQC (data)", + "SQG", + "PA", +}; + +static const char *mmhub_client_ids_raven[][2] = { + [0][0] = "MP1", + [1][0] = "MP0", + [2][0] = "VCN", + [3][0] = "VCNU", + [4][0] = "HDP", + [5][0] = "DCE", + [13][0] = "UTCL2", + [19][0] = "TLS", + [26][0] = "OSS", + [27][0] = "SDMA0", + [0][1] = "MP1", + [1][1] = "MP0", + [2][1] = "VCN", + [3][1] = "VCNU", + [4][1] = "HDP", + [5][1] = "XDP", + [6][1] = "DBGU0", + [7][1] = "DCE", + [8][1] = "DCEDWB0", + [9][1] = "DCEDWB1", + [26][1] = "OSS", + [27][1] = "SDMA0", +}; + +static const char *mmhub_client_ids_renoir[][2] = { + [0][0] = "MP1", + [1][0] = "MP0", + [2][0] = "HDP", + [4][0] = "DCEDMC", + [5][0] = "DCEVGA", + [13][0] = "UTCL2", + [19][0] = "TLS", + [26][0] = "OSS", + [27][0] = "SDMA0", + [28][0] = "VCN", + [29][0] = "VCNU", + [30][0] = "JPEG", + [0][1] = "MP1", + [1][1] = "MP0", + [2][1] = "HDP", + [3][1] = "XDP", + [6][1] = "DBGU0", + [7][1] = "DCEDMC", + [8][1] = "DCEVGA", + [9][1] = "DCEDWB", + [26][1] = "OSS", + [27][1] = "SDMA0", + [28][1] = "VCN", + [29][1] = "VCNU", + [30][1] = "JPEG", +}; + +static const char *mmhub_client_ids_vega10[][2] = { + [0][0] = "MP0", + [1][0] = "UVD", + [2][0] = "UVDU", + [3][0] = "HDP", + [13][0] = "UTCL2", + [14][0] = "OSS", + [15][0] = "SDMA1", + [32+0][0] = "VCE0", + [32+1][0] = "VCE0U", + [32+2][0] = "XDMA", + [32+3][0] = "DCE", + [32+4][0] = "MP1", + [32+14][0] = "SDMA0", + [0][1] = "MP0", + [1][1] = "UVD", + [2][1] = "UVDU", + [3][1] = "DBGU0", + [4][1] = "HDP", + [5][1] = "XDP", + [14][1] = "OSS", + [15][1] = "SDMA0", + [32+0][1] = "VCE0", + [32+1][1] = "VCE0U", + [32+2][1] = "XDMA", + [32+3][1] = "DCE", + [32+4][1] = "DCEDWB", + [32+5][1] = "MP1", + [32+6][1] = "DBGU1", + [32+14][1] = "SDMA1", +}; + +static const char *mmhub_client_ids_vega12[][2] = { + [0][0] = "MP0", + [1][0] = "VCE0", + [2][0] = "VCE0U", + [3][0] = "HDP", + [13][0] = "UTCL2", + [14][0] = "OSS", + [15][0] = "SDMA1", + [32+0][0] = "DCE", + [32+1][0] = "XDMA", + [32+2][0] = "UVD", + [32+3][0] = "UVDU", + [32+4][0] = "MP1", + [32+15][0] = "SDMA0", + [0][1] = "MP0", + [1][1] = "VCE0", + [2][1] = "VCE0U", + [3][1] = "DBGU0", + [4][1] = "HDP", + [5][1] = "XDP", + [14][1] = "OSS", + [15][1] = "SDMA0", + [32+0][1] = "DCE", + [32+1][1] = "DCEDWB", + [32+2][1] = "XDMA", + [32+3][1] = "UVD", + [32+4][1] = "UVDU", + [32+5][1] = "MP1", + [32+6][1] = "DBGU1", + [32+15][1] = "SDMA1", +}; + +static const char *mmhub_client_ids_vega20[][2] = { + [0][0] = "XDMA", + [1][0] = "DCE", + [2][0] = "VCE0", + [3][0] = "VCE0U", + [4][0] = "UVD", + [5][0] = "UVD1U", + [13][0] = "OSS", + [14][0] = "HDP", + [15][0] = "SDMA0", + [32+0][0] = "UVD", + [32+1][0] = "UVDU", + [32+2][0] = "MP1", + [32+3][0] = "MP0", + [32+12][0] = "UTCL2", + [32+14][0] = "SDMA1", + [0][1] = "XDMA", + [1][1] = "DCE", + [2][1] = "DCEDWB", + [3][1] = "VCE0", + [4][1] = "VCE0U", + [5][1] = "UVD1", + [6][1] = "UVD1U", + [7][1] = "DBGU0", + [8][1] = "XDP", + [13][1] = "OSS", + [14][1] = "HDP", + [15][1] = "SDMA0", + [32+0][1] = "UVD", + [32+1][1] = "UVDU", + [32+2][1] = "DBGU1", + [32+3][1] = "MP1", + [32+4][1] = "MP0", + [32+14][1] = "SDMA1", +}; + +static const char *mmhub_client_ids_arcturus[][2] = { + [2][0] = "MP1", + [3][0] = "MP0", + [10][0] = "UTCL2", + [13][0] = "OSS", + [14][0] = "HDP", + [15][0] = "SDMA0", + [32+15][0] = "SDMA1", + [64+15][0] = "SDMA2", + [96+15][0] = "SDMA3", + [128+15][0] = "SDMA4", + [160+11][0] = "JPEG", + [160+12][0] = "VCN", + [160+13][0] = "VCNU", + [160+15][0] = "SDMA5", + [192+10][0] = "UTCL2", + [192+11][0] = "JPEG1", + [192+12][0] = "VCN1", + [192+13][0] = "VCN1U", + [192+15][0] = "SDMA6", + [224+15][0] = "SDMA7", + [0][1] = "DBGU1", + [1][1] = "XDP", + [2][1] = "MP1", + [3][1] = "MP0", + [13][1] = "OSS", + [14][1] = "HDP", + [15][1] = "SDMA0", + [32+15][1] = "SDMA1", + [32+15][1] = "SDMA1", + [64+15][1] = "SDMA2", + [96+15][1] = "SDMA3", + [128+15][1] = "SDMA4", + [160+11][1] = "JPEG", + [160+12][1] = "VCN", + [160+13][1] = "VCNU", + [160+15][1] = "SDMA5", + [192+11][1] = "JPEG1", + [192+12][1] = "VCN1", + [192+13][1] = "VCN1U", + [192+15][1] = "SDMA6", + [224+15][1] = "SDMA7", +}; static const u32 golden_settings_vega10_hdp[] = { @@ -300,9 +516,10 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, { struct amdgpu_vmhub *hub; bool retry_fault = !!(entry->src_data[1] & 0x80); - uint32_t status = 0; + uint32_t status = 0, cid = 0, rw = 0; u64 addr; char hub_name[10]; + const char *mmhub_cid; addr = (u64)entry->src_data[0] << 12; addr |= ((u64)entry->src_data[1] & 0xf) << 44; @@ -337,6 +554,10 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, RREG32(hub->vm_l2_pro_fault_status); status = RREG32(hub->vm_l2_pro_fault_status); + cid = REG_GET_FIELD(status, + VM_L2_PROTECTION_FAULT_STATUS, CID); + rw = REG_GET_FIELD(status, + VM_L2_PROTECTION_FAULT_STATUS, RW); WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); } @@ -359,9 +580,37 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, dev_err(adev->dev, "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", status); - dev_err(adev->dev, "\t Faulty UTCL2 client ID: 0x%lx\n", - REG_GET_FIELD(status, - VM_L2_PROTECTION_FAULT_STATUS, CID)); + if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { + dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", + cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], + cid); + } else { + switch (adev->asic_type) { + case CHIP_VEGA10: + mmhub_cid = mmhub_client_ids_vega10[cid][rw]; + break; + case CHIP_VEGA12: + mmhub_cid = mmhub_client_ids_vega12[cid][rw]; + break; + case CHIP_VEGA20: + mmhub_cid = mmhub_client_ids_vega20[cid][rw]; + break; + case CHIP_ARCTURUS: + mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; + break; + case CHIP_RAVEN: + mmhub_cid = mmhub_client_ids_raven[cid][rw]; + break; + case CHIP_RENOIR: + mmhub_cid = mmhub_client_ids_renoir[cid][rw]; + break; + default: + mmhub_cid = NULL; + break; + } + dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", + mmhub_cid ? mmhub_cid : "unknown", cid); + } dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); @@ -374,10 +623,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); - dev_err(adev->dev, "\t RW: 0x%lx\n", - REG_GET_FIELD(status, - VM_L2_PROTECTION_FAULT_STATUS, RW)); - + dev_err(adev->dev, "\t RW: 0x%x\n", rw); } } @@ -500,13 +746,14 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, * as GFXOFF under bare metal */ if (adev->gfx.kiq.ring.sched.ready && - (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && - !adev->in_gpu_reset) { + (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && + down_read_trylock(&adev->reset_sem)) { uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 1 << vmid); + up_read(&adev->reset_sem); return; } @@ -596,10 +843,10 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, struct amdgpu_ring *ring = &adev->gfx.kiq.ring; struct amdgpu_kiq *kiq = &adev->gfx.kiq; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EIO; - if (ring->sched.ready) { + if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { /* Vega20+XGMI caches PTEs in TC and TLB. Add a * heavy-weight TLB flush (type 2), which flushes * both. Due to a race condition with concurrent @@ -626,6 +873,7 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, if (r) { amdgpu_ring_undo(ring); spin_unlock(&adev->gfx.kiq.ring_lock); + up_read(&adev->reset_sem); return -ETIME; } @@ -633,10 +881,11 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, spin_unlock(&adev->gfx.kiq.ring_lock); r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); if (r < 1) { - DRM_ERROR("wait for kiq fence error: %ld.\n", r); + dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); + up_read(&adev->reset_sem); return -ETIME; } - + up_read(&adev->reset_sem); return 0; } @@ -826,6 +1075,41 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, *flags |= AMDGPU_PTE_SNOOPED; } +static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) +{ + u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); + unsigned size; + + if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { + size = AMDGPU_VBIOS_VGA_ALLOCATION; + } else { + u32 viewport; + + switch (adev->asic_type) { + case CHIP_RAVEN: + case CHIP_RENOIR: + viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); + size = (REG_GET_FIELD(viewport, + HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * + REG_GET_FIELD(viewport, + HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * + 4); + break; + case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: + default: + viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); + size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * + REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * + 4); + break; + } + } + + return size; +} + static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, @@ -833,7 +1117,8 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, .map_mtype = gmc_v9_0_map_mtype, .get_vm_pde = gmc_v9_0_get_vm_pde, - .get_vm_pte = gmc_v9_0_get_vm_pte + .get_vm_pte = gmc_v9_0_get_vm_pte, + .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, }; static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) @@ -871,13 +1156,11 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) { switch (adev->asic_type) { - case CHIP_VEGA20: - adev->mmhub.funcs = &mmhub_v1_0_funcs; - break; case CHIP_ARCTURUS: adev->mmhub.funcs = &mmhub_v9_4_funcs; break; default: + adev->mmhub.funcs = &mmhub_v1_0_funcs; break; } } @@ -901,38 +1184,12 @@ static int gmc_v9_0_early_init(void *handle) return 0; } -static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev) -{ - - /* - * TODO: - * Currently there is a bug where some memory client outside - * of the driver writes to first 8M of VRAM on S3 resume, - * this overrides GART which by default gets placed in first 8M and - * causes VM_FAULTS once GTT is accessed. - * Keep the stolen memory reservation until the while this is not solved. - * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init - */ - switch (adev->asic_type) { - case CHIP_VEGA10: - case CHIP_RAVEN: - case CHIP_ARCTURUS: - case CHIP_RENOIR: - return true; - case CHIP_VEGA12: - case CHIP_VEGA20: - default: - return false; - } -} - static int gmc_v9_0_late_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; int r; - if (!gmc_v9_0_keep_stolen_memory(adev)) - amdgpu_bo_late_init(adev); + amdgpu_bo_late_init(adev); r = amdgpu_gmc_allocate_vm_inv_eng(adev); if (r) @@ -969,10 +1226,8 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, { u64 base = 0; - if (adev->asic_type == CHIP_ARCTURUS) - base = mmhub_v9_4_get_fb_location(adev); - else if (!amdgpu_sriov_vf(adev)) - base = mmhub_v1_0_get_fb_location(adev); + if (!amdgpu_sriov_vf(adev)) + base = adev->mmhub.funcs->get_fb_location(adev); /* add the xgmi offset of the physical node */ base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; @@ -1066,50 +1321,18 @@ static int gmc_v9_0_gart_init(struct amdgpu_device *adev) return amdgpu_gart_table_vram_alloc(adev); } -static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) +/** + * gmc_v9_0_save_registers - saves regs + * + * @adev: amdgpu_device pointer + * + * This saves potential register values that should be + * restored upon resume + */ +static void gmc_v9_0_save_registers(struct amdgpu_device *adev) { - u32 d1vga_control; - unsigned size; - - /* - * TODO Remove once GART corruption is resolved - * Check related code in gmc_v9_0_sw_fini - * */ - if (gmc_v9_0_keep_stolen_memory(adev)) - return 9 * 1024 * 1024; - - d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); - if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ - } else { - u32 viewport; - - switch (adev->asic_type) { - case CHIP_RAVEN: - case CHIP_RENOIR: - viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); - size = (REG_GET_FIELD(viewport, - HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * - REG_GET_FIELD(viewport, - HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * - 4); - break; - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - default: - viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); - size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * - REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * - 4); - break; - } - } - /* return 0 if the pre-OS buffer uses up most of vram */ - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) - return 0; - - return size; + if (adev->asic_type == CHIP_RAVEN) + adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); } static int gmc_v9_0_sw_init(void *handle) @@ -1118,10 +1341,8 @@ static int gmc_v9_0_sw_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; gfxhub_v1_0_init(adev); - if (adev->asic_type == CHIP_ARCTURUS) - mmhub_v9_4_init(adev); - else - mmhub_v1_0_init(adev); + + adev->mmhub.funcs->init(adev); spin_lock_init(&adev->gmc.invalidate_lock); @@ -1242,7 +1463,7 @@ static int gmc_v9_0_sw_init(void *handle) if (r) return r; - adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); + amdgpu_gmc_get_vbios_allocations(adev); /* Memory manager */ r = amdgpu_bo_init(adev); @@ -1268,21 +1489,18 @@ static int gmc_v9_0_sw_init(void *handle) amdgpu_vm_manager_init(adev); + gmc_v9_0_save_registers(adev); + return 0; } static int gmc_v9_0_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - void *stolen_vga_buf; amdgpu_gmc_ras_fini(adev); amdgpu_gem_force_release(adev); amdgpu_vm_manager_fini(adev); - - if (gmc_v9_0_keep_stolen_memory(adev)) - amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf); - amdgpu_gart_table_vram_free(adev); amdgpu_bo_fini(adev); amdgpu_gart_fini(adev); @@ -1297,7 +1515,7 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) case CHIP_VEGA10: if (amdgpu_sriov_vf(adev)) break; - /* fall through */ + fallthrough; case CHIP_VEGA20: soc15_program_register_sequence(adev, golden_settings_mmhub_1_0_0, @@ -1326,10 +1544,10 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) * * This restores register values, saved at suspend. */ -static void gmc_v9_0_restore_registers(struct amdgpu_device *adev) +void gmc_v9_0_restore_registers(struct amdgpu_device *adev) { if (adev->asic_type == CHIP_RAVEN) - WREG32(mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); + WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); } /** @@ -1353,10 +1571,7 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) if (r) return r; - if (adev->asic_type == CHIP_ARCTURUS) - r = mmhub_v9_4_gart_enable(adev); - else - r = mmhub_v1_0_gart_enable(adev); + r = adev->mmhub.funcs->gart_enable(adev); if (r) return r; @@ -1391,11 +1606,10 @@ static int gmc_v9_0_hw_init(void *handle) golden_settings_vega10_hdp, ARRAY_SIZE(golden_settings_vega10_hdp)); + if (adev->mmhub.funcs->update_power_gating) + adev->mmhub.funcs->update_power_gating(adev, true); + switch (adev->asic_type) { - case CHIP_RAVEN: - /* TODO for renoir */ - mmhub_v1_0_update_power_gating(adev, true); - break; case CHIP_ARCTURUS: WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); break; @@ -1421,10 +1635,7 @@ static int gmc_v9_0_hw_init(void *handle) if (!amdgpu_sriov_vf(adev)) { gfxhub_v1_0_set_fault_enable_default(adev, value); - if (adev->asic_type == CHIP_ARCTURUS) - mmhub_v9_4_set_fault_enable_default(adev, value); - else - mmhub_v1_0_set_fault_enable_default(adev, value); + adev->mmhub.funcs->set_fault_enable_default(adev, value); } for (i = 0; i < adev->num_vmhubs; ++i) gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); @@ -1438,20 +1649,6 @@ static int gmc_v9_0_hw_init(void *handle) } /** - * gmc_v9_0_save_registers - saves regs - * - * @adev: amdgpu_device pointer - * - * This saves potential register values that should be - * restored upon resume - */ -static void gmc_v9_0_save_registers(struct amdgpu_device *adev) -{ - if (adev->asic_type == CHIP_RAVEN) - adev->gmc.sdpif_register = RREG32(mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); -} - -/** * gmc_v9_0_gart_disable - gart disable * * @adev: amdgpu_device pointer @@ -1461,10 +1658,7 @@ static void gmc_v9_0_save_registers(struct amdgpu_device *adev) static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) { gfxhub_v1_0_gart_disable(adev); - if (adev->asic_type == CHIP_ARCTURUS) - mmhub_v9_4_gart_disable(adev); - else - mmhub_v1_0_gart_disable(adev); + adev->mmhub.funcs->gart_disable(adev); amdgpu_gart_table_vram_unpin(adev); } @@ -1494,8 +1688,6 @@ static int gmc_v9_0_suspend(void *handle) if (r) return r; - gmc_v9_0_save_registers(adev); - return 0; } @@ -1504,7 +1696,6 @@ static int gmc_v9_0_resume(void *handle) int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - gmc_v9_0_restore_registers(adev); r = gmc_v9_0_hw_init(adev); if (r) return r; @@ -1537,10 +1728,7 @@ static int gmc_v9_0_set_clockgating_state(void *handle, { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (adev->asic_type == CHIP_ARCTURUS) - mmhub_v9_4_set_clockgating(adev, state); - else - mmhub_v1_0_set_clockgating(adev, state); + adev->mmhub.funcs->set_clockgating(adev, state); athub_v1_0_set_clockgating(adev, state); @@ -1551,10 +1739,7 @@ static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (adev->asic_type == CHIP_ARCTURUS) - mmhub_v9_4_get_clockgating(adev, flags); - else - mmhub_v1_0_get_clockgating(adev, flags); + adev->mmhub.funcs->get_clockgating(adev, flags); athub_v1_0_get_clockgating(adev, flags); } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.h index e0585e8c6c1b..c415c439f690 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.h @@ -26,4 +26,6 @@ extern const struct amd_ip_funcs gmc_v9_0_ip_funcs; extern const struct amdgpu_ip_block_version gmc_v9_0_ip_block; + +void gmc_v9_0_restore_registers(struct amdgpu_device *adev); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c index 7a51c615d22d..845306f63cdb 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c @@ -55,22 +55,18 @@ static int amdgpu_ih_clientid_jpeg[] = { static int jpeg_v2_5_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (adev->asic_type == CHIP_ARCTURUS) { - u32 harvest; - int i; - - adev->jpeg.num_jpeg_inst = JPEG25_MAX_HW_INSTANCES_ARCTURUS; - for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) { - harvest = RREG32_SOC15(JPEG, i, mmCC_UVD_HARVESTING); - if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK) - adev->jpeg.harvest_config |= 1 << i; - } + u32 harvest; + int i; - if (adev->jpeg.harvest_config == (AMDGPU_JPEG_HARVEST_JPEG0 | - AMDGPU_JPEG_HARVEST_JPEG1)) - return -ENOENT; - } else - adev->jpeg.num_jpeg_inst = 1; + adev->jpeg.num_jpeg_inst = JPEG25_MAX_HW_INSTANCES_ARCTURUS; + for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) { + harvest = RREG32_SOC15(JPEG, i, mmCC_UVD_HARVESTING); + if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK) + adev->jpeg.harvest_config |= 1 << i; + } + if (adev->jpeg.harvest_config == (AMDGPU_JPEG_HARVEST_JPEG0 | + AMDGPU_JPEG_HARVEST_JPEG1)) + return -ENOENT; jpeg_v2_5_set_dec_ring_funcs(adev); jpeg_v2_5_set_irq_funcs(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c index c41e5590a701..3a0dff53654d 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c @@ -460,15 +460,10 @@ static bool jpeg_v3_0_is_idle(void *handle) static int jpeg_v3_0_wait_for_idle(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int ret; - ret = SOC15_WAIT_ON_RREG(JPEG, 0, mmUVD_JRBC_STATUS, + return SOC15_WAIT_ON_RREG(JPEG, 0, mmUVD_JRBC_STATUS, UVD_JRBC_STATUS__RB_JOB_DONE_MASK, UVD_JRBC_STATUS__RB_JOB_DONE_MASK); - if (ret) - return ret; - - return ret; } static int jpeg_v3_0_set_clockgating_state(void *handle, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c index dffcb93ecee5..45a902b1acb7 100755..100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c @@ -34,7 +34,7 @@ #define mmDAGB0_CNTL_MISC2_RV 0x008f #define mmDAGB0_CNTL_MISC2_RV_BASE_IDX 0 -u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev) +static u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev) { u64 base = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE); u64 top = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP); @@ -51,7 +51,7 @@ u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev) return base; } -void mmhub_v1_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, +static void mmhub_v1_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, uint64_t page_table_base) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; @@ -297,20 +297,19 @@ static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev) } } -void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev, +static void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev, bool enable) { if (amdgpu_sriov_vf(adev)) return; if (enable && adev->pg_flags & AMD_PG_SUPPORT_MMHUB) { - if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->set_powergating_by_smu) - amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GMC, true); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GMC, true); } } -int mmhub_v1_0_gart_enable(struct amdgpu_device *adev) +static int mmhub_v1_0_gart_enable(struct amdgpu_device *adev) { if (amdgpu_sriov_vf(adev)) { /* @@ -338,7 +337,7 @@ int mmhub_v1_0_gart_enable(struct amdgpu_device *adev) return 0; } -void mmhub_v1_0_gart_disable(struct amdgpu_device *adev) +static void mmhub_v1_0_gart_disable(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; u32 tmp; @@ -373,7 +372,7 @@ void mmhub_v1_0_gart_disable(struct amdgpu_device *adev) * @adev: amdgpu_device pointer * @value: true redirects VM faults to the default page */ -void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) +static void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) { u32 tmp; @@ -415,7 +414,7 @@ void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp); } -void mmhub_v1_0_init(struct amdgpu_device *adev) +static void mmhub_v1_0_init(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; @@ -525,7 +524,7 @@ static void mmhub_v1_0_update_medium_grain_light_sleep(struct amdgpu_device *ade WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data); } -int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, +static int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, enum amd_clockgating_state state) { if (amdgpu_sriov_vf(adev)) @@ -549,7 +548,7 @@ int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, return 0; } -void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) +static void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) { int data, data1; @@ -781,4 +780,13 @@ const struct amdgpu_mmhub_funcs mmhub_v1_0_funcs = { .ras_late_init = amdgpu_mmhub_ras_late_init, .query_ras_error_count = mmhub_v1_0_query_ras_error_count, .reset_ras_error_count = mmhub_v1_0_reset_ras_error_count, + .get_fb_location = mmhub_v1_0_get_fb_location, + .init = mmhub_v1_0_init, + .gart_enable = mmhub_v1_0_gart_enable, + .set_fault_enable_default = mmhub_v1_0_set_fault_enable_default, + .gart_disable = mmhub_v1_0_gart_disable, + .set_clockgating = mmhub_v1_0_set_clockgating, + .get_clockgating = mmhub_v1_0_get_clockgating, + .setup_vm_pt_regs = mmhub_v1_0_setup_vm_pt_regs, + .update_power_gating = mmhub_v1_0_update_power_gating, }; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h index c43319e8f945..d77f5b65a618 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h @@ -25,18 +25,4 @@ extern const struct amdgpu_mmhub_funcs mmhub_v1_0_funcs; -u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev); -int mmhub_v1_0_gart_enable(struct amdgpu_device *adev); -void mmhub_v1_0_gart_disable(struct amdgpu_device *adev); -void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, - bool value); -void mmhub_v1_0_init(struct amdgpu_device *adev); -int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, - enum amd_clockgating_state state); -void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags); -void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev, - bool enable); -void mmhub_v1_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, - uint64_t page_table_base); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index 757fa8e83f5b..2d88278c50bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -36,7 +36,130 @@ #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid 0x0070 #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid_BASE_IDX 0 -void mmhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, +static const char *mmhub_client_ids_navi1x[][2] = { + [3][0] = "DCEDMC", + [4][0] = "DCEVGA", + [5][0] = "MP0", + [6][0] = "MP1", + [13][0] = "VMC", + [14][0] = "HDP", + [15][0] = "OSS", + [16][0] = "VCNU", + [17][0] = "JPEG", + [18][0] = "VCN", + [3][1] = "DCEDMC", + [4][1] = "DCEXFC", + [5][1] = "DCEVGA", + [6][1] = "DCEDWB", + [7][1] = "MP0", + [8][1] = "MP1", + [9][1] = "DBGU1", + [10][1] = "DBGU0", + [11][1] = "XDP", + [14][1] = "HDP", + [15][1] = "OSS", + [16][1] = "VCNU", + [17][1] = "JPEG", + [18][1] = "VCN", +}; + +static const char *mmhub_client_ids_sienna_cichlid[][2] = { + [3][0] = "DCEDMC", + [4][0] = "DCEVGA", + [5][0] = "MP0", + [6][0] = "MP1", + [8][0] = "VMC", + [9][0] = "VCNU0", + [10][0] = "JPEG", + [12][0] = "VCNU1", + [13][0] = "VCN1", + [14][0] = "HDP", + [15][0] = "OSS", + [32+11][0] = "VCN0", + [0][1] = "DBGU0", + [1][1] = "DBGU1", + [2][1] = "DCEDWB", + [3][1] = "DCEDMC", + [4][1] = "DCEVGA", + [5][1] = "MP0", + [6][1] = "MP1", + [7][1] = "XDP", + [9][1] = "VCNU0", + [10][1] = "JPEG", + [11][1] = "VCN0", + [12][1] = "VCNU1", + [13][1] = "VCN1", + [14][1] = "HDP", + [15][1] = "OSS", +}; + +static uint32_t mmhub_v2_0_get_invalidate_req(unsigned int vmid, + uint32_t flush_type) +{ + u32 req = 0; + + /* invalidate using legacy mode on vmid*/ + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, + PER_VMID_INVALIDATE_REQ, 1 << vmid); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); + req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, + CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); + + return req; +} + +static void +mmhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev, + uint32_t status) +{ + uint32_t cid, rw; + const char *mmhub_cid = NULL; + + cid = REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, CID); + rw = REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, RW); + + dev_err(adev->dev, + "MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", + status); + switch (adev->asic_type) { + case CHIP_NAVI10: + case CHIP_NAVI12: + case CHIP_NAVI14: + mmhub_cid = mmhub_client_ids_navi1x[cid][rw]; + break; + case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: + mmhub_cid = mmhub_client_ids_sienna_cichlid[cid][rw]; + break; + default: + mmhub_cid = NULL; + break; + } + dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", + mmhub_cid ? mmhub_cid : "unknown", cid); + dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); + dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); + dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", + REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); + dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", + REG_GET_FIELD(status, + MMVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); + dev_err(adev->dev, "\t RW: 0x%x\n", rw); +} + +static void mmhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, uint64_t page_table_base) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; @@ -134,6 +257,12 @@ static void mmhub_v2_0_init_cache_regs(struct amdgpu_device *adev) { uint32_t tmp; + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + /* Setup L2 cache */ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL); tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1); @@ -189,6 +318,12 @@ static void mmhub_v2_0_enable_system_domain(struct amdgpu_device *adev) static void mmhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) { + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 0xFFFFFFFF); @@ -268,7 +403,7 @@ static void mmhub_v2_0_program_invalidation(struct amdgpu_device *adev) } } -int mmhub_v2_0_gart_enable(struct amdgpu_device *adev) +static int mmhub_v2_0_gart_enable(struct amdgpu_device *adev) { /* GART Enable. */ mmhub_v2_0_init_gart_aperture_regs(adev); @@ -284,7 +419,7 @@ int mmhub_v2_0_gart_enable(struct amdgpu_device *adev) return 0; } -void mmhub_v2_0_gart_disable(struct amdgpu_device *adev) +static void mmhub_v2_0_gart_disable(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; u32 tmp; @@ -315,9 +450,16 @@ void mmhub_v2_0_gart_disable(struct amdgpu_device *adev) * @adev: amdgpu_device pointer * @value: true redirects VM faults to the default page */ -void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) +static void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) { u32 tmp; + + /* These registers are not accessible to VF-SRIOV. + * The PF will program them instead. + */ + if (amdgpu_sriov_vf(adev)) + return; + tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL); tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); @@ -351,7 +493,12 @@ void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL, tmp); } -void mmhub_v2_0_init(struct amdgpu_device *adev) +static const struct amdgpu_vmhub_funcs mmhub_v2_0_vmhub_funcs = { + .print_l2_protection_fault_status = mmhub_v2_0_print_l2_protection_fault_status, + .get_invalidate_req = mmhub_v2_0_get_invalidate_req, +}; + +static void mmhub_v2_0_init(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; @@ -381,6 +528,16 @@ void mmhub_v2_0_init(struct amdgpu_device *adev) mmMMVM_INVALIDATE_ENG0_REQ; hub->eng_addr_distance = mmMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; + + hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | + MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; + + hub->vmhub_funcs = &mmhub_v2_0_vmhub_funcs; } static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, @@ -471,7 +628,7 @@ static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *ade } } -int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, +static int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, enum amd_clockgating_state state) { if (amdgpu_sriov_vf(adev)) @@ -495,7 +652,7 @@ int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, return 0; } -void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) +static void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) { int data, data1; @@ -528,3 +685,14 @@ void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK) *flags |= AMD_CG_SUPPORT_MC_LS; } + +const struct amdgpu_mmhub_funcs mmhub_v2_0_funcs = { + .ras_late_init = amdgpu_mmhub_ras_late_init, + .init = mmhub_v2_0_init, + .gart_enable = mmhub_v2_0_gart_enable, + .set_fault_enable_default = mmhub_v2_0_set_fault_enable_default, + .gart_disable = mmhub_v2_0_gart_disable, + .set_clockgating = mmhub_v2_0_set_clockgating, + .get_clockgating = mmhub_v2_0_get_clockgating, + .setup_vm_pt_regs = mmhub_v2_0_setup_vm_pt_regs, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h index 3ea4344f0315..f80f461d67da 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h @@ -23,15 +23,6 @@ #ifndef __MMHUB_V2_0_H__ #define __MMHUB_V2_0_H__ -int mmhub_v2_0_gart_enable(struct amdgpu_device *adev); -void mmhub_v2_0_gart_disable(struct amdgpu_device *adev); -void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, - bool value); -void mmhub_v2_0_init(struct amdgpu_device *adev); -int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, - enum amd_clockgating_state state); -void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags); -void mmhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, - uint64_t page_table_base); +extern const struct amdgpu_mmhub_funcs mmhub_v2_0_funcs; #endif diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c index 9979f54fef57..6c6ad529c65c 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c @@ -36,7 +36,7 @@ #define MMHUB_NUM_INSTANCES 2 #define MMHUB_INSTANCE_REGISTER_OFFSET 0x3000 -u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev) +static u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev) { /* The base should be same b/t 2 mmhubs on Acrturus. Read one here. */ u64 base = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE); @@ -97,7 +97,7 @@ static void mmhub_v9_4_init_gart_aperture_regs(struct amdgpu_device *adev, (u32)(adev->gmc.gart_end >> 44)); } -void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, +static void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, uint64_t page_table_base) { int i; @@ -375,7 +375,7 @@ static void mmhub_v9_4_program_invalidation(struct amdgpu_device *adev, } } -int mmhub_v9_4_gart_enable(struct amdgpu_device *adev) +static int mmhub_v9_4_gart_enable(struct amdgpu_device *adev) { int i; @@ -397,7 +397,7 @@ int mmhub_v9_4_gart_enable(struct amdgpu_device *adev) return 0; } -void mmhub_v9_4_gart_disable(struct amdgpu_device *adev) +static void mmhub_v9_4_gart_disable(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; u32 tmp; @@ -442,7 +442,7 @@ void mmhub_v9_4_gart_disable(struct amdgpu_device *adev) * @adev: amdgpu_device pointer * @value: true redirects VM faults to the default page */ -void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value) +static void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value) { u32 tmp; int i; @@ -500,7 +500,7 @@ void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value) } } -void mmhub_v9_4_init(struct amdgpu_device *adev) +static void mmhub_v9_4_init(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub[MMHUB_NUM_INSTANCES] = {&adev->vmhub[AMDGPU_MMHUB_0], &adev->vmhub[AMDGPU_MMHUB_1]}; @@ -630,7 +630,7 @@ static void mmhub_v9_4_update_medium_grain_light_sleep(struct amdgpu_device *ade } } -int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev, +static int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev, enum amd_clockgating_state state) { if (amdgpu_sriov_vf(adev)) @@ -650,7 +650,7 @@ int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev, return 0; } -void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags) +static void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags) { int data, data1; @@ -1628,4 +1628,12 @@ const struct amdgpu_mmhub_funcs mmhub_v9_4_funcs = { .ras_late_init = amdgpu_mmhub_ras_late_init, .query_ras_error_count = mmhub_v9_4_query_ras_error_count, .reset_ras_error_count = mmhub_v9_4_reset_ras_error_count, + .get_fb_location = mmhub_v9_4_get_fb_location, + .init = mmhub_v9_4_init, + .gart_enable = mmhub_v9_4_gart_enable, + .set_fault_enable_default = mmhub_v9_4_set_fault_enable_default, + .gart_disable = mmhub_v9_4_gart_disable, + .set_clockgating = mmhub_v9_4_set_clockgating, + .get_clockgating = mmhub_v9_4_get_clockgating, + .setup_vm_pt_regs = mmhub_v9_4_setup_vm_pt_regs, }; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.h index 1b979773776c..92404a8f66f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.h +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.h @@ -25,16 +25,4 @@ extern const struct amdgpu_mmhub_funcs mmhub_v9_4_funcs; -u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev); -int mmhub_v9_4_gart_enable(struct amdgpu_device *adev); -void mmhub_v9_4_gart_disable(struct amdgpu_device *adev); -void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, - bool value); -void mmhub_v9_4_init(struct amdgpu_device *adev); -int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev, - enum amd_clockgating_state state); -void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags); -void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, - uint64_t page_table_base); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index 5fd67e1cc2a0..9c07014d9bd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -238,19 +238,15 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; - int locked; /* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. - * - * we can unlock the lock_reset to allow "amdgpu_job_timedout" - * to run gpu_recover() after FLR_NOTIFICATION_CMPL received - * which means host side had finished this VF's FLR. */ - locked = mutex_trylock(&adev->lock_reset); - if (locked) - adev->in_gpu_reset = true; + if (!down_read_trylock(&adev->reset_sem)) + return; + + atomic_set(&adev->in_gpu_reset, 1); do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,14 +257,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: - if (locked) { - adev->in_gpu_reset = false; - mutex_unlock(&adev->lock_reset); - } + atomic_set(&adev->in_gpu_reset, 0); + up_read(&adev->reset_sem); /* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev) - && adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT) + && (amdgpu_device_has_job_running(adev) || adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)) amdgpu_device_gpu_recover(adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index ce2bf1fb79ed..9c23abf9b140 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -259,19 +259,15 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT; - int locked; /* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. - * - * we can unlock the lock_reset to allow "amdgpu_job_timedout" - * to run gpu_recover() after FLR_NOTIFICATION_CMPL received - * which means host side had finished this VF's FLR. */ - locked = mutex_trylock(&adev->lock_reset); - if (locked) - adev->in_gpu_reset = true; + if (!down_read_trylock(&adev->reset_sem)) + return; + + atomic_set(&adev->in_gpu_reset, 1); do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,14 +278,13 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: - if (locked) { - adev->in_gpu_reset = false; - mutex_unlock(&adev->lock_reset); - } + atomic_set(&adev->in_gpu_reset, 0); + up_read(&adev->reset_sem); /* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev) - && (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT || + && (amdgpu_device_has_job_running(adev) || + adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT || adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT || adev->compute_timeout == MAX_SCHEDULE_TIMEOUT || adev->video_timeout == MAX_SCHEDULE_TIMEOUT)) diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c index 350f1bf063c6..74b1e7dc49a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c @@ -306,7 +306,8 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) } else { WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); } - navi10_ih_reroute_ih(adev); + if (adev->irq.ih1.ring_size) + navi10_ih_reroute_ih(adev); if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) { if (ih->use_bus_addr) { @@ -668,19 +669,26 @@ static int navi10_ih_sw_init(void *handle) adev->irq.ih.use_doorbell = true; adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); - if (r) - return r; + adev->irq.ih1.ring_size = 0; + adev->irq.ih2.ring_size = 0; - adev->irq.ih1.use_doorbell = true; - adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; + if (adev->asic_type < CHIP_NAVI10) { + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); - if (r) - return r; + adev->irq.ih1.use_doorbell = true; + adev->irq.ih1.doorbell_index = + (adev->doorbell_index.ih + 1) << 1; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; - adev->irq.ih2.use_doorbell = true; - adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + adev->irq.ih2.use_doorbell = true; + adev->irq.ih2.doorbell_index = + (adev->doorbell_index.ih + 2) << 1; + } r = amdgpu_irq_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index e629156173d3..eadc9526d33f 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -302,6 +302,7 @@ static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device uint32_t bif_doorbell_intr_cntl; struct ras_manager *obj = amdgpu_ras_find_obj(adev, adev->nbio.ras_if); struct ras_err_data err_data = {0, 0, 0, NULL}; + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); if (REG_GET_FIELD(bif_doorbell_intr_cntl, @@ -312,28 +313,31 @@ static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device RAS_CNTLR_INTERRUPT_CLEAR, 1); WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); - /* - * clear error status after ras_controller_intr according to - * hw team and count ue number for query - */ - nbio_v7_4_query_ras_error_count(adev, &err_data); - - /* logging on error counter and printing for awareness */ - obj->err_data.ue_count += err_data.ue_count; - obj->err_data.ce_count += err_data.ce_count; - - if (err_data.ce_count) - dev_info(adev->dev, "%ld correctable hardware " - "errors detected in %s block, " - "no user action is needed.\n", - obj->err_data.ce_count, - adev->nbio.ras_if->name); - - if (err_data.ue_count) - dev_info(adev->dev, "%ld uncorrectable hardware " - "errors detected in %s block\n", - obj->err_data.ue_count, - adev->nbio.ras_if->name); + if (!ras->disable_ras_err_cnt_harvest) { + /* + * clear error status after ras_controller_intr + * according to hw team and count ue number + * for query + */ + nbio_v7_4_query_ras_error_count(adev, &err_data); + + /* logging on error cnt and printing for awareness */ + obj->err_data.ue_count += err_data.ue_count; + obj->err_data.ce_count += err_data.ce_count; + + if (err_data.ce_count) + dev_info(adev->dev, "%ld correctable hardware " + "errors detected in %s block, " + "no user action is needed.\n", + obj->err_data.ce_count, + adev->nbio.ras_if->name); + + if (err_data.ue_count) + dev_info(adev->dev, "%ld uncorrectable hardware " + "errors detected in %s block\n", + obj->err_data.ue_count, + adev->nbio.ras_if->name); + } dev_info(adev->dev, "RAS controller interrupt triggered " "by NBIF error\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index da8024c2826e..4d1402356262 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -364,6 +364,7 @@ nv_asic_reset_method(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: return AMD_RESET_METHOD_MODE1; default: if (smu_baco_is_support(smu)) @@ -379,7 +380,7 @@ static int nv_asic_reset(struct amdgpu_device *adev) struct smu_context *smu = &adev->smu; if (nv_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { - dev_info(adev->dev, "GPU BACO reset\n"); + dev_info(adev->dev, "BACO reset\n"); ret = smu_baco_enter(smu); if (ret) @@ -387,8 +388,10 @@ static int nv_asic_reset(struct amdgpu_device *adev) ret = smu_baco_exit(smu); if (ret) return ret; - } else + } else { + dev_info(adev->dev, "MODE1 reset\n"); ret = nv_asic_mode1_reset(adev); + } return ret; } @@ -689,6 +692,10 @@ static void nv_init_doorbell_index(struct amdgpu_device *adev) adev->doorbell_index.sdma_doorbell_range = 20; } +static void nv_pre_asic_init(struct amdgpu_device *adev) +{ +} + static const struct amdgpu_asic_funcs nv_asic_funcs = { .read_disabled_bios = &nv_read_disabled_bios, @@ -708,6 +715,7 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = .need_reset_on_init = &nv_need_reset_on_init, .get_pcie_replay_count = &nv_get_pcie_replay_count, .supports_baco = &nv_asic_supports_baco, + .pre_asic_init = &nv_pre_asic_init, }; static int nv_common_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index d488d250805d..e16874f30d5d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -179,12 +179,11 @@ static int psp_v11_0_init_microcode(struct psp_context *psp) } break; case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: err = psp_init_ta_microcode(&adev->psp, chip_name); if (err) return err; break; - case CHIP_NAVY_FLOUNDER: - break; default: BUG(); } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index 6c9614f77d33..75489313dbad 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -38,6 +38,8 @@ #include "oss/osssys_4_0_sh_mask.h" MODULE_FIRMWARE("amdgpu/renoir_asd.bin"); +MODULE_FIRMWARE("amdgpu/renoir_ta.bin"); + /* address block */ #define smnMP1_FIRMWARE_FLAGS 0x3010024 @@ -45,7 +47,10 @@ static int psp_v12_0_init_microcode(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; const char *chip_name; + char fw_name[30]; int err = 0; + const struct ta_firmware_header_v1_0 *ta_hdr; + DRM_DEBUG("\n"); switch (adev->asic_type) { case CHIP_RENOIR: @@ -56,6 +61,55 @@ static int psp_v12_0_init_microcode(struct psp_context *psp) } err = psp_init_asd_microcode(psp, chip_name); + if (err) + goto out; + + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name); + err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev); + if (err) { + release_firmware(adev->psp.ta_fw); + adev->psp.ta_fw = NULL; + dev_info(adev->dev, + "psp v12.0: Failed to load firmware \"%s\"\n", + fw_name); + } else { + err = amdgpu_ucode_validate(adev->psp.ta_fw); + if (err) + goto out2; + + ta_hdr = (const struct ta_firmware_header_v1_0 *) + adev->psp.ta_fw->data; + adev->psp.ta_hdcp_ucode_version = + le32_to_cpu(ta_hdr->ta_hdcp_ucode_version); + adev->psp.ta_hdcp_ucode_size = + le32_to_cpu(ta_hdr->ta_hdcp_size_bytes); + adev->psp.ta_hdcp_start_addr = + (uint8_t *)ta_hdr + + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); + + adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); + + adev->psp.ta_dtm_ucode_version = + le32_to_cpu(ta_hdr->ta_dtm_ucode_version); + adev->psp.ta_dtm_ucode_size = + le32_to_cpu(ta_hdr->ta_dtm_size_bytes); + adev->psp.ta_dtm_start_addr = + (uint8_t *)adev->psp.ta_hdcp_start_addr + + le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); + } + + return 0; + +out2: + release_firmware(adev->psp.ta_fw); + adev->psp.ta_fw = NULL; +out: + if (err) { + dev_err(adev->dev, + "psp v12.0: Failed to load firmware \"%s\"\n", + fw_name); + } + return err; } diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 1b449291f068..455d5e366c69 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -52,6 +52,8 @@ #include "bif/bif_3_0_d.h" #include "bif/bif_3_0_sh_mask.h" +#include "amdgpu_dm.h" + static const u32 tahiti_golden_registers[] = { mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011, @@ -1215,10 +1217,100 @@ static bool si_read_bios_from_rom(struct amdgpu_device *adev, return true; } -//xxx: not implemented +static void si_set_clk_bypass_mode(struct amdgpu_device *adev) +{ + u32 tmp, i; + + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_BYPASS_EN; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + + tmp = RREG32(CG_SPLL_FUNC_CNTL_2); + tmp |= SPLL_CTLREQ_CHG; + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + for (i = 0; i < adev->usec_timeout; i++) { + if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS) + break; + udelay(1); + } + + tmp = RREG32(CG_SPLL_FUNC_CNTL_2); + tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE); + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + tmp = RREG32(MPLL_CNTL_MODE); + tmp &= ~MPLL_MCLK_SEL; + WREG32(MPLL_CNTL_MODE, tmp); +} + +static void si_spll_powerdown(struct amdgpu_device *adev) +{ + u32 tmp; + + tmp = RREG32(SPLL_CNTL_MODE); + tmp |= SPLL_SW_DIR_CONTROL; + WREG32(SPLL_CNTL_MODE, tmp); + + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_RESET; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_SLEEP; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + + tmp = RREG32(SPLL_CNTL_MODE); + tmp &= ~SPLL_SW_DIR_CONTROL; + WREG32(SPLL_CNTL_MODE, tmp); +} + +static int si_gpu_pci_config_reset(struct amdgpu_device *adev) +{ + u32 i; + int r = -EINVAL; + + dev_info(adev->dev, "GPU pci config reset\n"); + + /* set mclk/sclk to bypass */ + si_set_clk_bypass_mode(adev); + /* powerdown spll */ + si_spll_powerdown(adev); + /* disable BM */ + pci_clear_master(adev->pdev); + /* reset */ + amdgpu_device_pci_config_reset(adev); + + udelay(100); + + /* wait for asic to come out of reset */ + for (i = 0; i < adev->usec_timeout; i++) { + if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) { + /* enable BM */ + pci_set_master(adev->pdev); + adev->has_hw_reset = true; + r = 0; + break; + } + udelay(1); + } + + return r; +} + static int si_asic_reset(struct amdgpu_device *adev) { - return 0; + int r; + + dev_info(adev->dev, "PCI CONFIG reset\n"); + + amdgpu_atombios_scratch_regs_engine_hung(adev, true); + + r = si_gpu_pci_config_reset(adev); + + amdgpu_atombios_scratch_regs_engine_hung(adev, false); + + return r; } static bool si_asic_supports_baco(struct amdgpu_device *adev) @@ -1779,6 +1871,10 @@ static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk) return 0; } +static void si_pre_asic_init(struct amdgpu_device *adev) +{ +} + static const struct amdgpu_asic_funcs si_asic_funcs = { .read_disabled_bios = &si_read_disabled_bios, @@ -1800,6 +1896,7 @@ static const struct amdgpu_asic_funcs si_asic_funcs = .need_reset_on_init = &si_need_reset_on_init, .get_pcie_replay_count = &si_get_pcie_replay_count, .supports_baco = &si_asic_supports_baco, + .pre_asic_init = &si_pre_asic_init, }; static uint32_t si_get_rev_id(struct amdgpu_device *adev) @@ -2546,6 +2643,10 @@ int si_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &si_smu_ip_block); if (adev->enable_virtual_display) amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif else amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block); amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); @@ -2560,6 +2661,10 @@ int si_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &si_smu_ip_block); if (adev->enable_virtual_display) amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif else amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block); amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); diff --git a/drivers/gpu/drm/amd/amdgpu/smu_v11_0_i2c.c b/drivers/gpu/drm/amd/amdgpu/smu_v11_0_i2c.c index d55bf64770c4..7fb240c4990c 100644 --- a/drivers/gpu/drm/amd/amdgpu/smu_v11_0_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/smu_v11_0_i2c.c @@ -508,14 +508,9 @@ static bool smu_v11_0_i2c_bus_lock(struct i2c_adapter *control) struct amdgpu_device *adev = to_amdgpu_device(control); /* Send PPSMC_MSG_RequestI2CBus */ - if (!adev->powerplay.pp_funcs->smu_i2c_bus_access) - goto Fail; - - - if (!adev->powerplay.pp_funcs->smu_i2c_bus_access(adev->powerplay.pp_handle, true)) + if (!amdgpu_dpm_smu_i2c_bus_access(adev, true)) return true; -Fail: return false; } @@ -523,16 +518,10 @@ static bool smu_v11_0_i2c_bus_unlock(struct i2c_adapter *control) { struct amdgpu_device *adev = to_amdgpu_device(control); - /* Send PPSMC_MSG_RequestI2CBus */ - if (!adev->powerplay.pp_funcs->smu_i2c_bus_access) - goto Fail; - /* Send PPSMC_MSG_ReleaseI2CBus */ - if (!adev->powerplay.pp_funcs->smu_i2c_bus_access(adev->powerplay.pp_handle, - false)) + if (!amdgpu_dpm_smu_i2c_bus_access(adev, false)) return true; -Fail: return false; } diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 84d811b6e48b..2f93c475d6d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -580,10 +580,13 @@ static int soc15_asic_reset(struct amdgpu_device *adev) switch (soc15_asic_reset_method(adev)) { case AMD_RESET_METHOD_BACO: + dev_info(adev->dev, "BACO reset\n"); return soc15_asic_baco_reset(adev); case AMD_RESET_METHOD_MODE2: + dev_info(adev->dev, "MODE2 reset\n"); return amdgpu_dpm_mode2_reset(adev); default: + dev_info(adev->dev, "MODE1 reset\n"); return soc15_asic_mode1_reset(adev); } } @@ -1026,6 +1029,11 @@ static uint64_t soc15_get_pcie_replay_count(struct amdgpu_device *adev) return (nak_r + nak_g); } +static void soc15_pre_asic_init(struct amdgpu_device *adev) +{ + gmc_v9_0_restore_registers(adev); +} + static const struct amdgpu_asic_funcs soc15_asic_funcs = { .read_disabled_bios = &soc15_read_disabled_bios, @@ -1046,6 +1054,7 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs = .need_reset_on_init = &soc15_need_reset_on_init, .get_pcie_replay_count = &soc15_get_pcie_replay_count, .supports_baco = &soc15_supports_baco, + .pre_asic_init = &soc15_pre_asic_init, }; static const struct amdgpu_asic_funcs vega20_asic_funcs = @@ -1069,6 +1078,7 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = .need_reset_on_init = &soc15_need_reset_on_init, .get_pcie_replay_count = &soc15_get_pcie_replay_count, .supports_baco = &soc15_supports_baco, + .pre_asic_init = &soc15_pre_asic_init, }; static int soc15_common_early_init(void *handle) @@ -1449,7 +1459,8 @@ static void soc15_update_hdp_light_sleep(struct amdgpu_device *adev, bool enable uint32_t def, data; if (adev->asic_type == CHIP_VEGA20 || - adev->asic_type == CHIP_ARCTURUS) { + adev->asic_type == CHIP_ARCTURUS || + adev->asic_type == CHIP_RENOIR) { def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) diff --git a/drivers/gpu/drm/amd/amdgpu/ta_rap_if.h b/drivers/gpu/drm/amd/amdgpu/ta_rap_if.h new file mode 100644 index 000000000000..f14833fae07c --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/ta_rap_if.h @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _TA_RAP_IF_H +#define _TA_RAP_IF_H + +/* Responses have bit 31 set */ +#define RSP_ID_MASK (1U << 31) +#define RSP_ID(cmdId) (((uint32_t)(cmdId)) | RSP_ID_MASK) + +enum ta_rap_status { + TA_RAP_STATUS__SUCCESS = 1, + TA_RAP_STATUS__ERROR_GENERIC_FAILURE = 2, + TA_RAP_STATUS__ERROR_CMD_NOT_SUPPORTED = 3, + TA_RAP_STATUS__ERROR_INVALID_VALIDATION_METHOD = 4, + TA_RAP_STATUS__ERROR_NULL_POINTER = 5, + TA_RAP_STATUS__ERROR_NOT_INITIALIZED = 6, + TA_RAP_STATUS__ERROR_VALIDATION_FAILED = 7, + TA_RAP_STATUS__ERROR_ASIC_NOT_SUPPORTED = 8, + TA_RAP_STATUS__ERROR_OPERATION_NOT_PERMISSABLE = 9, + TA_RAP_STATUS__ERROR_ALREADY_INIT = 10, +}; + +enum ta_rap_cmd { + TA_CMD_RAP__INITIALIZE = 1, + TA_CMD_RAP__VALIDATE_L0 = 2, +}; + +enum ta_rap_validation_method { + METHOD_A = 1, +}; + +struct ta_rap_cmd_input_data { + uint8_t reserved[8]; +}; + +struct ta_rap_cmd_output_data { + uint32_t last_subsection; + uint32_t num_total_validate; + uint32_t num_valid; + uint32_t last_validate_addr; + uint32_t last_validate_val; + uint32_t last_validate_val_exptd; +}; + +union ta_rap_cmd_input { + struct ta_rap_cmd_input_data input; +}; + +union ta_rap_cmd_output { + struct ta_rap_cmd_output_data output; +}; + +struct ta_rap_shared_memory { + uint32_t cmd_id; + uint32_t validation_method_id; + uint32_t resp_id; + enum ta_rap_status rap_status; + union ta_rap_cmd_input rap_in_message; + union ta_rap_cmd_output rap_out_message; + uint8_t reserved[64]; +}; + +#endif // #define _TA_RAP_IF_H diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c index 418cf097c918..5288617ca552 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c @@ -32,20 +32,6 @@ #define UMC_6_INST_DIST 0x40000 -/* - * (addr / 256) * 8192, the higher 26 bits in ErrorAddr - * is the index of 8KB block - */ -#define ADDR_OF_8KB_BLOCK(addr) (((addr) & ~0xffULL) << 5) -/* channel index is the index of 256B block */ -#define ADDR_OF_256B_BLOCK(channel_index) ((channel_index) << 8) -/* offset in 256B block */ -#define OFFSET_IN_256B_BLOCK(addr) ((addr) & 0xffULL) - -#define LOOP_UMC_INST(umc_inst) for ((umc_inst) = 0; (umc_inst) < adev->umc.umc_inst_num; (umc_inst)++) -#define LOOP_UMC_CH_INST(ch_inst) for ((ch_inst) = 0; (ch_inst) < adev->umc.channel_inst_num; (ch_inst)++) -#define LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) LOOP_UMC_INST((umc_inst)) LOOP_UMC_CH_INST((ch_inst)) - const uint32_t umc_v6_1_channel_idx_tbl[UMC_V6_1_UMC_INSTANCE_NUM][UMC_V6_1_CHANNEL_INSTANCE_NUM] = { {2, 18, 11, 27}, {4, 20, 13, 29}, diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c new file mode 100644 index 000000000000..5665c77a9d58 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c @@ -0,0 +1,331 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "umc_v8_7.h" +#include "amdgpu_ras.h" +#include "amdgpu.h" + +#include "rsmu/rsmu_0_0_2_offset.h" +#include "rsmu/rsmu_0_0_2_sh_mask.h" +#include "umc/umc_8_7_0_offset.h" +#include "umc/umc_8_7_0_sh_mask.h" + +#define UMC_8_INST_DIST 0x40000 + +const uint32_t + umc_v8_7_channel_idx_tbl[UMC_V8_7_UMC_INSTANCE_NUM][UMC_V8_7_CHANNEL_INSTANCE_NUM] = { + {2, 11}, {4, 13}, + {1, 8}, {7, 14}, + {10, 3}, {12, 5}, + {9, 0}, {15, 6} +}; + +static inline uint32_t get_umc_8_reg_offset(struct amdgpu_device *adev, + uint32_t umc_inst, + uint32_t ch_inst) +{ + return adev->umc.channel_offs*ch_inst + UMC_8_INST_DIST*umc_inst; +} + +static void umc_v8_7_clear_error_count_per_channel(struct amdgpu_device *adev, + uint32_t umc_reg_offset) +{ + uint32_t ecc_err_cnt_addr; + uint32_t ecc_err_cnt_sel, ecc_err_cnt_sel_addr; + + ecc_err_cnt_sel_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCntSel); + ecc_err_cnt_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCnt); + + /* select the lower chip */ + ecc_err_cnt_sel = RREG32_PCIE((ecc_err_cnt_sel_addr + + umc_reg_offset) * 4); + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, + UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 0); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, + ecc_err_cnt_sel); + + /* clear lower chip error count */ + WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4, + UMC_V8_7_CE_CNT_INIT); + + /* select the higher chip */ + ecc_err_cnt_sel = RREG32_PCIE((ecc_err_cnt_sel_addr + + umc_reg_offset) * 4); + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, + UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 1); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, + ecc_err_cnt_sel); + + /* clear higher chip error count */ + WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4, + UMC_V8_7_CE_CNT_INIT); +} + +static void umc_v8_7_clear_error_count(struct amdgpu_device *adev) +{ + uint32_t umc_inst = 0; + uint32_t ch_inst = 0; + uint32_t umc_reg_offset = 0; + + LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { + umc_reg_offset = get_umc_8_reg_offset(adev, + umc_inst, + ch_inst); + + umc_v8_7_clear_error_count_per_channel(adev, + umc_reg_offset); + } +} + +static void umc_v8_7_query_correctable_error_count(struct amdgpu_device *adev, + uint32_t umc_reg_offset, + unsigned long *error_count) +{ + uint32_t ecc_err_cnt_sel, ecc_err_cnt_sel_addr; + uint32_t ecc_err_cnt, ecc_err_cnt_addr; + uint64_t mc_umc_status; + uint32_t mc_umc_status_addr; + + /* UMC 8_7_2 registers */ + ecc_err_cnt_sel_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCntSel); + ecc_err_cnt_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCnt); + mc_umc_status_addr = + SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_STATUST0); + + /* select the lower chip and check the error count */ + ecc_err_cnt_sel = RREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4); + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 0); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, ecc_err_cnt_sel); + + ecc_err_cnt = RREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4); + *error_count += + (REG_GET_FIELD(ecc_err_cnt, UMCCH0_0_GeccErrCnt, GeccErrCnt) - + UMC_V8_7_CE_CNT_INIT); + + /* select the higher chip and check the err counter */ + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 1); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, ecc_err_cnt_sel); + + ecc_err_cnt = RREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4); + *error_count += + (REG_GET_FIELD(ecc_err_cnt, UMCCH0_0_GeccErrCnt, GeccErrCnt) - + UMC_V8_7_CE_CNT_INIT); + + /* check for SRAM correctable error + MCUMC_STATUS is a 64 bit register */ + mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4); + if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, ErrorCodeExt) == 6 && + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1) + *error_count += 1; +} + +static void umc_v8_7_querry_uncorrectable_error_count(struct amdgpu_device *adev, + uint32_t umc_reg_offset, + unsigned long *error_count) +{ + uint64_t mc_umc_status; + uint32_t mc_umc_status_addr; + + mc_umc_status_addr = SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_STATUST0); + + /* check the MCUMC_STATUS */ + mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4); + if ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) && + (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 || + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 || + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 || + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1)) + *error_count += 1; +} + +static void umc_v8_7_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status) +{ + struct ras_err_data* err_data = (struct ras_err_data*)ras_error_status; + + uint32_t umc_inst = 0; + uint32_t ch_inst = 0; + uint32_t umc_reg_offset = 0; + + LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { + umc_reg_offset = get_umc_8_reg_offset(adev, + umc_inst, + ch_inst); + + umc_v8_7_query_correctable_error_count(adev, + umc_reg_offset, + &(err_data->ce_count)); + umc_v8_7_querry_uncorrectable_error_count(adev, + umc_reg_offset, + &(err_data->ue_count)); + } + + umc_v8_7_clear_error_count(adev); +} + +static void umc_v8_7_query_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, + uint32_t umc_reg_offset, + uint32_t ch_inst, + uint32_t umc_inst) +{ + uint32_t lsb, mc_umc_status_addr; + uint64_t mc_umc_status, err_addr, retired_page, mc_umc_addrt0; + struct eeprom_table_record *err_rec; + uint32_t channel_index = adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; + + mc_umc_status_addr = + SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_STATUST0); + mc_umc_addrt0 = + SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_ADDRT0); + + mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4); + + if (mc_umc_status == 0) + return; + + if (!err_data->err_addr) { + /* clear umc status */ + WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL); + return; + } + + err_rec = &err_data->err_addr[err_data->err_addr_cnt]; + + /* calculate error address if ue/ce error is detected */ + if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && + (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { + + err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4); + /* the lowest lsb bits should be ignored */ + lsb = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, LSB); + err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); + err_addr &= ~((0x1ULL << lsb) - 1); + + /* translate umc channel address to soc pa, 3 parts are included */ + retired_page = ADDR_OF_8KB_BLOCK(err_addr) | + ADDR_OF_256B_BLOCK(channel_index) | + OFFSET_IN_256B_BLOCK(err_addr); + + /* we only save ue error information currently, ce is skipped */ + if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) + == 1) { + err_rec->address = err_addr; + /* page frame address is saved */ + err_rec->retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT; + err_rec->ts = (uint64_t)ktime_get_real_seconds(); + err_rec->err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE; + err_rec->cu = 0; + err_rec->mem_channel = channel_index; + err_rec->mcumc_id = umc_inst; + + err_data->err_addr_cnt++; + } + } + + /* clear umc status */ + WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL); +} + +static void umc_v8_7_query_ras_error_address(struct amdgpu_device *adev, + void *ras_error_status) +{ + struct ras_err_data* err_data = (struct ras_err_data*)ras_error_status; + + uint32_t umc_inst = 0; + uint32_t ch_inst = 0; + uint32_t umc_reg_offset = 0; + + LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { + umc_reg_offset = get_umc_8_reg_offset(adev, + umc_inst, + ch_inst); + + umc_v8_7_query_error_address(adev, + err_data, + umc_reg_offset, + ch_inst, + umc_inst); + } +} + +static void umc_v8_7_err_cnt_init_per_channel(struct amdgpu_device *adev, + uint32_t umc_reg_offset) +{ + uint32_t ecc_err_cnt_sel, ecc_err_cnt_sel_addr; + uint32_t ecc_err_cnt_addr; + + ecc_err_cnt_sel_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCntSel); + ecc_err_cnt_addr = + SOC15_REG_OFFSET(UMC, 0, mmUMCCH0_0_GeccErrCnt); + + /* select the lower chip and check the error count */ + ecc_err_cnt_sel = RREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4); + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 0); + /* set ce error interrupt type to APIC based interrupt */ + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, UMCCH0_0_GeccErrCntSel, + GeccErrInt, 0x1); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, ecc_err_cnt_sel); + /* set error count to initial value */ + WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V8_7_CE_CNT_INIT); + + /* select the higher chip and check the err counter */ + ecc_err_cnt_sel = REG_SET_FIELD(ecc_err_cnt_sel, UMCCH0_0_GeccErrCntSel, + GeccErrCntCsSel, 1); + WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, ecc_err_cnt_sel); + WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V8_7_CE_CNT_INIT); +} + +static void umc_v8_7_err_cnt_init(struct amdgpu_device *adev) +{ + uint32_t umc_inst = 0; + uint32_t ch_inst = 0; + uint32_t umc_reg_offset = 0; + + LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { + umc_reg_offset = get_umc_8_reg_offset(adev, + umc_inst, + ch_inst); + + umc_v8_7_err_cnt_init_per_channel(adev, umc_reg_offset); + } +} + +const struct amdgpu_umc_funcs umc_v8_7_funcs = { + .err_cnt_init = umc_v8_7_err_cnt_init, + .ras_late_init = amdgpu_umc_ras_late_init, + .query_ras_error_count = umc_v8_7_query_ras_error_count, + .query_ras_error_address = umc_v8_7_query_ras_error_address, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_7.h b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.h new file mode 100644 index 000000000000..d4d0468e3df5 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.h @@ -0,0 +1,51 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __UMC_V8_7_H__ +#define __UMC_V8_7_H__ + +#include "soc15_common.h" +#include "amdgpu.h" + +/* HBM Memory Channel Width */ +#define UMC_V8_7_HBM_MEMORY_CHANNEL_WIDTH 128 +/* number of umc channel instance with memory map register access */ +#define UMC_V8_7_CHANNEL_INSTANCE_NUM 2 +/* number of umc instance with memory map register access */ +#define UMC_V8_7_UMC_INSTANCE_NUM 8 +/* total channel instances in one umc block */ +#define UMC_V8_7_TOTAL_CHANNEL_NUM (UMC_V8_7_CHANNEL_INSTANCE_NUM * UMC_V8_7_UMC_INSTANCE_NUM) +/* UMC regiser per channel offset */ +#define UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA 0x400 + +/* EccErrCnt max value */ +#define UMC_V8_7_CE_CNT_MAX 0xffff +/* umc ce interrupt threshold */ +#define UMC_V8_7_CE_INT_THRESHOLD 0xffff +/* umc ce count initial value */ +#define UMC_V8_7_CE_CNT_INIT (UMC_V8_7_CE_CNT_MAX - UMC_V8_7_CE_INT_THRESHOLD) + +extern const struct amdgpu_umc_funcs umc_v8_7_funcs; +extern const uint32_t + umc_v8_7_channel_idx_tbl[UMC_V8_7_UMC_INSTANCE_NUM][UMC_V8_7_CHANNEL_INSTANCE_NUM]; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index e99bef6e2354..139fac0d8e76 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -80,23 +80,18 @@ static int vcn_v2_5_early_init(void *handle) adev->vcn.harvest_config = 0; adev->vcn.num_enc_rings = 1; } else { - if (adev->asic_type == CHIP_ARCTURUS) { - u32 harvest; - int i; - - adev->vcn.num_vcn_inst = VCN25_MAX_HW_INSTANCES_ARCTURUS; - for (i = 0; i < adev->vcn.num_vcn_inst; i++) { - harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING); - if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK) - adev->vcn.harvest_config |= 1 << i; - } - - if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 | - AMDGPU_VCN_HARVEST_VCN1)) - /* both instances are harvested, disable the block */ - return -ENOENT; - } else - adev->vcn.num_vcn_inst = 1; + u32 harvest; + int i; + adev->vcn.num_vcn_inst = VCN25_MAX_HW_INSTANCES_ARCTURUS; + for (i = 0; i < adev->vcn.num_vcn_inst; i++) { + harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING); + if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK) + adev->vcn.harvest_config |= 1 << i; + } + if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 | + AMDGPU_VCN_HARVEST_VCN1)) + /* both instances are harvested, disable the block */ + return -ENOENT; adev->vcn.num_enc_rings = 2; } @@ -1108,7 +1103,7 @@ static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev, { uint32_t data = 0, loop = 0, size = 0; uint64_t addr = table->gpu_addr; - struct mmsch_v1_1_init_header *header = NULL;; + struct mmsch_v1_1_init_header *header = NULL; header = (struct mmsch_v1_1_init_header *)table->cpu_addr; size = header->total_size; diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index 63e5547cfb16..589d6cd8adec 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -198,7 +198,7 @@ static int vcn_v3_0_sw_init(void *handle) } else { ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i; } - if (i != 0) + if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 0) ring->no_scheduler = true; sprintf(ring->name, "vcn_dec_%d", i); r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0, @@ -222,7 +222,7 @@ static int vcn_v3_0_sw_init(void *handle) } else { ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i; } - if (i != 1) + if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 1) ring->no_scheduler = true; sprintf(ring->name, "vcn_enc_%d.%d", i, j); r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0, diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index f6f2ed0830b1..9bcd0eebc6d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -752,8 +752,10 @@ static int vi_asic_reset(struct amdgpu_device *adev) int r; if (vi_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { + dev_info(adev->dev, "BACO reset\n"); r = amdgpu_dpm_baco_reset(adev); } else { + dev_info(adev->dev, "PCI CONFIG reset\n"); r = vi_asic_pci_config_reset(adev); } @@ -1066,6 +1068,10 @@ static bool vi_need_reset_on_init(struct amdgpu_device *adev) return false; } +static void vi_pre_asic_init(struct amdgpu_device *adev) +{ +} + static const struct amdgpu_asic_funcs vi_asic_funcs = { .read_disabled_bios = &vi_read_disabled_bios, @@ -1086,6 +1092,7 @@ static const struct amdgpu_asic_funcs vi_asic_funcs = .need_reset_on_init = &vi_need_reset_on_init, .get_pcie_replay_count = &vi_get_pcie_replay_count, .supports_baco = &vi_asic_supports_baco, + .pre_asic_init = &vi_pre_asic_init, }; #define CZ_REV_BRISTOL(rev) \ @@ -1507,8 +1514,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_MC, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) { @@ -1526,8 +1532,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_SDMA, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) { @@ -1545,8 +1550,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_HDP, pp_support_state, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } @@ -1560,8 +1564,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_BIF, PP_STATE_SUPPORT_LS, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) { if (state == AMD_CG_STATE_UNGATE) @@ -1573,8 +1576,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_BIF, PP_STATE_SUPPORT_CG, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) { @@ -1588,8 +1590,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_DRM, PP_STATE_SUPPORT_LS, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) { @@ -1603,8 +1604,7 @@ static int vi_common_set_clockgating_state_by_smu(void *handle, PP_BLOCK_SYS_ROM, PP_STATE_SUPPORT_CG, pp_state); - if (adev->powerplay.pp_funcs->set_clockgating_by_smu) - amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); + amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index 577d901fdb63..affbca7c0050 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -911,7 +911,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = { 0x705d0000, 0x807c817c, 0x8070ff70, 0x00000080, 0xbf0a7b7c, 0xbf85fff8, - 0xbf82014f, 0xbef4037e, + 0xbf820151, 0xbef4037e, 0x8775ff7f, 0x0000ffff, 0x8875ff75, 0x00040000, 0xbef60380, 0xbef703ff, @@ -1024,61 +1024,62 @@ static const uint32_t cwsr_trap_nv1x_hex[] = { 0xbe883108, 0xbe8a310a, 0xbe8c310c, 0xbe8e310e, 0xbf06807c, 0xbf84fff0, - 0xb9782a05, 0x80788178, - 0xbf0d9972, 0xbf850002, - 0x8f788978, 0xbf820001, - 0x8f788a78, 0xb96e1e06, - 0x8f6e8a6e, 0x80786e78, - 0x8078ff78, 0x00000200, - 0xbef603ff, 0x01000000, - 0xf4211bfa, 0xf0000000, - 0x80788478, 0xf4211b3a, + 0xba80f801, 0x00000000, + 0xbf8a0000, 0xb9782a05, + 0x80788178, 0xbf0d9972, + 0xbf850002, 0x8f788978, + 0xbf820001, 0x8f788a78, + 0xb96e1e06, 0x8f6e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0xbef603ff, + 0x01000000, 0xf4211bfa, 0xf0000000, 0x80788478, - 0xf4211b7a, 0xf0000000, - 0x80788478, 0xf4211c3a, + 0xf4211b3a, 0xf0000000, + 0x80788478, 0xf4211b7a, 0xf0000000, 0x80788478, - 0xf4211c7a, 0xf0000000, - 0x80788478, 0xf4211eba, + 0xf4211c3a, 0xf0000000, + 0x80788478, 0xf4211c7a, 0xf0000000, 0x80788478, - 0xf4211efa, 0xf0000000, - 0x80788478, 0xf4211e7a, + 0xf4211eba, 0xf0000000, + 0x80788478, 0xf4211efa, 0xf0000000, 0x80788478, - 0xf4211cfa, 0xf0000000, - 0x80788478, 0xf4211bba, + 0xf4211e7a, 0xf0000000, + 0x80788478, 0xf4211cfa, 0xf0000000, 0x80788478, - 0xbf8cc07f, 0xb9eef814, 0xf4211bba, 0xf0000000, 0x80788478, 0xbf8cc07f, - 0xb9eef815, 0xbefc036f, - 0xbefe0370, 0xbeff0371, - 0x876f7bff, 0x000003ff, - 0xb9ef4803, 0xb9f9f816, - 0x876f7bff, 0xfffff800, - 0x906f8b6f, 0xb9efa2c3, - 0xb9f3f801, 0xb96e2a05, - 0x806e816e, 0xbf0d9972, - 0xbf850002, 0x8f6e896e, - 0xbf820001, 0x8f6e8a6e, - 0x806eff6e, 0x00000200, - 0x806e746e, 0x826f8075, - 0x876fff6f, 0x0000ffff, - 0xf4091c37, 0xfa000050, - 0xf4091d37, 0xfa000060, - 0xf4011e77, 0xfa000074, - 0xbf8cc07f, 0x876fff6d, - 0xfc000000, 0x906f9a6f, - 0x8f6f906f, 0xbeee0380, + 0xb9eef814, 0xf4211bba, + 0xf0000000, 0x80788478, + 0xbf8cc07f, 0xb9eef815, + 0xbefc036f, 0xbefe0370, + 0xbeff0371, 0x876f7bff, + 0x000003ff, 0xb9ef4803, + 0xb9f9f816, 0x876f7bff, + 0xfffff800, 0x906f8b6f, + 0xb9efa2c3, 0xb9f3f801, + 0xb96e2a05, 0x806e816e, + 0xbf0d9972, 0xbf850002, + 0x8f6e896e, 0xbf820001, + 0x8f6e8a6e, 0x806eff6e, + 0x00000200, 0x806e746e, + 0x826f8075, 0x876fff6f, + 0x0000ffff, 0xf4091c37, + 0xfa000050, 0xf4091d37, + 0xfa000060, 0xf4011e77, + 0xfa000074, 0xbf8cc07f, + 0x876fff6d, 0xfc000000, + 0x906f9a6f, 0x8f6f906f, + 0xbeee0380, 0x886e6f6e, + 0x876fff6d, 0x02000000, + 0x906f996f, 0x8f6f8f6f, 0x886e6f6e, 0x876fff6d, - 0x02000000, 0x906f996f, - 0x8f6f8f6f, 0x886e6f6e, - 0x876fff6d, 0x01000000, - 0x906f986f, 0x8f6f996f, - 0x886e6f6e, 0x876fff7a, - 0x00800000, 0x906f976f, - 0xb9eef807, 0x876dff6d, - 0x0000ffff, 0x87fe7e7e, - 0x87ea6a6a, 0xb9faf802, - 0xbf8a0000, 0xbe80226c, + 0x01000000, 0x906f986f, + 0x8f6f996f, 0x886e6f6e, + 0x876fff7a, 0x00800000, + 0x906f976f, 0xb9eef807, + 0x876dff6d, 0x0000ffff, + 0x87fe7e7e, 0x87ea6a6a, + 0xb9faf802, 0xbe80226c, 0xbf810000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, @@ -1807,7 +1808,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0x705d0000, 0x807c817c, 0x8070ff70, 0x00000080, 0xbf0a7b7c, 0xbf85fff8, - 0xbf82013a, 0xbef4037e, + 0xbf82013c, 0xbef4037e, 0x8775ff7f, 0x0000ffff, 0x8875ff75, 0x00040000, 0xbef60380, 0xbef703ff, @@ -1920,50 +1921,51 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xbe883108, 0xbe8a310a, 0xbe8c310c, 0xbe8e310e, 0xbf06807c, 0xbf84fff0, - 0xb9782a05, 0x80788178, - 0xbf0d9972, 0xbf850002, - 0x8f788978, 0xbf820001, - 0x8f788a78, 0xb96e1e06, - 0x8f6e8a6e, 0x80786e78, - 0x8078ff78, 0x00000200, - 0xbef603ff, 0x01000000, - 0xf4211bfa, 0xf0000000, - 0x80788478, 0xf4211b3a, + 0xba80f801, 0x00000000, + 0xbf8a0000, 0xb9782a05, + 0x80788178, 0xbf0d9972, + 0xbf850002, 0x8f788978, + 0xbf820001, 0x8f788a78, + 0xb96e1e06, 0x8f6e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0xbef603ff, + 0x01000000, 0xf4211bfa, 0xf0000000, 0x80788478, - 0xf4211b7a, 0xf0000000, - 0x80788478, 0xf4211c3a, + 0xf4211b3a, 0xf0000000, + 0x80788478, 0xf4211b7a, 0xf0000000, 0x80788478, - 0xf4211c7a, 0xf0000000, - 0x80788478, 0xf4211eba, + 0xf4211c3a, 0xf0000000, + 0x80788478, 0xf4211c7a, 0xf0000000, 0x80788478, - 0xf4211efa, 0xf0000000, - 0x80788478, 0xf4211e7a, + 0xf4211eba, 0xf0000000, + 0x80788478, 0xf4211efa, 0xf0000000, 0x80788478, - 0xf4211cfa, 0xf0000000, - 0x80788478, 0xf4211bba, + 0xf4211e7a, 0xf0000000, + 0x80788478, 0xf4211cfa, 0xf0000000, 0x80788478, - 0xbf8cc07f, 0xb9eef814, 0xf4211bba, 0xf0000000, 0x80788478, 0xbf8cc07f, - 0xb9eef815, 0xbefc036f, - 0xbefe0370, 0xbeff0371, - 0x876f7bff, 0x000003ff, - 0xb9ef4803, 0x876f7bff, - 0xfffff800, 0x906f8b6f, - 0xb9efa2c3, 0xb9f3f801, - 0xb96e2a05, 0x806e816e, - 0xbf0d9972, 0xbf850002, - 0x8f6e896e, 0xbf820001, - 0x8f6e8a6e, 0x806eff6e, - 0x00000200, 0x806e746e, - 0x826f8075, 0x876fff6f, - 0x0000ffff, 0xf4091c37, - 0xfa000050, 0xf4091d37, - 0xfa000060, 0xf4011e77, - 0xfa000074, 0xbf8cc07f, - 0x876dff6d, 0x0000ffff, - 0x87fe7e7e, 0x87ea6a6a, - 0xb9faf802, 0xbf8a0000, + 0xb9eef814, 0xf4211bba, + 0xf0000000, 0x80788478, + 0xbf8cc07f, 0xb9eef815, + 0xbefc036f, 0xbefe0370, + 0xbeff0371, 0x876f7bff, + 0x000003ff, 0xb9ef4803, + 0x876f7bff, 0xfffff800, + 0x906f8b6f, 0xb9efa2c3, + 0xb9f3f801, 0xb96e2a05, + 0x806e816e, 0xbf0d9972, + 0xbf850002, 0x8f6e896e, + 0xbf820001, 0x8f6e8a6e, + 0x806eff6e, 0x00000200, + 0x806e746e, 0x826f8075, + 0x876fff6f, 0x0000ffff, + 0xf4091c37, 0xfa000050, + 0xf4091d37, 0xfa000060, + 0xf4011e77, 0xfa000074, + 0xbf8cc07f, 0x876dff6d, + 0x0000ffff, 0x87fe7e7e, + 0x87ea6a6a, 0xb9faf802, 0xbe80226c, 0xbf810000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm index 5b220f2a7501..5081f91190b8 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm @@ -894,6 +894,11 @@ L_RESTORE_SGPR: s_cmp_eq_u32 m0, 0 //scc = (m0 < s_sgpr_save_num) ? 1 : 0 s_cbranch_scc0 L_RESTORE_SGPR_LOOP + // s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception. + // Clear DEBUG_EN before and restore MODE after the barrier. + s_setreg_imm32_b32 hwreg(HW_REG_MODE), 0 + s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG + /* restore HW registers */ L_RESTORE_HWREG: // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR) @@ -976,8 +981,6 @@ L_RESTORE_HWREG: s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu - s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG - s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution L_END_PGM: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index e9b96ad3d9a5..b7b16adb0615 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1254,7 +1254,7 @@ bool kfd_dev_is_large_bar(struct kfd_dev *dev) return true; } - if (dev->device_info->needs_iommu_device) + if (dev->use_iommu_v2) return false; amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 6a250f8fcfb8..3fac06b281ce 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -742,6 +742,22 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, return 0; } +static bool kfd_ignore_crat(void) +{ + bool ret; + + if (ignore_crat) + return true; + +#ifndef KFD_SUPPORT_IOMMU_V2 + ret = true; +#else + ret = false; +#endif + + return ret; +} + /* * kfd_create_crat_image_acpi - Allocates memory for CRAT image and * copies CRAT from ACPI (if available). @@ -776,7 +792,7 @@ int kfd_create_crat_image_acpi(void **crat_image, size_t *size) return -EINVAL; } - if (ignore_crat) { + if (kfd_ignore_crat()) { pr_info("CRAT table disabled by module option\n"); return -ENODATA; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 4bfedaab183f..0e71a0543f98 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -29,6 +29,7 @@ #include "cwsr_trap_handler.h" #include "kfd_iommu.h" #include "amdgpu_amdkfd.h" +#include "kfd_smi_events.h" #define MQD_SIZE_ALIGNED 768 @@ -115,6 +116,7 @@ static const struct kfd_device_info carrizo_device_info = { .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; +#endif static const struct kfd_device_info raven_device_info = { .asic_family = CHIP_RAVEN, @@ -133,7 +135,6 @@ static const struct kfd_device_info raven_device_info = { .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; -#endif static const struct kfd_device_info hawaii_device_info = { .asic_family = CHIP_HAWAII, @@ -711,11 +712,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, goto kfd_doorbell_error; } - if (kfd->kfd2kgd->get_hive_id) - kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd); + kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd); - if (kfd->kfd2kgd->get_unique_id) - kfd->unique_id = kfd->kfd2kgd->get_unique_id(kfd->kgd); + kfd->unique_id = amdgpu_amdkfd_get_unique_id(kfd->kgd); if (kfd_interrupt_init(kfd)) { dev_err(kfd_device, "Error initializing interrupts\n"); @@ -737,6 +736,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, goto gws_error; } + /* If CRAT is broken, won't set iommu enabled */ + kfd_double_confirm_iommu_support(kfd); + if (kfd_iommu_device_init(kfd)) { dev_err(kfd_device, "Error initializing iommuv2\n"); goto device_iommu_error; @@ -810,6 +812,8 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd) if (!kfd->init_complete) return 0; + kfd_smi_event_update_gpu_reset(kfd, false); + kfd->dqm->ops.pre_reset(kfd->dqm); kgd2kfd_suspend(kfd, false); @@ -838,6 +842,8 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd) atomic_set(&kfd->sram_ecc_flag, 0); + kfd_smi_event_update_gpu_reset(kfd, true); + return 0; } @@ -1245,6 +1251,12 @@ void kfd_dec_compute_active(struct kfd_dev *kfd) WARN_ONCE(count < 0, "Compute profile ref. count error"); } +void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) +{ + if (kfd) + kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask); +} + #if defined(CONFIG_DEBUG_FS) /* This function will send a package to HIQ to hang the HWS diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index e0e60b0d0669..560adc57a050 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -153,30 +153,6 @@ static void decrement_queue_count(struct device_queue_manager *dqm, dqm->active_cp_queue_count--; } -int read_sdma_queue_counter(uint64_t q_rptr, uint64_t *val) -{ - int ret; - uint64_t tmp = 0; - - if (!val) - return -EINVAL; - /* - * SDMA activity counter is stored at queue's RPTR + 0x8 location. - */ - if (!access_ok((const void __user *)(q_rptr + - sizeof(uint64_t)), sizeof(uint64_t))) { - pr_err("Can't access sdma queue activity counter\n"); - return -EFAULT; - } - - ret = get_user(tmp, (uint64_t *)(q_rptr + sizeof(uint64_t))); - if (!ret) { - *val = tmp; - } - - return ret; -} - static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) { struct kfd_dev *dev = qpd->dqm->dev; @@ -552,7 +528,7 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm, /* Get the SDMA queue stats */ if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - retval = read_sdma_queue_counter((uint64_t)q->properties.read_ptr, + retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, &sdma_val); if (retval) pr_err("Failed to read SDMA queue counter for queue: %d\n", @@ -1473,7 +1449,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, /* Get the SDMA queue stats */ if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - retval = read_sdma_queue_counter((uint64_t)q->properties.read_ptr, + retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, &sdma_val); if (retval) pr_err("Failed to read SDMA queue counter for queue: %d\n", diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 49d8e324c636..16262e5d93f5 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -251,5 +251,11 @@ static inline void dqm_unlock(struct device_queue_manager *dqm) mutex_unlock(&dqm->lock_hidden); } -int read_sdma_queue_counter(uint64_t q_rptr, uint64_t *val); +static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) +{ + /* + * SDMA activity counter is stored at queue's RPTR + 0x8 location. + */ + return get_user(*val, q_rptr + 1); +} #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c index 95a82ac455f2..309f63a0b34a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c @@ -62,7 +62,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm, SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; if (amdgpu_noretry && - !dqm->dev->device_info->needs_iommu_device) + !dqm->dev->use_iommu_v2) qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c index c1166c40ac15..3c22909470f2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c @@ -321,7 +321,7 @@ static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id) pdd->lds_base = MAKE_LDS_APP_BASE_VI(); pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); - if (!pdd->dev->device_info->needs_iommu_device) { + if (!pdd->dev->use_iommu_v2) { /* dGPUs: SVM aperture starting at 0 * with small reserved space for kernel. * Set them to CANONICAL addresses. @@ -425,7 +425,7 @@ int kfd_init_apertures(struct kfd_process *process) return -EINVAL; } - if (!dev->device_info->needs_iommu_device) { + if (!dev->use_iommu_v2) { /* dGPUs: the reserved space for kernel * before SVM */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c index 7c8786b9eb0a..5a64915abaf7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c @@ -41,7 +41,7 @@ int kfd_iommu_check_device(struct kfd_dev *kfd) struct amd_iommu_device_info iommu_info; int err; - if (!kfd->device_info->needs_iommu_device) + if (!kfd->use_iommu_v2) return -ENODEV; iommu_info.flags = 0; @@ -63,7 +63,7 @@ int kfd_iommu_device_init(struct kfd_dev *kfd) unsigned int pasid_limit; int err; - if (!kfd->device_info->needs_iommu_device) + if (!kfd->use_iommu_v2) return 0; iommu_info.flags = 0; @@ -109,7 +109,7 @@ int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd) struct kfd_process *p = pdd->process; int err; - if (!dev->device_info->needs_iommu_device || pdd->bound == PDD_BOUND) + if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND) return 0; if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) { @@ -284,7 +284,7 @@ static void kfd_unbind_processes_from_device(struct kfd_dev *kfd) */ void kfd_iommu_suspend(struct kfd_dev *kfd) { - if (!kfd->device_info->needs_iommu_device) + if (!kfd->use_iommu_v2) return; kfd_unbind_processes_from_device(kfd); @@ -304,7 +304,7 @@ int kfd_iommu_resume(struct kfd_dev *kfd) unsigned int pasid_limit; int err; - if (!kfd->device_info->needs_iommu_device) + if (!kfd->use_iommu_v2) return 0; pasid_limit = kfd_get_pasid_limit(); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 6727e9de5b8b..023629f28495 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -297,6 +297,9 @@ struct kfd_dev { bool pci_atomic_requested; + /* Use IOMMU v2 flag */ + bool use_iommu_v2; + /* SRAM ECC flag */ atomic_t sram_ecc_flag; @@ -309,6 +312,8 @@ struct kfd_dev { /* Clients watching SMI events */ struct list_head smi_clients; spinlock_t smi_lock; + + uint32_t reset_seq_num; }; enum kfd_mempool { @@ -892,6 +897,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd); int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev); int kfd_numa_node_to_apic_id(int numa_node_id); +void kfd_double_confirm_iommu_support(struct kfd_dev *gpu); /* Interrupts */ int kfd_interrupt_init(struct kfd_dev *dev); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 40695d52e9a8..a0e12a79ab7d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -87,7 +87,7 @@ struct kfd_sdma_activity_handler_workarea { }; struct temp_sdma_queue_list { - uint64_t rptr; + uint64_t __user *rptr; uint64_t sdma_val; unsigned int queue_id; struct list_head list; @@ -159,7 +159,7 @@ static void kfd_sdma_activity_worker(struct work_struct *work) } INIT_LIST_HEAD(&sdma_q->list); - sdma_q->rptr = (uint64_t)q->properties.read_ptr; + sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; sdma_q->queue_id = q->properties.queue_id; list_add_tail(&sdma_q->list, &sdma_q_list.list); } @@ -218,7 +218,7 @@ static void kfd_sdma_activity_worker(struct work_struct *work) continue; list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { - if (((uint64_t)q->properties.read_ptr == sdma_q->rptr) && + if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && (sdma_q->queue_id == q->properties.queue_id)) { list_del(&sdma_q->list); kfree(sdma_q); @@ -270,6 +270,7 @@ static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, kfd_sdma_activity_worker); sdma_activity_work_handler.pdd = pdd; + sdma_activity_work_handler.sdma_activity_counter = 0; schedule_work(&sdma_activity_work_handler.sdma_activity_work); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c index 7b348bf9df21..17d1736367ea 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c @@ -24,6 +24,7 @@ #include <linux/wait.h> #include <linux/anon_inodes.h> #include <uapi/linux/kfd_ioctl.h> +#include "amdgpu.h" #include "amdgpu_vm.h" #include "kfd_priv.h" #include "kfd_smi_events.h" @@ -148,15 +149,94 @@ static int kfd_smi_ev_release(struct inode *inode, struct file *filep) return 0; } +static void add_event_to_kfifo(struct kfd_dev *dev, unsigned int smi_event, + char *event_msg, int len) +{ + struct kfd_smi_client *client; + + rcu_read_lock(); + + list_for_each_entry_rcu(client, &dev->smi_clients, list) { + if (!(READ_ONCE(client->events) & + KFD_SMI_EVENT_MASK_FROM_INDEX(smi_event))) + continue; + spin_lock(&client->lock); + if (kfifo_avail(&client->fifo) >= len) { + kfifo_in(&client->fifo, event_msg, len); + wake_up_all(&client->wait_queue); + } else { + pr_debug("smi_event(EventID: %u): no space left\n", + smi_event); + } + spin_unlock(&client->lock); + } + + rcu_read_unlock(); +} + +void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset) +{ + /* + * GpuReset msg = Reset seq number (incremented for + * every reset message sent before GPU reset). + * 1 byte event + 1 byte space + 8 bytes seq num + + * 1 byte \n + 1 byte \0 = 12 + */ + char fifo_in[12]; + int len; + unsigned int event; + + if (list_empty(&dev->smi_clients)) + return; + + memset(fifo_in, 0x0, sizeof(fifo_in)); + + if (post_reset) { + event = KFD_SMI_EVENT_GPU_POST_RESET; + } else { + event = KFD_SMI_EVENT_GPU_PRE_RESET; + ++(dev->reset_seq_num); + } + + len = snprintf(fifo_in, sizeof(fifo_in), "%x %x\n", event, + dev->reset_seq_num); + + add_event_to_kfifo(dev, event, fifo_in, len); +} + +void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev, + uint32_t throttle_bitmask) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)dev->kgd; + /* + * ThermalThrottle msg = throttle_bitmask(8): + * thermal_interrupt_count(16): + * 1 byte event + 1 byte space + 8 byte throttle_bitmask + + * 1 byte : + 16 byte thermal_interupt_counter + 1 byte \n + + * 1 byte \0 = 29 + */ + char fifo_in[29]; + int len; + + if (list_empty(&dev->smi_clients)) + return; + + len = snprintf(fifo_in, sizeof(fifo_in), "%x %x:%llx\n", + KFD_SMI_EVENT_THERMAL_THROTTLE, throttle_bitmask, + atomic64_read(&adev->smu.throttle_int_counter)); + + add_event_to_kfifo(dev, KFD_SMI_EVENT_THERMAL_THROTTLE, fifo_in, len); +} + void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid) { struct amdgpu_device *adev = (struct amdgpu_device *)dev->kgd; struct amdgpu_task_info task_info; /* VmFault msg = (hex)uint32_pid(8) + :(1) + task name(16) = 25 */ - /* 16 bytes event + 1 byte space + 25 bytes msg + 1 byte \n = 43 + /* 1 byte event + 1 byte space + 25 bytes msg + 1 byte \n + + * 1 byte \0 = 29 */ - char fifo_in[43]; - struct kfd_smi_client *client; + char fifo_in[29]; int len; if (list_empty(&dev->smi_clients)) @@ -168,25 +248,10 @@ void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid) if (!task_info.pid) return; - len = snprintf(fifo_in, 43, "%x %x:%s\n", KFD_SMI_EVENT_VMFAULT, + len = snprintf(fifo_in, sizeof(fifo_in), "%x %x:%s\n", KFD_SMI_EVENT_VMFAULT, task_info.pid, task_info.task_name); - rcu_read_lock(); - - list_for_each_entry_rcu(client, &dev->smi_clients, list) { - if (!(READ_ONCE(client->events) & KFD_SMI_EVENT_VMFAULT)) - continue; - spin_lock(&client->lock); - if (kfifo_avail(&client->fifo) >= len) { - kfifo_in(&client->fifo, fifo_in, len); - wake_up_all(&client->wait_queue); - } - else - pr_debug("smi_event(vmfault): no space left\n"); - spin_unlock(&client->lock); - } - - rcu_read_unlock(); + add_event_to_kfifo(dev, KFD_SMI_EVENT_VMFAULT, fifo_in, len); } int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h index a9cb218fef96..b9b0438202e2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h @@ -25,5 +25,8 @@ int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd); void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid); +void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev, + uint32_t throttle_bitmask); +void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset); #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index f185f6cbc05c..2b31c3066aaa 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -446,7 +446,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count", dev->node_props.cpu_cores_count); sysfs_show_32bit_prop(buffer, offs, "simd_count", - dev->node_props.simd_count); + dev->gpu ? dev->node_props.simd_count : 0); sysfs_show_32bit_prop(buffer, offs, "mem_banks_count", dev->node_props.mem_banks_count); sysfs_show_32bit_prop(buffer, offs, "caches_count", @@ -1139,7 +1139,7 @@ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) /* Discrete GPUs need their own topology device list * entries. Don't assign them to CPU/APU nodes. */ - if (!gpu->device_info->needs_iommu_device && + if (!gpu->use_iommu_v2 && dev->node_props.cpu_cores_count) continue; @@ -1239,7 +1239,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu) void *crat_image = NULL; size_t image_size = 0; int proximity_domain; - struct amdgpu_ras *ctx; + struct amdgpu_device *adev; INIT_LIST_HEAD(&temp_topology_device_list); @@ -1388,7 +1388,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu) * Overwrite ATS capability according to needs_iommu_device to fix * potential missing corresponding bit in CRAT of BIOS. */ - if (dev->gpu->device_info->needs_iommu_device) + if (dev->gpu->use_iommu_v2) dev->node_props.capability |= HSA_CAP_ATS_PRESENT; else dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT; @@ -1404,19 +1404,17 @@ int kfd_topology_add_device(struct kfd_dev *gpu) dev->node_props.max_waves_per_simd = 10; } - ctx = amdgpu_ras_get_context((struct amdgpu_device *)(dev->gpu->kgd)); - if (ctx) { - /* kfd only concerns sram ecc on GFX/SDMA and HBM ecc on UMC */ - dev->node_props.capability |= - (((ctx->features & BIT(AMDGPU_RAS_BLOCK__SDMA)) != 0) || - ((ctx->features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0)) ? - HSA_CAP_SRAM_EDCSUPPORTED : 0; - dev->node_props.capability |= ((ctx->features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? - HSA_CAP_MEM_EDCSUPPORTED : 0; - - dev->node_props.capability |= (ctx->features != 0) ? + adev = (struct amdgpu_device *)(dev->gpu->kgd); + /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */ + dev->node_props.capability |= + ((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ? + HSA_CAP_SRAM_EDCSUPPORTED : 0; + dev->node_props.capability |= ((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? + HSA_CAP_MEM_EDCSUPPORTED : 0; + + if (adev->asic_type != CHIP_VEGA10) + dev->node_props.capability |= (adev->ras_features != 0) ? HSA_CAP_RASEVENTNOTIFY : 0; - } kfd_debug_print_topology(); @@ -1515,6 +1513,29 @@ int kfd_numa_node_to_apic_id(int numa_node_id) return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id)); } +void kfd_double_confirm_iommu_support(struct kfd_dev *gpu) +{ + struct kfd_topology_device *dev; + + gpu->use_iommu_v2 = false; + + if (!gpu->device_info->needs_iommu_device) + return; + + down_read(&topology_lock); + + /* Only use IOMMUv2 if there is an APU topology node with no GPU + * assigned yet. This GPU will be assigned to it. + */ + list_for_each_entry(dev, &topology_device_list, list) + if (dev->node_props.cpu_cores_count && + dev->node_props.simd_count && + !dev->gpu) + gpu->use_iommu_v2 = true; + + up_read(&topology_lock); +} + #if defined(CONFIG_DEBUG_FS) int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 34ae4f3a32f4..f24abf428534 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -6,7 +6,7 @@ config DRM_AMD_DC bool "AMD DC - Enable new display engine" default y select SND_HDA_COMPONENT if SND_HDA_CORE - select DRM_AMD_DC_DCN if (X86 || PPC64) && !(KCOV_INSTRUMENT_ALL && KCOV_ENABLE_COMPARISONS) + select DRM_AMD_DC_DCN if (X86 || PPC64 || (ARM64 && KERNEL_MODE_NEON)) && !(KCOV_INSTRUMENT_ALL && KCOV_ENABLE_COMPARISONS) help Choose this option if you want to use the new display engine support for AMDGPU. This adds required support for Vega and @@ -31,6 +31,14 @@ config DRM_AMD_DC_HDCP help Choose this option if you want to support HDCP authentication. +config DRM_AMD_DC_SI + bool "AMD DC support for Southern Islands ASICs" + default n + help + Choose this option to enable new AMD DC support for SI asics + by default. This includes Tahiti, Pitcairn, Cape Verde, Oland. + Hainan is not supported by AMD DC and it has no physical DCE6. + config DEBUG_KERNEL_DC bool "Enable kgdb break in DC" depends on DRM_AMD_DC diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index fd96fafec5b8..ec29376667df 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -207,7 +207,7 @@ static void amdgpu_dm_set_psr_caps(struct dc_link *link); static bool amdgpu_dm_psr_enable(struct dc_stream_state *stream); static bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream); static bool amdgpu_dm_psr_disable(struct dc_stream_state *stream); - +static bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm); /* * dm_vblank_get_counter @@ -304,7 +304,7 @@ static struct amdgpu_crtc * get_crtc_by_otg_inst(struct amdgpu_device *adev, int otg_inst) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; @@ -356,7 +356,7 @@ static void dm_pflip_high_irq(void *interrupt_params) return; } - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED){ DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d !=AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p] \n", @@ -364,7 +364,7 @@ static void dm_pflip_high_irq(void *interrupt_params) AMDGPU_FLIP_SUBMITTED, amdgpu_crtc->crtc_id, amdgpu_crtc); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return; } @@ -416,7 +416,7 @@ static void dm_pflip_high_irq(void *interrupt_params) e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); e->pipe = amdgpu_crtc->crtc_id; - list_add_tail(&e->base.link, &adev->ddev->vblank_event_list); + list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); e = NULL; } @@ -429,7 +429,7 @@ static void dm_pflip_high_irq(void *interrupt_params) amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", amdgpu_crtc->crtc_id, amdgpu_crtc, @@ -465,7 +465,7 @@ static void dm_vupdate_high_irq(void *interrupt_params) /* BTR processing for pre-DCE12 ASICs */ if (acrtc_state->stream && adev->family < AMDGPU_FAMILY_AI) { - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); mod_freesync_handle_v_update( adev->dm.freesync_module, acrtc_state->stream, @@ -475,7 +475,7 @@ static void dm_vupdate_high_irq(void *interrupt_params) adev->dm.dc, acrtc_state->stream, &acrtc_state->vrr_params.adjust); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } } } @@ -525,7 +525,7 @@ static void dm_crtc_high_irq(void *interrupt_params) if (adev->family < AMDGPU_FAMILY_AI) return; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); if (acrtc_state->stream && acrtc_state->vrr_params.supported && acrtc_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE) { @@ -558,7 +558,7 @@ static void dm_crtc_high_irq(void *interrupt_params) acrtc->pflip_status = AMDGPU_FLIP_NONE; } - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } static int dm_set_clockgating_state(void *handle, @@ -580,7 +580,7 @@ static int dm_early_init(void* handle); static void amdgpu_dm_fbc_init(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct dm_comressor_info *compressor = &adev->dm.compressor; struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector); struct drm_display_mode *mode; @@ -622,7 +622,7 @@ static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port, unsigned char *buf, int max_bytes) { struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_connector *connector; struct drm_connector_list_iter conn_iter; struct amdgpu_dm_connector *aconnector; @@ -661,7 +661,7 @@ static int amdgpu_dm_audio_component_bind(struct device *kdev, struct device *hda_kdev, void *data) { struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_audio_component *acomp = data; acomp->ops = &amdgpu_dm_audio_component_ops; @@ -675,7 +675,7 @@ static void amdgpu_dm_audio_component_unbind(struct device *kdev, struct device *hda_kdev, void *data) { struct drm_device *dev = dev_get_drvdata(kdev); - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct drm_audio_component *acomp = data; acomp->ops = NULL; @@ -886,7 +886,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) #endif int r; - adev->dm.ddev = adev->ddev; + adev->dm.ddev = adev_to_drm(adev); adev->dm.adev = adev; /* Zero all the fields */ @@ -1022,10 +1022,10 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) /* TODO: Add_display_info? */ /* TODO use dynamic cursor width */ - adev->ddev->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size; - adev->ddev->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size; + adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size; + adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size; - if (drm_vblank_init(adev->ddev, adev->dm.display_indexes_num)) { + if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) { DRM_ERROR( "amdgpu: failed to initialize sw for display support.\n"); goto error; @@ -1102,6 +1102,12 @@ static int load_dmcu_fw(struct amdgpu_device *adev) const struct dmcu_firmware_header_v1_0 *hdr; switch(adev->asic_type) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: +#endif case CHIP_BONAIRE: case CHIP_HAWAII: case CHIP_KAVERI: @@ -1420,7 +1426,7 @@ static int dm_late_init(void *handle) bool ret = true; if (!adev->dm.fw_dmcu && !adev->dm.dmub_fw) - return detect_mst_link_for_all_connectors(adev->ddev); + return detect_mst_link_for_all_connectors(adev_to_drm(adev)); dmcu = adev->dm.dc->res_pool->dmcu; @@ -1445,12 +1451,12 @@ static int dm_late_init(void *handle) if (dmcu) ret = dmcu_load_iram(dmcu, params); else if (adev->dm.dc->ctx->dmub_srv) - ret = dmub_init_abm_config(adev->dm.dc->res_pool->abm, params); + ret = dmub_init_abm_config(adev->dm.dc->res_pool, params); if (!ret) return -EINVAL; - return detect_mst_link_for_all_connectors(adev->ddev); + return detect_mst_link_for_all_connectors(adev_to_drm(adev)); } static void s3_handle_mst(struct drm_device *dev, bool suspend) @@ -1688,7 +1694,7 @@ static int dm_suspend(void *handle) struct amdgpu_display_manager *dm = &adev->dm; int ret = 0; - if (adev->in_gpu_reset) { + if (amdgpu_in_reset(adev)) { mutex_lock(&dm->dc_lock); dm->cached_dc_state = dc_copy_state(dm->dc->current_state); @@ -1702,9 +1708,9 @@ static int dm_suspend(void *handle) } WARN_ON(adev->dm.cached_state); - adev->dm.cached_state = drm_atomic_helper_suspend(adev->ddev); + adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev)); - s3_handle_mst(adev->ddev, true); + s3_handle_mst(adev_to_drm(adev), true); amdgpu_dm_irq_suspend(adev); @@ -1858,7 +1864,7 @@ cleanup: static int dm_resume(void *handle) { struct amdgpu_device *adev = handle; - struct drm_device *ddev = adev->ddev; + struct drm_device *ddev = adev_to_drm(adev); struct amdgpu_display_manager *dm = &adev->dm; struct amdgpu_dm_connector *aconnector; struct drm_connector *connector; @@ -1874,7 +1880,7 @@ static int dm_resume(void *handle) struct dc_state *dc_state; int i, r, j; - if (adev->in_gpu_reset) { + if (amdgpu_in_reset(adev)) { dc_state = dm->cached_dc_state; r = dm_dmub_hw_init(adev); @@ -2080,7 +2086,7 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) return; conn_base = &aconnector->base; - adev = conn_base->dev->dev_private; + adev = drm_to_adev(conn_base->dev); dm = &adev->dm; caps = &dm->backlight_caps; caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps; @@ -2231,6 +2237,7 @@ void amdgpu_dm_update_connector_after_detect( drm_connector_update_edid_property(connector, aconnector->edid); + drm_add_edid_modes(connector, aconnector->edid); if (aconnector->dc_link->aux_mode) drm_dp_cec_set_edid(&aconnector->dm_dp_aux.aux, @@ -2269,7 +2276,7 @@ static void handle_hpd_irq(void *param) struct drm_device *dev = connector->dev; enum dc_connection_type new_connection_type = dc_connection_none; #ifdef CONFIG_DRM_AMD_DC_HDCP - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); #endif /* @@ -2402,7 +2409,7 @@ static void handle_hpd_rx_irq(void *param) enum dc_connection_type new_connection_type = dc_connection_none; #ifdef CONFIG_DRM_AMD_DC_HDCP union hpd_irq_data hpd_irq_data; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); memset(&hpd_irq_data, 0, sizeof(hpd_irq_data)); #endif @@ -2473,7 +2480,7 @@ static void handle_hpd_rx_irq(void *param) static void register_hpd_handlers(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct amdgpu_dm_connector *aconnector; const struct dc_link *dc_link; @@ -2510,6 +2517,89 @@ static void register_hpd_handlers(struct amdgpu_device *adev) } } +#if defined(CONFIG_DRM_AMD_DC_SI) +/* Register IRQ sources and initialize IRQ callbacks */ +static int dce60_register_irq_handlers(struct amdgpu_device *adev) +{ + struct dc *dc = adev->dm.dc; + struct common_irq_params *c_irq_params; + struct dc_interrupt_params int_params = {0}; + int r; + int i; + unsigned client_id = AMDGPU_IRQ_CLIENTID_LEGACY; + + int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; + int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; + + /* + * Actions of amdgpu_irq_add_id(): + * 1. Register a set() function with base driver. + * Base driver will call set() function to enable/disable an + * interrupt in DC hardware. + * 2. Register amdgpu_dm_irq_handler(). + * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts + * coming from DC hardware. + * amdgpu_dm_irq_handler() will re-direct the interrupt to DC + * for acknowledging and handling. */ + + /* Use VBLANK interrupt */ + for (i = 0; i < adev->mode_info.num_crtc; i++) { + r = amdgpu_irq_add_id(adev, client_id, i+1 , &adev->crtc_irq); + if (r) { + DRM_ERROR("Failed to add crtc irq id!\n"); + return r; + } + + int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; + int_params.irq_source = + dc_interrupt_to_irq_source(dc, i+1 , 0); + + c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; + + c_irq_params->adev = adev; + c_irq_params->irq_src = int_params.irq_source; + + amdgpu_dm_irq_register_interrupt(adev, &int_params, + dm_crtc_high_irq, c_irq_params); + } + + /* Use GRPH_PFLIP interrupt */ + for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; + i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { + r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); + if (r) { + DRM_ERROR("Failed to add page flip irq id!\n"); + return r; + } + + int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; + int_params.irq_source = + dc_interrupt_to_irq_source(dc, i, 0); + + c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; + + c_irq_params->adev = adev; + c_irq_params->irq_src = int_params.irq_source; + + amdgpu_dm_irq_register_interrupt(adev, &int_params, + dm_pflip_high_irq, c_irq_params); + + } + + /* HPD */ + r = amdgpu_irq_add_id(adev, client_id, + VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); + if (r) { + DRM_ERROR("Failed to add hpd irq id!\n"); + return r; + } + + register_hpd_handlers(adev); + + return 0; +} +#endif + /* Register IRQ sources and initialize IRQ callbacks */ static int dce110_register_irq_handlers(struct amdgpu_device *adev) { @@ -2740,7 +2830,7 @@ static int dm_atomic_get_state(struct drm_atomic_state *state, struct dm_atomic_state **dm_state) { struct drm_device *dev = state->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_display_manager *dm = &adev->dm; struct drm_private_state *priv_state; @@ -2760,7 +2850,7 @@ static struct dm_atomic_state * dm_atomic_get_new_state(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_display_manager *dm = &adev->dm; struct drm_private_obj *obj; struct drm_private_state *new_obj_state; @@ -2774,24 +2864,6 @@ dm_atomic_get_new_state(struct drm_atomic_state *state) return NULL; } -static struct dm_atomic_state * -dm_atomic_get_old_state(struct drm_atomic_state *state) -{ - struct drm_device *dev = state->dev; - struct amdgpu_device *adev = dev->dev_private; - struct amdgpu_display_manager *dm = &adev->dm; - struct drm_private_obj *obj; - struct drm_private_state *old_obj_state; - int i; - - for_each_old_private_obj_in_state(state, obj, old_obj_state, i) { - if (obj->funcs == dm->atomic_obj.funcs) - return to_dm_atomic_state(old_obj_state); - } - - return NULL; -} - static struct drm_private_state * dm_atomic_duplicate_state(struct drm_private_obj *obj) { @@ -2839,18 +2911,18 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) adev->mode_info.mode_config_initialized = true; - adev->ddev->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs; - adev->ddev->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs; + adev_to_drm(adev)->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs; + adev_to_drm(adev)->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs; - adev->ddev->mode_config.max_width = 16384; - adev->ddev->mode_config.max_height = 16384; + adev_to_drm(adev)->mode_config.max_width = 16384; + adev_to_drm(adev)->mode_config.max_height = 16384; - adev->ddev->mode_config.preferred_depth = 24; - adev->ddev->mode_config.prefer_shadow = 1; + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; /* indicates support for immediate flip */ - adev->ddev->mode_config.async_page_flip = true; + adev_to_drm(adev)->mode_config.async_page_flip = true; - adev->ddev->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; state = kzalloc(sizeof(*state), GFP_KERNEL); if (!state) @@ -2864,18 +2936,24 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) dc_resource_state_copy_construct_current(adev->dm.dc, state->context); - drm_atomic_private_obj_init(adev->ddev, + drm_atomic_private_obj_init(adev_to_drm(adev), &adev->dm.atomic_obj, &state->base, &dm_atomic_state_funcs); r = amdgpu_display_modeset_create_props(adev); - if (r) + if (r) { + dc_release_state(state->context); + kfree(state); return r; + } r = amdgpu_dm_audio_init(adev); - if (r) + if (r) { + dc_release_state(state->context); + kfree(state); return r; + } return 0; } @@ -2892,6 +2970,8 @@ static void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm) #if defined(CONFIG_ACPI) struct amdgpu_dm_backlight_caps caps; + memset(&caps, 0, sizeof(caps)); + if (dm->backlight_caps.caps_valid) return; @@ -2930,51 +3010,50 @@ static int set_backlight_via_aux(struct dc_link *link, uint32_t brightness) return rc ? 0 : 1; } -static u32 convert_brightness(const struct amdgpu_dm_backlight_caps *caps, - const uint32_t user_brightness) +static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, + unsigned *min, unsigned *max) { - u32 min, max, conversion_pace; - u32 brightness = user_brightness; - if (!caps) - goto out; + return 0; - if (!caps->aux_support) { - max = caps->max_input_signal; - min = caps->min_input_signal; - /* - * The brightness input is in the range 0-255 - * It needs to be rescaled to be between the - * requested min and max input signal - * It also needs to be scaled up by 0x101 to - * match the DC interface which has a range of - * 0 to 0xffff - */ - conversion_pace = 0x101; - brightness = - user_brightness - * conversion_pace - * (max - min) - / AMDGPU_MAX_BL_LEVEL - + min * conversion_pace; + if (caps->aux_support) { + // Firmware limits are in nits, DC API wants millinits. + *max = 1000 * caps->aux_max_input_signal; + *min = 1000 * caps->aux_min_input_signal; } else { - /* TODO - * We are doing a linear interpolation here, which is OK but - * does not provide the optimal result. We probably want - * something close to the Perceptual Quantizer (PQ) curve. - */ - max = caps->aux_max_input_signal; - min = caps->aux_min_input_signal; - - brightness = (AMDGPU_MAX_BL_LEVEL - user_brightness) * min - + user_brightness * max; - // Multiple the value by 1000 since we use millinits - brightness *= 1000; - brightness = DIV_ROUND_CLOSEST(brightness, AMDGPU_MAX_BL_LEVEL); + // Firmware limits are 8-bit, PWM control is 16-bit. + *max = 0x101 * caps->max_input_signal; + *min = 0x101 * caps->min_input_signal; } + return 1; +} + +static u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps, + uint32_t brightness) +{ + unsigned min, max; -out: - return brightness; + if (!get_brightness_range(caps, &min, &max)) + return brightness; + + // Rescale 0..255 to min..max + return min + DIV_ROUND_CLOSEST((max - min) * brightness, + AMDGPU_MAX_BL_LEVEL); +} + +static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps, + uint32_t brightness) +{ + unsigned min, max; + + if (!get_brightness_range(caps, &min, &max)) + return brightness; + + if (brightness < min) + return 0; + // Rescale min..max to 0..255 + return DIV_ROUND_CLOSEST(AMDGPU_MAX_BL_LEVEL * (brightness - min), + max - min); } static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) @@ -2990,7 +3069,7 @@ static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) link = (struct dc_link *)dm->backlight_link; - brightness = convert_brightness(&caps, bd->props.brightness); + brightness = convert_brightness_from_user(&caps, bd->props.brightness); // Change brightness based on AUX property if (caps.aux_support) return set_backlight_via_aux(link, brightness); @@ -3007,7 +3086,7 @@ static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) if (ret == DC_ERROR_UNEXPECTED) return bd->props.brightness; - return ret; + return convert_brightness_to_user(&dm->backlight_caps, ret); } static const struct backlight_ops amdgpu_dm_backlight_ops = { @@ -3029,13 +3108,13 @@ amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm) props.type = BACKLIGHT_RAW; snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d", - dm->adev->ddev->primary->index); + adev_to_drm(dm->adev)->primary->index); dm->backlight_dev = backlight_device_register(bl_name, - dm->adev->ddev->dev, - dm, - &amdgpu_dm_backlight_ops, - &props); + adev_to_drm(dm->adev)->dev, + dm, + &amdgpu_dm_backlight_ops, + &props); if (IS_ERR(dm->backlight_dev)) DRM_ERROR("DM: Backlight registration failed!\n"); @@ -3241,6 +3320,17 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) /* Software is initialized. Now we can register interrupt handlers. */ switch (adev->asic_type) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + if (dce60_register_irq_handlers(dm->adev)) { + DRM_ERROR("DM: Failed to initialize IRQ\n"); + goto fail; + } + break; +#endif case CHIP_BONAIRE: case CHIP_HAWAII: case CHIP_KAVERI: @@ -3341,14 +3431,14 @@ static ssize_t s3_debug_store(struct device *device, int ret; int s3_state; struct drm_device *drm_dev = dev_get_drvdata(device); - struct amdgpu_device *adev = drm_dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(drm_dev); ret = kstrtoint(buf, 0, &s3_state); if (ret == 0) { if (s3_state) { dm_resume(adev); - drm_kms_helper_hotplug_event(adev->ddev); + drm_kms_helper_hotplug_event(adev_to_drm(adev)); } else dm_suspend(adev); } @@ -3365,6 +3455,20 @@ static int dm_early_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; switch (adev->asic_type) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + adev->mode_info.num_crtc = 6; + adev->mode_info.num_hpd = 6; + adev->mode_info.num_dig = 6; + break; + case CHIP_OLAND: + adev->mode_info.num_crtc = 2; + adev->mode_info.num_hpd = 2; + adev->mode_info.num_dig = 2; + break; +#endif case CHIP_BONAIRE: case CHIP_HAWAII: adev->mode_info.num_crtc = 6; @@ -3461,7 +3565,7 @@ static int dm_early_init(void *handle) */ #if defined(CONFIG_DEBUG_KERNEL_DC) device_create_file( - adev->ddev->dev, + adev_to_drm(adev)->dev, &dev_attr_s3_debug); #endif @@ -3472,21 +3576,12 @@ static bool modeset_required(struct drm_crtc_state *crtc_state, struct dc_stream_state *new_stream, struct dc_stream_state *old_stream) { - if (!drm_atomic_crtc_needs_modeset(crtc_state)) - return false; - - if (!crtc_state->enable) - return false; - - return crtc_state->active; + return crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state); } static bool modereset_required(struct drm_crtc_state *crtc_state) { - if (!drm_atomic_crtc_needs_modeset(crtc_state)) - return false; - - return !crtc_state->enable || !crtc_state->active; + return !crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state); } static void amdgpu_dm_encoder_destroy(struct drm_encoder *encoder) @@ -3559,8 +3654,17 @@ static int fill_dc_scaling_info(const struct drm_plane_state *state, static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb, uint64_t *tiling_flags, bool *tmz_surface) { - struct amdgpu_bo *rbo = gem_to_amdgpu_bo(amdgpu_fb->base.obj[0]); - int r = amdgpu_bo_reserve(rbo, false); + struct amdgpu_bo *rbo; + int r; + + if (!amdgpu_fb) { + *tiling_flags = 0; + *tmz_surface = false; + return 0; + } + + rbo = gem_to_amdgpu_bo(amdgpu_fb->base.obj[0]); + r = amdgpu_bo_reserve(rbo, false); if (unlikely(r)) { /* Don't show error message when returning -ERESTARTSYS */ @@ -3983,13 +4087,10 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev, struct drm_crtc_state *crtc_state) { struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); - const struct amdgpu_framebuffer *amdgpu_fb = - to_amdgpu_framebuffer(plane_state->fb); + struct dm_plane_state *dm_plane_state = to_dm_plane_state(plane_state); struct dc_scaling_info scaling_info; struct dc_plane_info plane_info; - uint64_t tiling_flags; int ret; - bool tmz_surface = false; bool force_disable_dcc = false; ret = fill_dc_scaling_info(plane_state, &scaling_info); @@ -4001,15 +4102,12 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev, dc_plane_state->clip_rect = scaling_info.clip_rect; dc_plane_state->scaling_quality = scaling_info.scaling_quality; - ret = get_fb_info(amdgpu_fb, &tiling_flags, &tmz_surface); - if (ret) - return ret; - force_disable_dcc = adev->asic_type == CHIP_RAVEN && adev->in_suspend; - ret = fill_dc_plane_info_and_addr(adev, plane_state, tiling_flags, + ret = fill_dc_plane_info_and_addr(adev, plane_state, + dm_plane_state->tiling_flags, &plane_info, &dc_plane_state->address, - tmz_surface, + dm_plane_state->tmz_surface, force_disable_dcc); if (ret) return ret; @@ -4591,7 +4689,10 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, dc_link_get_link_cap(aconnector->dc_link)); #if defined(CONFIG_DRM_AMD_DC_DCN) - if (dsc_caps.is_dsc_supported) + if (dsc_caps.is_dsc_supported) { + /* Set DSC policy according to dsc_clock_en */ + dc_dsc_policy_set_enable_dsc_when_not_needed(aconnector->dsc_settings.dsc_clock_en); + if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0], &dsc_caps, aconnector->dc_link->ctx->dc->debug.dsc_min_slice_height_override, @@ -4599,6 +4700,21 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, &stream->timing, &stream->timing.dsc_cfg)) stream->timing.flags.DSC = 1; + /* Overwrite the stream flag if DSC is enabled through debugfs */ + if (aconnector->dsc_settings.dsc_clock_en) + stream->timing.flags.DSC = 1; + + if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_slice_width) + stream->timing.dsc_cfg.num_slices_h = DIV_ROUND_UP(stream->timing.h_addressable, + aconnector->dsc_settings.dsc_slice_width); + + if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_slice_height) + stream->timing.dsc_cfg.num_slices_v = DIV_ROUND_UP(stream->timing.v_addressable, + aconnector->dsc_settings.dsc_slice_height); + + if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel) + stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel; + } #endif } @@ -4612,7 +4728,8 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, update_stream_signal(stream, sink); if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) - mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket, false, false); + mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket); + if (stream->link->psr_settings.psr_feature_enabled) { // // should decide stream support vsc sdp colorimetry capability @@ -4710,7 +4827,7 @@ static inline int dm_set_vupdate_irq(struct drm_crtc *crtc, bool enable) { enum dc_irq_source irq_source; struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); int rc; irq_source = IRQ_TYPE_VUPDATE + acrtc->otg_inst; @@ -4726,7 +4843,7 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) { enum dc_irq_source irq_source; struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state); int rc = 0; @@ -4805,7 +4922,7 @@ int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector, uint64_t val) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct dm_connector_state *dm_old_state = to_dm_connector_state(connector->state); struct dm_connector_state *dm_new_state = @@ -4860,7 +4977,7 @@ int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector, uint64_t *val) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct dm_connector_state *dm_state = to_dm_connector_state(state); int ret = -EINVAL; @@ -4910,9 +5027,10 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) { struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); const struct dc_link *link = aconnector->dc_link; - struct amdgpu_device *adev = connector->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(connector->dev); struct amdgpu_display_manager *dm = &adev->dm; + drm_atomic_private_obj_fini(&aconnector->mst_mgr.base); #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) ||\ defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) @@ -5095,7 +5213,7 @@ create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, const struct dc_stream_state *old_stream) { struct drm_connector *connector = &aconnector->base; - struct amdgpu_device *adev = connector->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(connector->dev); struct dc_stream_state *stream; const struct drm_connector_state *drm_state = dm_state ? &dm_state->base : NULL; int requested_bpc = drm_state ? drm_state->max_requested_bpc : 8; @@ -5372,7 +5490,7 @@ static void dm_update_crtc_active_planes(struct drm_crtc *crtc, static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dc *dc = adev->dm.dc; struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(state); int ret = -EINVAL; @@ -5592,6 +5710,10 @@ dm_drm_plane_duplicate_state(struct drm_plane *plane) dc_plane_state_retain(dm_plane_state->dc_state); } + /* Framebuffer hasn't been updated yet, so retain old flags. */ + dm_plane_state->tiling_flags = old_dm_plane_state->tiling_flags; + dm_plane_state->tmz_surface = old_dm_plane_state->tmz_surface; + return &dm_plane_state->base; } @@ -5626,14 +5748,8 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane, struct list_head list; struct ttm_validate_buffer tv; struct ww_acquire_ctx ticket; - uint64_t tiling_flags; uint32_t domain; int r; - bool tmz_surface = false; - bool force_disable_dcc = false; - - dm_plane_state_old = to_dm_plane_state(plane->state); - dm_plane_state_new = to_dm_plane_state(new_state); if (!new_state->fb) { DRM_DEBUG_DRIVER("No FB bound\n"); @@ -5677,27 +5793,35 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane, return r; } - amdgpu_bo_get_tiling_flags(rbo, &tiling_flags); - - tmz_surface = amdgpu_bo_encrypted(rbo); - ttm_eu_backoff_reservation(&ticket, &list); afb->address = amdgpu_bo_gpu_offset(rbo); amdgpu_bo_ref(rbo); + /** + * We don't do surface updates on planes that have been newly created, + * but we also don't have the afb->address during atomic check. + * + * Fill in buffer attributes depending on the address here, but only on + * newly created planes since they're not being used by DC yet and this + * won't modify global state. + */ + dm_plane_state_old = to_dm_plane_state(plane->state); + dm_plane_state_new = to_dm_plane_state(new_state); + if (dm_plane_state_new->dc_state && - dm_plane_state_old->dc_state != dm_plane_state_new->dc_state) { - struct dc_plane_state *plane_state = dm_plane_state_new->dc_state; + dm_plane_state_old->dc_state != dm_plane_state_new->dc_state) { + struct dc_plane_state *plane_state = + dm_plane_state_new->dc_state; + bool force_disable_dcc = !plane_state->dcc.enable; - force_disable_dcc = adev->asic_type == CHIP_RAVEN && adev->in_suspend; fill_plane_buffer_attributes( adev, afb, plane_state->format, plane_state->rotation, - tiling_flags, &plane_state->tiling_info, - &plane_state->plane_size, &plane_state->dcc, - &plane_state->address, tmz_surface, - force_disable_dcc); + dm_plane_state_new->tiling_flags, + &plane_state->tiling_info, &plane_state->plane_size, + &plane_state->dcc, &plane_state->address, + dm_plane_state_new->tmz_surface, force_disable_dcc); } return 0; @@ -5738,7 +5862,7 @@ static int dm_plane_helper_check_state(struct drm_plane_state *state, static int dm_plane_atomic_check(struct drm_plane *plane, struct drm_plane_state *state) { - struct amdgpu_device *adev = plane->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(plane->dev); struct dc *dc = adev->dm.dc; struct dm_plane_state *dm_plane_state; struct dc_scaling_info scaling_info; @@ -5907,7 +6031,7 @@ static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, num_formats = get_plane_formats(plane, plane_cap, formats, ARRAY_SIZE(formats)); - res = drm_universal_plane_init(dm->adev->ddev, plane, possible_crtcs, + res = drm_universal_plane_init(adev_to_drm(dm->adev), plane, possible_crtcs, &dm_plane_funcs, formats, num_formats, NULL, plane->type, NULL); if (res) @@ -5941,8 +6065,9 @@ static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270; - drm_plane_create_rotation_property(plane, DRM_MODE_ROTATE_0, - supported_rotations); + if (dm->adev->asic_type >= CHIP_BONAIRE) + drm_plane_create_rotation_property(plane, DRM_MODE_ROTATE_0, + supported_rotations); drm_plane_helper_add(plane, &dm_plane_helper_funcs); @@ -6212,7 +6337,7 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, struct dc_link *link, int link_index) { - struct amdgpu_device *adev = dm->ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dm->ddev); /* * Some of the properties below require access to state, like bpc. @@ -6463,7 +6588,7 @@ static int amdgpu_dm_encoder_init(struct drm_device *dev, struct amdgpu_encoder *aencoder, uint32_t link_index) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int res = drm_encoder_init(dev, &aencoder->base, @@ -6648,7 +6773,7 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc, static void handle_cursor_update(struct drm_plane *plane, struct drm_plane_state *old_plane_state) { - struct amdgpu_device *adev = plane->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(plane->dev); struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(plane->state->fb); struct drm_crtc *crtc = afb ? plane->state->crtc : old_plane_state->crtc; struct dm_crtc_state *crtc_state = crtc ? to_dm_crtc_state(crtc->state) : NULL; @@ -6750,7 +6875,7 @@ static void update_freesync_state_on_stream( if (!new_stream->timing.h_total || !new_stream->timing.v_total) return; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); vrr_params = new_crtc_state->vrr_params; if (surface) { @@ -6803,7 +6928,7 @@ static void update_freesync_state_on_stream( (int)new_crtc_state->base.vrr_enabled, (int)vrr_params.state); - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } static void pre_update_freesync_state_on_stream( @@ -6826,7 +6951,7 @@ static void pre_update_freesync_state_on_stream( if (!new_stream->timing.h_total || !new_stream->timing.v_total) return; - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); vrr_params = new_crtc_state->vrr_params; if (new_crtc_state->vrr_supported && @@ -6849,7 +6974,7 @@ static void pre_update_freesync_state_on_stream( sizeof(vrr_params.adjust)) != 0); new_crtc_state->vrr_params = vrr_params; - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, @@ -6919,8 +7044,6 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, long r; unsigned long flags; struct amdgpu_bo *abo; - uint64_t tiling_flags; - bool tmz_surface = false; uint32_t target_vblank, last_flip_vblank; bool vrr_active = amdgpu_dm_vrr_active(acrtc_state); bool pflip_present = false; @@ -7004,28 +7127,12 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, if (unlikely(r <= 0)) DRM_ERROR("Waiting for fences timed out!"); - /* - * TODO This might fail and hence better not used, wait - * explicitly on fences instead - * and in general should be called for - * blocking commit to as per framework helpers - */ - r = amdgpu_bo_reserve(abo, true); - if (unlikely(r != 0)) - DRM_ERROR("failed to reserve buffer before flip\n"); - - amdgpu_bo_get_tiling_flags(abo, &tiling_flags); - - tmz_surface = amdgpu_bo_encrypted(abo); - - amdgpu_bo_unreserve(abo); - fill_dc_plane_info_and_addr( - dm->adev, new_plane_state, tiling_flags, + dm->adev, new_plane_state, + dm_new_plane_state->tiling_flags, &bundle->plane_infos[planes_count], &bundle->flip_addrs[planes_count].address, - tmz_surface, - false); + dm_new_plane_state->tmz_surface, false); DRM_DEBUG_DRIVER("plane: id=%d dcc_en=%d\n", new_plane_state->plane->index, @@ -7203,9 +7310,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, * on some ASICs). */ if (dm_old_crtc_state->active_planes != acrtc_state->active_planes) - dm_update_pflip_irq_state( - (struct amdgpu_device *)dev->dev_private, - acrtc_attach); + dm_update_pflip_irq_state(drm_to_adev(dev), + acrtc_attach); if ((acrtc_state->update_type > UPDATE_TYPE_FAST) && acrtc_state->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED && @@ -7235,7 +7341,7 @@ cleanup: static void amdgpu_dm_commit_audio(struct drm_device *dev, struct drm_atomic_state *state) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_dm_connector *aconnector; struct drm_connector *connector; struct drm_connector_state *old_con_state, *new_con_state; @@ -7327,7 +7433,7 @@ static int amdgpu_dm_atomic_commit(struct drm_device *dev, { struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state, *new_crtc_state; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); int i; /* @@ -7374,7 +7480,7 @@ static int amdgpu_dm_atomic_commit(struct drm_device *dev, static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_display_manager *dm = &adev->dm; struct dm_atomic_state *dm_state; struct dc_state *dc_state = NULL, *dc_state_temp = NULL; @@ -7387,6 +7493,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) struct drm_connector_state *old_con_state, *new_con_state; struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state; int crtc_disable_count = 0; + bool mode_set_reset_required = false; drm_atomic_helper_update_legacy_modeset_state(dev, state); @@ -7463,19 +7570,21 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) acrtc->enabled = true; acrtc->hw_mode = new_crtc_state->mode; crtc->hwmode = new_crtc_state->mode; + mode_set_reset_required = true; } else if (modereset_required(new_crtc_state)) { DRM_DEBUG_DRIVER("Atomic commit: RESET. crtc id %d:[%p]\n", acrtc->crtc_id, acrtc); /* i.e. reset mode */ - if (dm_old_crtc_state->stream) { - if (dm_old_crtc_state->stream->link->psr_settings.psr_allow_active) - amdgpu_dm_psr_disable(dm_old_crtc_state->stream); - + if (dm_old_crtc_state->stream) remove_stream(adev, acrtc, dm_old_crtc_state->stream); - } + mode_set_reset_required = true; } } /* for_each_crtc_in_state() */ if (dc_state) { + /* if there mode set or reset, disable eDP PSR */ + if (mode_set_reset_required) + amdgpu_dm_psr_disable_all(dm); + dm_enable_per_frame_crtc_master_sync(dc_state); mutex_lock(&dm->dc_lock); WARN_ON(!dc_commit_state(dm->dc, dc_state)); @@ -7677,7 +7786,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) * send vblank event on all events not handled in flip and * mark consumed event for drm_atomic_helper_commit_hw_done */ - spin_lock_irqsave(&adev->ddev->event_lock, flags); + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { if (new_crtc_state->event) @@ -7685,7 +7794,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) new_crtc_state->event = NULL; } - spin_unlock_irqrestore(&adev->ddev->event_lock, flags); + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); /* Signal HW programming completion */ drm_atomic_helper_commit_hw_done(state); @@ -7957,6 +8066,13 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm, goto fail; } + /* + * TODO: Check VSDB bits to decide whether this should + * be enabled or not. + */ + new_stream->triggered_crtc_reset.enabled = + dm->force_timing_sync; + dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level; ret = fill_hdr_info_packet(drm_new_conn_state, @@ -8076,8 +8192,7 @@ skip_modeset: * We want to do dc stream updates that do not require a * full modeset below. */ - if (!(enable && aconnector && new_crtc_state->enable && - new_crtc_state->active)) + if (!(enable && aconnector && new_crtc_state->active)) return 0; /* * Given above conditions, the dc state cannot be NULL because: @@ -8168,6 +8283,8 @@ static bool should_reset_plane(struct drm_atomic_state *state, * TODO: Come up with a more elegant solution for this. */ for_each_oldnew_plane_in_state(state, other, old_other_state, new_other_state, i) { + struct dm_plane_state *old_dm_plane_state, *new_dm_plane_state; + if (other->type == DRM_PLANE_TYPE_CURSOR) continue; @@ -8178,9 +8295,45 @@ static bool should_reset_plane(struct drm_atomic_state *state, if (old_other_state->crtc != new_other_state->crtc) return true; - /* TODO: Remove this once we can handle fast format changes. */ - if (old_other_state->fb && new_other_state->fb && - old_other_state->fb->format != new_other_state->fb->format) + /* Src/dst size and scaling updates. */ + if (old_other_state->src_w != new_other_state->src_w || + old_other_state->src_h != new_other_state->src_h || + old_other_state->crtc_w != new_other_state->crtc_w || + old_other_state->crtc_h != new_other_state->crtc_h) + return true; + + /* Rotation / mirroring updates. */ + if (old_other_state->rotation != new_other_state->rotation) + return true; + + /* Blending updates. */ + if (old_other_state->pixel_blend_mode != + new_other_state->pixel_blend_mode) + return true; + + /* Alpha updates. */ + if (old_other_state->alpha != new_other_state->alpha) + return true; + + /* Colorspace changes. */ + if (old_other_state->color_range != new_other_state->color_range || + old_other_state->color_encoding != new_other_state->color_encoding) + return true; + + /* Framebuffer checks fall at the end. */ + if (!old_other_state->fb || !new_other_state->fb) + continue; + + /* Pixel format changes can require bandwidth updates. */ + if (old_other_state->fb->format != new_other_state->fb->format) + return true; + + old_dm_plane_state = to_dm_plane_state(old_other_state); + new_dm_plane_state = to_dm_plane_state(new_other_state); + + /* Tiling and DCC changes also require bandwidth updates. */ + if (old_dm_plane_state->tiling_flags != + new_dm_plane_state->tiling_flags) return true; } @@ -8260,8 +8413,7 @@ static int dm_update_plane_state(struct dc *dc, dm_old_plane_state->dc_state, dm_state->context)) { - ret = EINVAL; - return ret; + return -EINVAL; } @@ -8302,7 +8454,7 @@ static int dm_update_plane_state(struct dc *dc, plane->base.id, new_plane_crtc->base.id); ret = fill_dc_plane_attributes( - new_plane_crtc->dev->dev_private, + drm_to_adev(new_plane_crtc->dev), dc_new_plane_state, new_plane_state, new_crtc_state); @@ -8348,169 +8500,6 @@ static int dm_update_plane_state(struct dc *dc, return ret; } -static int -dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, - struct drm_atomic_state *state, - enum surface_update_type *out_type) -{ - struct dc *dc = dm->dc; - struct dm_atomic_state *dm_state = NULL, *old_dm_state = NULL; - int i, j, num_plane, ret = 0; - struct drm_plane_state *old_plane_state, *new_plane_state; - struct dm_plane_state *new_dm_plane_state, *old_dm_plane_state; - struct drm_crtc *new_plane_crtc; - struct drm_plane *plane; - - struct drm_crtc *crtc; - struct drm_crtc_state *new_crtc_state, *old_crtc_state; - struct dm_crtc_state *new_dm_crtc_state, *old_dm_crtc_state; - struct dc_stream_status *status = NULL; - enum surface_update_type update_type = UPDATE_TYPE_FAST; - struct surface_info_bundle { - struct dc_surface_update surface_updates[MAX_SURFACES]; - struct dc_plane_info plane_infos[MAX_SURFACES]; - struct dc_scaling_info scaling_infos[MAX_SURFACES]; - struct dc_flip_addrs flip_addrs[MAX_SURFACES]; - struct dc_stream_update stream_update; - } *bundle; - - bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - - if (!bundle) { - DRM_ERROR("Failed to allocate update bundle\n"); - /* Set type to FULL to avoid crashing in DC*/ - update_type = UPDATE_TYPE_FULL; - goto cleanup; - } - - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - - memset(bundle, 0, sizeof(struct surface_info_bundle)); - - new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); - old_dm_crtc_state = to_dm_crtc_state(old_crtc_state); - num_plane = 0; - - if (new_dm_crtc_state->stream != old_dm_crtc_state->stream) { - update_type = UPDATE_TYPE_FULL; - goto cleanup; - } - - if (!new_dm_crtc_state->stream) - continue; - - for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, j) { - const struct amdgpu_framebuffer *amdgpu_fb = - to_amdgpu_framebuffer(new_plane_state->fb); - struct dc_plane_info *plane_info = &bundle->plane_infos[num_plane]; - struct dc_flip_addrs *flip_addr = &bundle->flip_addrs[num_plane]; - struct dc_scaling_info *scaling_info = &bundle->scaling_infos[num_plane]; - uint64_t tiling_flags; - bool tmz_surface = false; - - new_plane_crtc = new_plane_state->crtc; - new_dm_plane_state = to_dm_plane_state(new_plane_state); - old_dm_plane_state = to_dm_plane_state(old_plane_state); - - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - if (new_dm_plane_state->dc_state != old_dm_plane_state->dc_state) { - update_type = UPDATE_TYPE_FULL; - goto cleanup; - } - - if (crtc != new_plane_crtc) - continue; - - bundle->surface_updates[num_plane].surface = - new_dm_plane_state->dc_state; - - if (new_crtc_state->mode_changed) { - bundle->stream_update.dst = new_dm_crtc_state->stream->dst; - bundle->stream_update.src = new_dm_crtc_state->stream->src; - } - - if (new_crtc_state->color_mgmt_changed) { - bundle->surface_updates[num_plane].gamma = - new_dm_plane_state->dc_state->gamma_correction; - bundle->surface_updates[num_plane].in_transfer_func = - new_dm_plane_state->dc_state->in_transfer_func; - bundle->surface_updates[num_plane].gamut_remap_matrix = - &new_dm_plane_state->dc_state->gamut_remap_matrix; - bundle->stream_update.gamut_remap = - &new_dm_crtc_state->stream->gamut_remap_matrix; - bundle->stream_update.output_csc_transform = - &new_dm_crtc_state->stream->csc_color_matrix; - bundle->stream_update.out_transfer_func = - new_dm_crtc_state->stream->out_transfer_func; - } - - ret = fill_dc_scaling_info(new_plane_state, - scaling_info); - if (ret) - goto cleanup; - - bundle->surface_updates[num_plane].scaling_info = scaling_info; - - if (amdgpu_fb) { - ret = get_fb_info(amdgpu_fb, &tiling_flags, &tmz_surface); - if (ret) - goto cleanup; - - ret = fill_dc_plane_info_and_addr( - dm->adev, new_plane_state, tiling_flags, - plane_info, - &flip_addr->address, tmz_surface, - false); - if (ret) - goto cleanup; - - bundle->surface_updates[num_plane].plane_info = plane_info; - bundle->surface_updates[num_plane].flip_addr = flip_addr; - } - - num_plane++; - } - - if (num_plane == 0) - continue; - - ret = dm_atomic_get_state(state, &dm_state); - if (ret) - goto cleanup; - - old_dm_state = dm_atomic_get_old_state(state); - if (!old_dm_state) { - ret = -EINVAL; - goto cleanup; - } - - status = dc_stream_get_status_from_state(old_dm_state->context, - new_dm_crtc_state->stream); - bundle->stream_update.stream = new_dm_crtc_state->stream; - /* - * TODO: DC modifies the surface during this call so we need - * to lock here - find a way to do this without locking. - */ - mutex_lock(&dm->dc_lock); - update_type = dc_check_update_surfaces_for_stream( - dc, bundle->surface_updates, num_plane, - &bundle->stream_update, status); - mutex_unlock(&dm->dc_lock); - - if (update_type > UPDATE_TYPE_MED) { - update_type = UPDATE_TYPE_FULL; - goto cleanup; - } - } - -cleanup: - kfree(bundle); - - *out_type = update_type; - return ret; -} #if defined(CONFIG_DRM_AMD_DC_DCN) static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc) { @@ -8551,8 +8540,7 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm * acquired. For full updates case which removes/adds/updates streams on one * CRTC while flipping on another CRTC, acquiring global lock will guarantee * that any such full update commit will wait for completion of any outstanding - * flip using DRMs synchronization events. See - * dm_determine_update_type_for_commit() + * flip using DRMs synchronization events. * * Note that DM adds the affected connectors for all CRTCs in state, when that * might not seem necessary. This is because DC stream creation requires the @@ -8564,7 +8552,7 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm static int amdgpu_dm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state) { - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct dm_atomic_state *dm_state = NULL; struct dc *dc = adev->dm.dc; struct drm_connector *connector; @@ -8573,15 +8561,8 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, struct drm_crtc_state *old_crtc_state, *new_crtc_state; struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; - enum surface_update_type update_type = UPDATE_TYPE_FAST; - enum surface_update_type overall_update_type = UPDATE_TYPE_FAST; enum dc_status status; int ret, i; - - /* - * This bool will be set for true for any modeset/reset - * or plane update which implies non fast surface update. - */ bool lock_and_validation_needed = false; ret = drm_atomic_helper_check_modeset(dev, state); @@ -8676,6 +8657,17 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } } + /* Prepass for updating tiling flags on new planes. */ + for_each_new_plane_in_state(state, plane, new_plane_state, i) { + struct dm_plane_state *new_dm_plane_state = to_dm_plane_state(new_plane_state); + struct amdgpu_framebuffer *new_afb = to_amdgpu_framebuffer(new_plane_state->fb); + + ret = get_fb_info(new_afb, &new_dm_plane_state->tiling_flags, + &new_dm_plane_state->tmz_surface); + if (ret) + goto fail; + } + /* Remove exiting planes if they are modified */ for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { ret = dm_update_plane_state(dc, state, plane, @@ -8764,27 +8756,23 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, if (!is_scaling_state_different(dm_new_con_state, dm_old_con_state)) continue; - overall_update_type = UPDATE_TYPE_FULL; lock_and_validation_needed = true; } - ret = dm_determine_update_type_for_commit(&adev->dm, state, &update_type); - if (ret) - goto fail; - - if (overall_update_type < update_type) - overall_update_type = update_type; - - /* - * lock_and_validation_needed was an old way to determine if we need to set - * the global lock. Leaving it in to check if we broke any corner cases - * lock_and_validation_needed true = UPDATE_TYPE_FULL or UPDATE_TYPE_MED - * lock_and_validation_needed false = UPDATE_TYPE_FAST + /** + * Streams and planes are reset when there are changes that affect + * bandwidth. Anything that affects bandwidth needs to go through + * DC global validation to ensure that the configuration can be applied + * to hardware. + * + * We have to currently stall out here in atomic_check for outstanding + * commits to finish in this case because our IRQ handlers reference + * DRM state directly - we can end up disabling interrupts too early + * if we don't. + * + * TODO: Remove this stall and drop DM state private objects. */ - if (lock_and_validation_needed && overall_update_type <= UPDATE_TYPE_FAST) - WARN(1, "Global lock should be Set, overall_update_type should be UPDATE_TYPE_MED or UPDATE_TYPE_FULL"); - - if (overall_update_type > UPDATE_TYPE_FAST) { + if (lock_and_validation_needed) { ret = dm_atomic_get_state(state, &dm_state); if (ret) goto fail; @@ -8866,7 +8854,9 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, struct dm_crtc_state *dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_new_crtc_state->update_type = (int)overall_update_type; + dm_new_crtc_state->update_type = lock_and_validation_needed ? + UPDATE_TYPE_FULL : + UPDATE_TYPE_FAST; } /* Must be success */ @@ -8915,7 +8905,7 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, struct dm_connector_state *dm_con_state = NULL; struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); bool freesync_capable = false; if (!connector->state) { @@ -9114,3 +9104,34 @@ static bool amdgpu_dm_psr_disable(struct dc_stream_state *stream) return dc_link_set_psr_allow_active(stream->link, false, true); } + +/* + * amdgpu_dm_psr_disable() - disable psr f/w + * if psr is enabled on any stream + * + * Return: true if success + */ +static bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm) +{ + DRM_DEBUG_DRIVER("Disabling psr if psr is enabled on any stream\n"); + return dc_set_psr_allow_active(dm->dc, false); +} + +void amdgpu_dm_trigger_timing_sync(struct drm_device *dev) +{ + struct amdgpu_device *adev = drm_to_adev(dev); + struct dc *dc = adev->dm.dc; + int i; + + mutex_lock(&adev->dm.dc_lock); + if (dc->current_state) { + for (i = 0; i < dc->current_state->stream_count; ++i) + dc->current_state->streams[i] + ->triggered_crtc_reset.enabled = + adev->dm.force_timing_sync; + + dm_enable_per_frame_crtc_master_sync(dc->current_state); + dc_trigger_sync(dc, dc->current_state); + } + mutex_unlock(&adev->dm.dc_lock); +} diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index dd1559c743c2..a7856ae2e5f5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -340,6 +340,14 @@ struct amdgpu_display_manager { * fake encoders used for DP MST. */ struct amdgpu_encoder mst_encoders[AMDGPU_DM_MAX_CRTC]; + bool force_timing_sync; +}; + +struct dsc_preferred_settings { + bool dsc_clock_en; + uint32_t dsc_slice_width; + uint32_t dsc_slice_height; + uint32_t dsc_bits_per_pixel; }; struct amdgpu_dm_connector { @@ -389,6 +397,7 @@ struct amdgpu_dm_connector { uint32_t debugfs_dpcd_size; #endif bool force_yuv420_output; + struct dsc_preferred_settings dsc_settings; }; #define to_amdgpu_dm_connector(x) container_of(x, struct amdgpu_dm_connector, base) @@ -403,6 +412,8 @@ struct dc_plane_state; struct dm_plane_state { struct drm_plane_state base; struct dc_plane_state *dc_state; + uint64_t tiling_flags; + bool tmz_surface; }; struct dm_crtc_state { @@ -485,6 +496,8 @@ void dm_restore_drm_connector_state(struct drm_device *dev, void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, struct edid *edid); +void amdgpu_dm_trigger_timing_sync(struct drm_device *dev); + #define MAX_COLOR_LUT_ENTRIES 4096 /* Legacy gamm LUT users such as X doesn't like large LUT sizes */ #define MAX_COLOR_LEGACY_LUT_ENTRIES 256 diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index b321ff654df4..5df05f0d18bc 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -308,8 +308,7 @@ static int __set_input_tf(struct dc_transfer_func *func, int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc) { struct dc_stream_state *stream = crtc->stream; - struct amdgpu_device *adev = - (struct amdgpu_device *)crtc->base.state->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->base.state->dev); bool has_rom = adev->asic_type <= CHIP_RAVEN; struct drm_color_ctm *ctm = NULL; const struct drm_color_lut *degamma_lut, *regamma_lut; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index eaad9099bc0b..d0699e98db92 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -101,7 +101,7 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc, struct dm_crtc_state *dm_crtc_state, enum amdgpu_dm_pipe_crc_source source) { - struct amdgpu_device *adev = crtc->dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dc_stream_state *stream_state = dm_crtc_state->stream; bool enable = amdgpu_dm_is_valid_crc_source(source); int ret = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index e5a6d9115949..94fcb086154c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -49,6 +49,10 @@ struct dmub_debugfs_trace_entry { uint32_t param1; }; +static inline const char *yesno(bool v) +{ + return v ? "yes" : "no"; +} /* parse_write_buffer_into_params - Helper function to parse debugfs write buffer into an array * @@ -980,6 +984,161 @@ static ssize_t dp_dpcd_data_read(struct file *f, char __user *buf, return read_size - r; } +/* function: Read link's DSC & FEC capabilities + * + * + * Access it with the following command (you need to specify + * connector like DP-1): + * + * cat /sys/kernel/debug/dri/0/DP-X/dp_dsc_fec_support + * + */ +static int dp_dsc_fec_support_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct drm_modeset_acquire_ctx ctx; + struct drm_device *dev = connector->dev; + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + int ret = 0; + bool try_again = false; + bool is_fec_supported = false; + bool is_dsc_supported = false; + struct dpcd_caps dpcd_caps; + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + do { + try_again = false; + ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx); + if (ret) { + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) { + try_again = true; + continue; + } + } + break; + } + if (connector->status != connector_status_connected) { + ret = -ENODEV; + break; + } + dpcd_caps = aconnector->dc_link->dpcd_caps; + if (aconnector->port) { + /* aconnector sets dsc_aux during get_modes call + * if MST connector has it means it can either + * enable DSC on the sink device or on MST branch + * its connected to. + */ + if (aconnector->dsc_aux) { + is_fec_supported = true; + is_dsc_supported = true; + } + } else { + is_fec_supported = dpcd_caps.fec_cap.raw & 0x1; + is_dsc_supported = dpcd_caps.dsc_caps.dsc_basic_caps.raw[0] & 0x1; + } + } while (try_again); + + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + + seq_printf(m, "FEC_Sink_Support: %s\n", yesno(is_fec_supported)); + seq_printf(m, "DSC_Sink_Support: %s\n", yesno(is_dsc_supported)); + + return ret; +} + +/* function: Trigger virtual HPD redetection on connector + * + * This function will perform link rediscovery, link disable + * and enable, and dm connector state update. + * + * Retrigger HPD on an existing connector by echoing 1 into + * its respectful "trigger_hotplug" debugfs entry: + * + * echo 1 > /sys/kernel/debug/dri/0/DP-X/trigger_hotplug + * + */ +static ssize_t dp_trigger_hotplug(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; + struct drm_connector *connector = &aconnector->base; + struct drm_device *dev = connector->dev; + enum dc_connection_type new_connection_type = dc_connection_none; + char *wr_buf = NULL; + uint32_t wr_buf_size = 42; + int max_param_num = 1; + long param[1] = {0}; + uint8_t param_nums = 0; + + if (!aconnector || !aconnector->dc_link) + return -EINVAL; + + if (size == 0) + return -EINVAL; + + wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL); + + if (!wr_buf) { + DRM_DEBUG_DRIVER("no memory to allocate write buffer\n"); + return -ENOSPC; + } + + if (parse_write_buffer_into_params(wr_buf, wr_buf_size, + (long *)param, buf, + max_param_num, + ¶m_nums)) + return -EINVAL; + + if (param_nums <= 0) { + DRM_DEBUG_DRIVER("user data not be read\n"); + kfree(wr_buf); + return -EINVAL; + } + + if (param[0] == 1) { + mutex_lock(&aconnector->hpd_lock); + + if (!dc_link_detect_sink(aconnector->dc_link, &new_connection_type) && + new_connection_type != dc_connection_none) + goto unlock; + + if (!dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD)) + goto unlock; + + amdgpu_dm_update_connector_after_detect(aconnector); + + drm_modeset_lock_all(dev); + dm_restore_drm_connector_state(dev, connector); + drm_modeset_unlock_all(dev); + + drm_kms_helper_hotplug_event(dev); + +unlock: + mutex_unlock(&aconnector->hpd_lock); + } + + kfree(wr_buf); + return size; +} + +/* function: read DSC status on the connector + * + * The read function: dp_dsc_clock_en_read + * returns current status of DSC clock on the connector. + * The return is a boolean flag: 1 or 0. + * + * Access it with the following command (you need to specify + * connector like DP-1): + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_clock_en + * + * Expected output: + * 1 - means that DSC is currently enabled + * 0 - means that DSC is disabled + */ static ssize_t dp_dsc_clock_en_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1037,6 +1196,95 @@ static ssize_t dp_dsc_clock_en_read(struct file *f, char __user *buf, return result; } +/* function: write force DSC on the connector + * + * The write function: dp_dsc_clock_en_write + * enables to force DSC on the connector. + * User can write to either force enable DSC + * on the next modeset or set it to driver default + * + * Writing DSC settings is done with the following command: + * - To force enable DSC (you need to specify + * connector like DP-1): + * + * echo 0x1 > /sys/kernel/debug/dri/0/DP-X/dsc_clock_en + * + * - To return to default state set the flag to zero and + * let driver deal with DSC automatically + * (you need to specify connector like DP-1): + * + * echo 0x0 > /sys/kernel/debug/dri/0/DP-X/dsc_clock_en + * + */ +static ssize_t dp_dsc_clock_en_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; + struct pipe_ctx *pipe_ctx; + int i; + char *wr_buf = NULL; + uint32_t wr_buf_size = 42; + int max_param_num = 1; + long param[1] = {0}; + uint8_t param_nums = 0; + + if (size == 0) + return -EINVAL; + + wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL); + + if (!wr_buf) { + DRM_DEBUG_DRIVER("no memory to allocate write buffer\n"); + return -ENOSPC; + } + + if (parse_write_buffer_into_params(wr_buf, wr_buf_size, + (long *)param, buf, + max_param_num, + ¶m_nums)) { + kfree(wr_buf); + return -EINVAL; + } + + if (param_nums <= 0) { + DRM_DEBUG_DRIVER("user data not be read\n"); + kfree(wr_buf); + return -EINVAL; + } + + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx && pipe_ctx->stream && + pipe_ctx->stream->link == aconnector->dc_link) + break; + } + + if (!pipe_ctx || !pipe_ctx->stream) + goto done; + + aconnector->dsc_settings.dsc_clock_en = param[0]; + +done: + kfree(wr_buf); + return size; +} + +/* function: read DSC slice width parameter on the connector + * + * The read function: dp_dsc_slice_width_read + * returns dsc slice width used in the current configuration + * The return is an integer: 0 or other positive number + * + * Access the status with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_slice_width + * + * 0 - means that DSC is disabled + * + * Any other number more than zero represents the + * slice width currently used by DSC in pixels + * + */ static ssize_t dp_dsc_slice_width_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1094,6 +1342,98 @@ static ssize_t dp_dsc_slice_width_read(struct file *f, char __user *buf, return result; } +/* function: write DSC slice width parameter + * + * The write function: dp_dsc_slice_width_write + * overwrites automatically generated DSC configuration + * of slice width. + * + * The user has to write the slice width divisible by the + * picture width. + * + * Also the user has to write width in hexidecimal + * rather than in decimal. + * + * Writing DSC settings is done with the following command: + * - To force overwrite slice width: (example sets to 1920 pixels) + * + * echo 0x780 > /sys/kernel/debug/dri/0/DP-X/dsc_slice_width + * + * - To stop overwriting and let driver find the optimal size, + * set the width to zero: + * + * echo 0x0 > /sys/kernel/debug/dri/0/DP-X/dsc_slice_width + * + */ +static ssize_t dp_dsc_slice_width_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; + struct pipe_ctx *pipe_ctx; + int i; + char *wr_buf = NULL; + uint32_t wr_buf_size = 42; + int max_param_num = 1; + long param[1] = {0}; + uint8_t param_nums = 0; + + if (size == 0) + return -EINVAL; + + wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL); + + if (!wr_buf) { + DRM_DEBUG_DRIVER("no memory to allocate write buffer\n"); + return -ENOSPC; + } + + if (parse_write_buffer_into_params(wr_buf, wr_buf_size, + (long *)param, buf, + max_param_num, + ¶m_nums)) { + kfree(wr_buf); + return -EINVAL; + } + + if (param_nums <= 0) { + DRM_DEBUG_DRIVER("user data not be read\n"); + kfree(wr_buf); + return -EINVAL; + } + + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx && pipe_ctx->stream && + pipe_ctx->stream->link == aconnector->dc_link) + break; + } + + if (!pipe_ctx || !pipe_ctx->stream) + goto done; + + aconnector->dsc_settings.dsc_slice_width = param[0]; + +done: + kfree(wr_buf); + return size; +} + +/* function: read DSC slice height parameter on the connector + * + * The read function: dp_dsc_slice_height_read + * returns dsc slice height used in the current configuration + * The return is an integer: 0 or other positive number + * + * Access the status with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_slice_height + * + * 0 - means that DSC is disabled + * + * Any other number more than zero represents the + * slice height currently used by DSC in pixels + * + */ static ssize_t dp_dsc_slice_height_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1151,6 +1491,94 @@ static ssize_t dp_dsc_slice_height_read(struct file *f, char __user *buf, return result; } +/* function: write DSC slice height parameter + * + * The write function: dp_dsc_slice_height_write + * overwrites automatically generated DSC configuration + * of slice height. + * + * The user has to write the slice height divisible by the + * picture height. + * + * Also the user has to write height in hexidecimal + * rather than in decimal. + * + * Writing DSC settings is done with the following command: + * - To force overwrite slice height (example sets to 128 pixels): + * + * echo 0x80 > /sys/kernel/debug/dri/0/DP-X/dsc_slice_height + * + * - To stop overwriting and let driver find the optimal size, + * set the height to zero: + * + * echo 0x0 > /sys/kernel/debug/dri/0/DP-X/dsc_slice_height + * + */ +static ssize_t dp_dsc_slice_height_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; + struct pipe_ctx *pipe_ctx; + int i; + char *wr_buf = NULL; + uint32_t wr_buf_size = 42; + int max_param_num = 1; + uint8_t param_nums = 0; + long param[1] = {0}; + + if (size == 0) + return -EINVAL; + + wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL); + + if (!wr_buf) { + DRM_DEBUG_DRIVER("no memory to allocate write buffer\n"); + return -ENOSPC; + } + + if (parse_write_buffer_into_params(wr_buf, wr_buf_size, + (long *)param, buf, + max_param_num, + ¶m_nums)) { + kfree(wr_buf); + return -EINVAL; + } + + if (param_nums <= 0) { + DRM_DEBUG_DRIVER("user data not be read\n"); + kfree(wr_buf); + return -EINVAL; + } + + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx && pipe_ctx->stream && + pipe_ctx->stream->link == aconnector->dc_link) + break; + } + + if (!pipe_ctx || !pipe_ctx->stream) + goto done; + + aconnector->dsc_settings.dsc_slice_height = param[0]; + +done: + kfree(wr_buf); + return size; +} + +/* function: read DSC target rate on the connector in bits per pixel + * + * The read function: dp_dsc_bits_per_pixel_read + * returns target rate of compression in bits per pixel + * The return is an integer: 0 or other positive integer + * + * Access it with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_bits_per_pixel + * + * 0 - means that DSC is disabled + */ static ssize_t dp_dsc_bits_per_pixel_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1208,6 +1636,94 @@ static ssize_t dp_dsc_bits_per_pixel_read(struct file *f, char __user *buf, return result; } +/* function: write DSC target rate in bits per pixel + * + * The write function: dp_dsc_bits_per_pixel_write + * overwrites automatically generated DSC configuration + * of DSC target bit rate. + * + * Also the user has to write bpp in hexidecimal + * rather than in decimal. + * + * Writing DSC settings is done with the following command: + * - To force overwrite rate (example sets to 256 bpp x 1/16): + * + * echo 0x100 > /sys/kernel/debug/dri/0/DP-X/dsc_bits_per_pixel + * + * - To stop overwriting and let driver find the optimal rate, + * set the rate to zero: + * + * echo 0x0 > /sys/kernel/debug/dri/0/DP-X/dsc_bits_per_pixel + * + */ +static ssize_t dp_dsc_bits_per_pixel_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; + struct pipe_ctx *pipe_ctx; + int i; + char *wr_buf = NULL; + uint32_t wr_buf_size = 42; + int max_param_num = 1; + uint8_t param_nums = 0; + long param[1] = {0}; + + if (size == 0) + return -EINVAL; + + wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL); + + if (!wr_buf) { + DRM_DEBUG_DRIVER("no memory to allocate write buffer\n"); + return -ENOSPC; + } + + if (parse_write_buffer_into_params(wr_buf, wr_buf_size, + (long *)param, buf, + max_param_num, + ¶m_nums)) { + kfree(wr_buf); + return -EINVAL; + } + + if (param_nums <= 0) { + DRM_DEBUG_DRIVER("user data not be read\n"); + kfree(wr_buf); + return -EINVAL; + } + + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx && pipe_ctx->stream && + pipe_ctx->stream->link == aconnector->dc_link) + break; + } + + if (!pipe_ctx || !pipe_ctx->stream) + goto done; + + aconnector->dsc_settings.dsc_bits_per_pixel = param[0]; + +done: + kfree(wr_buf); + return size; +} + +/* function: read DSC picture width parameter on the connector + * + * The read function: dp_dsc_pic_width_read + * returns dsc picture width used in the current configuration + * It is the same as h_addressable of the current + * display's timing + * The return is an integer: 0 or other positive integer + * If 0 then DSC is disabled. + * + * Access it with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_pic_width + * + * 0 - means that DSC is disabled + */ static ssize_t dp_dsc_pic_width_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1322,6 +1838,21 @@ static ssize_t dp_dsc_pic_height_read(struct file *f, char __user *buf, return result; } +/* function: read DSC chunk size parameter on the connector + * + * The read function: dp_dsc_chunk_size_read + * returns dsc chunk size set in the current configuration + * The value is calculated automatically by DSC code + * and depends on slice parameters and bpp target rate + * The return is an integer: 0 or other positive integer + * If 0 then DSC is disabled. + * + * Access it with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_chunk_size + * + * 0 - means that DSC is disabled + */ static ssize_t dp_dsc_chunk_size_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1379,6 +1910,21 @@ static ssize_t dp_dsc_chunk_size_read(struct file *f, char __user *buf, return result; } +/* function: read DSC slice bpg offset on the connector + * + * The read function: dp_dsc_slice_bpg_offset_read + * returns dsc bpg slice offset set in the current configuration + * The value is calculated automatically by DSC code + * and depends on slice parameters and bpp target rate + * The return is an integer: 0 or other positive integer + * If 0 then DSC is disabled. + * + * Access it with the following command: + * + * cat /sys/kernel/debug/dri/0/DP-X/dsc_slice_bpg_offset + * + * 0 - means that DSC is disabled + */ static ssize_t dp_dsc_slice_bpg_offset_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { @@ -1436,6 +1982,7 @@ static ssize_t dp_dsc_slice_bpg_offset_read(struct file *f, char __user *buf, return result; } +DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support); DEFINE_SHOW_ATTRIBUTE(dmub_fw_state); DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer); DEFINE_SHOW_ATTRIBUTE(output_bpc); @@ -1446,24 +1993,28 @@ DEFINE_SHOW_ATTRIBUTE(hdcp_sink_capability); static const struct file_operations dp_dsc_clock_en_debugfs_fops = { .owner = THIS_MODULE, .read = dp_dsc_clock_en_read, + .write = dp_dsc_clock_en_write, .llseek = default_llseek }; static const struct file_operations dp_dsc_slice_width_debugfs_fops = { .owner = THIS_MODULE, .read = dp_dsc_slice_width_read, + .write = dp_dsc_slice_width_write, .llseek = default_llseek }; static const struct file_operations dp_dsc_slice_height_debugfs_fops = { .owner = THIS_MODULE, .read = dp_dsc_slice_height_read, + .write = dp_dsc_slice_height_write, .llseek = default_llseek }; static const struct file_operations dp_dsc_bits_per_pixel_debugfs_fops = { .owner = THIS_MODULE, .read = dp_dsc_bits_per_pixel_read, + .write = dp_dsc_bits_per_pixel_write, .llseek = default_llseek }; @@ -1491,6 +2042,12 @@ static const struct file_operations dp_dsc_slice_bpg_offset_debugfs_fops = { .llseek = default_llseek }; +static const struct file_operations dp_trigger_hotplug_debugfs_fops = { + .owner = THIS_MODULE, + .write = dp_trigger_hotplug, + .llseek = default_llseek +}; + static const struct file_operations dp_link_settings_debugfs_fops = { .owner = THIS_MODULE, .read = dp_link_settings_read, @@ -1557,7 +2114,8 @@ static const struct { {"dsc_pic_width", &dp_dsc_pic_width_debugfs_fops}, {"dsc_pic_height", &dp_dsc_pic_height_debugfs_fops}, {"dsc_chunk_size", &dp_dsc_chunk_size_debugfs_fops}, - {"dsc_slice_bpg", &dp_dsc_slice_bpg_offset_debugfs_fops} + {"dsc_slice_bpg", &dp_dsc_slice_bpg_offset_debugfs_fops}, + {"dp_dsc_fec_support", &dp_dsc_fec_support_fops} }; #ifdef CONFIG_DRM_AMD_DC_HDCP @@ -1721,7 +2279,7 @@ static int current_backlight_read(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_display_manager *dm = &adev->dm; unsigned int backlight = dc_link_get_backlight_level(dm->backlight_link); @@ -1739,7 +2297,7 @@ static int target_backlight_read(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_display_manager *dm = &adev->dm; unsigned int backlight = dc_link_get_target_backlight_pwm(dm->backlight_link); @@ -1778,6 +2336,38 @@ static const struct drm_info_list amdgpu_dm_debugfs_list[] = { }; /* + * Sets the force_timing_sync debug optino from the given string. + * All connected displays will be force synchronized immediately. + * Usage: echo 1 > /sys/kernel/debug/dri/0/amdgpu_dm_force_timing_sync + */ +static int force_timing_sync_set(void *data, u64 val) +{ + struct amdgpu_device *adev = data; + + adev->dm.force_timing_sync = (bool)val; + + amdgpu_dm_trigger_timing_sync(adev_to_drm(adev)); + + return 0; +} + +/* + * Gets the force_timing_sync debug option value into the given buffer. + * Usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_force_timing_sync + */ +static int force_timing_sync_get(void *data, u64 *val) +{ + struct amdgpu_device *adev = data; + + *val = adev->dm.force_timing_sync; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(force_timing_sync_ops, force_timing_sync_get, + force_timing_sync_set, "%llu\n"); + +/* * Sets the DC visual confirm debug option from the given string. * Example usage: echo 1 > /sys/kernel/debug/dri/0/amdgpu_visual_confirm */ @@ -1815,7 +2405,7 @@ int dtn_debugfs_init(struct amdgpu_device *adev) .llseek = default_llseek }; - struct drm_minor *minor = adev->ddev->primary; + struct drm_minor *minor = adev_to_drm(adev)->primary; struct dentry *root = minor->debugfs_root; int ret; @@ -1836,5 +2426,8 @@ int dtn_debugfs_init(struct amdgpu_device *adev) debugfs_create_file_unsafe("amdgpu_dm_dmub_fw_state", 0644, root, adev, &dmub_fw_state_fops); + debugfs_create_file_unsafe("amdgpu_dm_force_timing_sync", 0644, root, + adev, &force_timing_sync_ops); + return 0; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index cbcf504f73a5..357778556b06 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -719,7 +719,7 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) */ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; @@ -755,7 +755,7 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) */ void amdgpu_dm_hpd_fini(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index a61a294caebe..adbb44822e94 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -68,7 +68,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, result = dc_link_aux_transfer_raw(TO_DM_AUX(aux)->ddc_service, &payload, &operation_result); - if (payload.write) + if (payload.write && result >= 0) result = msg->size; if (result < 0) @@ -242,7 +242,7 @@ dm_mst_atomic_best_encoder(struct drm_connector *connector, struct drm_connector_state *connector_state) { struct drm_device *dev = connector->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_crtc *acrtc = to_amdgpu_crtc(connector_state->crtc); return &adev->dm.mst_encoders[acrtc->crtc_id].base; @@ -311,7 +311,7 @@ static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = { void dm_dp_create_fake_mst_encoders(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); int i; for (i = 0; i < adev->dm.display_indexes_num; i++) { @@ -338,7 +338,7 @@ dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, { struct amdgpu_dm_connector *master = container_of(mgr, struct amdgpu_dm_connector, mst_mgr); struct drm_device *dev = master->base.dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); struct amdgpu_dm_connector *aconnector; struct drm_connector *connector; int i; @@ -427,7 +427,7 @@ void amdgpu_dm_initialize_dp_connector(struct amdgpu_display_manager *dm, aconnector->mst_mgr.cbs = &dm_mst_cbs; drm_dp_mst_topology_mgr_init( &aconnector->mst_mgr, - dm->adev->ddev, + adev_to_drm(dm->adev), &aconnector->dm_dp_aux.aux, 16, 4, @@ -453,6 +453,10 @@ struct dsc_mst_fairness_params { struct dc_dsc_bw_range bw_range; bool compression_possible; struct drm_dp_mst_port *port; + bool clock_overwrite; + uint32_t slice_width_overwrite; + uint32_t slice_height_overwrite; + uint32_t bpp_overwrite; }; struct dsc_mst_fairness_vars { @@ -486,7 +490,21 @@ static void set_dsc_configs_from_fairness_vars(struct dsc_mst_fairness_params *p params[i].timing, ¶ms[i].timing->dsc_cfg)) { params[i].timing->flags.DSC = 1; - params[i].timing->dsc_cfg.bits_per_pixel = vars[i].bpp_x16; + + if (params[i].bpp_overwrite) + params[i].timing->dsc_cfg.bits_per_pixel = params[i].bpp_overwrite; + else + params[i].timing->dsc_cfg.bits_per_pixel = vars[i].bpp_x16; + + if (params[i].slice_width_overwrite) + params[i].timing->dsc_cfg.num_slices_h = DIV_ROUND_UP( + params[i].timing->h_addressable, + params[i].slice_width_overwrite); + + if (params[i].slice_height_overwrite) + params[i].timing->dsc_cfg.num_slices_v = DIV_ROUND_UP( + params[i].timing->v_addressable, + params[i].slice_height_overwrite); } else { params[i].timing->flags.DSC = 0; } @@ -618,7 +636,9 @@ static void try_disable_dsc(struct drm_atomic_state *state, int remaining_to_try = 0; for (i = 0; i < count; i++) { - if (vars[i].dsc_enabled && vars[i].bpp_x16 == params[i].bw_range.max_target_bpp_x16) { + if (vars[i].dsc_enabled + && vars[i].bpp_x16 == params[i].bw_range.max_target_bpp_x16 + && !params[i].clock_overwrite) { kbps_increase[i] = params[i].bw_range.stream_kbps - params[i].bw_range.max_kbps; tried[i] = false; remaining_to_try += 1; @@ -679,6 +699,7 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, struct dsc_mst_fairness_vars vars[MAX_PIPES]; struct amdgpu_dm_connector *aconnector; int count = 0; + bool debugfs_overwrite = false; memset(params, 0, sizeof(params)); @@ -697,6 +718,12 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, params[count].sink = stream->sink; aconnector = (struct amdgpu_dm_connector *)stream->dm_stream_context; params[count].port = aconnector->port; + params[count].clock_overwrite = aconnector->dsc_settings.dsc_clock_en; + if (params[count].clock_overwrite) + debugfs_overwrite = true; + params[count].slice_width_overwrite = aconnector->dsc_settings.dsc_slice_width; + params[count].slice_height_overwrite = aconnector->dsc_settings.dsc_slice_height; + params[count].bpp_overwrite = aconnector->dsc_settings.dsc_bits_per_pixel; params[count].compression_possible = stream->sink->dsc_caps.dsc_dec_caps.is_dsc_supported; dc_dsc_get_policy_for_timing(params[count].timing, &dsc_policy); if (!dc_dsc_compute_bandwidth_range( @@ -722,7 +749,7 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state, dm_mst_get_pbn_divider(dc_link)) < 0) return false; } - if (!drm_dp_mst_atomic_check(state)) { + if (!drm_dp_mst_atomic_check(state) && !debugfs_overwrite) { set_dsc_configs_from_fairness_vars(params, vars, count); return true; } diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile index e0f4f1be1618..047b1e2dd8f1 100644 --- a/drivers/gpu/drm/amd/display/dc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/Makefile @@ -43,6 +43,10 @@ DC_LIBS += dce110 DC_LIBS += dce100 DC_LIBS += dce80 +ifdef CONFIG_DRM_AMD_DC_SI +DC_LIBS += dce60 +endif + ifdef CONFIG_DRM_AMD_DC_HDCP DC_LIBS += hdcp endif diff --git a/drivers/gpu/drm/amd/display/dc/bios/Makefile b/drivers/gpu/drm/amd/display/dc/bios/Makefile index 239e86bbec5a..ed6b5e9763f6 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/Makefile +++ b/drivers/gpu/drm/amd/display/dc/bios/Makefile @@ -32,6 +32,15 @@ AMD_DAL_BIOS = $(addprefix $(AMDDALPATH)/dc/bios/,$(BIOS)) AMD_DISPLAY_FILES += $(AMD_DAL_BIOS) ############################################################################### +# DCE 6x +############################################################################### +# All DCE6.x are derived from DCE6.0, so 6.0 MUST be defined if ANY of +# DCE6.x is compiled. +ifdef CONFIG_DRM_AMD_DC_SI +AMD_DISPLAY_FILES += $(AMDDALPATH)/dc/bios/dce60/command_table_helper_dce60.o +endif + +############################################################################### # DCE 8x ############################################################################### # All DCE8.x are derived from DCE8.0, so 8.0 MUST be defined if ANY of diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index 078b7e344185..2d5c7daaee23 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -1108,6 +1108,18 @@ static enum bp_result bios_parser_enable_disp_power_gating( action); } +static enum bp_result bios_parser_enable_lvtma_control( + struct dc_bios *dcb, + uint8_t uc_pwr_on) +{ + struct bios_parser *bp = BP_FROM_DCB(dcb); + + if (!bp->cmd_tbl.enable_lvtma_control) + return BP_RESULT_FAILURE; + + return bp->cmd_tbl.enable_lvtma_control(bp, uc_pwr_on); +} + static bool bios_parser_is_accelerated_mode( struct dc_bios *dcb) { @@ -2208,7 +2220,9 @@ static const struct dc_vbios_funcs vbios_funcs = { .get_board_layout_info = bios_get_board_layout_info, .pack_data_tables = bios_parser_pack_data_tables, - .get_atom_dc_golden_table = bios_get_atom_dc_golden_table + .get_atom_dc_golden_table = bios_get_atom_dc_golden_table, + + .enable_lvtma_control = bios_parser_enable_lvtma_control }; static bool bios_parser2_construct( diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 5815983caaf8..070459e3e407 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -1877,9 +1877,7 @@ static enum bp_result set_crtc_using_dtd_timing_v3( * but it is 4 either from Edid data (spec CEA 861) * or CEA timing table. */ - params.usV_SyncOffset = - cpu_to_le16(le16_to_cpu(params.usV_SyncOffset) + 1); - + le16_add_cpu(¶ms.usV_SyncOffset, 1); } } diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c index bed91572f82a..25bdf1c38e0a 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c @@ -569,10 +569,7 @@ static enum bp_result set_crtc_using_dtd_timing_v3( * but it is 4 either from Edid data (spec CEA 861) * or CEA timing table. */ - params.v_syncoffset = - cpu_to_le16(le16_to_cpu(params.v_syncoffset) + - 1); - + le16_add_cpu(¶ms.v_syncoffset, 1); } } @@ -904,6 +901,61 @@ static unsigned int get_smu_clock_info_v3_1(struct bios_parser *bp, uint8_t id) return 0; } +/****************************************************************************** + ****************************************************************************** + ** + ** LVTMA CONTROL + ** + ****************************************************************************** + *****************************************************************************/ + +static enum bp_result enable_lvtma_control( + struct bios_parser *bp, + uint8_t uc_pwr_on); + +static void init_enable_lvtma_control(struct bios_parser *bp) +{ + /* TODO add switch for table vrsion */ + bp->cmd_tbl.enable_lvtma_control = enable_lvtma_control; + +} + +static void enable_lvtma_control_dmcub( + struct dc_dmub_srv *dmcub, + uint8_t uc_pwr_on) +{ + + union dmub_rb_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + + cmd.cmd_common.header.type = DMUB_CMD__VBIOS; + cmd.cmd_common.header.sub_type = + DMUB_CMD__VBIOS_LVTMA_CONTROL; + cmd.cmd_common.cmd_buffer[0] = + uc_pwr_on; + + dc_dmub_srv_cmd_queue(dmcub, &cmd); + dc_dmub_srv_cmd_execute(dmcub); + dc_dmub_srv_wait_idle(dmcub); + +} + +static enum bp_result enable_lvtma_control( + struct bios_parser *bp, + uint8_t uc_pwr_on) +{ + enum bp_result result = BP_RESULT_FAILURE; + + if (bp->base.ctx->dc->ctx->dmub_srv && + bp->base.ctx->dc->debug.dmub_command_table) { + enable_lvtma_control_dmcub(bp->base.ctx->dmub_srv, + uc_pwr_on); + return BP_RESULT_OK; + } + return result; +} + void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp) { init_dig_encoder_control(bp); @@ -919,4 +971,5 @@ void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp) init_set_dce_clock(bp); init_get_smu_clock_info(bp); + init_enable_lvtma_control(bp); } diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.h b/drivers/gpu/drm/amd/display/dc/bios/command_table2.h index 7a2af24dfe60..7bdce013cde5 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.h +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.h @@ -94,7 +94,8 @@ struct cmd_tbl { struct bp_set_dce_clock_parameters *bp_params); unsigned int (*get_smu_clock_info)( struct bios_parser *bp, uint8_t id); - + enum bp_result (*enable_lvtma_control)(struct bios_parser *bp, + uint8_t uc_pwr_on); }; void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp); diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c index 253bbb1eea60..48b4ef03fc8f 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c @@ -36,6 +36,14 @@ bool dal_bios_parser_init_cmd_tbl_helper( enum dce_version dce) { switch (dce) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case DCE_VERSION_6_0: + case DCE_VERSION_6_1: + case DCE_VERSION_6_4: + *h = dal_cmd_tbl_helper_dce60_get_table(); + return true; +#endif + case DCE_VERSION_8_0: case DCE_VERSION_8_1: case DCE_VERSION_8_3: diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.h b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.h index 4c3789df253d..dfd30aaf4032 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.h +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.h @@ -26,6 +26,9 @@ #ifndef __DAL_COMMAND_TABLE_HELPER_H__ #define __DAL_COMMAND_TABLE_HELPER_H__ +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/command_table_helper_dce60.h" +#endif #include "dce80/command_table_helper_dce80.h" #include "dce110/command_table_helper_dce110.h" #include "dce112/command_table_helper_dce112.h" diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c index 21ff6b686f5f..74c498b6774d 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c @@ -37,6 +37,14 @@ bool dal_bios_parser_init_cmd_tbl_helper2( enum dce_version dce) { switch (dce) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case DCE_VERSION_6_0: + case DCE_VERSION_6_1: + case DCE_VERSION_6_4: + *h = dal_cmd_tbl_helper_dce60_get_table(); + return true; +#endif + case DCE_VERSION_8_0: case DCE_VERSION_8_1: case DCE_VERSION_8_3: diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.h b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.h index 785fcb20a1b9..66e0a3e73768 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.h +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.h @@ -26,6 +26,9 @@ #ifndef __DAL_COMMAND_TABLE_HELPER2_H__ #define __DAL_COMMAND_TABLE_HELPER2_H__ +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/command_table_helper_dce60.h" +#endif #include "dce80/command_table_helper_dce80.h" #include "dce110/command_table_helper_dce110.h" #include "dce112/command_table_helper2_dce112.h" diff --git a/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.c b/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.c new file mode 100644 index 000000000000..710221b4f5c5 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.c @@ -0,0 +1,354 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dm_services.h" + +#include "atom.h" + +#include "include/grph_object_id.h" +#include "include/grph_object_defs.h" +#include "include/bios_parser_types.h" + +#include "../command_table_helper.h" + +static uint8_t encoder_action_to_atom(enum bp_encoder_control_action action) +{ + uint8_t atom_action = 0; + + switch (action) { + case ENCODER_CONTROL_ENABLE: + atom_action = ATOM_ENABLE; + break; + case ENCODER_CONTROL_DISABLE: + atom_action = ATOM_DISABLE; + break; + case ENCODER_CONTROL_SETUP: + atom_action = ATOM_ENCODER_CMD_SETUP; + break; + case ENCODER_CONTROL_INIT: + atom_action = ATOM_ENCODER_INIT; + break; + default: + BREAK_TO_DEBUGGER(); /* Unhandle action in driver.!! */ + break; + } + + return atom_action; +} + +static bool engine_bp_to_atom(enum engine_id id, uint32_t *atom_engine_id) +{ + bool result = false; + + if (atom_engine_id != NULL) + switch (id) { + case ENGINE_ID_DIGA: + *atom_engine_id = ASIC_INT_DIG1_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGB: + *atom_engine_id = ASIC_INT_DIG2_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGC: + *atom_engine_id = ASIC_INT_DIG3_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGD: + *atom_engine_id = ASIC_INT_DIG4_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGE: + *atom_engine_id = ASIC_INT_DIG5_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGF: + *atom_engine_id = ASIC_INT_DIG6_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DIGG: + *atom_engine_id = ASIC_INT_DIG7_ENCODER_ID; + result = true; + break; + case ENGINE_ID_DACA: + *atom_engine_id = ASIC_INT_DAC1_ENCODER_ID; + result = true; + break; + default: + break; + } + + return result; +} + +static bool clock_source_id_to_atom( + enum clock_source_id id, + uint32_t *atom_pll_id) +{ + bool result = true; + + if (atom_pll_id != NULL) + switch (id) { + case CLOCK_SOURCE_ID_PLL0: + *atom_pll_id = ATOM_PPLL0; + break; + case CLOCK_SOURCE_ID_PLL1: + *atom_pll_id = ATOM_PPLL1; + break; + case CLOCK_SOURCE_ID_PLL2: + *atom_pll_id = ATOM_PPLL2; + break; + case CLOCK_SOURCE_ID_EXTERNAL: + *atom_pll_id = ATOM_PPLL_INVALID; + break; + case CLOCK_SOURCE_ID_DFS: + *atom_pll_id = ATOM_EXT_PLL1; + break; + case CLOCK_SOURCE_ID_VCE: + /* for VCE encoding, + * we need to pass in ATOM_PPLL_INVALID + */ + *atom_pll_id = ATOM_PPLL_INVALID; + break; + case CLOCK_SOURCE_ID_DP_DTO: + /* When programming DP DTO PLL ID should be invalid */ + *atom_pll_id = ATOM_PPLL_INVALID; + break; + case CLOCK_SOURCE_ID_UNDEFINED: + BREAK_TO_DEBUGGER(); /* check when this will happen! */ + *atom_pll_id = ATOM_PPLL_INVALID; + result = false; + break; + default: + result = false; + break; + } + + return result; +} + +static uint8_t clock_source_id_to_atom_phy_clk_src_id( + enum clock_source_id id) +{ + uint8_t atom_phy_clk_src_id = 0; + + switch (id) { + case CLOCK_SOURCE_ID_PLL0: + atom_phy_clk_src_id = ATOM_TRANSMITTER_CONFIG_V5_P0PLL; + break; + case CLOCK_SOURCE_ID_PLL1: + atom_phy_clk_src_id = ATOM_TRANSMITTER_CONFIG_V5_P1PLL; + break; + case CLOCK_SOURCE_ID_PLL2: + atom_phy_clk_src_id = ATOM_TRANSMITTER_CONFIG_V5_P2PLL; + break; + case CLOCK_SOURCE_ID_EXTERNAL: + atom_phy_clk_src_id = ATOM_TRANSMITTER_CONFIG_V5_REFCLK_SRC_EXT; + break; + default: + atom_phy_clk_src_id = ATOM_TRANSMITTER_CONFIG_V5_P1PLL; + break; + } + + return atom_phy_clk_src_id >> 2; +} + +static uint8_t signal_type_to_atom_dig_mode(enum signal_type s) +{ + uint8_t atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_DP; + + switch (s) { + case SIGNAL_TYPE_DISPLAY_PORT: + case SIGNAL_TYPE_EDP: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_DP; + break; + case SIGNAL_TYPE_LVDS: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_LVDS; + break; + case SIGNAL_TYPE_DVI_SINGLE_LINK: + case SIGNAL_TYPE_DVI_DUAL_LINK: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_DVI; + break; + case SIGNAL_TYPE_HDMI_TYPE_A: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_HDMI; + break; + case SIGNAL_TYPE_DISPLAY_PORT_MST: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_DP_MST; + break; + default: + atom_dig_mode = ATOM_TRANSMITTER_DIGMODE_V5_DVI; + break; + } + + return atom_dig_mode; +} + +static uint8_t hpd_sel_to_atom(enum hpd_source_id id) +{ + uint8_t atom_hpd_sel = 0; + + switch (id) { + case HPD_SOURCEID1: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD1_SEL; + break; + case HPD_SOURCEID2: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD2_SEL; + break; + case HPD_SOURCEID3: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD3_SEL; + break; + case HPD_SOURCEID4: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD4_SEL; + break; + case HPD_SOURCEID5: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD5_SEL; + break; + case HPD_SOURCEID6: + atom_hpd_sel = ATOM_TRANSMITTER_CONFIG_V5_HPD6_SEL; + break; + case HPD_SOURCEID_UNKNOWN: + default: + atom_hpd_sel = 0; + break; + } + return atom_hpd_sel >> 4; +} + +static uint8_t dig_encoder_sel_to_atom(enum engine_id id) +{ + uint8_t atom_dig_encoder_sel = 0; + + switch (id) { + case ENGINE_ID_DIGA: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGA_SEL; + break; + case ENGINE_ID_DIGB: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGB_SEL; + break; + case ENGINE_ID_DIGC: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGC_SEL; + break; + case ENGINE_ID_DIGD: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGD_SEL; + break; + case ENGINE_ID_DIGE: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGE_SEL; + break; + case ENGINE_ID_DIGF: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGF_SEL; + break; + case ENGINE_ID_DIGG: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGG_SEL; + break; + default: + atom_dig_encoder_sel = ATOM_TRANMSITTER_V5__DIGA_SEL; + break; + } + + return atom_dig_encoder_sel; +} + +static uint8_t phy_id_to_atom(enum transmitter t) +{ + uint8_t atom_phy_id; + + switch (t) { + case TRANSMITTER_UNIPHY_A: + atom_phy_id = ATOM_PHY_ID_UNIPHYA; + break; + case TRANSMITTER_UNIPHY_B: + atom_phy_id = ATOM_PHY_ID_UNIPHYB; + break; + case TRANSMITTER_UNIPHY_C: + atom_phy_id = ATOM_PHY_ID_UNIPHYC; + break; + case TRANSMITTER_UNIPHY_D: + atom_phy_id = ATOM_PHY_ID_UNIPHYD; + break; + case TRANSMITTER_UNIPHY_E: + atom_phy_id = ATOM_PHY_ID_UNIPHYE; + break; + case TRANSMITTER_UNIPHY_F: + atom_phy_id = ATOM_PHY_ID_UNIPHYF; + break; + case TRANSMITTER_UNIPHY_G: + atom_phy_id = ATOM_PHY_ID_UNIPHYG; + break; + default: + atom_phy_id = ATOM_PHY_ID_UNIPHYA; + break; + } + return atom_phy_id; +} + +static uint8_t disp_power_gating_action_to_atom( + enum bp_pipe_control_action action) +{ + uint8_t atom_pipe_action = 0; + + switch (action) { + case ASIC_PIPE_DISABLE: + atom_pipe_action = ATOM_DISABLE; + break; + case ASIC_PIPE_ENABLE: + atom_pipe_action = ATOM_ENABLE; + break; + case ASIC_PIPE_INIT: + atom_pipe_action = ATOM_INIT; + break; + default: + BREAK_TO_DEBUGGER(); /* Unhandle action in driver! */ + break; + } + + return atom_pipe_action; +} + +static const struct command_table_helper command_table_helper_funcs = { + .controller_id_to_atom = dal_cmd_table_helper_controller_id_to_atom, + .encoder_action_to_atom = encoder_action_to_atom, + .engine_bp_to_atom = engine_bp_to_atom, + .clock_source_id_to_atom = clock_source_id_to_atom, + .clock_source_id_to_atom_phy_clk_src_id = + clock_source_id_to_atom_phy_clk_src_id, + .signal_type_to_atom_dig_mode = signal_type_to_atom_dig_mode, + .hpd_sel_to_atom = hpd_sel_to_atom, + .dig_encoder_sel_to_atom = dig_encoder_sel_to_atom, + .phy_id_to_atom = phy_id_to_atom, + .disp_power_gating_action_to_atom = disp_power_gating_action_to_atom, + .assign_control_parameter = + dal_cmd_table_helper_assign_control_parameter, + .clock_source_id_to_ref_clk_src = + dal_cmd_table_helper_clock_source_id_to_ref_clk_src, + .transmitter_bp_to_atom = dal_cmd_table_helper_transmitter_bp_to_atom, + .encoder_id_to_atom = dal_cmd_table_helper_encoder_id_to_atom, + .encoder_mode_bp_to_atom = + dal_cmd_table_helper_encoder_mode_bp_to_atom, +}; + +const struct command_table_helper *dal_cmd_tbl_helper_dce60_get_table(void) +{ + return &command_table_helper_funcs; +} diff --git a/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.h b/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.h new file mode 100644 index 000000000000..f733be553d5a --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/bios/dce60/command_table_helper_dce60.h @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DAL_COMMAND_TABLE_HELPER_DCE60_H__ +#define __DAL_COMMAND_TABLE_HELPER_DCE60_H__ + +struct command_table_helper; + +const struct command_table_helper *dal_cmd_tbl_helper_dce60_get_table(void); + +#endif diff --git a/drivers/gpu/drm/amd/display/dc/calcs/Makefile b/drivers/gpu/drm/amd/display/dc/calcs/Makefile index 4674aca8f206..64f515d74410 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/Makefile +++ b/drivers/gpu/drm/amd/display/dc/calcs/Makefile @@ -33,6 +33,10 @@ ifdef CONFIG_PPC64 calcs_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +calcs_rcflags := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -53,6 +57,9 @@ endif CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calcs.o := $(calcs_ccflags) CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calc_auto.o := $(calcs_ccflags) CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calc_math.o := $(calcs_ccflags) -Wno-tautological-compare +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calcs.o := $(calcs_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calc_auto.o := $(calcs_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calc_math.o := $(calcs_rcflags) BW_CALCS = dce_calcs.o bw_fixed.o custom_float.o diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile index 6874276bb2a1..1a495759a034 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile @@ -30,6 +30,17 @@ AMD_DAL_CLK_MGR = $(addprefix $(AMDDALPATH)/dc/clk_mgr/,$(CLK_MGR)) AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR) +ifdef CONFIG_DRM_AMD_DC_SI +############################################################################### +# DCE 60 +############################################################################### +CLK_MGR_DCE60 = dce60_clk_mgr.o + +AMD_DAL_CLK_MGR_DCE60 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce60/,$(CLK_MGR_DCE60)) + +AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE60) +endif + ############################################################################### # DCE 100 and DCE8x ############################################################################### @@ -93,6 +104,13 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/clk_mgr/dcn21/rn_clk_mgr.o := $(call cc-option,-mno-gnu-attribute) endif +# prevent build errors: +# ...: '-mgeneral-regs-only' is incompatible with the use of floating-point types +# this file is unused on arm64, just like on ppc64 +ifdef CONFIG_ARM64 +CFLAGS_REMOVE_$(AMDDALPATH)/dc/clk_mgr/dcn21/rn_clk_mgr.o := -mgeneral-regs-only +endif + AMD_DAL_CLK_MGR_DCN21 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn21/,$(CLK_MGR_DCN21)) AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN21) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index 6a345d43028c..efb909ef7a0f 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -34,6 +34,7 @@ #include "dce110/dce110_clk_mgr.h" #include "dce112/dce112_clk_mgr.h" #include "dce120/dce120_clk_mgr.h" +#include "dce60/dce60_clk_mgr.h" #include "dcn10/rv1_clk_mgr.h" #include "dcn10/rv2_clk_mgr.h" #include "dcn20/dcn20_clk_mgr.h" @@ -123,6 +124,11 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p } switch (asic_id.chip_family) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case FAMILY_SI: + dce60_clk_mgr_construct(ctx, clk_mgr); + break; +#endif case FAMILY_CI: case FAMILY_KV: dce_clk_mgr_construct(ctx, clk_mgr); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c new file mode 100644 index 000000000000..c11c6b3a787d --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c @@ -0,0 +1,174 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + + +#include "dccg.h" +#include "clk_mgr_internal.h" +#include "dce100/dce_clk_mgr.h" +#include "dce110/dce110_clk_mgr.h" +#include "dce60_clk_mgr.h" +#include "reg_helper.h" +#include "dmcu.h" +#include "core_types.h" +#include "dal_asic_id.h" + +/* + * Currently the register shifts and masks in this file are used for dce60 + * which has no DPREFCLK_CNTL register + * TODO: remove this when DENTIST_DISPCLK_CNTL + * is moved to dccg, where it belongs + */ +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + +#define REG(reg) \ + (clk_mgr->regs->reg) + +#undef FN +#define FN(reg_name, field_name) \ + clk_mgr->clk_mgr_shift->field_name, clk_mgr->clk_mgr_mask->field_name + +/* set register offset */ +#define SR(reg_name)\ + .reg_name = mm ## reg_name + +static const struct clk_mgr_registers disp_clk_regs = { + CLK_COMMON_REG_LIST_DCE60_BASE() +}; + +static const struct clk_mgr_shift disp_clk_shift = { + CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(__SHIFT) +}; + +static const struct clk_mgr_mask disp_clk_mask = { + CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(_MASK) +}; + + +/* Max clock values for each state indexed by "enum clocks_state": */ +static const struct state_dependent_clocks dce60_max_clks_by_state[] = { +/* ClocksStateInvalid - should not be used */ +{ .display_clk_khz = 0, .pixel_clk_khz = 0 }, +/* ClocksStateUltraLow - not expected to be used for DCE 6.0 */ +{ .display_clk_khz = 0, .pixel_clk_khz = 0 }, +/* ClocksStateLow */ +{ .display_clk_khz = 352000, .pixel_clk_khz = 330000}, +/* ClocksStateNominal */ +{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 }, +/* ClocksStatePerformance */ +{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 } }; + +int dce60_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base) +{ + struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); + int dprefclk_wdivider; + int dp_ref_clk_khz; + int target_div; + + /* DCE6 has no DPREFCLK_CNTL to read DP Reference Clock source */ + + /* Read the mmDENTIST_DISPCLK_CNTL to get the currently + * programmed DID DENTIST_DPREFCLK_WDIVIDER*/ + REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, &dprefclk_wdivider); + + /* Convert DENTIST_DPREFCLK_WDIVIDERto actual divider*/ + target_div = dentist_get_divider_from_did(dprefclk_wdivider); + + /* Calculate the current DFS clock, in kHz.*/ + dp_ref_clk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR + * clk_mgr->base.dentist_vco_freq_khz) / target_div; + + return dce_adjust_dp_ref_freq_for_ss(clk_mgr, dp_ref_clk_khz); +} + +static void dce60_pplib_apply_display_requirements( + struct dc *dc, + struct dc_state *context) +{ + struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg; + + pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context); + + dce110_fill_display_configs(context, pp_display_cfg); + + if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0) + dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg); +} + +static void dce60_update_clocks(struct clk_mgr *clk_mgr_base, + struct dc_state *context, + bool safe_to_lower) +{ + struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base); + struct dm_pp_power_level_change_request level_change_req; + int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz; + + /*TODO: W/A for dal3 linux, investigate why this works */ + if (!clk_mgr_dce->dfs_bypass_active) + patched_disp_clk = patched_disp_clk * 115 / 100; + + level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context); + /* get max clock state from PPLIB */ + if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower) + || level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) { + if (dm_pp_apply_power_level_change_request(clk_mgr_base->ctx, &level_change_req)) + clk_mgr_dce->cur_min_clks_state = level_change_req.power_level; + } + + if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) { + patched_disp_clk = dce_set_clock(clk_mgr_base, patched_disp_clk); + clk_mgr_base->clks.dispclk_khz = patched_disp_clk; + } + dce60_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context); +} + + + + + + + + +static struct clk_mgr_funcs dce60_funcs = { + .get_dp_ref_clk_frequency = dce60_get_dp_ref_freq_khz, + .update_clocks = dce60_update_clocks +}; + +void dce60_clk_mgr_construct( + struct dc_context *ctx, + struct clk_mgr_internal *clk_mgr) +{ + dce_clk_mgr_construct(ctx, clk_mgr); + + memcpy(clk_mgr->max_clks_by_state, + dce60_max_clks_by_state, + sizeof(dce60_max_clks_by_state)); + + clk_mgr->regs = &disp_clk_regs; + clk_mgr->clk_mgr_shift = &disp_clk_shift; + clk_mgr->clk_mgr_mask = &disp_clk_mask; + clk_mgr->base.funcs = &dce60_funcs; +} + diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h new file mode 100644 index 000000000000..eca3e5168089 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + + +#ifndef DAL_DC_DCE_DCE60_CLK_MGR_H_ +#define DAL_DC_DCE_DCE60_CLK_MGR_H_ + +#include "dc.h" + +void dce60_clk_mgr_construct( + struct dc_context *ctx, + struct clk_mgr_internal *clk_mgr_dce); + +#endif /* DAL_DC_DCE_DCE60_CLK_MGR_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c index c664404a75d4..543afa34d87a 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c @@ -94,6 +94,15 @@ int rn_get_active_display_cnt_wa( return display_count; } +void rn_set_low_power_state(struct clk_mgr *clk_mgr_base) +{ + struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); + + rn_vbios_smu_set_dcn_low_power_state(clk_mgr, DCN_PWR_STATE_LOW_POWER); + /* update power state */ + clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_LOW_POWER; +} + void rn_update_clocks(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool safe_to_lower) @@ -516,6 +525,7 @@ static struct clk_mgr_funcs dcn21_funcs = { .init_clocks = rn_init_clocks, .enable_pme_wa = rn_enable_pme_wa, .are_clock_states_equal = rn_are_clock_states_equal, + .set_low_power_state = rn_set_low_power_state, .notify_wm_ranges = rn_notify_wm_ranges, .notify_link_rate_change = rn_notify_link_rate_change, }; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c index 9133646f6d5f..b0e9b0509568 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c @@ -554,8 +554,7 @@ void dcn3_clk_mgr_construct( void dcn3_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr) { - if (clk_mgr->base.bw_params) - kfree(clk_mgr->base.bw_params); + kfree(clk_mgr->base.bw_params); if (clk_mgr->wm_range_table) dm_helpers_free_gpu_mem(clk_mgr->base.ctx, DC_MEM_ALLOC_TYPE_GART, diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 92eb1ca1634f..dc463d99ef50 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1238,6 +1238,14 @@ bool dc_enable_stereo( return ret; } +void dc_trigger_sync(struct dc *dc, struct dc_state *context) +{ + if (context->stream_count > 1 && !dc->debug.disable_timing_sync) { + enable_timing_multisync(dc, context); + program_timing_sync(dc, context); + } +} + /* * Applies given context to HW and copy it into current context. * It's up to the user to release the src context afterwards. @@ -1297,10 +1305,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c if (result != DC_OK) return result; - if (context->stream_count > 1 && !dc->debug.disable_timing_sync) { - enable_timing_multisync(dc, context); - program_timing_sync(dc, context); - } + dc_trigger_sync(dc, context); /* Program all planes within new context*/ if (dc->hwss.program_front_end_for_ctx) { @@ -2295,6 +2300,7 @@ static void commit_planes_for_stream(struct dc *dc, enum surface_update_type update_type, struct dc_state *context) { + bool mpcc_disconnected = false; int i, j; struct pipe_ctx *top_pipe_to_program = NULL; @@ -2325,6 +2331,15 @@ static void commit_planes_for_stream(struct dc *dc, context_clock_trace(dc, context); } + if (update_type != UPDATE_TYPE_FAST && dc->hwss.interdependent_update_lock && + dc->hwss.disconnect_pipes && dc->hwss.wait_for_pending_cleared){ + dc->hwss.interdependent_update_lock(dc, context, true); + mpcc_disconnected = dc->hwss.disconnect_pipes(dc, context); + dc->hwss.interdependent_update_lock(dc, context, false); + if (mpcc_disconnected) + dc->hwss.wait_for_pending_cleared(dc, context); + } + for (j = 0; j < dc->res_pool->pipe_count; j++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j]; @@ -2621,7 +2636,7 @@ void dc_commit_updates_for_stream(struct dc *dc, copy_stream_update_to_stream(dc, context, stream, stream_update); - if (update_type > UPDATE_TYPE_FAST) { + if (update_type >= UPDATE_TYPE_FULL) { if (!dc->res_pool->funcs->validate_bandwidth(dc, context, false)) { DC_ERROR("Mode validation failed for stream update!\n"); dc_release_state(context); @@ -2933,6 +2948,30 @@ void dc_get_clock(struct dc *dc, enum dc_clock_type clock_type, struct dc_clock_ dc->hwss.get_clock(dc, clock_type, clock_cfg); } +/* enable/disable eDP PSR without specify stream for eDP */ +bool dc_set_psr_allow_active(struct dc *dc, bool enable) +{ + int i; + + for (i = 0; i < dc->current_state->stream_count ; i++) { + struct dc_link *link; + struct dc_stream_state *stream = dc->current_state->streams[i]; + + link = stream->link; + if (!link) + continue; + + if (link->psr_settings.psr_feature_enabled) { + if (enable && !link->psr_settings.psr_allow_active) + return dc_link_set_psr_allow_active(link, true, false); + else if (!enable && link->psr_settings.psr_allow_active) + return dc_link_set_psr_allow_active(link, false, false); + } + } + + return true; +} + #if defined(CONFIG_DRM_AMD_DC_DCN3_0) void dc_allow_idle_optimizations(struct dc *dc, bool allow) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 4bd6e03a7ef3..437d1a7a16fe 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -763,6 +763,7 @@ static bool detect_dp(struct dc_link *link, sink_caps->signal = dp_passive_dongle_detection(link->ddc, sink_caps, audio_support); + link->dpcd_caps.dongle_type = sink_caps->dongle_type; } return true; @@ -3289,7 +3290,6 @@ void core_link_disable_stream(struct pipe_ctx *pipe_ctx) #if defined(CONFIG_DRM_AMD_DC_HDCP) update_psp_stream_config(pipe_ctx, true); #endif - dc->hwss.blank_stream(pipe_ctx); if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 9bc03f26efda..d1d95d3e248a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -2431,6 +2431,12 @@ static bool decide_edp_link_settings(struct dc_link *link, struct dc_link_settin return false; } +static bool decide_mst_link_settings(const struct dc_link *link, struct dc_link_settings *link_setting) +{ + *link_setting = link->verified_link_cap; + return true; +} + void decide_link_settings(struct dc_stream_state *stream, struct dc_link_settings *link_setting) { @@ -2456,11 +2462,9 @@ void decide_link_settings(struct dc_stream_state *stream, * TODO: add MST specific link training routine */ if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - *link_setting = link->verified_link_cap; - return; - } - - if (link->connector_signal == SIGNAL_TYPE_EDP) { + if (decide_mst_link_settings(link, link_setting)) + return; + } else if (link->connector_signal == SIGNAL_TYPE_EDP) { if (decide_edp_link_settings(link, link_setting, req_bw)) return; } else if (decide_dp_link_settings(link, link_setting, req_bw)) @@ -4409,9 +4413,9 @@ bool dc_link_get_backlight_level_nits(struct dc_link *link, link->connector_signal != SIGNAL_TYPE_DISPLAY_PORT)) return false; - if (!core_link_read_dpcd(link, DP_SOURCE_BACKLIGHT_CURRENT_PEAK, + if (core_link_read_dpcd(link, DP_SOURCE_BACKLIGHT_CURRENT_PEAK, dpcd_backlight_get.raw, - sizeof(union dpcd_source_backlight_get))) + sizeof(union dpcd_source_backlight_get)) != DC_OK) return false; *backlight_millinits_avg = @@ -4450,9 +4454,9 @@ bool dc_link_read_default_bl_aux(struct dc_link *link, uint32_t *backlight_milli link->connector_signal != SIGNAL_TYPE_DISPLAY_PORT)) return false; - if (!core_link_read_dpcd(link, DP_SOURCE_BACKLIGHT_LEVEL, + if (core_link_read_dpcd(link, DP_SOURCE_BACKLIGHT_LEVEL, (uint8_t *) backlight_millinits, - sizeof(uint32_t))) + sizeof(uint32_t)) != DC_OK) return false; return true; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 7b5f90ebb133..c6b737dd8425 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -42,6 +42,9 @@ #include "virtual/virtual_stream_encoder.h" #include "dpcd_defs.h" +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/dce60_resource.h" +#endif #include "dce80/dce80_resource.h" #include "dce100/dce100_resource.h" #include "dce110/dce110_resource.h" @@ -63,6 +66,18 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id) enum dce_version dc_version = DCE_VERSION_UNKNOWN; switch (asic_id.chip_family) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case FAMILY_SI: + if (ASIC_REV_IS_TAHITI_P(asic_id.hw_internal_rev) || + ASIC_REV_IS_PITCAIRN_PM(asic_id.hw_internal_rev) || + ASIC_REV_IS_CAPEVERDE_M(asic_id.hw_internal_rev)) + dc_version = DCE_VERSION_6_0; + else if (ASIC_REV_IS_OLAND_M(asic_id.hw_internal_rev)) + dc_version = DCE_VERSION_6_4; + else + dc_version = DCE_VERSION_6_1; + break; +#endif case FAMILY_CI: dc_version = DCE_VERSION_8_0; break; @@ -129,6 +144,20 @@ struct resource_pool *dc_create_resource_pool(struct dc *dc, struct resource_pool *res_pool = NULL; switch (dc_version) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case DCE_VERSION_6_0: + res_pool = dce60_create_resource_pool( + init_data->num_virtual_links, dc); + break; + case DCE_VERSION_6_1: + res_pool = dce61_create_resource_pool( + init_data->num_virtual_links, dc); + break; + case DCE_VERSION_6_4: + res_pool = dce64_create_resource_pool( + init_data->num_virtual_links, dc); + break; +#endif case DCE_VERSION_8_0: res_pool = dce80_create_resource_pool( init_data->num_virtual_links, dc); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 0257a900fe2b..f42a17d765e3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -708,3 +708,4 @@ void dc_stream_log(const struct dc *dc, const struct dc_stream_state *stream) "\tlink: %d\n", stream->link->link_index); } + diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index f50ef4255020..1d9c8e09c08b 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -42,7 +42,7 @@ #include "inc/hw/dmcu.h" #include "dml/display_mode_lib.h" -#define DC_VER "3.2.95" +#define DC_VER "3.2.99" #define MAX_SURFACES 3 #define MAX_PLANES 6 @@ -1265,6 +1265,9 @@ void dc_unlock_memory_clock_frequency(struct dc *dc); void dc_lock_memory_clock_frequency(struct dc *dc); #endif + +bool dc_set_psr_allow_active(struct dc *dc, bool enable); + /******************************************************************************* * DSC Interfaces ******************************************************************************/ diff --git a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h index d06d07042a12..0811f941f430 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h @@ -136,6 +136,10 @@ struct dc_vbios_funcs { enum bp_result (*get_atom_dc_golden_table)( struct dc_bios *dcb); + + enum bp_result (*enable_lvtma_control)( + struct dc_bios *bios, + uint8_t uc_pwr_on); }; struct bios_registers { diff --git a/drivers/gpu/drm/amd/display/dc/dc_dsc.h b/drivers/gpu/drm/amd/display/dc/dc_dsc.h index 3800340a5b4f..768ab38d41cf 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dsc.h +++ b/drivers/gpu/drm/amd/display/dc/dc_dsc.h @@ -51,6 +51,7 @@ struct dc_dsc_policy { int min_slice_height; // Must not be less than 8 uint32_t max_target_bpp; uint32_t min_target_bpp; + bool enable_dsc_when_not_needed; }; bool dc_dsc_parse_dsc_dpcd(const struct dc *dc, @@ -80,4 +81,6 @@ void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing *timing, void dc_dsc_policy_set_max_target_bpp_limit(uint32_t limit); +void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable); + #endif diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 633442bc7ef2..c246af7c584b 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -233,7 +233,7 @@ struct dc_stream_state { union stream_update_flags update_flags; }; -#define ABM_LEVEL_IMMEDIATE_DISABLE 0xFFFFFFFF +#define ABM_LEVEL_IMMEDIATE_DISABLE 255 struct dc_stream_update { struct dc_stream_state *stream; @@ -397,6 +397,8 @@ bool dc_enable_stereo( struct dc_stream_state *streams[], uint8_t stream_count); +/* Triggers multi-stream synchronization. */ +void dc_trigger_sync(struct dc *dc, struct dc_state *context); enum surface_update_type dc_check_update_surfaces_for_stream( struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index 946ba929c6f6..aa8e0955db48 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -233,6 +233,7 @@ struct dc_panel_patch { unsigned int skip_scdc_overwrite; unsigned int delay_ignore_msa; unsigned int disable_fec; + unsigned int extra_t3_ms; }; struct dc_edid_caps { diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h index a44effcda49f..e84d21605854 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h @@ -46,6 +46,8 @@ SR(BL1_PWM_USER_LEVEL), \ SR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES), \ SR(DC_ABM1_HGLS_REG_READ_PROGRESS), \ + SR(DC_ABM1_ACE_OFFSET_SLOPE_0), \ + SR(DC_ABM1_ACE_THRES_12), \ SR(BIOS_SCRATCH_2) #define ABM_DCN10_REG_LIST(id)\ @@ -60,6 +62,8 @@ SRI(BL1_PWM_USER_LEVEL, ABM, id), \ SRI(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id), \ SRI(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id), \ + SRI(DC_ABM1_ACE_OFFSET_SLOPE_0, ABM, id), \ + SRI(DC_ABM1_ACE_THRES_12, ABM, id), \ NBIO_SR(BIOS_SCRATCH_2) #define ABM_DCN20_REG_LIST() \ @@ -74,10 +78,12 @@ SR(BL1_PWM_USER_LEVEL), \ SR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES), \ SR(DC_ABM1_HGLS_REG_READ_PROGRESS), \ + SR(DC_ABM1_ACE_OFFSET_SLOPE_0), \ + SR(DC_ABM1_ACE_THRES_12), \ NBIO_SR(BIOS_SCRATCH_2) #if defined(CONFIG_DRM_AMD_DC_DCN3_0) -#define ABM_DCN301_REG_LIST(id)\ +#define ABM_DCN30_REG_LIST(id)\ ABM_COMMON_REG_LIST_DCE_BASE(), \ SRI(DC_ABM1_HG_SAMPLE_RATE, ABM, id), \ SRI(DC_ABM1_LS_SAMPLE_RATE, ABM, id), \ @@ -89,6 +95,8 @@ SRI(BL1_PWM_USER_LEVEL, ABM, id), \ SRI(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id), \ SRI(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id), \ + SRI(DC_ABM1_ACE_OFFSET_SLOPE_0, ABM, id), \ + SRI(DC_ABM1_ACE_THRES_12, ABM, id), \ NBIO_SR(BIOS_SCRATCH_2) #endif @@ -208,6 +216,8 @@ struct dce_abm_registers { uint32_t BL1_PWM_USER_LEVEL; uint32_t DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES; uint32_t DC_ABM1_HGLS_REG_READ_PROGRESS; + uint32_t DC_ABM1_ACE_OFFSET_SLOPE_0; + uint32_t DC_ABM1_ACE_THRES_12; uint32_t MASTER_COMM_CNTL_REG; uint32_t MASTER_COMM_CMD_REG; uint32_t MASTER_COMM_DATA_REG1; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c index 408046579712..2a2a0fdb9253 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c @@ -867,6 +867,98 @@ void dce_aud_wall_dto_setup( } } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_aud_wall_dto_setup( + struct audio *audio, + enum signal_type signal, + const struct audio_crtc_info *crtc_info, + const struct audio_pll_info *pll_info) +{ + struct dce_audio *aud = DCE_AUD(audio); + + struct azalia_clock_info clock_info = { 0 }; + + if (dc_is_hdmi_signal(signal)) { + uint32_t src_sel; + + /*DTO0 Programming goal: + -generate 24MHz, 128*Fs from 24MHz + -use DTO0 when an active HDMI port is connected + (optionally a DP is connected) */ + + /* calculate DTO settings */ + get_azalia_clock_info_hdmi( + crtc_info->requested_pixel_clock_100Hz, + crtc_info->calculated_pixel_clock_100Hz, + &clock_info); + + DC_LOG_HW_AUDIO("\n%s:Input::requested_pixel_clock_100Hz = %d"\ + "calculated_pixel_clock_100Hz =%d\n"\ + "audio_dto_module = %d audio_dto_phase =%d \n\n", __func__,\ + crtc_info->requested_pixel_clock_100Hz,\ + crtc_info->calculated_pixel_clock_100Hz,\ + clock_info.audio_dto_module,\ + clock_info.audio_dto_phase); + + /* On TN/SI, Program DTO source select and DTO select before + programming DTO modulo and DTO phase. These bits must be + programmed first, otherwise there will be no HDMI audio at boot + up. This is a HW sequence change (different from old ASICs). + Caution when changing this programming sequence. + + HDMI enabled, using DTO0 + program master CRTC for DTO0 */ + src_sel = pll_info->dto_source - DTO_SOURCE_ID0; + REG_UPDATE_2(DCCG_AUDIO_DTO_SOURCE, + DCCG_AUDIO_DTO0_SOURCE_SEL, src_sel, + DCCG_AUDIO_DTO_SEL, 0); + + /* module */ + REG_UPDATE(DCCG_AUDIO_DTO0_MODULE, + DCCG_AUDIO_DTO0_MODULE, clock_info.audio_dto_module); + + /* phase */ + REG_UPDATE(DCCG_AUDIO_DTO0_PHASE, + DCCG_AUDIO_DTO0_PHASE, clock_info.audio_dto_phase); + } else { + /*DTO1 Programming goal: + -generate 24MHz, 128*Fs from 24MHz (DCE6 does not support 512*Fs) + -default is to used DTO1, and switch to DTO0 when an audio + master HDMI port is connected + -use as default for DP + + calculate DTO settings */ + get_azalia_clock_info_dp( + crtc_info->requested_pixel_clock_100Hz, + pll_info, + &clock_info); + + /* Program DTO select before programming DTO modulo and DTO + phase. default to use DTO1 */ + + REG_UPDATE(DCCG_AUDIO_DTO_SOURCE, + DCCG_AUDIO_DTO_SEL, 1); + + /* DCCG_AUDIO_DTO2_USE_512FBR_DTO, 1) + * Cannot select 512fs for DP + * + * DCE6 has no DCCG_AUDIO_DTO2_USE_512FBR_DTO mask + */ + + /* module */ + REG_UPDATE(DCCG_AUDIO_DTO1_MODULE, + DCCG_AUDIO_DTO1_MODULE, clock_info.audio_dto_module); + + /* phase */ + REG_UPDATE(DCCG_AUDIO_DTO1_PHASE, + DCCG_AUDIO_DTO1_PHASE, clock_info.audio_dto_phase); + + /* DCE6 has no DCCG_AUDIO_DTO2_USE_512FBR_DTO mask in DCCG_AUDIO_DTO_SOURCE reg */ + + } +} +#endif + static bool dce_aud_endpoint_valid(struct audio *audio) { uint32_t value; @@ -926,6 +1018,19 @@ static const struct audio_funcs funcs = { .az_configure = dce_aud_az_configure, .destroy = dce_aud_destroy, }; + +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct audio_funcs dce60_funcs = { + .endpoint_valid = dce_aud_endpoint_valid, + .hw_init = dce_aud_hw_init, + .wall_dto_setup = dce60_aud_wall_dto_setup, + .az_enable = dce_aud_az_enable, + .az_disable = dce_aud_az_disable, + .az_configure = dce_aud_az_configure, + .destroy = dce_aud_destroy, +}; +#endif + void dce_aud_destroy(struct audio **audio) { struct dce_audio *aud = DCE_AUD(*audio); @@ -959,3 +1064,29 @@ struct audio *dce_audio_create( return &audio->base; } +#if defined(CONFIG_DRM_AMD_DC_SI) +struct audio *dce60_audio_create( + struct dc_context *ctx, + unsigned int inst, + const struct dce_audio_registers *reg, + const struct dce_audio_shift *shifts, + const struct dce_audio_mask *masks + ) +{ + struct dce_audio *audio = kzalloc(sizeof(*audio), GFP_KERNEL); + + if (audio == NULL) { + ASSERT_CRITICAL(audio); + return NULL; + } + + audio->base.ctx = ctx; + audio->base.inst = inst; + audio->base.funcs = &dce60_funcs; + + audio->regs = reg; + audio->shifts = shifts; + audio->masks = masks; + return &audio->base; +} +#endif diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h index 1392fab0860b..5622d5e32d81 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h @@ -64,6 +64,20 @@ SF(AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\ SF(AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define AUD_DCE60_MASK_SH_LIST(mask_sh)\ + SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\ + SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\ + SF(DCCG_AUDIO_DTO0_MODULE, DCCG_AUDIO_DTO0_MODULE, mask_sh),\ + SF(DCCG_AUDIO_DTO0_PHASE, DCCG_AUDIO_DTO0_PHASE, mask_sh),\ + SF(DCCG_AUDIO_DTO1_MODULE, DCCG_AUDIO_DTO1_MODULE, mask_sh),\ + SF(DCCG_AUDIO_DTO1_PHASE, DCCG_AUDIO_DTO1_PHASE, mask_sh),\ + SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, AUDIO_RATE_CAPABILITIES, mask_sh),\ + SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, CLKSTOP, mask_sh),\ + SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, EPSS, mask_sh), \ + SF(AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\ + SF(AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh) +#endif struct dce_audio_registers { uint32_t AZALIA_F0_CODEC_ENDPOINT_INDEX; @@ -135,6 +149,15 @@ struct audio *dce_audio_create( const struct dce_audio_shift *shifts, const struct dce_audio_mask *masks); +#if defined(CONFIG_DRM_AMD_DC_SI) +struct audio *dce60_audio_create( + struct dc_context *ctx, + unsigned int inst, + const struct dce_audio_registers *reg, + const struct dce_audio_shift *shifts, + const struct dce_audio_mask *masks); +#endif + void dce_aud_destroy(struct audio **audio); void dce_aud_hw_init(struct audio *audio); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h index 5e044c2d3d6d..93e7f34d4775 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h @@ -46,6 +46,24 @@ SR(SMU_INTERRUPT_CONTROL), \ SR(DC_DMCU_SCRATCH) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define DMCU_DCE60_REG_LIST() \ + SR(DMCU_CTRL), \ + SR(DMCU_STATUS), \ + SR(DMCU_RAM_ACCESS_CTRL), \ + SR(DMCU_IRAM_WR_CTRL), \ + SR(DMCU_IRAM_WR_DATA), \ + SR(MASTER_COMM_DATA_REG1), \ + SR(MASTER_COMM_DATA_REG2), \ + SR(MASTER_COMM_DATA_REG3), \ + SR(MASTER_COMM_CMD_REG), \ + SR(MASTER_COMM_CNTL_REG), \ + SR(DMCU_IRAM_RD_CTRL), \ + SR(DMCU_IRAM_RD_DATA), \ + SR(DMCU_INTERRUPT_TO_UC_EN_MASK), \ + SR(DC_DMCU_SCRATCH) +#endif + #define DMCU_DCE80_REG_LIST() \ SR(DMCU_CTRL), \ SR(DMCU_STATUS), \ @@ -104,6 +122,25 @@ STATIC_SCREEN4_INT_TO_UC_EN, mask_sh), \ DMCU_SF(SMU_INTERRUPT_CONTROL, DC_SMU_INT_ENABLE, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define DMCU_MASK_SH_LIST_DCE60(mask_sh) \ + DMCU_SF(DMCU_CTRL, \ + DMCU_ENABLE, mask_sh), \ + DMCU_SF(DMCU_STATUS, \ + UC_IN_STOP_MODE, mask_sh), \ + DMCU_SF(DMCU_STATUS, \ + UC_IN_RESET, mask_sh), \ + DMCU_SF(DMCU_RAM_ACCESS_CTRL, \ + IRAM_HOST_ACCESS_EN, mask_sh), \ + DMCU_SF(DMCU_RAM_ACCESS_CTRL, \ + IRAM_WR_ADDR_AUTO_INC, mask_sh), \ + DMCU_SF(DMCU_RAM_ACCESS_CTRL, \ + IRAM_RD_ADDR_AUTO_INC, mask_sh), \ + DMCU_SF(MASTER_COMM_CMD_REG, \ + MASTER_COMM_CMD_REG_BYTE0, mask_sh), \ + DMCU_SF(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, mask_sh) +#endif + #define DMCU_MASK_SH_LIST_DCE80(mask_sh) \ DMCU_SF(DMCU_CTRL, \ DMCU_ENABLE, mask_sh), \ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c index e1c5839a80dc..4202fadb2c0e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c @@ -85,6 +85,15 @@ void dce_pipe_control_lock(struct dc *dc, } } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_pipe_control_lock(struct dc *dc, + struct pipe_ctx *pipe, + bool lock) +{ + /* DCE6 has no BLND_V_UPDATE_LOCK register */ +} +#endif + void dce_set_blender_mode(struct dce_hwseq *hws, unsigned int blnd_inst, enum blnd_mode mode) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h index 66b88d6ba398..70bbc1311327 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h @@ -110,6 +110,12 @@ SR(BLNDV_CONTROL),\ HWSEQ_PIXEL_RATE_REG_LIST(CRTC) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define HWSEQ_DCE6_REG_LIST() \ + HWSEQ_DCEF_REG_LIST_DCE8(), \ + HWSEQ_PIXEL_RATE_REG_LIST(CRTC) +#endif + #define HWSEQ_DCE8_REG_LIST() \ HWSEQ_DCEF_REG_LIST_DCE8(), \ HWSEQ_BLND_REG_LIST(), \ @@ -488,6 +494,12 @@ struct dce_hwseq_registers { HWS_SF1(blk, PHYPLL_PIXEL_RATE_CNTL, PHYPLL_PIXEL_RATE_SOURCE, mask_sh),\ HWS_SF1(blk, PHYPLL_PIXEL_RATE_CNTL, PIXEL_RATE_PLL_SOURCE, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define HWSEQ_DCE6_MASK_SH_LIST(mask_sh)\ + .DCFE_CLOCK_ENABLE = CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_CLOCK_ENABLE ## mask_sh, \ + HWSEQ_PIXEL_RATE_MASK_SH_LIST(mask_sh, CRTC0_) +#endif + #define HWSEQ_DCE8_MASK_SH_LIST(mask_sh)\ .DCFE_CLOCK_ENABLE = CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_CLOCK_ENABLE ## mask_sh, \ HWS_SF(BLND_, V_UPDATE_LOCK, BLND_DCP_GRPH_V_UPDATE_LOCK, mask_sh),\ @@ -836,6 +848,12 @@ void dce_pipe_control_lock(struct dc *dc, void dce_set_blender_mode(struct dce_hwseq *hws, unsigned int blnd_inst, enum blnd_mode mode); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_pipe_control_lock(struct dc *dc, + struct pipe_ctx *pipe, + bool lock); +#endif + void dce_clock_gating_power_up(struct dce_hwseq *hws, bool enable); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c index ce30dbf579d4..80569a2734eb 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c @@ -231,6 +231,22 @@ static void dce_ipp_set_degamma( CURSOR2_DEGAMMA_MODE, degamma_type); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_ipp_set_degamma( + struct input_pixel_processor *ipp, + enum ipp_degamma_mode mode) +{ + struct dce_ipp *ipp_dce = TO_DCE_IPP(ipp); + uint32_t degamma_type = (mode == IPP_DEGAMMA_MODE_HW_sRGB) ? 1 : 0; + + ASSERT(mode == IPP_DEGAMMA_MODE_BYPASS || mode == IPP_DEGAMMA_MODE_HW_sRGB); + /* DCE6 does not have CURSOR2_DEGAMMA_MODE bit in DEGAMMA_CONTROL reg */ + REG_SET_2(DEGAMMA_CONTROL, 0, + GRPH_DEGAMMA_MODE, degamma_type, + CURSOR_DEGAMMA_MODE, degamma_type); +} +#endif + static const struct ipp_funcs dce_ipp_funcs = { .ipp_cursor_set_attributes = dce_ipp_cursor_set_attributes, .ipp_cursor_set_position = dce_ipp_cursor_set_position, @@ -239,6 +255,17 @@ static const struct ipp_funcs dce_ipp_funcs = { .ipp_set_degamma = dce_ipp_set_degamma }; +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct ipp_funcs dce60_ipp_funcs = { + .ipp_cursor_set_attributes = dce_ipp_cursor_set_attributes, + .ipp_cursor_set_position = dce_ipp_cursor_set_position, + .ipp_program_prescale = dce_ipp_program_prescale, + .ipp_program_input_lut = dce_ipp_program_input_lut, + .ipp_set_degamma = dce60_ipp_set_degamma +}; +#endif + + /*****************************************/ /* Constructor, Destructor */ /*****************************************/ @@ -260,6 +287,25 @@ void dce_ipp_construct( ipp_dce->ipp_mask = ipp_mask; } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_ipp_construct( + struct dce_ipp *ipp_dce, + struct dc_context *ctx, + int inst, + const struct dce_ipp_registers *regs, + const struct dce_ipp_shift *ipp_shift, + const struct dce_ipp_mask *ipp_mask) +{ + ipp_dce->base.ctx = ctx; + ipp_dce->base.inst = inst; + ipp_dce->base.funcs = &dce60_ipp_funcs; + + ipp_dce->regs = regs; + ipp_dce->ipp_shift = ipp_shift; + ipp_dce->ipp_mask = ipp_mask; +} +#endif + void dce_ipp_destroy(struct input_pixel_processor **ipp) { kfree(TO_DCE_IPP(*ipp)); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.h b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.h index ca04e97d44c3..0028d4bdd81b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.h @@ -147,6 +147,46 @@ IPP_SF(DCP0_DEGAMMA_CONTROL, CURSOR_DEGAMMA_MODE, mask_sh), \ IPP_SF(DCP0_DEGAMMA_CONTROL, CURSOR2_DEGAMMA_MODE, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define IPP_DCE60_MASK_SH_LIST_DCE_COMMON_BASE(mask_sh) \ + IPP_SF(CUR_UPDATE, CURSOR_UPDATE_LOCK, mask_sh), \ + IPP_SF(CUR_CONTROL, CURSOR_EN, mask_sh), \ + IPP_SF(CUR_CONTROL, CURSOR_MODE, mask_sh), \ + IPP_SF(CUR_CONTROL, CURSOR_2X_MAGNIFY, mask_sh), \ + IPP_SF(CUR_CONTROL, CUR_INV_TRANS_CLAMP, mask_sh), \ + IPP_SF(CUR_POSITION, CURSOR_X_POSITION, mask_sh), \ + IPP_SF(CUR_POSITION, CURSOR_Y_POSITION, mask_sh), \ + IPP_SF(CUR_HOT_SPOT, CURSOR_HOT_SPOT_X, mask_sh), \ + IPP_SF(CUR_HOT_SPOT, CURSOR_HOT_SPOT_Y, mask_sh), \ + IPP_SF(CUR_COLOR1, CUR_COLOR1_BLUE, mask_sh), \ + IPP_SF(CUR_COLOR1, CUR_COLOR1_GREEN, mask_sh), \ + IPP_SF(CUR_COLOR1, CUR_COLOR1_RED, mask_sh), \ + IPP_SF(CUR_COLOR2, CUR_COLOR2_BLUE, mask_sh), \ + IPP_SF(CUR_COLOR2, CUR_COLOR2_GREEN, mask_sh), \ + IPP_SF(CUR_COLOR2, CUR_COLOR2_RED, mask_sh), \ + IPP_SF(CUR_SIZE, CURSOR_WIDTH, mask_sh), \ + IPP_SF(CUR_SIZE, CURSOR_HEIGHT, mask_sh), \ + IPP_SF(CUR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, mask_sh), \ + IPP_SF(CUR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, mask_sh), \ + IPP_SF(PRESCALE_GRPH_CONTROL, GRPH_PRESCALE_BYPASS, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_R, GRPH_PRESCALE_SCALE_R, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_R, GRPH_PRESCALE_BIAS_R, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_G, GRPH_PRESCALE_SCALE_G, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_G, GRPH_PRESCALE_BIAS_G, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_B, GRPH_PRESCALE_SCALE_B, mask_sh), \ + IPP_SF(PRESCALE_VALUES_GRPH_B, GRPH_PRESCALE_BIAS_B, mask_sh), \ + IPP_SF(INPUT_GAMMA_CONTROL, GRPH_INPUT_GAMMA_MODE, mask_sh), \ + IPP_SF(DC_LUT_WRITE_EN_MASK, DC_LUT_WRITE_EN_MASK, mask_sh), \ + IPP_SF(DC_LUT_RW_MODE, DC_LUT_RW_MODE, mask_sh), \ + IPP_SF(DC_LUT_CONTROL, DC_LUT_DATA_R_FORMAT, mask_sh), \ + IPP_SF(DC_LUT_CONTROL, DC_LUT_DATA_G_FORMAT, mask_sh), \ + IPP_SF(DC_LUT_CONTROL, DC_LUT_DATA_B_FORMAT, mask_sh), \ + IPP_SF(DC_LUT_RW_INDEX, DC_LUT_RW_INDEX, mask_sh), \ + IPP_SF(DC_LUT_SEQ_COLOR, DC_LUT_SEQ_COLOR, mask_sh), \ + IPP_SF(DEGAMMA_CONTROL, GRPH_DEGAMMA_MODE, mask_sh), \ + IPP_SF(DEGAMMA_CONTROL, CURSOR_DEGAMMA_MODE, mask_sh) +#endif + #define IPP_REG_FIELD_LIST(type) \ type CURSOR_UPDATE_LOCK; \ type CURSOR_EN; \ @@ -233,6 +273,15 @@ void dce_ipp_construct(struct dce_ipp *ipp_dce, const struct dce_ipp_shift *ipp_shift, const struct dce_ipp_mask *ipp_mask); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_ipp_construct(struct dce_ipp *ipp_dce, + struct dc_context *ctx, + int inst, + const struct dce_ipp_registers *regs, + const struct dce_ipp_shift *ipp_shift, + const struct dce_ipp_mask *ipp_mask); +#endif + void dce_ipp_destroy(struct input_pixel_processor **ipp); #endif /* _DCE_IPP_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 8d8c84c81b34..b409f6b2bfd8 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -425,6 +425,59 @@ static void set_dp_phy_pattern_hbr2_compliance_cp2520_2( enable_phy_bypass_mode(enc110, false); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_set_dp_phy_pattern_hbr2_compliance_cp2520_2( + struct dce110_link_encoder *enc110, + unsigned int cp2520_pattern) +{ + + /* previously there is a register DP_HBR2_EYE_PATTERN + * that is enabled to get the pattern. + * But it does not work with the latest spec change, + * so we are programming the following registers manually. + * + * The following settings have been confirmed + * by Nick Chorney and Sandra Liu */ + + /* Disable PHY Bypass mode to setup the test pattern */ + + enable_phy_bypass_mode(enc110, false); + + /* Setup DIG encoder in DP SST mode */ + enc110->base.funcs->setup(&enc110->base, SIGNAL_TYPE_DISPLAY_PORT); + + /* ensure normal panel mode. */ + setup_panel_mode(enc110, DP_PANEL_MODE_DEFAULT); + + /* no vbid after BS (SR) + * DP_LINK_FRAMING_CNTL changed history Sandra Liu + * 11000260 / 11000104 / 110000FC */ + REG_UPDATE_3(DP_LINK_FRAMING_CNTL, + DP_IDLE_BS_INTERVAL, 0xFC, + DP_VBID_DISABLE, 1, + DP_VID_ENHANCED_FRAME_MODE, 1); + + /* DCE6 has no DP_DPHY_SCRAM_CNTL register, skip swap BS with SR */ + + /* select cp2520 patterns */ + if (REG(DP_DPHY_HBR2_PATTERN_CONTROL)) + REG_UPDATE(DP_DPHY_HBR2_PATTERN_CONTROL, + DP_DPHY_HBR2_PATTERN_CONTROL, cp2520_pattern); + else + /* pre-DCE11 can only generate CP2520 pattern 2 */ + ASSERT(cp2520_pattern == 2); + + /* set link training complete */ + set_link_training_complete(enc110, true); + + /* disable video stream */ + REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, 0); + + /* Disable PHY Bypass mode to setup the test pattern */ + enable_phy_bypass_mode(enc110, false); +} +#endif + static void set_dp_phy_pattern_passthrough_mode( struct dce110_link_encoder *enc110, enum dp_panel_mode panel_mode) @@ -452,6 +505,35 @@ static void set_dp_phy_pattern_passthrough_mode( disable_prbs_mode(enc110); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_set_dp_phy_pattern_passthrough_mode( + struct dce110_link_encoder *enc110, + enum dp_panel_mode panel_mode) +{ + /* program correct panel mode */ + setup_panel_mode(enc110, panel_mode); + + /* restore LINK_FRAMING_CNTL + * in case we were doing HBR2 compliance pattern before + */ + REG_UPDATE_3(DP_LINK_FRAMING_CNTL, + DP_IDLE_BS_INTERVAL, 0x2000, + DP_VBID_DISABLE, 0, + DP_VID_ENHANCED_FRAME_MODE, 1); + + /* DCE6 has no DP_DPHY_SCRAM_CNTL register, skip DPHY_SCRAMBLER_BS_COUNT restore */ + + /* set link training complete */ + set_link_training_complete(enc110, true); + + /* Disable PHY Bypass mode to setup the test pattern */ + enable_phy_bypass_mode(enc110, false); + + /* Disable PRBS mode */ + disable_prbs_mode(enc110); +} +#endif + /* return value is bit-vector */ static uint8_t get_frontend_source( enum engine_id engine) @@ -490,6 +572,20 @@ static void configure_encoder( REG_UPDATE(DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_ADVANCE, 1); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_configure_encoder( + struct dce110_link_encoder *enc110, + const struct dc_link_settings *link_settings) +{ + /* set number of lanes */ + + REG_SET(DP_CONFIG, 0, + DP_UDI_LANES, link_settings->lane_count - LANE_COUNT_ONE); + + /* DCE6 has no DP_DPHY_SCRAM_CNTL register, skip setup scrambler */ +} +#endif + static void aux_initialize( struct dce110_link_encoder *enc110) { @@ -1059,6 +1155,87 @@ void dce110_link_encoder_enable_dp_mst_output( BREAK_TO_DEBUGGER(); } } + +#if defined(CONFIG_DRM_AMD_DC_SI) +/* enables DP PHY output */ +void dce60_link_encoder_enable_dp_output( + struct link_encoder *enc, + const struct dc_link_settings *link_settings, + enum clock_source_id clock_source) +{ + struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); + struct bp_transmitter_control cntl = { 0 }; + enum bp_result result; + + /* Enable the PHY */ + + /* number_of_lanes is used for pixel clock adjust, + * but it's not passed to asic_control. + * We need to set number of lanes manually. + */ + dce60_configure_encoder(enc110, link_settings); + cntl.connector_obj_id = enc110->base.connector; + cntl.action = TRANSMITTER_CONTROL_ENABLE; + cntl.engine_id = enc->preferred_engine; + cntl.transmitter = enc110->base.transmitter; + cntl.pll_id = clock_source; + cntl.signal = SIGNAL_TYPE_DISPLAY_PORT; + cntl.lanes_number = link_settings->lane_count; + cntl.hpd_sel = enc110->base.hpd_source; + cntl.pixel_clock = link_settings->link_rate + * LINK_RATE_REF_FREQ_IN_KHZ; + /* TODO: check if undefined works */ + cntl.color_depth = COLOR_DEPTH_UNDEFINED; + + result = link_transmitter_control(enc110, &cntl); + + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n", + __func__); + BREAK_TO_DEBUGGER(); + } +} + +/* enables DP PHY output in MST mode */ +void dce60_link_encoder_enable_dp_mst_output( + struct link_encoder *enc, + const struct dc_link_settings *link_settings, + enum clock_source_id clock_source) +{ + struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); + struct bp_transmitter_control cntl = { 0 }; + enum bp_result result; + + /* Enable the PHY */ + + /* number_of_lanes is used for pixel clock adjust, + * but it's not passed to asic_control. + * We need to set number of lanes manually. + */ + dce60_configure_encoder(enc110, link_settings); + + cntl.action = TRANSMITTER_CONTROL_ENABLE; + cntl.engine_id = ENGINE_ID_UNKNOWN; + cntl.transmitter = enc110->base.transmitter; + cntl.pll_id = clock_source; + cntl.signal = SIGNAL_TYPE_DISPLAY_PORT_MST; + cntl.lanes_number = link_settings->lane_count; + cntl.hpd_sel = enc110->base.hpd_source; + cntl.pixel_clock = link_settings->link_rate + * LINK_RATE_REF_FREQ_IN_KHZ; + /* TODO: check if undefined works */ + cntl.color_depth = COLOR_DEPTH_UNDEFINED; + + result = link_transmitter_control(enc110, &cntl); + + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n", + __func__); + BREAK_TO_DEBUGGER(); + } +} +#endif + /* * @brief * Disable transmitter and its encoder @@ -1208,6 +1385,63 @@ void dce110_link_encoder_dp_set_phy_pattern( } } +#if defined(CONFIG_DRM_AMD_DC_SI) +/* set DP PHY test and training patterns */ +void dce60_link_encoder_dp_set_phy_pattern( + struct link_encoder *enc, + const struct encoder_set_dp_phy_pattern_param *param) +{ + struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); + + switch (param->dp_phy_pattern) { + case DP_TEST_PATTERN_TRAINING_PATTERN1: + dce110_link_encoder_set_dp_phy_pattern_training_pattern(enc, 0); + break; + case DP_TEST_PATTERN_TRAINING_PATTERN2: + dce110_link_encoder_set_dp_phy_pattern_training_pattern(enc, 1); + break; + case DP_TEST_PATTERN_TRAINING_PATTERN3: + dce110_link_encoder_set_dp_phy_pattern_training_pattern(enc, 2); + break; + case DP_TEST_PATTERN_TRAINING_PATTERN4: + dce110_link_encoder_set_dp_phy_pattern_training_pattern(enc, 3); + break; + case DP_TEST_PATTERN_D102: + set_dp_phy_pattern_d102(enc110); + break; + case DP_TEST_PATTERN_SYMBOL_ERROR: + set_dp_phy_pattern_symbol_error(enc110); + break; + case DP_TEST_PATTERN_PRBS7: + set_dp_phy_pattern_prbs7(enc110); + break; + case DP_TEST_PATTERN_80BIT_CUSTOM: + set_dp_phy_pattern_80bit_custom( + enc110, param->custom_pattern); + break; + case DP_TEST_PATTERN_CP2520_1: + dce60_set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc110, 1); + break; + case DP_TEST_PATTERN_CP2520_2: + dce60_set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc110, 2); + break; + case DP_TEST_PATTERN_CP2520_3: + dce60_set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc110, 3); + break; + case DP_TEST_PATTERN_VIDEO_MODE: { + dce60_set_dp_phy_pattern_passthrough_mode( + enc110, param->dp_panel_mode); + break; + } + + default: + /* invalid phy pattern */ + ASSERT_CRITICAL(false); + break; + } +} +#endif + static void fill_stream_allocation_row_info( const struct link_mst_stream_allocation *stream_allocation, uint32_t *src, @@ -1407,3 +1641,138 @@ void dce110_link_encoder_get_max_link_cap(struct link_encoder *enc, *link_settings = max_link_cap; } + +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct link_encoder_funcs dce60_lnk_enc_funcs = { + .validate_output_with_stream = + dce110_link_encoder_validate_output_with_stream, + .hw_init = dce110_link_encoder_hw_init, + .setup = dce110_link_encoder_setup, + .enable_tmds_output = dce110_link_encoder_enable_tmds_output, + .enable_dp_output = dce60_link_encoder_enable_dp_output, + .enable_dp_mst_output = dce60_link_encoder_enable_dp_mst_output, + .enable_lvds_output = dce110_link_encoder_enable_lvds_output, + .disable_output = dce110_link_encoder_disable_output, + .dp_set_lane_settings = dce110_link_encoder_dp_set_lane_settings, + .dp_set_phy_pattern = dce60_link_encoder_dp_set_phy_pattern, + .update_mst_stream_allocation_table = + dce110_link_encoder_update_mst_stream_allocation_table, + .psr_program_dp_dphy_fast_training = + dce110_psr_program_dp_dphy_fast_training, + .psr_program_secondary_packet = dce110_psr_program_secondary_packet, + .connect_dig_be_to_fe = dce110_link_encoder_connect_dig_be_to_fe, + .enable_hpd = dce110_link_encoder_enable_hpd, + .disable_hpd = dce110_link_encoder_disable_hpd, + .is_dig_enabled = dce110_is_dig_enabled, + .destroy = dce110_link_encoder_destroy, + .get_max_link_cap = dce110_link_encoder_get_max_link_cap +}; + +void dce60_link_encoder_construct( + struct dce110_link_encoder *enc110, + const struct encoder_init_data *init_data, + const struct encoder_feature_support *enc_features, + const struct dce110_link_enc_registers *link_regs, + const struct dce110_link_enc_aux_registers *aux_regs, + const struct dce110_link_enc_hpd_registers *hpd_regs) +{ + struct bp_encoder_cap_info bp_cap_info = {0}; + const struct dc_vbios_funcs *bp_funcs = init_data->ctx->dc_bios->funcs; + enum bp_result result = BP_RESULT_OK; + + enc110->base.funcs = &dce60_lnk_enc_funcs; + enc110->base.ctx = init_data->ctx; + enc110->base.id = init_data->encoder; + + enc110->base.hpd_source = init_data->hpd_source; + enc110->base.connector = init_data->connector; + + enc110->base.preferred_engine = ENGINE_ID_UNKNOWN; + + enc110->base.features = *enc_features; + + enc110->base.transmitter = init_data->transmitter; + + /* set the flag to indicate whether driver poll the I2C data pin + * while doing the DP sink detect + */ + +/* if (dal_adapter_service_is_feature_supported(as, + FEATURE_DP_SINK_DETECT_POLL_DATA_PIN)) + enc110->base.features.flags.bits. + DP_SINK_DETECT_POLL_DATA_PIN = true;*/ + + enc110->base.output_signals = + SIGNAL_TYPE_DVI_SINGLE_LINK | + SIGNAL_TYPE_DVI_DUAL_LINK | + SIGNAL_TYPE_LVDS | + SIGNAL_TYPE_DISPLAY_PORT | + SIGNAL_TYPE_DISPLAY_PORT_MST | + SIGNAL_TYPE_EDP | + SIGNAL_TYPE_HDMI_TYPE_A; + + /* For DCE 8.0 and 8.1, by design, UNIPHY is hardwired to DIG_BE. + * SW always assign DIG_FE 1:1 mapped to DIG_FE for non-MST UNIPHY. + * SW assign DIG_FE to non-MST UNIPHY first and MST last. So prefer + * DIG is per UNIPHY and used by SST DP, eDP, HDMI, DVI and LVDS. + * Prefer DIG assignment is decided by board design. + * For DCE 8.0, there are only max 6 UNIPHYs, we assume board design + * and VBIOS will filter out 7 UNIPHY for DCE 8.0. + * By this, adding DIGG should not hurt DCE 8.0. + * This will let DCE 8.1 share DCE 8.0 as much as possible + */ + + enc110->link_regs = link_regs; + enc110->aux_regs = aux_regs; + enc110->hpd_regs = hpd_regs; + + switch (enc110->base.transmitter) { + case TRANSMITTER_UNIPHY_A: + enc110->base.preferred_engine = ENGINE_ID_DIGA; + break; + case TRANSMITTER_UNIPHY_B: + enc110->base.preferred_engine = ENGINE_ID_DIGB; + break; + case TRANSMITTER_UNIPHY_C: + enc110->base.preferred_engine = ENGINE_ID_DIGC; + break; + case TRANSMITTER_UNIPHY_D: + enc110->base.preferred_engine = ENGINE_ID_DIGD; + break; + case TRANSMITTER_UNIPHY_E: + enc110->base.preferred_engine = ENGINE_ID_DIGE; + break; + case TRANSMITTER_UNIPHY_F: + enc110->base.preferred_engine = ENGINE_ID_DIGF; + break; + case TRANSMITTER_UNIPHY_G: + enc110->base.preferred_engine = ENGINE_ID_DIGG; + break; + default: + ASSERT_CRITICAL(false); + enc110->base.preferred_engine = ENGINE_ID_UNKNOWN; + } + + /* default to one to mirror Windows behavior */ + enc110->base.features.flags.bits.HDMI_6GB_EN = 1; + + result = bp_funcs->get_encoder_cap_info(enc110->base.ctx->dc_bios, + enc110->base.id, &bp_cap_info); + + /* Override features with DCE-specific values */ + if (BP_RESULT_OK == result) { + enc110->base.features.flags.bits.IS_HBR2_CAPABLE = + bp_cap_info.DP_HBR2_EN; + enc110->base.features.flags.bits.IS_HBR3_CAPABLE = + bp_cap_info.DP_HBR3_EN; + enc110->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN; + } else { + DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n", + __func__, + result); + } + if (enc110->base.ctx->dc->debug.hdmi20_disable) { + enc110->base.features.flags.bits.HDMI_6GB_EN = 0; + } +} +#endif diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h index 66027d496778..cb714a48b171 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h @@ -76,6 +76,34 @@ SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \ SR(DCI_MEM_PWR_STATUS) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define LE_DCE60_REG_LIST(id)\ + SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \ + SR(DMCU_RAM_ACCESS_CTRL), \ + SR(DMCU_IRAM_RD_CTRL), \ + SR(DMCU_IRAM_RD_DATA), \ + SR(DMCU_INTERRUPT_TO_UC_EN_MASK), \ + SRI(DIG_BE_CNTL, DIG, id), \ + SRI(DIG_BE_EN_CNTL, DIG, id), \ + SRI(DP_CONFIG, DP, id), \ + SRI(DP_DPHY_CNTL, DP, id), \ + SRI(DP_DPHY_PRBS_CNTL, DP, id), \ + SRI(DP_DPHY_SYM0, DP, id), \ + SRI(DP_DPHY_SYM1, DP, id), \ + SRI(DP_DPHY_SYM2, DP, id), \ + SRI(DP_DPHY_TRAINING_PATTERN_SEL, DP, id), \ + SRI(DP_LINK_CNTL, DP, id), \ + SRI(DP_LINK_FRAMING_CNTL, DP, id), \ + SRI(DP_MSE_SAT0, DP, id), \ + SRI(DP_MSE_SAT1, DP, id), \ + SRI(DP_MSE_SAT2, DP, id), \ + SRI(DP_MSE_SAT_UPDATE, DP, id), \ + SRI(DP_SEC_CNTL, DP, id), \ + SRI(DP_VID_STREAM_CNTL, DP, id), \ + SRI(DP_DPHY_FAST_TRAINING, DP, id), \ + SRI(DP_SEC_CNTL1, DP, id) +#endif + #define LE_DCE80_REG_LIST(id)\ SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \ LE_COMMON_REG_LIST_BASE(id) @@ -171,6 +199,16 @@ void dce110_link_encoder_construct( const struct dce110_link_enc_aux_registers *aux_regs, const struct dce110_link_enc_hpd_registers *hpd_regs); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_link_encoder_construct( + struct dce110_link_encoder *enc110, + const struct encoder_init_data *init_data, + const struct encoder_feature_support *enc_features, + const struct dce110_link_enc_registers *link_regs, + const struct dce110_link_enc_aux_registers *aux_regs, + const struct dce110_link_enc_hpd_registers *hpd_regs); +#endif + bool dce110_link_encoder_validate_dvi_output( const struct dce110_link_encoder *enc110, enum signal_type connector_signal, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c index 51481e922eb9..79a6f261a0da 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c @@ -174,6 +174,22 @@ static void program_urgency_watermark( URGENCY_HIGH_WATERMARK, urgency_high_wm); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_program_urgency_watermark( + struct dce_mem_input *dce_mi, + uint32_t wm_select, + uint32_t urgency_low_wm, + uint32_t urgency_high_wm) +{ + REG_UPDATE(DPG_PIPE_ARBITRATION_CONTROL3, + URGENCY_WATERMARK_MASK, wm_select); + + REG_SET_2(DPG_PIPE_URGENCY_CONTROL, 0, + URGENCY_LOW_WATERMARK, urgency_low_wm, + URGENCY_HIGH_WATERMARK, urgency_high_wm); +} +#endif + static void dce120_program_urgency_watermark( struct dce_mem_input *dce_mi, uint32_t wm_select, @@ -193,6 +209,25 @@ static void dce120_program_urgency_watermark( } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_program_nbp_watermark( + struct dce_mem_input *dce_mi, + uint32_t wm_select, + uint32_t nbp_wm) +{ + REG_UPDATE(DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, + NB_PSTATE_CHANGE_WATERMARK_MASK, wm_select); + + REG_UPDATE_3(DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, + NB_PSTATE_CHANGE_ENABLE, 1, + NB_PSTATE_CHANGE_URGENT_DURING_REQUEST, 1, + NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST, 1); + + REG_UPDATE(DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, + NB_PSTATE_CHANGE_WATERMARK, nbp_wm); +} +#endif + static void program_nbp_watermark( struct dce_mem_input *dce_mi, uint32_t wm_select, @@ -225,6 +260,20 @@ static void program_nbp_watermark( } } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_program_stutter_watermark( + struct dce_mem_input *dce_mi, + uint32_t wm_select, + uint32_t stutter_mark) +{ + REG_UPDATE(DPG_PIPE_STUTTER_CONTROL, + STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK, wm_select); + + REG_UPDATE(DPG_PIPE_STUTTER_CONTROL, + STUTTER_EXIT_SELF_REFRESH_WATERMARK, stutter_mark); +} +#endif + static void dce120_program_stutter_watermark( struct dce_mem_input *dce_mi, uint32_t wm_select, @@ -286,6 +335,34 @@ static void dce_mi_program_display_marks( program_stutter_watermark(dce_mi, 1, stutter_exit.d_mark); /* set d */ } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_mi_program_display_marks( + struct mem_input *mi, + struct dce_watermarks nbp, + struct dce_watermarks stutter_exit, + struct dce_watermarks stutter_enter, + struct dce_watermarks urgent, + uint32_t total_dest_line_time_ns) +{ + struct dce_mem_input *dce_mi = TO_DCE_MEM_INPUT(mi); + uint32_t stutter_en = mi->ctx->dc->debug.disable_stutter ? 0 : 1; + + dce60_program_urgency_watermark(dce_mi, 2, /* set a */ + urgent.a_mark, total_dest_line_time_ns); + dce60_program_urgency_watermark(dce_mi, 1, /* set d */ + urgent.d_mark, total_dest_line_time_ns); + + REG_UPDATE_2(DPG_PIPE_STUTTER_CONTROL, + STUTTER_ENABLE, stutter_en, + STUTTER_IGNORE_FBC, 1); + dce60_program_nbp_watermark(dce_mi, 2, nbp.a_mark); /* set a */ + dce60_program_nbp_watermark(dce_mi, 1, nbp.d_mark); /* set d */ + + dce60_program_stutter_watermark(dce_mi, 2, stutter_exit.a_mark); /* set a */ + dce60_program_stutter_watermark(dce_mi, 1, stutter_exit.d_mark); /* set d */ +} +#endif + static void dce112_mi_program_display_marks(struct mem_input *mi, struct dce_watermarks nbp, struct dce_watermarks stutter_exit, @@ -369,7 +446,7 @@ static void program_tiling( */ } - if (dce_mi->masks->GRPH_ARRAY_MODE) { /* GFX8 */ + if (dce_mi->masks->GRPH_MICRO_TILE_MODE) { /* GFX8 */ REG_UPDATE_9(GRPH_CONTROL, GRPH_NUM_BANKS, info->gfx8.num_banks, GRPH_BANK_WIDTH, info->gfx8.bank_width, @@ -385,6 +462,23 @@ static void program_tiling( GRPH_Z, 0); */ } + + if (dce_mi->masks->GRPH_ARRAY_MODE) { /* GFX6 but reuses gfx8 struct */ + REG_UPDATE_8(GRPH_CONTROL, + GRPH_NUM_BANKS, info->gfx8.num_banks, + GRPH_BANK_WIDTH, info->gfx8.bank_width, + GRPH_BANK_HEIGHT, info->gfx8.bank_height, + GRPH_MACRO_TILE_ASPECT, info->gfx8.tile_aspect, + GRPH_TILE_SPLIT, info->gfx8.tile_split, + /* DCE6 has no GRPH_MICRO_TILE_MODE mask */ + GRPH_PIPE_CONFIG, info->gfx8.pipe_config, + GRPH_ARRAY_MODE, info->gfx8.array_mode, + GRPH_COLOR_EXPANSION_MODE, 1); + /* 01 - DCP_GRPH_COLOR_EXPANSION_MODE_ZEXP: zero expansion for YCbCr */ + /* + GRPH_Z, 0); + */ + } } @@ -429,6 +523,36 @@ static void program_size_and_rotation( GRPH_ROTATION_ANGLE, rotation_angles[rotation]); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_program_size( + struct dce_mem_input *dce_mi, + enum dc_rotation_angle rotation, /* not used in DCE6 */ + const struct plane_size *plane_size) +{ + struct rect hw_rect = plane_size->surface_size; + /* DCE6 has no HW rotation, skip rotation_angles declaration */ + + /* DCE6 has no HW rotation, skip ROTATION_ANGLE_* processing */ + + REG_SET(GRPH_X_START, 0, + GRPH_X_START, hw_rect.x); + + REG_SET(GRPH_Y_START, 0, + GRPH_Y_START, hw_rect.y); + + REG_SET(GRPH_X_END, 0, + GRPH_X_END, hw_rect.width); + + REG_SET(GRPH_Y_END, 0, + GRPH_Y_END, hw_rect.height); + + REG_SET(GRPH_PITCH, 0, + GRPH_PITCH, plane_size->surface_pitch); + + /* DCE6 has no HW_ROTATION register, skip setting rotation_angles */ +} +#endif + static void program_grph_pixel_format( struct dce_mem_input *dce_mi, enum surface_pixel_format format) @@ -521,6 +645,28 @@ static void dce_mi_program_surface_config( program_grph_pixel_format(dce_mi, format); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_mi_program_surface_config( + struct mem_input *mi, + enum surface_pixel_format format, + union dc_tiling_info *tiling_info, + struct plane_size *plane_size, + enum dc_rotation_angle rotation, /* not used in DCE6 */ + struct dc_plane_dcc_param *dcc, + bool horizontal_mirror) +{ + struct dce_mem_input *dce_mi = TO_DCE_MEM_INPUT(mi); + REG_UPDATE(GRPH_ENABLE, GRPH_ENABLE, 1); + + program_tiling(dce_mi, tiling_info); + dce60_program_size(dce_mi, rotation, plane_size); + + if (format >= SURFACE_PIXEL_FORMAT_GRPH_BEGIN && + format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) + program_grph_pixel_format(dce_mi, format); +} +#endif + static uint32_t get_dmif_switch_time_us( uint32_t h_total, uint32_t v_total, @@ -741,6 +887,20 @@ static const struct mem_input_funcs dce_mi_funcs = { .mem_input_is_flip_pending = dce_mi_is_flip_pending }; +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct mem_input_funcs dce60_mi_funcs = { + .mem_input_program_display_marks = dce60_mi_program_display_marks, + .allocate_mem_input = dce_mi_allocate_dmif, + .free_mem_input = dce_mi_free_dmif, + .mem_input_program_surface_flip_and_addr = + dce_mi_program_surface_flip_and_addr, + .mem_input_program_pte_vm = dce_mi_program_pte_vm, + .mem_input_program_surface_config = + dce60_mi_program_surface_config, + .mem_input_is_flip_pending = dce_mi_is_flip_pending +}; +#endif + static const struct mem_input_funcs dce112_mi_funcs = { .mem_input_program_display_marks = dce112_mi_program_display_marks, .allocate_mem_input = dce_mi_allocate_dmif, @@ -783,6 +943,20 @@ void dce_mem_input_construct( dce_mi->masks = mi_mask; } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_mem_input_construct( + struct dce_mem_input *dce_mi, + struct dc_context *ctx, + int inst, + const struct dce_mem_input_registers *regs, + const struct dce_mem_input_shift *mi_shift, + const struct dce_mem_input_mask *mi_mask) +{ + dce_mem_input_construct(dce_mi, ctx, inst, regs, mi_shift, mi_mask); + dce_mi->base.funcs = &dce60_mi_funcs; +} +#endif + void dce112_mem_input_construct( struct dce_mem_input *dce_mi, struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h index d15b0d7f47fc..23db5c72f07e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h @@ -58,6 +58,31 @@ SRI(DVMM_PTE_CONTROL, DCP, id),\ SRI(DVMM_PTE_ARB_CONTROL, DCP, id) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define MI_DCE6_REG_LIST(id)\ + SRI(GRPH_ENABLE, DCP, id),\ + SRI(GRPH_CONTROL, DCP, id),\ + SRI(GRPH_X_START, DCP, id),\ + SRI(GRPH_Y_START, DCP, id),\ + SRI(GRPH_X_END, DCP, id),\ + SRI(GRPH_Y_END, DCP, id),\ + SRI(GRPH_PITCH, DCP, id),\ + SRI(GRPH_SWAP_CNTL, DCP, id),\ + SRI(PRESCALE_GRPH_CONTROL, DCP, id),\ + SRI(GRPH_UPDATE, DCP, id),\ + SRI(GRPH_FLIP_CONTROL, DCP, id),\ + SRI(GRPH_PRIMARY_SURFACE_ADDRESS, DCP, id),\ + SRI(GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, DCP, id),\ + SRI(GRPH_SECONDARY_SURFACE_ADDRESS, DCP, id),\ + SRI(GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, DCP, id),\ + SRI(DPG_PIPE_ARBITRATION_CONTROL1, DMIF_PG, id),\ + SRI(DPG_PIPE_ARBITRATION_CONTROL3, DMIF_PG, id),\ + SRI(DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, DMIF_PG, id),\ + SRI(DPG_PIPE_URGENCY_CONTROL, DMIF_PG, id),\ + SRI(DPG_PIPE_STUTTER_CONTROL, DMIF_PG, id),\ + SRI(DMIF_BUFFER_CONTROL, PIPE, id) +#endif + #define MI_DCE8_REG_LIST(id)\ MI_DCE_BASE_REG_LIST(id),\ SRI(DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, DMIF_PG, id) @@ -104,6 +129,9 @@ struct dce_mem_input_registers { uint32_t GRPH_SECONDARY_SURFACE_ADDRESS_HIGH; /* DMIF_PG */ uint32_t DPG_PIPE_ARBITRATION_CONTROL1; +#if defined(CONFIG_DRM_AMD_DC_SI) + uint32_t DPG_PIPE_ARBITRATION_CONTROL3; +#endif uint32_t DPG_WATERMARK_MASK_CONTROL; uint32_t DPG_PIPE_URGENCY_CONTROL; uint32_t DPG_PIPE_URGENT_LEVEL_CONTROL; @@ -126,6 +154,18 @@ struct dce_mem_input_registers { #define SFB(blk_name, reg_name, field_name, post_fix)\ .field_name = blk_name ## reg_name ## __ ## field_name ## post_fix +#if defined(CONFIG_DRM_AMD_DC_SI) +#define MI_GFX6_TILE_MASK_SH_LIST(mask_sh, blk)\ + SFB(blk, GRPH_CONTROL, GRPH_NUM_BANKS, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_BANK_WIDTH, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_BANK_HEIGHT, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_MACRO_TILE_ASPECT, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_TILE_SPLIT, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_PIPE_CONFIG, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_ARRAY_MODE, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_COLOR_EXPANSION_MODE, mask_sh) +#endif + #define MI_GFX8_TILE_MASK_SH_LIST(mask_sh, blk)\ SFB(blk, GRPH_CONTROL, GRPH_NUM_BANKS, mask_sh),\ SFB(blk, GRPH_CONTROL, GRPH_BANK_WIDTH, mask_sh),\ @@ -162,6 +202,32 @@ struct dce_mem_input_registers { SFB(blk, GRPH_UPDATE, GRPH_UPDATE_LOCK, mask_sh),\ SFB(blk, GRPH_FLIP_CONTROL, GRPH_SURFACE_UPDATE_H_RETRACE_EN, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define MI_DCP_MASK_SH_LIST_DCE6(mask_sh, blk)\ + SFB(blk, GRPH_ENABLE, GRPH_ENABLE, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_DEPTH, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_FORMAT, mask_sh),\ + SFB(blk, GRPH_CONTROL, GRPH_NUM_BANKS, mask_sh),\ + SFB(blk, GRPH_X_START, GRPH_X_START, mask_sh),\ + SFB(blk, GRPH_Y_START, GRPH_Y_START, mask_sh),\ + SFB(blk, GRPH_X_END, GRPH_X_END, mask_sh),\ + SFB(blk, GRPH_Y_END, GRPH_Y_END, mask_sh),\ + SFB(blk, GRPH_PITCH, GRPH_PITCH, mask_sh),\ + SFB(blk, GRPH_SWAP_CNTL, GRPH_RED_CROSSBAR, mask_sh),\ + SFB(blk, GRPH_SWAP_CNTL, GRPH_BLUE_CROSSBAR, mask_sh),\ + SFB(blk, PRESCALE_GRPH_CONTROL, GRPH_PRESCALE_SELECT, mask_sh),\ + SFB(blk, PRESCALE_GRPH_CONTROL, GRPH_PRESCALE_R_SIGN, mask_sh),\ + SFB(blk, PRESCALE_GRPH_CONTROL, GRPH_PRESCALE_G_SIGN, mask_sh),\ + SFB(blk, PRESCALE_GRPH_CONTROL, GRPH_PRESCALE_B_SIGN, mask_sh),\ + SFB(blk, GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, mask_sh),\ + SFB(blk, GRPH_SECONDARY_SURFACE_ADDRESS, GRPH_SECONDARY_SURFACE_ADDRESS, mask_sh),\ + SFB(blk, GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, mask_sh),\ + SFB(blk, GRPH_PRIMARY_SURFACE_ADDRESS, GRPH_PRIMARY_SURFACE_ADDRESS, mask_sh),\ + SFB(blk, GRPH_UPDATE, GRPH_SURFACE_UPDATE_PENDING, mask_sh),\ + SFB(blk, GRPH_UPDATE, GRPH_UPDATE_LOCK, mask_sh),\ + SFB(blk, GRPH_FLIP_CONTROL, GRPH_SURFACE_UPDATE_H_RETRACE_EN, mask_sh) +#endif + #define MI_DCP_DCE11_MASK_SH_LIST(mask_sh, blk)\ SFB(blk, GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT, GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT, mask_sh) @@ -172,6 +238,33 @@ struct dce_mem_input_registers { SFB(blk, DVMM_PTE_ARB_CONTROL, DVMM_PTE_REQ_PER_CHUNK, mask_sh),\ SFB(blk, DVMM_PTE_ARB_CONTROL, DVMM_MAX_PTE_REQ_OUTSTANDING, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define MI_DMIF_PG_MASK_SH_LIST_DCE6(mask_sh, blk)\ + SFB(blk, DPG_PIPE_ARBITRATION_CONTROL1, PIXEL_DURATION, mask_sh),\ + SFB(blk, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, mask_sh),\ + SFB(blk, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, mask_sh),\ + SFB(blk, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE, mask_sh),\ + SFB(blk, DPG_PIPE_STUTTER_CONTROL, STUTTER_IGNORE_FBC, mask_sh),\ + SF(PIPE0_DMIF_BUFFER_CONTROL, DMIF_BUFFERS_ALLOCATED, mask_sh),\ + SF(PIPE0_DMIF_BUFFER_CONTROL, DMIF_BUFFERS_ALLOCATION_COMPLETED, mask_sh) + +#define MI_DMIF_PG_MASK_SH_DCE6(mask_sh, blk)\ + SFB(blk, DPG_PIPE_ARBITRATION_CONTROL3, URGENCY_WATERMARK_MASK, mask_sh),\ + SFB(blk, DPG_PIPE_STUTTER_CONTROL, STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK, mask_sh),\ + SFB(blk, DPG_PIPE_STUTTER_CONTROL, STUTTER_EXIT_SELF_REFRESH_WATERMARK, mask_sh),\ + SFB(blk, DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\ + SFB(blk, DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, NB_PSTATE_CHANGE_ENABLE, mask_sh),\ + SFB(blk, DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, NB_PSTATE_CHANGE_URGENT_DURING_REQUEST, mask_sh),\ + SFB(blk, DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST, mask_sh),\ + SFB(blk, DPG_PIPE_NB_PSTATE_CHANGE_CONTROL, NB_PSTATE_CHANGE_WATERMARK, mask_sh) + +#define MI_DCE6_MASK_SH_LIST(mask_sh)\ + MI_DCP_MASK_SH_LIST_DCE6(mask_sh, ),\ + MI_DMIF_PG_MASK_SH_LIST_DCE6(mask_sh, ),\ + MI_DMIF_PG_MASK_SH_DCE6(mask_sh, ),\ + MI_GFX6_TILE_MASK_SH_LIST(mask_sh, ) +#endif + #define MI_DMIF_PG_MASK_SH_LIST(mask_sh, blk)\ SFB(blk, DPG_PIPE_ARBITRATION_CONTROL1, PIXEL_DURATION, mask_sh),\ SFB(blk, DPG_WATERMARK_MASK_CONTROL, URGENCY_WATERMARK_MASK, mask_sh),\ @@ -345,6 +438,16 @@ void dce_mem_input_construct( const struct dce_mem_input_shift *mi_shift, const struct dce_mem_input_mask *mi_mask); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_mem_input_construct( + struct dce_mem_input *dce_mi, + struct dc_context *ctx, + int inst, + const struct dce_mem_input_registers *regs, + const struct dce_mem_input_shift *mi_shift, + const struct dce_mem_input_mask *mi_mask); +#endif + void dce112_mem_input_construct( struct dce_mem_input *dce_mi, struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c index 51081d9ae3fb..e459ae65aaf7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c @@ -141,6 +141,47 @@ static void set_truncation( params->flags.TRUNCATE_MODE); } +#if defined(CONFIG_DRM_AMD_DC_SI) +/** + * dce60_set_truncation + * 1) set truncation depth: 0 for 18 bpp or 1 for 24 bpp + * 2) enable truncation + * 3) HW remove 12bit FMT support for DCE11 power saving reason. + */ +static void dce60_set_truncation( + struct dce110_opp *opp110, + const struct bit_depth_reduction_params *params) +{ + /* DCE6 has no FMT_TRUNCATE_MODE bit in FMT_BIT_DEPTH_CONTROL reg */ + + /*Disable truncation*/ + REG_UPDATE_2(FMT_BIT_DEPTH_CONTROL, + FMT_TRUNCATE_EN, 0, + FMT_TRUNCATE_DEPTH, 0); + + if (params->pixel_encoding == PIXEL_ENCODING_YCBCR422) { + /* 8bpc trunc on YCbCr422*/ + if (params->flags.TRUNCATE_DEPTH == 1) + REG_UPDATE_2(FMT_BIT_DEPTH_CONTROL, + FMT_TRUNCATE_EN, 1, + FMT_TRUNCATE_DEPTH, 1); + else if (params->flags.TRUNCATE_DEPTH == 2) + /* 10bpc trunc on YCbCr422*/ + REG_UPDATE_2(FMT_BIT_DEPTH_CONTROL, + FMT_TRUNCATE_EN, 1, + FMT_TRUNCATE_DEPTH, 2); + return; + } + /* on other format-to do */ + if (params->flags.TRUNCATE_ENABLED == 0) + return; + /*Set truncation depth and Enable truncation*/ + REG_UPDATE_2(FMT_BIT_DEPTH_CONTROL, + FMT_TRUNCATE_EN, 1, + FMT_TRUNCATE_DEPTH, + params->flags.TRUNCATE_DEPTH); +} +#endif /** * set_spatial_dither @@ -373,6 +414,57 @@ void dce110_opp_set_clamping( } } +#if defined(CONFIG_DRM_AMD_DC_SI) +/** + * Set Clamping for DCE6 parts + * 1) Set clamping format based on bpc - 0 for 6bpc (No clamping) + * 1 for 8 bpc + * 2 for 10 bpc + * 3 for 12 bpc + * 7 for programable + * 2) Enable clamp if Limited range requested + */ +void dce60_opp_set_clamping( + struct dce110_opp *opp110, + const struct clamping_and_pixel_encoding_params *params) +{ + REG_SET_2(FMT_CLAMP_CNTL, 0, + FMT_CLAMP_DATA_EN, 0, + FMT_CLAMP_COLOR_FORMAT, 0); + + switch (params->clamping_level) { + case CLAMPING_FULL_RANGE: + break; + case CLAMPING_LIMITED_RANGE_8BPC: + REG_SET_2(FMT_CLAMP_CNTL, 0, + FMT_CLAMP_DATA_EN, 1, + FMT_CLAMP_COLOR_FORMAT, 1); + break; + case CLAMPING_LIMITED_RANGE_10BPC: + REG_SET_2(FMT_CLAMP_CNTL, 0, + FMT_CLAMP_DATA_EN, 1, + FMT_CLAMP_COLOR_FORMAT, 2); + break; + case CLAMPING_LIMITED_RANGE_12BPC: + REG_SET_2(FMT_CLAMP_CNTL, 0, + FMT_CLAMP_DATA_EN, 1, + FMT_CLAMP_COLOR_FORMAT, 3); + break; + case CLAMPING_LIMITED_RANGE_PROGRAMMABLE: + /*Set clamp control*/ + REG_SET_2(FMT_CLAMP_CNTL, 0, + FMT_CLAMP_DATA_EN, 1, + FMT_CLAMP_COLOR_FORMAT, 7); + + /* DCE6 does have FMT_CLAMP_COMPONENT_{R,G,B} registers */ + + break; + default: + break; + } +} +#endif + /** * set_pixel_encoding * @@ -408,6 +500,39 @@ static void set_pixel_encoding( } +#if defined(CONFIG_DRM_AMD_DC_SI) +/** + * dce60_set_pixel_encoding + * DCE6 has no FMT_SUBSAMPLING_{MODE,ORDER} bits in FMT_CONTROL reg + * Set Pixel Encoding + * 0: RGB 4:4:4 or YCbCr 4:4:4 or YOnly + * 1: YCbCr 4:2:2 + */ +static void dce60_set_pixel_encoding( + struct dce110_opp *opp110, + const struct clamping_and_pixel_encoding_params *params) +{ + if (opp110->opp_mask->FMT_CBCR_BIT_REDUCTION_BYPASS) + REG_UPDATE_2(FMT_CONTROL, + FMT_PIXEL_ENCODING, 0, + FMT_CBCR_BIT_REDUCTION_BYPASS, 0); + else + REG_UPDATE(FMT_CONTROL, + FMT_PIXEL_ENCODING, 0); + + if (params->pixel_encoding == PIXEL_ENCODING_YCBCR422) { + REG_UPDATE(FMT_CONTROL, + FMT_PIXEL_ENCODING, 1); + } + if (params->pixel_encoding == PIXEL_ENCODING_YCBCR420) { + REG_UPDATE_2(FMT_CONTROL, + FMT_PIXEL_ENCODING, 2, + FMT_CBCR_BIT_REDUCTION_BYPASS, 1); + } + +} +#endif + void dce110_opp_program_bit_depth_reduction( struct output_pixel_processor *opp, const struct bit_depth_reduction_params *params) @@ -419,6 +544,19 @@ void dce110_opp_program_bit_depth_reduction( set_temporal_dither(opp110, params); } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_opp_program_bit_depth_reduction( + struct output_pixel_processor *opp, + const struct bit_depth_reduction_params *params) +{ + struct dce110_opp *opp110 = TO_DCE110_OPP(opp); + + dce60_set_truncation(opp110, params); + set_spatial_dither(opp110, params); + set_temporal_dither(opp110, params); +} +#endif + void dce110_opp_program_clamping_and_pixel_encoding( struct output_pixel_processor *opp, const struct clamping_and_pixel_encoding_params *params) @@ -429,6 +567,19 @@ void dce110_opp_program_clamping_and_pixel_encoding( set_pixel_encoding(opp110, params); } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_opp_program_clamping_and_pixel_encoding( + struct output_pixel_processor *opp, + const struct clamping_and_pixel_encoding_params *params) +{ + struct dce110_opp *opp110 = TO_DCE110_OPP(opp); + + dce60_opp_set_clamping(opp110, params); + dce60_set_pixel_encoding(opp110, params); +} +#endif + + static void program_formatter_420_memory(struct output_pixel_processor *opp) { struct dce110_opp *opp110 = TO_DCE110_OPP(opp); @@ -526,7 +677,32 @@ void dce110_opp_program_fmt( return; } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_opp_program_fmt( + struct output_pixel_processor *opp, + struct bit_depth_reduction_params *fmt_bit_depth, + struct clamping_and_pixel_encoding_params *clamping) +{ + /* dithering is affected by <CrtcSourceSelect>, hence should be + * programmed afterwards */ + + if (clamping->pixel_encoding == PIXEL_ENCODING_YCBCR420) + program_formatter_420_memory(opp); + + dce60_opp_program_bit_depth_reduction( + opp, + fmt_bit_depth); + + dce60_opp_program_clamping_and_pixel_encoding( + opp, + clamping); + + if (clamping->pixel_encoding == PIXEL_ENCODING_YCBCR420) + program_formatter_reset_dig_resync_fifo(opp); + return; +} +#endif @@ -541,6 +717,15 @@ static const struct opp_funcs funcs = { .opp_program_bit_depth_reduction = dce110_opp_program_bit_depth_reduction }; +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct opp_funcs dce60_opp_funcs = { + .opp_set_dyn_expansion = dce110_opp_set_dyn_expansion, + .opp_destroy = dce110_opp_destroy, + .opp_program_fmt = dce60_opp_program_fmt, + .opp_program_bit_depth_reduction = dce60_opp_program_bit_depth_reduction +}; +#endif + void dce110_opp_construct(struct dce110_opp *opp110, struct dc_context *ctx, uint32_t inst, @@ -559,6 +744,26 @@ void dce110_opp_construct(struct dce110_opp *opp110, opp110->opp_mask = opp_mask; } +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_opp_construct(struct dce110_opp *opp110, + struct dc_context *ctx, + uint32_t inst, + const struct dce_opp_registers *regs, + const struct dce_opp_shift *opp_shift, + const struct dce_opp_mask *opp_mask) +{ + opp110->base.funcs = &dce60_opp_funcs; + + opp110->base.ctx = ctx; + + opp110->base.inst = inst; + + opp110->regs = regs; + opp110->opp_shift = opp_shift; + opp110->opp_mask = opp_mask; +} +#endif + void dce110_opp_destroy(struct output_pixel_processor **opp) { if (*opp) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h index 2ab0147cbd9d..4d484ef60f35 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h @@ -81,6 +81,17 @@ enum dce110_opp_reg_type { OPP_COMMON_REG_LIST_BASE(id), \ SRI(CONTROL, FMT_MEMORY, id) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define OPP_DCE_60_REG_LIST(id) \ + SRI(FMT_DYNAMIC_EXP_CNTL, FMT, id), \ + SRI(FMT_BIT_DEPTH_CONTROL, FMT, id), \ + SRI(FMT_CONTROL, FMT, id), \ + SRI(FMT_DITHER_RAND_R_SEED, FMT, id), \ + SRI(FMT_DITHER_RAND_G_SEED, FMT, id), \ + SRI(FMT_DITHER_RAND_B_SEED, FMT, id), \ + SRI(FMT_CLAMP_CNTL, FMT, id) +#endif + #define OPP_SF(reg_name, field_name, post_fix)\ .field_name = reg_name ## __ ## field_name ## post_fix @@ -192,6 +203,35 @@ enum dce110_opp_reg_type { OPP_SF(FMT0_FMT_CONTROL, FMT_SUBSAMPLING_ORDER, mask_sh),\ OPP_SF(FMT0_FMT_CONTROL, FMT_CBCR_BIT_REDUCTION_BYPASS, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define OPP_COMMON_MASK_SH_LIST_DCE_60(mask_sh)\ + OPP_SF(FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_EN, mask_sh),\ + OPP_SF(FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_MODE, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TRUNCATE_EN, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TRUNCATE_DEPTH, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_SPATIAL_DITHER_EN, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_SPATIAL_DITHER_DEPTH, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_SPATIAL_DITHER_MODE, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_HIGHPASS_RANDOM_ENABLE, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_FRAME_RANDOM_ENABLE, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_RGB_RANDOM_ENABLE, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ + OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\ + OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\ + OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_LEVEL, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_25FRC_SEL, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_50FRC_SEL, mask_sh),\ + OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_75FRC_SEL, mask_sh),\ + OPP_SF(FMT_CLAMP_CNTL, FMT_CLAMP_DATA_EN, mask_sh),\ + OPP_SF(FMT_CLAMP_CNTL, FMT_CLAMP_COLOR_FORMAT, mask_sh),\ + OPP_SF(FMT_CONTROL, FMT_PIXEL_ENCODING, mask_sh) +#endif + #define OPP_REG_FIELD_LIST(type) \ type FMT_DYNAMIC_EXP_EN; \ type FMT_DYNAMIC_EXP_MODE; \ @@ -279,6 +319,15 @@ void dce110_opp_construct(struct dce110_opp *opp110, const struct dce_opp_shift *opp_shift, const struct dce_opp_mask *opp_mask); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_opp_construct(struct dce110_opp *opp110, + struct dc_context *ctx, + uint32_t inst, + const struct dce_opp_registers *regs, + const struct dce_opp_shift *opp_shift, + const struct dce_opp_mask *opp_mask); +#endif + void dce110_opp_destroy(struct output_pixel_processor **opp); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c index 43781e77be43..df7f826eebd8 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c @@ -46,13 +46,14 @@ #define FN(reg_name, field_name) \ dce_panel_cntl->shift->field_name, dce_panel_cntl->mask->field_name -static unsigned int calculate_16_bit_backlight_from_pwm(struct dce_panel_cntl *dce_panel_cntl) +static unsigned int dce_get_16_bit_backlight_from_pwm(struct panel_cntl *panel_cntl) { uint64_t current_backlight; uint32_t round_result; uint32_t pwm_period_cntl, bl_period, bl_int_count; uint32_t bl_pwm_cntl, bl_pwm, fractional_duty_cycle_en; uint32_t bl_period_mask, bl_pwm_mask; + struct dce_panel_cntl *dce_panel_cntl = TO_DCE_PANEL_CNTL(panel_cntl); pwm_period_cntl = REG_READ(BL_PWM_PERIOD_CNTL); REG_GET(BL_PWM_PERIOD_CNTL, BL_PWM_PERIOD, &bl_period); @@ -75,7 +76,7 @@ static unsigned int calculate_16_bit_backlight_from_pwm(struct dce_panel_cntl *d else bl_pwm &= 0xFFFF; - current_backlight = bl_pwm << (1 + bl_int_count); + current_backlight = (uint64_t)bl_pwm << (1 + bl_int_count); if (bl_period == 0) bl_period = 0xFFFF; @@ -150,7 +151,7 @@ static uint32_t dce_panel_cntl_hw_init(struct panel_cntl *panel_cntl) REG_UPDATE(BL_PWM_GRP1_REG_LOCK, BL_PWM_GRP1_REG_LOCK, 0); - current_backlight = calculate_16_bit_backlight_from_pwm(dce_panel_cntl); + current_backlight = dce_get_16_bit_backlight_from_pwm(panel_cntl); return current_backlight; } @@ -273,6 +274,7 @@ static const struct panel_cntl_funcs dce_link_panel_cntl_funcs = { .is_panel_powered_on = dce_is_panel_powered_on, .store_backlight_level = dce_store_backlight_level, .driver_set_backlight = dce_driver_set_backlight, + .get_current_backlight = dce_get_16_bit_backlight_from_pwm, }; void dce_panel_cntl_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.h b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.h index 70ec691e14d2..99c68ca9c7e0 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.h @@ -49,7 +49,7 @@ #define DCN_PANEL_CNTL_REG_LIST()\ DCN_PANEL_CNTL_SR(PWRSEQ_CNTL, LVTMA), \ DCN_PANEL_CNTL_SR(PWRSEQ_STATE, LVTMA), \ - DCE_PANEL_CNTL_SR(PWRSEQ_REF_DIV, LVTMA), \ + DCN_PANEL_CNTL_SR(PWRSEQ_REF_DIV, LVTMA), \ SR(BL_PWM_CNTL), \ SR(BL_PWM_CNTL2), \ SR(BL_PWM_PERIOD_CNTL), \ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index ab63d0d0304c..2a32b66959ba 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -146,6 +146,33 @@ static bool setup_scaling_configuration( return true; } +#if defined(CONFIG_DRM_AMD_DC_SI) +static bool dce60_setup_scaling_configuration( + struct dce_transform *xfm_dce, + const struct scaler_data *data) +{ + REG_SET(SCL_BYPASS_CONTROL, 0, SCL_BYPASS_MODE, 0); + + if (data->taps.h_taps + data->taps.v_taps <= 2) { + /* Set bypass */ + + /* DCE6 has no SCL_MODE register, skip scale mode programming */ + + return false; + } + + REG_SET_2(SCL_TAP_CONTROL, 0, + SCL_H_NUM_OF_TAPS, data->taps.h_taps - 1, + SCL_V_NUM_OF_TAPS, data->taps.v_taps - 1); + + /* DCE6 has no SCL_MODE register, skip scale mode programming */ + + /* DCE6 has no SCL_BOUNDARY_MODE bit, skip replace out of bound pixels */ + + return true; +} +#endif + static void program_overscan( struct dce_transform *xfm_dce, const struct scaler_data *data) @@ -279,6 +306,36 @@ static void calculate_inits( inits->v_init.fraction = dc_fixpt_u0d19(v_init) << 5; } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_calculate_inits( + struct dce_transform *xfm_dce, + const struct scaler_data *data, + struct sclh_ratios_inits *inits) +{ + struct fixed31_32 v_init; + + inits->h_int_scale_ratio = + dc_fixpt_u2d19(data->ratios.horz) << 5; + inits->v_int_scale_ratio = + dc_fixpt_u2d19(data->ratios.vert) << 5; + + /* DCE6 h_init_luma setting inspired by DCE110 */ + inits->h_init_luma.integer = 1; + + /* DCE6 h_init_chroma setting inspired by DCE110 */ + inits->h_init_chroma.integer = 1; + + v_init = + dc_fixpt_div_int( + dc_fixpt_add( + data->ratios.vert, + dc_fixpt_from_int(data->taps.v_taps + 1)), + 2); + inits->v_init.integer = dc_fixpt_floor(v_init); + inits->v_init.fraction = dc_fixpt_u0d19(v_init) << 5; +} +#endif + static void program_scl_ratios_inits( struct dce_transform *xfm_dce, struct scl_ratios_inits *inits) @@ -301,6 +358,36 @@ static void program_scl_ratios_inits( REG_WRITE(SCL_AUTOMATIC_MODE_CONTROL, 0); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_program_scl_ratios_inits( + struct dce_transform *xfm_dce, + struct sclh_ratios_inits *inits) +{ + + REG_SET(SCL_HORZ_FILTER_SCALE_RATIO, 0, + SCL_H_SCALE_RATIO, inits->h_int_scale_ratio); + + REG_SET(SCL_VERT_FILTER_SCALE_RATIO, 0, + SCL_V_SCALE_RATIO, inits->v_int_scale_ratio); + + /* DCE6 has SCL_HORZ_FILTER_INIT_RGB_LUMA register */ + REG_SET_2(SCL_HORZ_FILTER_INIT_RGB_LUMA, 0, + SCL_H_INIT_INT_RGB_Y, inits->h_init_luma.integer, + SCL_H_INIT_FRAC_RGB_Y, inits->h_init_luma.fraction); + + /* DCE6 has SCL_HORZ_FILTER_INIT_CHROMA register */ + REG_SET_2(SCL_HORZ_FILTER_INIT_CHROMA, 0, + SCL_H_INIT_INT_CBCR, inits->h_init_chroma.integer, + SCL_H_INIT_FRAC_CBCR, inits->h_init_chroma.fraction); + + REG_SET_2(SCL_VERT_FILTER_INIT, 0, + SCL_V_INIT_INT, inits->v_init.integer, + SCL_V_INIT_FRAC, inits->v_init.fraction); + + REG_WRITE(SCL_AUTOMATIC_MODE_CONTROL, 0); +} +#endif + static const uint16_t *get_filter_coeffs_16p(int taps, struct fixed31_32 ratio) { if (taps == 4) @@ -399,6 +486,91 @@ static void dce_transform_set_scaler( REG_UPDATE(LB_DATA_FORMAT, ALPHA_EN, data->lb_params.alpha_en); } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_transform_set_scaler( + struct transform *xfm, + const struct scaler_data *data) +{ + struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); + bool is_scaling_required; + bool filter_updated = false; + const uint16_t *coeffs_v, *coeffs_h; + + /*Use whole line buffer memory always*/ + REG_SET(DC_LB_MEMORY_SPLIT, 0, + DC_LB_MEMORY_CONFIG, 0); + + REG_SET(DC_LB_MEM_SIZE, 0, + DC_LB_MEM_SIZE, xfm_dce->lb_memory_size); + + /* Clear SCL_F_SHARP_CONTROL value to 0 */ + REG_WRITE(SCL_F_SHARP_CONTROL, 0); + + /* 1. Program overscan */ + program_overscan(xfm_dce, data); + + /* 2. Program taps and configuration */ + is_scaling_required = dce60_setup_scaling_configuration(xfm_dce, data); + + if (is_scaling_required) { + /* 3. Calculate and program ratio, DCE6 filter initialization */ + struct sclh_ratios_inits inits = { 0 }; + + /* DCE6 has specific calculate_inits() function */ + dce60_calculate_inits(xfm_dce, data, &inits); + + /* DCE6 has specific program_scl_ratios_inits() function */ + dce60_program_scl_ratios_inits(xfm_dce, &inits); + + coeffs_v = get_filter_coeffs_16p(data->taps.v_taps, data->ratios.vert); + coeffs_h = get_filter_coeffs_16p(data->taps.h_taps, data->ratios.horz); + + if (coeffs_v != xfm_dce->filter_v || coeffs_h != xfm_dce->filter_h) { + /* 4. Program vertical filters */ + if (xfm_dce->filter_v == NULL) + REG_SET(SCL_VERT_FILTER_CONTROL, 0, + SCL_V_2TAP_HARDCODE_COEF_EN, 0); + program_multi_taps_filter( + xfm_dce, + data->taps.v_taps, + coeffs_v, + FILTER_TYPE_RGB_Y_VERTICAL); + program_multi_taps_filter( + xfm_dce, + data->taps.v_taps, + coeffs_v, + FILTER_TYPE_ALPHA_VERTICAL); + + /* 5. Program horizontal filters */ + if (xfm_dce->filter_h == NULL) + REG_SET(SCL_HORZ_FILTER_CONTROL, 0, + SCL_H_2TAP_HARDCODE_COEF_EN, 0); + program_multi_taps_filter( + xfm_dce, + data->taps.h_taps, + coeffs_h, + FILTER_TYPE_RGB_Y_HORIZONTAL); + program_multi_taps_filter( + xfm_dce, + data->taps.h_taps, + coeffs_h, + FILTER_TYPE_ALPHA_HORIZONTAL); + + xfm_dce->filter_v = coeffs_v; + xfm_dce->filter_h = coeffs_h; + filter_updated = true; + } + } + + /* 6. Program the viewport */ + program_viewport(xfm_dce, &data->viewport); + + /* DCE6 has no SCL_COEF_UPDATE_COMPLETE bit to flip to new coefficient memory */ + + /* DCE6 DATA_FORMAT register does not support ALPHA_EN */ +} +#endif + /***************************************************************************** * set_clamp * @@ -664,6 +836,67 @@ static void program_bit_depth_reduction( bit_depth_params->flags.HIGHPASS_RANDOM); } +#if defined(CONFIG_DRM_AMD_DC_SI) +/***************************************************************************** + * dce60_transform_bit_depth_reduction program + * + * @brief + * Programs the DCP bit depth reduction registers (Clamp, Round/Truncate, + * Dither) for dce + * + * @param depth : bit depth to set the clamp to (should match denorm) + * + ******************************************************************************/ +static void dce60_program_bit_depth_reduction( + struct dce_transform *xfm_dce, + enum dc_color_depth depth, + const struct bit_depth_reduction_params *bit_depth_params) +{ + enum dcp_out_trunc_round_depth trunc_round_depth; + enum dcp_out_trunc_round_mode trunc_mode; + bool spatial_dither_enable; + + ASSERT(depth < COLOR_DEPTH_121212); /* Invalid clamp bit depth */ + + spatial_dither_enable = bit_depth_params->flags.SPATIAL_DITHER_ENABLED; + /* Default to 12 bit truncation without rounding */ + trunc_round_depth = DCP_OUT_TRUNC_ROUND_DEPTH_12BIT; + trunc_mode = DCP_OUT_TRUNC_ROUND_MODE_TRUNCATE; + + if (bit_depth_params->flags.TRUNCATE_ENABLED) { + /* Don't enable dithering if truncation is enabled */ + spatial_dither_enable = false; + trunc_mode = bit_depth_params->flags.TRUNCATE_MODE ? + DCP_OUT_TRUNC_ROUND_MODE_ROUND : + DCP_OUT_TRUNC_ROUND_MODE_TRUNCATE; + + if (bit_depth_params->flags.TRUNCATE_DEPTH == 0 || + bit_depth_params->flags.TRUNCATE_DEPTH == 1) + trunc_round_depth = DCP_OUT_TRUNC_ROUND_DEPTH_8BIT; + else if (bit_depth_params->flags.TRUNCATE_DEPTH == 2) + trunc_round_depth = DCP_OUT_TRUNC_ROUND_DEPTH_10BIT; + else { + /* + * Invalid truncate/round depth. Setting here to 12bit + * to prevent use-before-initialize errors. + */ + trunc_round_depth = DCP_OUT_TRUNC_ROUND_DEPTH_12BIT; + BREAK_TO_DEBUGGER(); + } + } + + /* DCE6 has no OUT_CLAMP_CONTROL_* registers - set_clamp() is skipped */ + set_round(xfm_dce, trunc_mode, trunc_round_depth); + set_dither(xfm_dce, + spatial_dither_enable, + DCP_SPATIAL_DITHER_MODE_A_AA_A, + DCP_SPATIAL_DITHER_DEPTH_30BPP, + bit_depth_params->flags.FRAME_RANDOM, + bit_depth_params->flags.RGB_RANDOM, + bit_depth_params->flags.HIGHPASS_RANDOM); +} +#endif + static int dce_transform_get_max_num_of_supported_lines( struct dce_transform *xfm_dce, enum lb_pixel_depth depth, @@ -797,6 +1030,59 @@ static void dce_transform_set_pixel_storage_depth( } } +#if defined(CONFIG_DRM_AMD_DC_SI) +static void dce60_transform_set_pixel_storage_depth( + struct transform *xfm, + enum lb_pixel_depth depth, + const struct bit_depth_reduction_params *bit_depth_params) +{ + struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); + int pixel_depth, expan_mode; + enum dc_color_depth color_depth; + + switch (depth) { + case LB_PIXEL_DEPTH_18BPP: + color_depth = COLOR_DEPTH_666; + pixel_depth = 2; + expan_mode = 1; + break; + case LB_PIXEL_DEPTH_24BPP: + color_depth = COLOR_DEPTH_888; + pixel_depth = 1; + expan_mode = 1; + break; + case LB_PIXEL_DEPTH_30BPP: + color_depth = COLOR_DEPTH_101010; + pixel_depth = 0; + expan_mode = 1; + break; + case LB_PIXEL_DEPTH_36BPP: + color_depth = COLOR_DEPTH_121212; + pixel_depth = 3; + expan_mode = 0; + break; + default: + color_depth = COLOR_DEPTH_101010; + pixel_depth = 0; + expan_mode = 1; + BREAK_TO_DEBUGGER(); + break; + } + + set_denormalization(xfm_dce, color_depth); + dce60_program_bit_depth_reduction(xfm_dce, color_depth, bit_depth_params); + + /* DATA_FORMAT in DCE6 does not have PIXEL_DEPTH and PIXEL_EXPAN_MODE masks */ + + if (!(xfm_dce->lb_pixel_depth_supported & depth)) { + /*we should use unsupported capabilities + * unless it is required by w/a*/ + DC_LOG_WARNING("%s: Capability not supported", + __func__); + } +} +#endif + static void program_gamut_remap( struct dce_transform *xfm_dce, const uint16_t *reg_val) @@ -1335,6 +1621,21 @@ static const struct transform_funcs dce_transform_funcs = { .transform_get_optimal_number_of_taps = dce_transform_get_optimal_number_of_taps }; +#if defined(CONFIG_DRM_AMD_DC_SI) +static const struct transform_funcs dce60_transform_funcs = { + .transform_reset = dce_transform_reset, + .transform_set_scaler = dce60_transform_set_scaler, + .transform_set_gamut_remap = dce_transform_set_gamut_remap, + .opp_set_csc_adjustment = dce110_opp_set_csc_adjustment, + .opp_set_csc_default = dce110_opp_set_csc_default, + .opp_power_on_regamma_lut = dce110_opp_power_on_regamma_lut, + .opp_program_regamma_pwl = dce110_opp_program_regamma_pwl, + .opp_set_regamma_mode = dce110_opp_set_regamma_mode, + .transform_set_pixel_storage_depth = dce60_transform_set_pixel_storage_depth, + .transform_get_optimal_number_of_taps = dce_transform_get_optimal_number_of_taps +}; +#endif + /*****************************************/ /* Constructor, Destructor */ /*****************************************/ @@ -1365,3 +1666,32 @@ void dce_transform_construct( xfm_dce->lb_bits_per_entry = LB_BITS_PER_ENTRY; xfm_dce->lb_memory_size = LB_TOTAL_NUMBER_OF_ENTRIES; /*0x6B0*/ } + +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_transform_construct( + struct dce_transform *xfm_dce, + struct dc_context *ctx, + uint32_t inst, + const struct dce_transform_registers *regs, + const struct dce_transform_shift *xfm_shift, + const struct dce_transform_mask *xfm_mask) +{ + xfm_dce->base.ctx = ctx; + + xfm_dce->base.inst = inst; + xfm_dce->base.funcs = &dce60_transform_funcs; + + xfm_dce->regs = regs; + xfm_dce->xfm_shift = xfm_shift; + xfm_dce->xfm_mask = xfm_mask; + + xfm_dce->prescaler_on = true; + xfm_dce->lb_pixel_depth_supported = + LB_PIXEL_DEPTH_18BPP | + LB_PIXEL_DEPTH_24BPP | + LB_PIXEL_DEPTH_30BPP; + + xfm_dce->lb_bits_per_entry = LB_BITS_PER_ENTRY; + xfm_dce->lb_memory_size = LB_TOTAL_NUMBER_OF_ENTRIES; /*0x6B0*/ +} +#endif diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h index 948281d8b6af..cbce194ec7b8 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.h @@ -108,6 +108,68 @@ SRI(DCFE_MEM_PWR_CTRL, DCFE, id), \ SRI(DCFE_MEM_PWR_STATUS, DCFE, id) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define XFM_COMMON_REG_LIST_DCE60_BASE(id) \ + SRI(DATA_FORMAT, LB, id), \ + SRI(GAMUT_REMAP_CONTROL, DCP, id), \ + SRI(GAMUT_REMAP_C11_C12, DCP, id), \ + SRI(GAMUT_REMAP_C13_C14, DCP, id), \ + SRI(GAMUT_REMAP_C21_C22, DCP, id), \ + SRI(GAMUT_REMAP_C23_C24, DCP, id), \ + SRI(GAMUT_REMAP_C31_C32, DCP, id), \ + SRI(GAMUT_REMAP_C33_C34, DCP, id), \ + SRI(OUTPUT_CSC_C11_C12, DCP, id), \ + SRI(OUTPUT_CSC_C13_C14, DCP, id), \ + SRI(OUTPUT_CSC_C21_C22, DCP, id), \ + SRI(OUTPUT_CSC_C23_C24, DCP, id), \ + SRI(OUTPUT_CSC_C31_C32, DCP, id), \ + SRI(OUTPUT_CSC_C33_C34, DCP, id), \ + SRI(OUTPUT_CSC_CONTROL, DCP, id), \ + SRI(REGAMMA_CNTLA_START_CNTL, DCP, id), \ + SRI(REGAMMA_CNTLA_SLOPE_CNTL, DCP, id), \ + SRI(REGAMMA_CNTLA_END_CNTL1, DCP, id), \ + SRI(REGAMMA_CNTLA_END_CNTL2, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_0_1, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_2_3, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_4_5, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_6_7, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_8_9, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_10_11, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_12_13, DCP, id), \ + SRI(REGAMMA_CNTLA_REGION_14_15, DCP, id), \ + SRI(REGAMMA_LUT_WRITE_EN_MASK, DCP, id), \ + SRI(REGAMMA_LUT_INDEX, DCP, id), \ + SRI(REGAMMA_LUT_DATA, DCP, id), \ + SRI(REGAMMA_CONTROL, DCP, id), \ + SRI(DENORM_CONTROL, DCP, id), \ + SRI(DCP_SPATIAL_DITHER_CNTL, DCP, id), \ + SRI(OUT_ROUND_CONTROL, DCP, id), \ + SRI(SCL_TAP_CONTROL, SCL, id), \ + SRI(SCL_CONTROL, SCL, id), \ + SRI(SCL_BYPASS_CONTROL, SCL, id), \ + SRI(EXT_OVERSCAN_LEFT_RIGHT, SCL, id), \ + SRI(EXT_OVERSCAN_TOP_BOTTOM, SCL, id), \ + SRI(SCL_VERT_FILTER_CONTROL, SCL, id), \ + SRI(SCL_HORZ_FILTER_CONTROL, SCL, id), \ + SRI(SCL_COEF_RAM_SELECT, SCL, id), \ + SRI(SCL_COEF_RAM_TAP_DATA, SCL, id), \ + SRI(VIEWPORT_START, SCL, id), \ + SRI(VIEWPORT_SIZE, SCL, id), \ + SRI(SCL_HORZ_FILTER_SCALE_RATIO, SCL, id), \ + SRI(SCL_VERT_FILTER_SCALE_RATIO, SCL, id), \ + SRI(SCL_VERT_FILTER_INIT, SCL, id), \ + SRI(SCL_AUTOMATIC_MODE_CONTROL, SCL, id), \ + SRI(DC_LB_MEMORY_SPLIT, LB, id), \ + SRI(DC_LB_MEM_SIZE, LB, id), \ + SRI(DCFE_MEM_LIGHT_SLEEP_CNTL, CRTC, id), \ + SRI(SCL_UPDATE, SCL, id), \ + SRI(SCL_F_SHARP_CONTROL, SCL, id) + +#define XFM_COMMON_REG_LIST_DCE60(id) \ + XFM_COMMON_REG_LIST_DCE60_BASE(id), \ + SRI(DCFE_MEM_LIGHT_SLEEP_CNTL, CRTC, id) +#endif + #define XFM_SF(reg_name, field_name, post_fix)\ .field_name = reg_name ## __ ## field_name ## post_fix @@ -204,6 +266,83 @@ XFM_SF(DCFE_MEM_PWR_STATUS, DCP_REGAMMA_MEM_PWR_STATE, mask_sh),\ XFM_SF(SCL_MODE, SCL_PSCL_EN, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define XFM_COMMON_MASK_SH_LIST_DCE60(mask_sh) \ + XFM_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(mask_sh), \ + OPP_SF(DCFE_MEM_LIGHT_SLEEP_CNTL, REGAMMA_LUT_LIGHT_SLEEP_DIS, mask_sh),\ + OPP_SF(DCFE_MEM_LIGHT_SLEEP_CNTL, DCP_LUT_LIGHT_SLEEP_DIS, mask_sh),\ + OPP_SF(DCFE_MEM_LIGHT_SLEEP_CNTL, REGAMMA_LUT_MEM_PWR_STATE, mask_sh) + +#define XFM_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(mask_sh) \ + XFM_SF(OUT_ROUND_CONTROL, OUT_ROUND_TRUNC_MODE, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_SPATIAL_DITHER_EN, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_SPATIAL_DITHER_MODE, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_SPATIAL_DITHER_DEPTH, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_FRAME_RANDOM_ENABLE, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_RGB_RANDOM_ENABLE, mask_sh), \ + XFM_SF(DCP_SPATIAL_DITHER_CNTL, DCP_HIGHPASS_RANDOM_ENABLE, mask_sh), \ + XFM_SF(DENORM_CONTROL, DENORM_MODE, mask_sh), \ + XFM_SF(DATA_FORMAT, INTERLEAVE_EN, mask_sh), \ + XFM_SF(GAMUT_REMAP_C11_C12, GAMUT_REMAP_C11, mask_sh), \ + XFM_SF(GAMUT_REMAP_C11_C12, GAMUT_REMAP_C12, mask_sh), \ + XFM_SF(GAMUT_REMAP_C13_C14, GAMUT_REMAP_C13, mask_sh), \ + XFM_SF(GAMUT_REMAP_C13_C14, GAMUT_REMAP_C14, mask_sh), \ + XFM_SF(GAMUT_REMAP_C21_C22, GAMUT_REMAP_C21, mask_sh), \ + XFM_SF(GAMUT_REMAP_C21_C22, GAMUT_REMAP_C22, mask_sh), \ + XFM_SF(GAMUT_REMAP_C23_C24, GAMUT_REMAP_C23, mask_sh), \ + XFM_SF(GAMUT_REMAP_C23_C24, GAMUT_REMAP_C24, mask_sh), \ + XFM_SF(GAMUT_REMAP_C31_C32, GAMUT_REMAP_C31, mask_sh), \ + XFM_SF(GAMUT_REMAP_C31_C32, GAMUT_REMAP_C32, mask_sh), \ + XFM_SF(GAMUT_REMAP_C33_C34, GAMUT_REMAP_C33, mask_sh), \ + XFM_SF(GAMUT_REMAP_C33_C34, GAMUT_REMAP_C34, mask_sh), \ + XFM_SF(GAMUT_REMAP_CONTROL, GRPH_GAMUT_REMAP_MODE, mask_sh), \ + XFM_SF(OUTPUT_CSC_C11_C12, OUTPUT_CSC_C11, mask_sh),\ + XFM_SF(OUTPUT_CSC_C11_C12, OUTPUT_CSC_C12, mask_sh),\ + XFM_SF(OUTPUT_CSC_CONTROL, OUTPUT_CSC_GRPH_MODE, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_START_CNTL, REGAMMA_CNTLA_EXP_REGION_START, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_START_CNTL, REGAMMA_CNTLA_EXP_REGION_START_SEGMENT, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_SLOPE_CNTL, REGAMMA_CNTLA_EXP_REGION_LINEAR_SLOPE, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_END_CNTL1, REGAMMA_CNTLA_EXP_REGION_END, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_END_CNTL2, REGAMMA_CNTLA_EXP_REGION_END_BASE, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_END_CNTL2, REGAMMA_CNTLA_EXP_REGION_END_SLOPE, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_REGION_0_1, REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_REGION_0_1, REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_REGION_0_1, REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, mask_sh),\ + XFM_SF(REGAMMA_CNTLA_REGION_0_1, REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\ + XFM_SF(REGAMMA_LUT_WRITE_EN_MASK, REGAMMA_LUT_WRITE_EN_MASK, mask_sh),\ + XFM_SF(REGAMMA_CONTROL, GRPH_REGAMMA_MODE, mask_sh),\ + XFM_SF(SCL_TAP_CONTROL, SCL_H_NUM_OF_TAPS, mask_sh), \ + XFM_SF(SCL_TAP_CONTROL, SCL_V_NUM_OF_TAPS, mask_sh), \ + XFM_SF(SCL_BYPASS_CONTROL, SCL_BYPASS_MODE, mask_sh), \ + XFM_SF(EXT_OVERSCAN_LEFT_RIGHT, EXT_OVERSCAN_LEFT, mask_sh), \ + XFM_SF(EXT_OVERSCAN_LEFT_RIGHT, EXT_OVERSCAN_RIGHT, mask_sh), \ + XFM_SF(EXT_OVERSCAN_TOP_BOTTOM, EXT_OVERSCAN_TOP, mask_sh), \ + XFM_SF(EXT_OVERSCAN_TOP_BOTTOM, EXT_OVERSCAN_BOTTOM, mask_sh), \ + XFM_SF(SCL_COEF_RAM_SELECT, SCL_C_RAM_FILTER_TYPE, mask_sh), \ + XFM_SF(SCL_COEF_RAM_SELECT, SCL_C_RAM_PHASE, mask_sh), \ + XFM_SF(SCL_COEF_RAM_SELECT, SCL_C_RAM_TAP_PAIR_IDX, mask_sh), \ + XFM_SF(SCL_COEF_RAM_TAP_DATA, SCL_C_RAM_EVEN_TAP_COEF_EN, mask_sh), \ + XFM_SF(SCL_COEF_RAM_TAP_DATA, SCL_C_RAM_EVEN_TAP_COEF, mask_sh), \ + XFM_SF(SCL_COEF_RAM_TAP_DATA, SCL_C_RAM_ODD_TAP_COEF_EN, mask_sh), \ + XFM_SF(SCL_COEF_RAM_TAP_DATA, SCL_C_RAM_ODD_TAP_COEF, mask_sh), \ + XFM_SF(VIEWPORT_START, VIEWPORT_X_START, mask_sh), \ + XFM_SF(VIEWPORT_START, VIEWPORT_Y_START, mask_sh), \ + XFM_SF(VIEWPORT_SIZE, VIEWPORT_HEIGHT, mask_sh), \ + XFM_SF(VIEWPORT_SIZE, VIEWPORT_WIDTH, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_SCALE_RATIO, SCL_H_SCALE_RATIO, mask_sh), \ + XFM_SF(SCL_VERT_FILTER_SCALE_RATIO, SCL_V_SCALE_RATIO, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_INIT_RGB_LUMA, SCL_H_INIT_INT_RGB_Y, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_INIT_RGB_LUMA, SCL_H_INIT_FRAC_RGB_Y, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_INIT_CHROMA, SCL_H_INIT_INT_CBCR, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_INIT_CHROMA, SCL_H_INIT_FRAC_CBCR, mask_sh), \ + XFM_SF(SCL_VERT_FILTER_INIT, SCL_V_INIT_INT, mask_sh), \ + XFM_SF(SCL_VERT_FILTER_INIT, SCL_V_INIT_FRAC, mask_sh), \ + XFM_SF(SCL_HORZ_FILTER_CONTROL, SCL_H_FILTER_PICK_NEAREST, mask_sh), \ + XFM_SF(SCL_VERT_FILTER_CONTROL, SCL_V_FILTER_PICK_NEAREST, mask_sh), \ + XFM_SF(DC_LB_MEMORY_SPLIT, DC_LB_MEMORY_CONFIG, mask_sh), \ + XFM_SF(DC_LB_MEM_SIZE, DC_LB_MEM_SIZE, mask_sh) +#endif + #define XFM_COMMON_MASK_SH_LIST_SOC_BASE(mask_sh) \ XFM_SF(DCP0_OUT_CLAMP_CONTROL_B_CB, OUT_CLAMP_MIN_B_CB, mask_sh), \ XFM_SF(DCP0_OUT_CLAMP_CONTROL_B_CB, OUT_CLAMP_MAX_B_CB, mask_sh), \ @@ -302,6 +441,7 @@ type DCP_RGB_RANDOM_ENABLE; \ type DCP_HIGHPASS_RANDOM_ENABLE; \ type DENORM_MODE; \ + type INTERLEAVE_EN; \ type PIXEL_DEPTH; \ type PIXEL_EXPAN_MODE; \ type GAMUT_REMAP_C11; \ @@ -365,12 +505,20 @@ type SCL_V_SCALE_RATIO; \ type SCL_H_INIT_INT; \ type SCL_H_INIT_FRAC; \ + type SCL_H_INIT_INT_RGB_Y; \ + type SCL_H_INIT_FRAC_RGB_Y; \ + type SCL_H_INIT_INT_CBCR; \ + type SCL_H_INIT_FRAC_CBCR; \ type SCL_V_INIT_INT; \ type SCL_V_INIT_FRAC; \ + type DC_LB_MEMORY_CONFIG; \ + type DC_LB_MEM_SIZE; \ type LB_MEMORY_CONFIG; \ type LB_MEMORY_SIZE; \ type SCL_V_2TAP_HARDCODE_COEF_EN; \ type SCL_H_2TAP_HARDCODE_COEF_EN; \ + type SCL_V_FILTER_PICK_NEAREST; \ + type SCL_H_FILTER_PICK_NEAREST; \ type SCL_COEF_UPDATE_COMPLETE; \ type ALPHA_EN @@ -383,6 +531,9 @@ struct dce_transform_mask { }; struct dce_transform_registers { +#if defined(CONFIG_DRM_AMD_DC_SI) + uint32_t DATA_FORMAT; +#endif uint32_t LB_DATA_FORMAT; uint32_t GAMUT_REMAP_CONTROL; uint32_t GAMUT_REMAP_C11_C12; @@ -438,8 +589,16 @@ struct dce_transform_registers { uint32_t SCL_HORZ_FILTER_SCALE_RATIO; uint32_t SCL_VERT_FILTER_SCALE_RATIO; uint32_t SCL_HORZ_FILTER_INIT; +#if defined(CONFIG_DRM_AMD_DC_SI) + uint32_t SCL_HORZ_FILTER_INIT_RGB_LUMA; + uint32_t SCL_HORZ_FILTER_INIT_CHROMA; +#endif uint32_t SCL_VERT_FILTER_INIT; uint32_t SCL_AUTOMATIC_MODE_CONTROL; +#if defined(CONFIG_DRM_AMD_DC_SI) + uint32_t DC_LB_MEMORY_SPLIT; + uint32_t DC_LB_MEM_SIZE; +#endif uint32_t LB_MEMORY_CTRL; uint32_t SCL_UPDATE; uint32_t SCL_F_SHARP_CONTROL; @@ -457,6 +616,16 @@ struct scl_ratios_inits { struct init_int_and_frac v_init; }; +#if defined(CONFIG_DRM_AMD_DC_SI) +struct sclh_ratios_inits { + uint32_t h_int_scale_ratio; + uint32_t v_int_scale_ratio; + struct init_int_and_frac h_init_luma; + struct init_int_and_frac h_init_chroma; + struct init_int_and_frac v_init; +}; +#endif + enum ram_filter_type { FILTER_TYPE_RGB_Y_VERTICAL = 0, /* 0 - RGB/Y Vertical filter */ FILTER_TYPE_CBCR_VERTICAL = 1, /* 1 - CbCr Vertical filter */ @@ -489,6 +658,15 @@ void dce_transform_construct(struct dce_transform *xfm_dce, const struct dce_transform_shift *xfm_shift, const struct dce_transform_mask *xfm_mask); +#if defined(CONFIG_DRM_AMD_DC_SI) +void dce60_transform_construct(struct dce_transform *xfm_dce, + struct dc_context *ctx, + uint32_t inst, + const struct dce_transform_registers *regs, + const struct dce_transform_shift *xfm_shift, + const struct dce_transform_mask *xfm_mask); +#endif + bool dce_transform_get_optimal_number_of_taps( struct transform *xfm, struct scaler_data *scl_data, diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index 49380ed3aeae..0603ddca7bd0 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -720,6 +720,7 @@ void dce110_edp_wait_for_hpd_ready( struct dc_context *ctx = link->ctx; struct graphics_object_id connector = link->link_enc->connector; struct gpio *hpd; + struct dc_sink *sink = link->local_sink; bool edp_hpd_high = false; uint32_t time_elapsed = 0; uint32_t timeout = power_up ? @@ -752,6 +753,14 @@ void dce110_edp_wait_for_hpd_ready( return; } + if (sink != NULL) { + if (sink->edid_caps.panel_patch.extra_t3_ms > 0) { + int extra_t3_in_ms = sink->edid_caps.panel_patch.extra_t3_ms; + + msleep(extra_t3_in_ms); + } + } + dal_gpio_open(hpd, GPIO_MODE_INTERRUPT); /* wait until timeout or panel detected */ @@ -842,6 +851,17 @@ void dce110_edp_power_control( cntl.coherent = false; cntl.lanes_number = LANE_COUNT_FOUR; cntl.hpd_sel = link->link_enc->hpd_source; + + if (ctx->dc->ctx->dmub_srv && + ctx->dc->debug.dmub_command_table) { + if (cntl.action == TRANSMITTER_CONTROL_POWER_ON) + bp_result = ctx->dc_bios->funcs->enable_lvtma_control(ctx->dc_bios, + LVTMA_CONTROL_POWER_ON); + else + bp_result = ctx->dc_bios->funcs->enable_lvtma_control(ctx->dc_bios, + LVTMA_CONTROL_POWER_OFF); + } + bp_result = link_transmitter_control(ctx->dc_bios, &cntl); if (!power_up) @@ -919,8 +939,21 @@ void dce110_edp_backlight_control( /*edp 1.2*/ if (cntl.action == TRANSMITTER_CONTROL_BACKLIGHT_ON) edp_receiver_ready_T7(link); + + if (ctx->dc->ctx->dmub_srv && + ctx->dc->debug.dmub_command_table) { + if (cntl.action == TRANSMITTER_CONTROL_BACKLIGHT_ON) + ctx->dc_bios->funcs->enable_lvtma_control(ctx->dc_bios, + LVTMA_CONTROL_LCD_BLON); + else + ctx->dc_bios->funcs->enable_lvtma_control(ctx->dc_bios, + LVTMA_CONTROL_LCD_BLOFF); + } + link_transmitter_control(ctx->dc_bios, &cntl); + + if (enable && link->dpcd_sink_ext_caps.bits.oled) msleep(OLED_POST_T7_DELAY); diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c index 5d83e8174005..0853bc9917c7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c @@ -1017,7 +1017,7 @@ enum dc_status dce112_add_stream_to_ctx( struct dc_state *new_ctx, struct dc_stream_state *dc_stream) { - enum dc_status result = DC_ERROR_UNEXPECTED; + enum dc_status result; result = resource_map_pool_resources(dc, new_ctx, dc_stream); diff --git a/drivers/gpu/drm/amd/display/dc/dce60/Makefile b/drivers/gpu/drm/amd/display/dc/dce60/Makefile new file mode 100644 index 000000000000..7036c3bd0f87 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/Makefile @@ -0,0 +1,34 @@ +# +# Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +# Makefile for the 'controller' sub-component of DAL. +# It provides the control and status of HW CRTC block. + +DCE60 = dce60_timing_generator.o dce60_hw_sequencer.o \ + dce60_resource.o + +AMD_DAL_DCE60 = $(addprefix $(AMDDALPATH)/dc/dce60/,$(DCE60)) + +AMD_DISPLAY_FILES += $(AMD_DAL_DCE60) + + + diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.c new file mode 100644 index 000000000000..920c7ae29d53 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.c @@ -0,0 +1,432 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dm_services.h" +#include "dc.h" +#include "core_types.h" +#include "dce60_hw_sequencer.h" + +#include "dce/dce_hwseq.h" +#include "dce110/dce110_hw_sequencer.h" +#include "dce100/dce100_hw_sequencer.h" + +/* include DCE6 register header files */ +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + +#define DC_LOGGER_INIT() + +/******************************************************************************* + * Private definitions + ******************************************************************************/ + +/***************************PIPE_CONTROL***********************************/ + +/* + * Check if FBC can be enabled + */ +static bool dce60_should_enable_fbc(struct dc *dc, + struct dc_state *context, + uint32_t *pipe_idx) +{ + uint32_t i; + struct pipe_ctx *pipe_ctx = NULL; + struct resource_context *res_ctx = &context->res_ctx; + unsigned int underlay_idx = dc->res_pool->underlay_pipe_index; + + + ASSERT(dc->fbc_compressor); + + /* FBC memory should be allocated */ + if (!dc->ctx->fbc_gpu_addr) + return false; + + /* Only supports single display */ + if (context->stream_count != 1) + return false; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + if (res_ctx->pipe_ctx[i].stream) { + + pipe_ctx = &res_ctx->pipe_ctx[i]; + + if (!pipe_ctx) + continue; + + /* fbc not applicable on underlay pipe */ + if (pipe_ctx->pipe_idx != underlay_idx) { + *pipe_idx = i; + break; + } + } + } + + if (i == dc->res_pool->pipe_count) + return false; + + if (!pipe_ctx->stream->link) + return false; + + /* Only supports eDP */ + if (pipe_ctx->stream->link->connector_signal != SIGNAL_TYPE_EDP) + return false; + + /* PSR should not be enabled */ + if (pipe_ctx->stream->link->psr_settings.psr_feature_enabled) + return false; + + /* Nothing to compress */ + if (!pipe_ctx->plane_state) + return false; + + /* Only for non-linear tiling */ + if (pipe_ctx->plane_state->tiling_info.gfx8.array_mode == DC_ARRAY_LINEAR_GENERAL) + return false; + + return true; +} + +/* + * Enable FBC + */ +static void dce60_enable_fbc( + struct dc *dc, + struct dc_state *context) +{ + uint32_t pipe_idx = 0; + + if (dce60_should_enable_fbc(dc, context, &pipe_idx)) { + /* Program GRPH COMPRESSED ADDRESS and PITCH */ + struct compr_addr_and_pitch_params params = {0, 0, 0}; + struct compressor *compr = dc->fbc_compressor; + struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[pipe_idx]; + + params.source_view_width = pipe_ctx->stream->timing.h_addressable; + params.source_view_height = pipe_ctx->stream->timing.v_addressable; + params.inst = pipe_ctx->stream_res.tg->inst; + compr->compr_surface_address.quad_part = dc->ctx->fbc_gpu_addr; + + compr->funcs->surface_address_and_pitch(compr, ¶ms); + compr->funcs->set_fbc_invalidation_triggers(compr, 1); + + compr->funcs->enable_fbc(compr, ¶ms); + } +} + + +/******************************************************************************* + * Front End programming + ******************************************************************************/ + +static void dce60_set_default_colors(struct pipe_ctx *pipe_ctx) +{ + struct default_adjustment default_adjust = { 0 }; + + default_adjust.force_hw_default = false; + default_adjust.in_color_space = pipe_ctx->plane_state->color_space; + default_adjust.out_color_space = pipe_ctx->stream->output_color_space; + default_adjust.csc_adjust_type = GRAPHICS_CSC_ADJUST_TYPE_SW; + default_adjust.surface_pixel_format = pipe_ctx->plane_res.scl_data.format; + + /* display color depth */ + default_adjust.color_depth = + pipe_ctx->stream->timing.display_color_depth; + + /* Lb color depth */ + default_adjust.lb_color_depth = pipe_ctx->plane_res.scl_data.lb_params.depth; + + pipe_ctx->plane_res.xfm->funcs->opp_set_csc_default( + pipe_ctx->plane_res.xfm, &default_adjust); +} + +/******************************************************************************* + * In order to turn on surface we will program + * CRTC + * + * DCE6 has no bottom_pipe and no Blender HW + * We need to set 'blank_target' to false in order to turn on the display + * + * |-----------|------------|---------| + * |curr pipe | set_blank | | + * |Surface |blank_target| CRCT | + * |visibility | argument | | + * |-----------|------------|---------| + * | off | true | blank | + * | on | false | unblank | + * |-----------|------------|---------| + * + ******************************************************************************/ +static void dce60_program_surface_visibility(const struct dc *dc, + struct pipe_ctx *pipe_ctx) +{ + bool blank_target = false; + + /* DCE6 has no bottom_pipe and no Blender HW */ + + if (!pipe_ctx->plane_state->visible) + blank_target = true; + + /* DCE6 skip dce_set_blender_mode() but then proceed to 'unblank' CRTC */ + pipe_ctx->stream_res.tg->funcs->set_blank(pipe_ctx->stream_res.tg, blank_target); + +} + + +static void dce60_get_surface_visual_confirm_color(const struct pipe_ctx *pipe_ctx, + struct tg_color *color) +{ + uint32_t color_value = MAX_TG_COLOR_VALUE * (4 - pipe_ctx->stream_res.tg->inst) / 4; + + switch (pipe_ctx->plane_res.scl_data.format) { + case PIXEL_FORMAT_ARGB8888: + /* set boarder color to red */ + color->color_r_cr = color_value; + break; + + case PIXEL_FORMAT_ARGB2101010: + /* set boarder color to blue */ + color->color_b_cb = color_value; + break; + case PIXEL_FORMAT_420BPP8: + /* set boarder color to green */ + color->color_g_y = color_value; + break; + case PIXEL_FORMAT_420BPP10: + /* set boarder color to yellow */ + color->color_g_y = color_value; + color->color_r_cr = color_value; + break; + case PIXEL_FORMAT_FP16: + /* set boarder color to white */ + color->color_r_cr = color_value; + color->color_b_cb = color_value; + color->color_g_y = color_value; + break; + default: + break; + } +} + +static void dce60_program_scaler(const struct dc *dc, + const struct pipe_ctx *pipe_ctx) +{ + struct tg_color color = {0}; + + /* DCE6 skips DCN TOFPGA check for transform_set_pixel_storage_depth == NULL */ + + if (dc->debug.visual_confirm == VISUAL_CONFIRM_SURFACE) + dce60_get_surface_visual_confirm_color(pipe_ctx, &color); + else + color_space_to_black_color(dc, + pipe_ctx->stream->output_color_space, + &color); + + pipe_ctx->plane_res.xfm->funcs->transform_set_pixel_storage_depth( + pipe_ctx->plane_res.xfm, + pipe_ctx->plane_res.scl_data.lb_params.depth, + &pipe_ctx->stream->bit_depth_params); + + if (pipe_ctx->stream_res.tg->funcs->set_overscan_blank_color) { + /* + * The way 420 is packed, 2 channels carry Y component, 1 channel + * alternate between Cb and Cr, so both channels need the pixel + * value for Y + */ + if (pipe_ctx->stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) + color.color_r_cr = color.color_g_y; + + pipe_ctx->stream_res.tg->funcs->set_overscan_blank_color( + pipe_ctx->stream_res.tg, + &color); + } + + pipe_ctx->plane_res.xfm->funcs->transform_set_scaler(pipe_ctx->plane_res.xfm, + &pipe_ctx->plane_res.scl_data); +} + +static void +dce60_program_front_end_for_pipe( + struct dc *dc, struct pipe_ctx *pipe_ctx) +{ + struct mem_input *mi = pipe_ctx->plane_res.mi; + struct dc_plane_state *plane_state = pipe_ctx->plane_state; + struct xfm_grph_csc_adjustment adjust; + struct out_csc_color_matrix tbl_entry; + unsigned int i; + struct dce_hwseq *hws = dc->hwseq; + + DC_LOGGER_INIT(); + memset(&tbl_entry, 0, sizeof(tbl_entry)); + + memset(&adjust, 0, sizeof(adjust)); + adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS; + + dce_enable_fe_clock(dc->hwseq, mi->inst, true); + + dce60_set_default_colors(pipe_ctx); + if (pipe_ctx->stream->csc_color_matrix.enable_adjustment + == true) { + tbl_entry.color_space = + pipe_ctx->stream->output_color_space; + + for (i = 0; i < 12; i++) + tbl_entry.regval[i] = + pipe_ctx->stream->csc_color_matrix.matrix[i]; + + pipe_ctx->plane_res.xfm->funcs->opp_set_csc_adjustment + (pipe_ctx->plane_res.xfm, &tbl_entry); + } + + if (pipe_ctx->stream->gamut_remap_matrix.enable_remap == true) { + adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_SW; + + for (i = 0; i < CSC_TEMPERATURE_MATRIX_SIZE; i++) + adjust.temperature_matrix[i] = + pipe_ctx->stream->gamut_remap_matrix.matrix[i]; + } + + pipe_ctx->plane_res.xfm->funcs->transform_set_gamut_remap(pipe_ctx->plane_res.xfm, &adjust); + + pipe_ctx->plane_res.scl_data.lb_params.alpha_en = pipe_ctx->bottom_pipe != 0; + + dce60_program_scaler(dc, pipe_ctx); + + mi->funcs->mem_input_program_surface_config( + mi, + plane_state->format, + &plane_state->tiling_info, + &plane_state->plane_size, + plane_state->rotation, + NULL, + false); + if (mi->funcs->set_blank) + mi->funcs->set_blank(mi, pipe_ctx->plane_state->visible); + + if (dc->config.gpu_vm_support) + mi->funcs->mem_input_program_pte_vm( + pipe_ctx->plane_res.mi, + plane_state->format, + &plane_state->tiling_info, + plane_state->rotation); + + /* Moved programming gamma from dc to hwss */ + if (pipe_ctx->plane_state->update_flags.bits.full_update || + pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change || + pipe_ctx->plane_state->update_flags.bits.gamma_change) + hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state); + + if (pipe_ctx->plane_state->update_flags.bits.full_update) + hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream); + + DC_LOG_SURFACE( + "Pipe:%d %p: addr hi:0x%x, " + "addr low:0x%x, " + "src: %d, %d, %d," + " %d; dst: %d, %d, %d, %d;" + "clip: %d, %d, %d, %d\n", + pipe_ctx->pipe_idx, + (void *) pipe_ctx->plane_state, + pipe_ctx->plane_state->address.grph.addr.high_part, + pipe_ctx->plane_state->address.grph.addr.low_part, + pipe_ctx->plane_state->src_rect.x, + pipe_ctx->plane_state->src_rect.y, + pipe_ctx->plane_state->src_rect.width, + pipe_ctx->plane_state->src_rect.height, + pipe_ctx->plane_state->dst_rect.x, + pipe_ctx->plane_state->dst_rect.y, + pipe_ctx->plane_state->dst_rect.width, + pipe_ctx->plane_state->dst_rect.height, + pipe_ctx->plane_state->clip_rect.x, + pipe_ctx->plane_state->clip_rect.y, + pipe_ctx->plane_state->clip_rect.width, + pipe_ctx->plane_state->clip_rect.height); + + DC_LOG_SURFACE( + "Pipe %d: width, height, x, y\n" + "viewport:%d, %d, %d, %d\n" + "recout: %d, %d, %d, %d\n", + pipe_ctx->pipe_idx, + pipe_ctx->plane_res.scl_data.viewport.width, + pipe_ctx->plane_res.scl_data.viewport.height, + pipe_ctx->plane_res.scl_data.viewport.x, + pipe_ctx->plane_res.scl_data.viewport.y, + pipe_ctx->plane_res.scl_data.recout.width, + pipe_ctx->plane_res.scl_data.recout.height, + pipe_ctx->plane_res.scl_data.recout.x, + pipe_ctx->plane_res.scl_data.recout.y); +} + +static void dce60_apply_ctx_for_surface( + struct dc *dc, + const struct dc_stream_state *stream, + int num_planes, + struct dc_state *context) +{ + int i; + + if (num_planes == 0) + return; + + if (dc->fbc_compressor) + dc->fbc_compressor->funcs->disable_fbc(dc->fbc_compressor); + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; + + if (pipe_ctx->stream != stream) + continue; + + /* Need to allocate mem before program front end for Fiji */ + pipe_ctx->plane_res.mi->funcs->allocate_mem_input( + pipe_ctx->plane_res.mi, + pipe_ctx->stream->timing.h_total, + pipe_ctx->stream->timing.v_total, + pipe_ctx->stream->timing.pix_clk_100hz / 10, + context->stream_count); + + dce60_program_front_end_for_pipe(dc, pipe_ctx); + + dc->hwss.update_plane_addr(dc, pipe_ctx); + + dce60_program_surface_visibility(dc, pipe_ctx); + + } + + if (dc->fbc_compressor) + dce60_enable_fbc(dc, context); +} + +void dce60_hw_sequencer_construct(struct dc *dc) +{ + dce110_hw_sequencer_construct(dc); + + dc->hwseq->funcs.enable_display_power_gating = dce100_enable_display_power_gating; + dc->hwss.apply_ctx_for_surface = dce60_apply_ctx_for_surface; + dc->hwss.cursor_lock = dce60_pipe_control_lock; + dc->hwss.pipe_control_lock = dce60_pipe_control_lock; + dc->hwss.prepare_bandwidth = dce100_prepare_bandwidth; + dc->hwss.optimize_bandwidth = dce100_optimize_bandwidth; +} + diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.h new file mode 100644 index 000000000000..f3b2d8b60d5b --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_hw_sequencer.h @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DC_HWSS_DCE60_H__ +#define __DC_HWSS_DCE60_H__ + +#include "core_types.h" +#include "hw_sequencer_private.h" + +struct dc; + +void dce60_hw_sequencer_construct(struct dc *dc); + +#endif /* __DC_HWSS_DCE60_H__ */ + diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c new file mode 100644 index 000000000000..5a5a9cb77acb --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c @@ -0,0 +1,1527 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include <linux/slab.h> + +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + +#include "dm_services.h" + +#include "link_encoder.h" +#include "stream_encoder.h" + +#include "resource.h" +#include "include/irq_service_interface.h" +#include "irq/dce60/irq_service_dce60.h" +#include "dce110/dce110_timing_generator.h" +#include "dce110/dce110_resource.h" +#include "dce60/dce60_timing_generator.h" +#include "dce/dce_mem_input.h" +#include "dce/dce_link_encoder.h" +#include "dce/dce_stream_encoder.h" +#include "dce/dce_ipp.h" +#include "dce/dce_transform.h" +#include "dce/dce_opp.h" +#include "dce/dce_clock_source.h" +#include "dce/dce_audio.h" +#include "dce/dce_hwseq.h" +#include "dce60/dce60_hw_sequencer.h" +#include "dce100/dce100_resource.h" +#include "dce/dce_panel_cntl.h" + +#include "reg_helper.h" + +#include "dce/dce_dmcu.h" +#include "dce/dce_aux.h" +#include "dce/dce_abm.h" +#include "dce/dce_i2c.h" +/* TODO remove this include */ + +#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT +#include "gmc/gmc_6_0_d.h" +#include "gmc/gmc_6_0_sh_mask.h" +#endif + +#ifndef mmDP_DPHY_INTERNAL_CTRL +#define mmDP_DPHY_INTERNAL_CTRL 0x1CDE +#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x1CDE +#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x1FDE +#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x42DE +#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x45DE +#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x48DE +#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4BDE +#endif + + +#ifndef mmBIOS_SCRATCH_2 + #define mmBIOS_SCRATCH_2 0x05CB + #define mmBIOS_SCRATCH_3 0x05CC + #define mmBIOS_SCRATCH_6 0x05CF +#endif + +#ifndef mmDP_DPHY_FAST_TRAINING + #define mmDP_DPHY_FAST_TRAINING 0x1CCE + #define mmDP0_DP_DPHY_FAST_TRAINING 0x1CCE + #define mmDP1_DP_DPHY_FAST_TRAINING 0x1FCE + #define mmDP2_DP_DPHY_FAST_TRAINING 0x42CE + #define mmDP3_DP_DPHY_FAST_TRAINING 0x45CE + #define mmDP4_DP_DPHY_FAST_TRAINING 0x48CE + #define mmDP5_DP_DPHY_FAST_TRAINING 0x4BCE +#endif + + +#ifndef mmHPD_DC_HPD_CONTROL + #define mmHPD_DC_HPD_CONTROL 0x189A + #define mmHPD0_DC_HPD_CONTROL 0x189A + #define mmHPD1_DC_HPD_CONTROL 0x18A2 + #define mmHPD2_DC_HPD_CONTROL 0x18AA + #define mmHPD3_DC_HPD_CONTROL 0x18B2 + #define mmHPD4_DC_HPD_CONTROL 0x18BA + #define mmHPD5_DC_HPD_CONTROL 0x18C2 +#endif + +#define DCE11_DIG_FE_CNTL 0x4a00 +#define DCE11_DIG_BE_CNTL 0x4a47 +#define DCE11_DP_SEC 0x4ac3 + +static const struct dce110_timing_generator_offsets dce60_tg_offsets[] = { + { + .crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmGRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + }, + { + .crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + }, + { + .crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + }, + { + .crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + }, + { + .crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + }, + { + .crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL), + .dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL), + .dmif = (mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL3 + - mmDPG_PIPE_ARBITRATION_CONTROL3), + } +}; + +/* set register offset */ +#define SR(reg_name)\ + .reg_name = mm ## reg_name + +/* set register offset with instance */ +#define SRI(reg_name, block, id)\ + .reg_name = mm ## block ## id ## _ ## reg_name + +#define ipp_regs(id)\ +[id] = {\ + IPP_COMMON_REG_LIST_DCE_BASE(id)\ +} + +static const struct dce_ipp_registers ipp_regs[] = { + ipp_regs(0), + ipp_regs(1), + ipp_regs(2), + ipp_regs(3), + ipp_regs(4), + ipp_regs(5) +}; + +static const struct dce_ipp_shift ipp_shift = { + IPP_DCE60_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT) +}; + +static const struct dce_ipp_mask ipp_mask = { + IPP_DCE60_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) +}; + +#define transform_regs(id)\ +[id] = {\ + XFM_COMMON_REG_LIST_DCE60(id)\ +} + +static const struct dce_transform_registers xfm_regs[] = { + transform_regs(0), + transform_regs(1), + transform_regs(2), + transform_regs(3), + transform_regs(4), + transform_regs(5) +}; + +static const struct dce_transform_shift xfm_shift = { + XFM_COMMON_MASK_SH_LIST_DCE60(__SHIFT) +}; + +static const struct dce_transform_mask xfm_mask = { + XFM_COMMON_MASK_SH_LIST_DCE60(_MASK) +}; + +#define aux_regs(id)\ +[id] = {\ + AUX_REG_LIST(id)\ +} + +static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = { + aux_regs(0), + aux_regs(1), + aux_regs(2), + aux_regs(3), + aux_regs(4), + aux_regs(5) +}; + +#define hpd_regs(id)\ +[id] = {\ + HPD_REG_LIST(id)\ +} + +static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = { + hpd_regs(0), + hpd_regs(1), + hpd_regs(2), + hpd_regs(3), + hpd_regs(4), + hpd_regs(5) +}; + +#define link_regs(id)\ +[id] = {\ + LE_DCE60_REG_LIST(id)\ +} + +static const struct dce110_link_enc_registers link_enc_regs[] = { + link_regs(0), + link_regs(1), + link_regs(2), + link_regs(3), + link_regs(4), + link_regs(5) +}; + +#define stream_enc_regs(id)\ +[id] = {\ + SE_COMMON_REG_LIST_DCE_BASE(id),\ + .AFMT_CNTL = 0,\ +} + +static const struct dce110_stream_enc_registers stream_enc_regs[] = { + stream_enc_regs(0), + stream_enc_regs(1), + stream_enc_regs(2), + stream_enc_regs(3), + stream_enc_regs(4), + stream_enc_regs(5) +}; + +static const struct dce_stream_encoder_shift se_shift = { + SE_COMMON_MASK_SH_LIST_DCE80_100(__SHIFT) +}; + +static const struct dce_stream_encoder_mask se_mask = { + SE_COMMON_MASK_SH_LIST_DCE80_100(_MASK) +}; + +static const struct dce_panel_cntl_registers panel_cntl_regs[] = { + { DCE_PANEL_CNTL_REG_LIST() } +}; + +static const struct dce_panel_cntl_shift panel_cntl_shift = { + DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT) +}; + +static const struct dce_panel_cntl_mask panel_cntl_mask = { + DCE_PANEL_CNTL_MASK_SH_LIST(_MASK) +}; + +#define opp_regs(id)\ +[id] = {\ + OPP_DCE_60_REG_LIST(id),\ +} + +static const struct dce_opp_registers opp_regs[] = { + opp_regs(0), + opp_regs(1), + opp_regs(2), + opp_regs(3), + opp_regs(4), + opp_regs(5) +}; + +static const struct dce_opp_shift opp_shift = { + OPP_COMMON_MASK_SH_LIST_DCE_60(__SHIFT) +}; + +static const struct dce_opp_mask opp_mask = { + OPP_COMMON_MASK_SH_LIST_DCE_60(_MASK) +}; + +static const struct dce110_aux_registers_shift aux_shift = { + DCE10_AUX_MASK_SH_LIST(__SHIFT) +}; + +static const struct dce110_aux_registers_mask aux_mask = { + DCE10_AUX_MASK_SH_LIST(_MASK) +}; + +#define aux_engine_regs(id)\ +[id] = {\ + AUX_COMMON_REG_LIST(id), \ + .AUX_RESET_MASK = 0 \ +} + +static const struct dce110_aux_registers aux_engine_regs[] = { + aux_engine_regs(0), + aux_engine_regs(1), + aux_engine_regs(2), + aux_engine_regs(3), + aux_engine_regs(4), + aux_engine_regs(5) +}; + +#define audio_regs(id)\ +[id] = {\ + AUD_COMMON_REG_LIST(id)\ +} + +static const struct dce_audio_registers audio_regs[] = { + audio_regs(0), + audio_regs(1), + audio_regs(2), + audio_regs(3), + audio_regs(4), + audio_regs(5), +}; + +static const struct dce_audio_shift audio_shift = { + AUD_DCE60_MASK_SH_LIST(__SHIFT) +}; + +static const struct dce_audio_mask audio_mask = { + AUD_DCE60_MASK_SH_LIST(_MASK) +}; + +#define clk_src_regs(id)\ +[id] = {\ + CS_COMMON_REG_LIST_DCE_80(id),\ +} + + +static const struct dce110_clk_src_regs clk_src_regs[] = { + clk_src_regs(0), + clk_src_regs(1), + clk_src_regs(2) +}; + +static const struct dce110_clk_src_shift cs_shift = { + CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT) +}; + +static const struct dce110_clk_src_mask cs_mask = { + CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) +}; + +static const struct bios_registers bios_regs = { + .BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3, + .BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6 +}; + +static const struct resource_caps res_cap = { + .num_timing_generator = 6, + .num_audio = 6, + .num_stream_encoder = 6, + .num_pll = 2, + .num_ddc = 6, +}; + +static const struct resource_caps res_cap_61 = { + .num_timing_generator = 4, + .num_audio = 6, + .num_stream_encoder = 6, + .num_pll = 3, + .num_ddc = 6, +}; + +static const struct resource_caps res_cap_64 = { + .num_timing_generator = 2, + .num_audio = 2, + .num_stream_encoder = 2, + .num_pll = 2, + .num_ddc = 2, +}; + +static const struct dc_plane_cap plane_cap = { + .type = DC_PLANE_TYPE_DCE_RGB, + + .pixel_format_support = { + .argb8888 = true, + .nv12 = false, + .fp16 = false + }, + + .max_upscale_factor = { + .argb8888 = 16000, + .nv12 = 1, + .fp16 = 1 + }, + + .max_downscale_factor = { + .argb8888 = 250, + .nv12 = 1, + .fp16 = 1 + } +}; + +static const struct dce_dmcu_registers dmcu_regs = { + DMCU_DCE60_REG_LIST() +}; + +static const struct dce_dmcu_shift dmcu_shift = { + DMCU_MASK_SH_LIST_DCE60(__SHIFT) +}; + +static const struct dce_dmcu_mask dmcu_mask = { + DMCU_MASK_SH_LIST_DCE60(_MASK) +}; +static const struct dce_abm_registers abm_regs = { + ABM_DCE110_COMMON_REG_LIST() +}; + +static const struct dce_abm_shift abm_shift = { + ABM_MASK_SH_LIST_DCE110(__SHIFT) +}; + +static const struct dce_abm_mask abm_mask = { + ABM_MASK_SH_LIST_DCE110(_MASK) +}; + +#define CTX ctx +#define REG(reg) mm ## reg + +#ifndef mmCC_DC_HDMI_STRAPS +#define mmCC_DC_HDMI_STRAPS 0x1918 +#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40 +#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6 +#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700 +#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8 +#endif + +static int map_transmitter_id_to_phy_instance( + enum transmitter transmitter) +{ + switch (transmitter) { + case TRANSMITTER_UNIPHY_A: + return 0; + break; + case TRANSMITTER_UNIPHY_B: + return 1; + break; + case TRANSMITTER_UNIPHY_C: + return 2; + break; + case TRANSMITTER_UNIPHY_D: + return 3; + break; + case TRANSMITTER_UNIPHY_E: + return 4; + break; + case TRANSMITTER_UNIPHY_F: + return 5; + break; + case TRANSMITTER_UNIPHY_G: + return 6; + break; + default: + ASSERT(0); + return 0; + } +} + +static void read_dce_straps( + struct dc_context *ctx, + struct resource_straps *straps) +{ + REG_GET_2(CC_DC_HDMI_STRAPS, + HDMI_DISABLE, &straps->hdmi_disable, + AUDIO_STREAM_NUMBER, &straps->audio_stream_number); + + REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio); +} + +static struct audio *create_audio( + struct dc_context *ctx, unsigned int inst) +{ + return dce60_audio_create(ctx, inst, + &audio_regs[inst], &audio_shift, &audio_mask); +} + +static struct timing_generator *dce60_timing_generator_create( + struct dc_context *ctx, + uint32_t instance, + const struct dce110_timing_generator_offsets *offsets) +{ + struct dce110_timing_generator *tg110 = + kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL); + + if (!tg110) + return NULL; + + dce60_timing_generator_construct(tg110, ctx, instance, offsets); + return &tg110->base; +} + +static struct output_pixel_processor *dce60_opp_create( + struct dc_context *ctx, + uint32_t inst) +{ + struct dce110_opp *opp = + kzalloc(sizeof(struct dce110_opp), GFP_KERNEL); + + if (!opp) + return NULL; + + dce60_opp_construct(opp, + ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask); + return &opp->base; +} + +struct dce_aux *dce60_aux_engine_create( + struct dc_context *ctx, + uint32_t inst) +{ + struct aux_engine_dce110 *aux_engine = + kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL); + + if (!aux_engine) + return NULL; + + dce110_aux_engine_construct(aux_engine, ctx, inst, + SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD, + &aux_engine_regs[inst], + &aux_mask, + &aux_shift, + ctx->dc->caps.extended_aux_timeout_support); + + return &aux_engine->base; +} +#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) } + +static const struct dce_i2c_registers i2c_hw_regs[] = { + i2c_inst_regs(1), + i2c_inst_regs(2), + i2c_inst_regs(3), + i2c_inst_regs(4), + i2c_inst_regs(5), + i2c_inst_regs(6), +}; + +static const struct dce_i2c_shift i2c_shifts = { + I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT) +}; + +static const struct dce_i2c_mask i2c_masks = { + I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) +}; + +struct dce_i2c_hw *dce60_i2c_hw_create( + struct dc_context *ctx, + uint32_t inst) +{ + struct dce_i2c_hw *dce_i2c_hw = + kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL); + + if (!dce_i2c_hw) + return NULL; + + dce_i2c_hw_construct(dce_i2c_hw, ctx, inst, + &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks); + + return dce_i2c_hw; +} + +struct dce_i2c_sw *dce60_i2c_sw_create( + struct dc_context *ctx) +{ + struct dce_i2c_sw *dce_i2c_sw = + kzalloc(sizeof(struct dce_i2c_sw), GFP_KERNEL); + + if (!dce_i2c_sw) + return NULL; + + dce_i2c_sw_construct(dce_i2c_sw, ctx); + + return dce_i2c_sw; +} +static struct stream_encoder *dce60_stream_encoder_create( + enum engine_id eng_id, + struct dc_context *ctx) +{ + struct dce110_stream_encoder *enc110 = + kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL); + + if (!enc110) + return NULL; + + dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], + &se_shift, &se_mask); + return &enc110->base; +} + +#define SRII(reg_name, block, id)\ + .reg_name[id] = mm ## block ## id ## _ ## reg_name + +static const struct dce_hwseq_registers hwseq_reg = { + HWSEQ_DCE6_REG_LIST() +}; + +static const struct dce_hwseq_shift hwseq_shift = { + HWSEQ_DCE6_MASK_SH_LIST(__SHIFT) +}; + +static const struct dce_hwseq_mask hwseq_mask = { + HWSEQ_DCE6_MASK_SH_LIST(_MASK) +}; + +static struct dce_hwseq *dce60_hwseq_create( + struct dc_context *ctx) +{ + struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL); + + if (hws) { + hws->ctx = ctx; + hws->regs = &hwseq_reg; + hws->shifts = &hwseq_shift; + hws->masks = &hwseq_mask; + } + return hws; +} + +static const struct resource_create_funcs res_create_funcs = { + .read_dce_straps = read_dce_straps, + .create_audio = create_audio, + .create_stream_encoder = dce60_stream_encoder_create, + .create_hwseq = dce60_hwseq_create, +}; + +#define mi_inst_regs(id) { \ + MI_DCE6_REG_LIST(id), \ + .MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \ +} +static const struct dce_mem_input_registers mi_regs[] = { + mi_inst_regs(0), + mi_inst_regs(1), + mi_inst_regs(2), + mi_inst_regs(3), + mi_inst_regs(4), + mi_inst_regs(5), +}; + +static const struct dce_mem_input_shift mi_shifts = { + MI_DCE6_MASK_SH_LIST(__SHIFT), + .ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT +}; + +static const struct dce_mem_input_mask mi_masks = { + MI_DCE6_MASK_SH_LIST(_MASK), + .ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK +}; + +static struct mem_input *dce60_mem_input_create( + struct dc_context *ctx, + uint32_t inst) +{ + struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input), + GFP_KERNEL); + + if (!dce_mi) { + BREAK_TO_DEBUGGER(); + return NULL; + } + + dce60_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks); + dce_mi->wa.single_head_rdreq_dmif_limit = 2; + return &dce_mi->base; +} + +static void dce60_transform_destroy(struct transform **xfm) +{ + kfree(TO_DCE_TRANSFORM(*xfm)); + *xfm = NULL; +} + +static struct transform *dce60_transform_create( + struct dc_context *ctx, + uint32_t inst) +{ + struct dce_transform *transform = + kzalloc(sizeof(struct dce_transform), GFP_KERNEL); + + if (!transform) + return NULL; + + dce60_transform_construct(transform, ctx, inst, + &xfm_regs[inst], &xfm_shift, &xfm_mask); + transform->prescaler_on = false; + return &transform->base; +} + +static const struct encoder_feature_support link_enc_feature = { + .max_hdmi_deep_color = COLOR_DEPTH_121212, + .max_hdmi_pixel_clock = 297000, + .flags.bits.IS_HBR2_CAPABLE = true, + .flags.bits.IS_TPS3_CAPABLE = true +}; + +struct link_encoder *dce60_link_encoder_create( + const struct encoder_init_data *enc_init_data) +{ + struct dce110_link_encoder *enc110 = + kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL); + int link_regs_id; + + if (!enc110) + return NULL; + + link_regs_id = + map_transmitter_id_to_phy_instance(enc_init_data->transmitter); + + dce60_link_encoder_construct(enc110, + enc_init_data, + &link_enc_feature, + &link_enc_regs[link_regs_id], + &link_enc_aux_regs[enc_init_data->channel - 1], + &link_enc_hpd_regs[enc_init_data->hpd_source]); + return &enc110->base; +} + +static struct panel_cntl *dce60_panel_cntl_create(const struct panel_cntl_init_data *init_data) +{ + struct dce_panel_cntl *panel_cntl = + kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL); + + if (!panel_cntl) + return NULL; + + dce_panel_cntl_construct(panel_cntl, + init_data, + &panel_cntl_regs[init_data->inst], + &panel_cntl_shift, + &panel_cntl_mask); + + return &panel_cntl->base; +} + +struct clock_source *dce60_clock_source_create( + struct dc_context *ctx, + struct dc_bios *bios, + enum clock_source_id id, + const struct dce110_clk_src_regs *regs, + bool dp_clk_src) +{ + struct dce110_clk_src *clk_src = + kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL); + + if (!clk_src) + return NULL; + + if (dce110_clk_src_construct(clk_src, ctx, bios, id, + regs, &cs_shift, &cs_mask)) { + clk_src->base.dp_clk_src = dp_clk_src; + return &clk_src->base; + } + + kfree(clk_src); + BREAK_TO_DEBUGGER(); + return NULL; +} + +void dce60_clock_source_destroy(struct clock_source **clk_src) +{ + kfree(TO_DCE110_CLK_SRC(*clk_src)); + *clk_src = NULL; +} + +static struct input_pixel_processor *dce60_ipp_create( + struct dc_context *ctx, uint32_t inst) +{ + struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL); + + if (!ipp) { + BREAK_TO_DEBUGGER(); + return NULL; + } + + dce60_ipp_construct(ipp, ctx, inst, + &ipp_regs[inst], &ipp_shift, &ipp_mask); + return &ipp->base; +} + +static void dce60_resource_destruct(struct dce110_resource_pool *pool) +{ + unsigned int i; + + for (i = 0; i < pool->base.pipe_count; i++) { + if (pool->base.opps[i] != NULL) + dce110_opp_destroy(&pool->base.opps[i]); + + if (pool->base.transforms[i] != NULL) + dce60_transform_destroy(&pool->base.transforms[i]); + + if (pool->base.ipps[i] != NULL) + dce_ipp_destroy(&pool->base.ipps[i]); + + if (pool->base.mis[i] != NULL) { + kfree(TO_DCE_MEM_INPUT(pool->base.mis[i])); + pool->base.mis[i] = NULL; + } + + if (pool->base.timing_generators[i] != NULL) { + kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i])); + pool->base.timing_generators[i] = NULL; + } + } + + for (i = 0; i < pool->base.res_cap->num_ddc; i++) { + if (pool->base.engines[i] != NULL) + dce110_engine_destroy(&pool->base.engines[i]); + if (pool->base.hw_i2cs[i] != NULL) { + kfree(pool->base.hw_i2cs[i]); + pool->base.hw_i2cs[i] = NULL; + } + if (pool->base.sw_i2cs[i] != NULL) { + kfree(pool->base.sw_i2cs[i]); + pool->base.sw_i2cs[i] = NULL; + } + } + + for (i = 0; i < pool->base.stream_enc_count; i++) { + if (pool->base.stream_enc[i] != NULL) + kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i])); + } + + for (i = 0; i < pool->base.clk_src_count; i++) { + if (pool->base.clock_sources[i] != NULL) { + dce60_clock_source_destroy(&pool->base.clock_sources[i]); + } + } + + if (pool->base.abm != NULL) + dce_abm_destroy(&pool->base.abm); + + if (pool->base.dmcu != NULL) + dce_dmcu_destroy(&pool->base.dmcu); + + if (pool->base.dp_clock_source != NULL) + dce60_clock_source_destroy(&pool->base.dp_clock_source); + + for (i = 0; i < pool->base.audio_count; i++) { + if (pool->base.audios[i] != NULL) { + dce_aud_destroy(&pool->base.audios[i]); + } + } + + if (pool->base.irqs != NULL) { + dal_irq_service_destroy(&pool->base.irqs); + } +} + +bool dce60_validate_bandwidth( + struct dc *dc, + struct dc_state *context, + bool fast_validate) +{ + int i; + bool at_least_one_pipe = false; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + if (context->res_ctx.pipe_ctx[i].stream) + at_least_one_pipe = true; + } + + if (at_least_one_pipe) { + /* TODO implement when needed but for now hardcode max value*/ + context->bw_ctx.bw.dce.dispclk_khz = 681000; + context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ; + } else { + context->bw_ctx.bw.dce.dispclk_khz = 0; + context->bw_ctx.bw.dce.yclk_khz = 0; + } + + return true; +} + +static bool dce60_validate_surface_sets( + struct dc_state *context) +{ + int i; + + for (i = 0; i < context->stream_count; i++) { + if (context->stream_status[i].plane_count == 0) + continue; + + if (context->stream_status[i].plane_count > 1) + return false; + + if (context->stream_status[i].plane_states[0]->format + >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) + return false; + } + + return true; +} + +enum dc_status dce60_validate_global( + struct dc *dc, + struct dc_state *context) +{ + if (!dce60_validate_surface_sets(context)) + return DC_FAIL_SURFACE_VALIDATE; + + return DC_OK; +} + +static void dce60_destroy_resource_pool(struct resource_pool **pool) +{ + struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool); + + dce60_resource_destruct(dce110_pool); + kfree(dce110_pool); + *pool = NULL; +} + +static const struct resource_funcs dce60_res_pool_funcs = { + .destroy = dce60_destroy_resource_pool, + .link_enc_create = dce60_link_encoder_create, + .panel_cntl_create = dce60_panel_cntl_create, + .validate_bandwidth = dce60_validate_bandwidth, + .validate_plane = dce100_validate_plane, + .add_stream_to_ctx = dce100_add_stream_to_ctx, + .validate_global = dce60_validate_global, + .find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link +}; + +static bool dce60_construct( + uint8_t num_virtual_links, + struct dc *dc, + struct dce110_resource_pool *pool) +{ + unsigned int i; + struct dc_context *ctx = dc->ctx; + struct dc_bios *bp; + + ctx->dc_bios->regs = &bios_regs; + + pool->base.res_cap = &res_cap; + pool->base.funcs = &dce60_res_pool_funcs; + + + /************************************************* + * Resource + asic cap harcoding * + *************************************************/ + pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE; + pool->base.pipe_count = res_cap.num_timing_generator; + pool->base.timing_generator_count = res_cap.num_timing_generator; + dc->caps.max_downscale_ratio = 200; + dc->caps.i2c_speed_in_khz = 40; + dc->caps.max_cursor_size = 64; + dc->caps.dual_link_dvi = true; + dc->caps.extended_aux_timeout_support = false; + + /************************************************* + * Create resources * + *************************************************/ + + bp = ctx->dc_bios; + + if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clk_src_count = 2; + + } else { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clk_src_count = 1; + } + + if (pool->base.dp_clock_source == NULL) { + dm_error("DC: failed to create dp clock source!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + for (i = 0; i < pool->base.clk_src_count; i++) { + if (pool->base.clock_sources[i] == NULL) { + dm_error("DC: failed to create clock sources!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + } + + pool->base.dmcu = dce_dmcu_create(ctx, + &dmcu_regs, + &dmcu_shift, + &dmcu_mask); + if (pool->base.dmcu == NULL) { + dm_error("DC: failed to create dmcu!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + pool->base.abm = dce_abm_create(ctx, + &abm_regs, + &abm_shift, + &abm_mask); + if (pool->base.abm == NULL) { + dm_error("DC: failed to create abm!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + { + struct irq_service_init_data init_data; + init_data.ctx = dc->ctx; + pool->base.irqs = dal_irq_service_dce60_create(&init_data); + if (!pool->base.irqs) + goto res_create_fail; + } + + for (i = 0; i < pool->base.pipe_count; i++) { + pool->base.timing_generators[i] = dce60_timing_generator_create( + ctx, i, &dce60_tg_offsets[i]); + if (pool->base.timing_generators[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create tg!\n"); + goto res_create_fail; + } + + pool->base.mis[i] = dce60_mem_input_create(ctx, i); + if (pool->base.mis[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create memory input!\n"); + goto res_create_fail; + } + + pool->base.ipps[i] = dce60_ipp_create(ctx, i); + if (pool->base.ipps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create input pixel processor!\n"); + goto res_create_fail; + } + + pool->base.transforms[i] = dce60_transform_create(ctx, i); + if (pool->base.transforms[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create transform!\n"); + goto res_create_fail; + } + + pool->base.opps[i] = dce60_opp_create(ctx, i); + if (pool->base.opps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create output pixel processor!\n"); + goto res_create_fail; + } + } + + for (i = 0; i < pool->base.res_cap->num_ddc; i++) { + pool->base.engines[i] = dce60_aux_engine_create(ctx, i); + if (pool->base.engines[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create aux engine!!\n"); + goto res_create_fail; + } + pool->base.hw_i2cs[i] = dce60_i2c_hw_create(ctx, i); + if (pool->base.hw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create i2c engine!!\n"); + goto res_create_fail; + } + pool->base.sw_i2cs[i] = dce60_i2c_sw_create(ctx); + if (pool->base.sw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create sw i2c!!\n"); + goto res_create_fail; + } + } + + dc->caps.max_planes = pool->base.pipe_count; + + for (i = 0; i < dc->caps.max_planes; ++i) + dc->caps.planes[i] = plane_cap; + + dc->caps.disable_dp_clk_share = true; + + if (!resource_construct(num_virtual_links, dc, &pool->base, + &res_create_funcs)) + goto res_create_fail; + + /* Create hardware sequencer */ + dce60_hw_sequencer_construct(dc); + + return true; + +res_create_fail: + dce60_resource_destruct(pool); + return false; +} + +struct resource_pool *dce60_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc) +{ + struct dce110_resource_pool *pool = + kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL); + + if (!pool) + return NULL; + + if (dce60_construct(num_virtual_links, dc, pool)) + return &pool->base; + + BREAK_TO_DEBUGGER(); + return NULL; +} + +static bool dce61_construct( + uint8_t num_virtual_links, + struct dc *dc, + struct dce110_resource_pool *pool) +{ + unsigned int i; + struct dc_context *ctx = dc->ctx; + struct dc_bios *bp; + + ctx->dc_bios->regs = &bios_regs; + + pool->base.res_cap = &res_cap_61; + pool->base.funcs = &dce60_res_pool_funcs; + + + /************************************************* + * Resource + asic cap harcoding * + *************************************************/ + pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE; + pool->base.pipe_count = res_cap_61.num_timing_generator; + pool->base.timing_generator_count = res_cap_61.num_timing_generator; + dc->caps.max_downscale_ratio = 200; + dc->caps.i2c_speed_in_khz = 40; + dc->caps.max_cursor_size = 64; + dc->caps.is_apu = true; + + /************************************************* + * Create resources * + *************************************************/ + + bp = ctx->dc_bios; + + if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clock_sources[2] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); + pool->base.clk_src_count = 3; + + } else { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); + pool->base.clk_src_count = 2; + } + + if (pool->base.dp_clock_source == NULL) { + dm_error("DC: failed to create dp clock source!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + for (i = 0; i < pool->base.clk_src_count; i++) { + if (pool->base.clock_sources[i] == NULL) { + dm_error("DC: failed to create clock sources!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + } + + pool->base.dmcu = dce_dmcu_create(ctx, + &dmcu_regs, + &dmcu_shift, + &dmcu_mask); + if (pool->base.dmcu == NULL) { + dm_error("DC: failed to create dmcu!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + pool->base.abm = dce_abm_create(ctx, + &abm_regs, + &abm_shift, + &abm_mask); + if (pool->base.abm == NULL) { + dm_error("DC: failed to create abm!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + { + struct irq_service_init_data init_data; + init_data.ctx = dc->ctx; + pool->base.irqs = dal_irq_service_dce60_create(&init_data); + if (!pool->base.irqs) + goto res_create_fail; + } + + for (i = 0; i < pool->base.pipe_count; i++) { + pool->base.timing_generators[i] = dce60_timing_generator_create( + ctx, i, &dce60_tg_offsets[i]); + if (pool->base.timing_generators[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create tg!\n"); + goto res_create_fail; + } + + pool->base.mis[i] = dce60_mem_input_create(ctx, i); + if (pool->base.mis[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create memory input!\n"); + goto res_create_fail; + } + + pool->base.ipps[i] = dce60_ipp_create(ctx, i); + if (pool->base.ipps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create input pixel processor!\n"); + goto res_create_fail; + } + + pool->base.transforms[i] = dce60_transform_create(ctx, i); + if (pool->base.transforms[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create transform!\n"); + goto res_create_fail; + } + + pool->base.opps[i] = dce60_opp_create(ctx, i); + if (pool->base.opps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create output pixel processor!\n"); + goto res_create_fail; + } + } + + for (i = 0; i < pool->base.res_cap->num_ddc; i++) { + pool->base.engines[i] = dce60_aux_engine_create(ctx, i); + if (pool->base.engines[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create aux engine!!\n"); + goto res_create_fail; + } + pool->base.hw_i2cs[i] = dce60_i2c_hw_create(ctx, i); + if (pool->base.hw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create i2c engine!!\n"); + goto res_create_fail; + } + pool->base.sw_i2cs[i] = dce60_i2c_sw_create(ctx); + if (pool->base.sw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create sw i2c!!\n"); + goto res_create_fail; + } + } + + dc->caps.max_planes = pool->base.pipe_count; + + for (i = 0; i < dc->caps.max_planes; ++i) + dc->caps.planes[i] = plane_cap; + + dc->caps.disable_dp_clk_share = true; + + if (!resource_construct(num_virtual_links, dc, &pool->base, + &res_create_funcs)) + goto res_create_fail; + + /* Create hardware sequencer */ + dce60_hw_sequencer_construct(dc); + + return true; + +res_create_fail: + dce60_resource_destruct(pool); + return false; +} + +struct resource_pool *dce61_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc) +{ + struct dce110_resource_pool *pool = + kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL); + + if (!pool) + return NULL; + + if (dce61_construct(num_virtual_links, dc, pool)) + return &pool->base; + + BREAK_TO_DEBUGGER(); + return NULL; +} + +static bool dce64_construct( + uint8_t num_virtual_links, + struct dc *dc, + struct dce110_resource_pool *pool) +{ + unsigned int i; + struct dc_context *ctx = dc->ctx; + struct dc_bios *bp; + + ctx->dc_bios->regs = &bios_regs; + + pool->base.res_cap = &res_cap_64; + pool->base.funcs = &dce60_res_pool_funcs; + + + /************************************************* + * Resource + asic cap harcoding * + *************************************************/ + pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE; + pool->base.pipe_count = res_cap_64.num_timing_generator; + pool->base.timing_generator_count = res_cap_64.num_timing_generator; + dc->caps.max_downscale_ratio = 200; + dc->caps.i2c_speed_in_khz = 40; + dc->caps.max_cursor_size = 64; + dc->caps.is_apu = true; + + /************************************************* + * Create resources * + *************************************************/ + + bp = ctx->dc_bios; + + if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false); + pool->base.clk_src_count = 2; + + } else { + pool->base.dp_clock_source = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], true); + + pool->base.clock_sources[0] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false); + pool->base.clk_src_count = 1; + } + + if (pool->base.dp_clock_source == NULL) { + dm_error("DC: failed to create dp clock source!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + for (i = 0; i < pool->base.clk_src_count; i++) { + if (pool->base.clock_sources[i] == NULL) { + dm_error("DC: failed to create clock sources!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + } + + pool->base.dmcu = dce_dmcu_create(ctx, + &dmcu_regs, + &dmcu_shift, + &dmcu_mask); + if (pool->base.dmcu == NULL) { + dm_error("DC: failed to create dmcu!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + pool->base.abm = dce_abm_create(ctx, + &abm_regs, + &abm_shift, + &abm_mask); + if (pool->base.abm == NULL) { + dm_error("DC: failed to create abm!\n"); + BREAK_TO_DEBUGGER(); + goto res_create_fail; + } + + { + struct irq_service_init_data init_data; + init_data.ctx = dc->ctx; + pool->base.irqs = dal_irq_service_dce60_create(&init_data); + if (!pool->base.irqs) + goto res_create_fail; + } + + for (i = 0; i < pool->base.pipe_count; i++) { + pool->base.timing_generators[i] = dce60_timing_generator_create( + ctx, i, &dce60_tg_offsets[i]); + if (pool->base.timing_generators[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create tg!\n"); + goto res_create_fail; + } + + pool->base.mis[i] = dce60_mem_input_create(ctx, i); + if (pool->base.mis[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create memory input!\n"); + goto res_create_fail; + } + + pool->base.ipps[i] = dce60_ipp_create(ctx, i); + if (pool->base.ipps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create input pixel processor!\n"); + goto res_create_fail; + } + + pool->base.transforms[i] = dce60_transform_create(ctx, i); + if (pool->base.transforms[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create transform!\n"); + goto res_create_fail; + } + + pool->base.opps[i] = dce60_opp_create(ctx, i); + if (pool->base.opps[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error("DC: failed to create output pixel processor!\n"); + goto res_create_fail; + } + } + + for (i = 0; i < pool->base.res_cap->num_ddc; i++) { + pool->base.engines[i] = dce60_aux_engine_create(ctx, i); + if (pool->base.engines[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create aux engine!!\n"); + goto res_create_fail; + } + pool->base.hw_i2cs[i] = dce60_i2c_hw_create(ctx, i); + if (pool->base.hw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create i2c engine!!\n"); + goto res_create_fail; + } + pool->base.sw_i2cs[i] = dce60_i2c_sw_create(ctx); + if (pool->base.sw_i2cs[i] == NULL) { + BREAK_TO_DEBUGGER(); + dm_error( + "DC:failed to create sw i2c!!\n"); + goto res_create_fail; + } + } + + dc->caps.max_planes = pool->base.pipe_count; + + for (i = 0; i < dc->caps.max_planes; ++i) + dc->caps.planes[i] = plane_cap; + + dc->caps.disable_dp_clk_share = true; + + if (!resource_construct(num_virtual_links, dc, &pool->base, + &res_create_funcs)) + goto res_create_fail; + + /* Create hardware sequencer */ + dce60_hw_sequencer_construct(dc); + + return true; + +res_create_fail: + dce60_resource_destruct(pool); + return false; +} + +struct resource_pool *dce64_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc) +{ + struct dce110_resource_pool *pool = + kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL); + + if (!pool) + return NULL; + + if (dce64_construct(num_virtual_links, dc, pool)) + return &pool->base; + + BREAK_TO_DEBUGGER(); + return NULL; +} diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.h b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.h new file mode 100644 index 000000000000..5d653a76b0b0 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.h @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DC_RESOURCE_DCE60_H__ +#define __DC_RESOURCE_DCE60_H__ + +#include "core_types.h" + +struct dc; +struct resource_pool; + +struct resource_pool *dce60_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc); + +struct resource_pool *dce61_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc); + +struct resource_pool *dce64_create_resource_pool( + uint8_t num_virtual_links, + struct dc *dc); + +#endif /* __DC_RESOURCE_DCE60_H__ */ + diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c new file mode 100644 index 000000000000..fc1af0ff0ca4 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c @@ -0,0 +1,266 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dm_services.h" + +/* include DCE6 register header files */ +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + +#include "dc_types.h" + +#include "include/grph_object_id.h" +#include "include/logger_interface.h" +#include "../dce110/dce110_timing_generator.h" +#include "dce60_timing_generator.h" + +#include "timing_generator.h" + +enum black_color_format { + BLACK_COLOR_FORMAT_RGB_FULLRANGE = 0, /* used as index in array */ + BLACK_COLOR_FORMAT_RGB_LIMITED, + BLACK_COLOR_FORMAT_YUV_TV, + BLACK_COLOR_FORMAT_YUV_CV, + BLACK_COLOR_FORMAT_YUV_SUPER_AA, + + BLACK_COLOR_FORMAT_COUNT +}; + +static const struct dce110_timing_generator_offsets reg_offsets[] = { +{ + .crtc = (mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP0_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +}, +{ + .crtc = (mmCRTC1_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP1_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +}, +{ + .crtc = (mmCRTC2_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP2_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +}, +{ + .crtc = (mmCRTC3_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP3_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +}, +{ + .crtc = (mmCRTC4_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP4_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +}, +{ + .crtc = (mmCRTC5_DCFE_MEM_LIGHT_SLEEP_CNTL - mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL), + .dcp = (mmDCP5_GRPH_CONTROL - mmDCP0_GRPH_CONTROL), +} +}; + +#define NUMBER_OF_FRAME_TO_WAIT_ON_TRIGGERED_RESET 10 + +#define MAX_H_TOTAL (CRTC_H_TOTAL__CRTC_H_TOTAL_MASK + 1) +#define MAX_V_TOTAL (CRTC_V_TOTAL__CRTC_V_TOTAL_MASKhw + 1) + +#define CRTC_REG(reg) (reg + tg110->offsets.crtc) +#define DCP_REG(reg) (reg + tg110->offsets.dcp) +#define DMIF_REG(reg) (reg + tg110->offsets.dmif) + +static void program_pix_dur(struct timing_generator *tg, uint32_t pix_clk_100hz) +{ + uint64_t pix_dur; + uint32_t addr = mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL1 + + DCE110TG_FROM_TG(tg)->offsets.dmif; + uint32_t value = dm_read_reg(tg->ctx, addr); + + if (pix_clk_100hz == 0) + return; + + pix_dur = div_u64(10000000000ull, pix_clk_100hz); + + set_reg_field_value( + value, + pix_dur, + DPG_PIPE_ARBITRATION_CONTROL1, + PIXEL_DURATION); + + dm_write_reg(tg->ctx, addr, value); +} + +static void program_timing(struct timing_generator *tg, + const struct dc_crtc_timing *timing, + int vready_offset, + int vstartup_start, + int vupdate_offset, + int vupdate_width, + const enum signal_type signal, + bool use_vbios) +{ + if (!use_vbios) + program_pix_dur(tg, timing->pix_clk_100hz); + + dce110_tg_program_timing(tg, timing, 0, 0, 0, 0, 0, use_vbios); +} + +static void dce60_timing_generator_enable_advanced_request( + struct timing_generator *tg, + bool enable, + const struct dc_crtc_timing *timing) +{ + struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); + uint32_t addr = CRTC_REG(mmCRTC_START_LINE_CONTROL); + uint32_t value = dm_read_reg(tg->ctx, addr); + /* DCE6 has CRTC_PREFETCH_EN bit in CRTC_CONTROL register */ + uint32_t addr2 = CRTC_REG(mmCRTC_CONTROL); + uint32_t value2 = dm_read_reg(tg->ctx, addr2); + + /* DCE6 does not support CRTC_LEGACY_REQUESTOR_EN bit + so here is not possible to set bit based on enable argument */ + + if ((timing->v_sync_width + timing->v_front_porch) <= 3) { + set_reg_field_value( + value, + 3, + CRTC_START_LINE_CONTROL, + CRTC_ADVANCED_START_LINE_POSITION); + set_reg_field_value( + value2, + 0, + CRTC_CONTROL, + CRTC_PREFETCH_EN); + } else { + set_reg_field_value( + value, + 4, + CRTC_START_LINE_CONTROL, + CRTC_ADVANCED_START_LINE_POSITION); + set_reg_field_value( + value2, + 1, + CRTC_CONTROL, + CRTC_PREFETCH_EN); + } + + set_reg_field_value( + value, + 1, + CRTC_START_LINE_CONTROL, + CRTC_PROGRESSIVE_START_LINE_EARLY); + + set_reg_field_value( + value, + 1, + CRTC_START_LINE_CONTROL, + CRTC_INTERLACE_START_LINE_EARLY); + + dm_write_reg(tg->ctx, addr, value); + dm_write_reg(tg->ctx, addr2, value2); +} + +static bool dce60_is_tg_enabled(struct timing_generator *tg) +{ + uint32_t addr = 0; + uint32_t value = 0; + uint32_t field = 0; + struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); + + addr = CRTC_REG(mmCRTC_CONTROL); + value = dm_read_reg(tg->ctx, addr); + field = get_reg_field_value(value, CRTC_CONTROL, + CRTC_CURRENT_MASTER_EN_STATE); + return field == 1; +} + +bool dce60_configure_crc(struct timing_generator *tg, + const struct crc_params *params) +{ + /* Cannot configure crc on a CRTC that is disabled */ + if (!dce60_is_tg_enabled(tg)) + return false; + + /* DCE6 has no CRTC_CRC_CNTL register, nothing to do */ + + return true; +} + +static const struct timing_generator_funcs dce60_tg_funcs = { + .validate_timing = dce110_tg_validate_timing, + .program_timing = program_timing, + .enable_crtc = dce110_timing_generator_enable_crtc, + .disable_crtc = dce110_timing_generator_disable_crtc, + .is_counter_moving = dce110_timing_generator_is_counter_moving, + .get_position = dce110_timing_generator_get_position, + .get_frame_count = dce110_timing_generator_get_vblank_counter, + .get_scanoutpos = dce110_timing_generator_get_crtc_scanoutpos, + .set_early_control = dce110_timing_generator_set_early_control, + .wait_for_state = dce110_tg_wait_for_state, + .set_blank = dce110_tg_set_blank, + .is_blanked = dce110_tg_is_blanked, + .set_colors = dce110_tg_set_colors, + .set_overscan_blank_color = + dce110_timing_generator_set_overscan_color_black, + .set_blank_color = dce110_timing_generator_program_blank_color, + .disable_vga = dce110_timing_generator_disable_vga, + .did_triggered_reset_occur = + dce110_timing_generator_did_triggered_reset_occur, + .setup_global_swap_lock = + dce110_timing_generator_setup_global_swap_lock, + .enable_reset_trigger = dce110_timing_generator_enable_reset_trigger, + .disable_reset_trigger = dce110_timing_generator_disable_reset_trigger, + .tear_down_global_swap_lock = + dce110_timing_generator_tear_down_global_swap_lock, + .set_drr = dce110_timing_generator_set_drr, + .set_static_screen_control = + dce110_timing_generator_set_static_screen_control, + .set_test_pattern = dce110_timing_generator_set_test_pattern, + .arm_vert_intr = dce110_arm_vert_intr, + + /* DCE6.0 overrides */ + .enable_advanced_request = + dce60_timing_generator_enable_advanced_request, + .configure_crc = dce60_configure_crc, + .get_crc = dce110_get_crc, +}; + +void dce60_timing_generator_construct( + struct dce110_timing_generator *tg110, + struct dc_context *ctx, + uint32_t instance, + const struct dce110_timing_generator_offsets *offsets) +{ + tg110->controller_id = CONTROLLER_ID_D0 + instance; + tg110->base.inst = instance; + tg110->offsets = *offsets; + tg110->derived_offsets = reg_offsets[instance]; + + tg110->base.funcs = &dce60_tg_funcs; + + tg110->base.ctx = ctx; + tg110->base.bp = ctx->dc_bios; + + tg110->max_h_total = CRTC_H_TOTAL__CRTC_H_TOTAL_MASK + 1; + tg110->max_v_total = CRTC_V_TOTAL__CRTC_V_TOTAL_MASK + 1; + + tg110->min_h_blank = 56; + tg110->min_h_front_porch = 4; + tg110->min_h_back_porch = 4; +} + diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.h b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.h new file mode 100644 index 000000000000..81d831233cc5 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.h @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DC_TIMING_GENERATOR_DCE60_H__ +#define __DC_TIMING_GENERATOR_DCE60_H__ + +#include "timing_generator.h" +#include "../include/grph_object_id.h" + +/* DCE6.0 implementation inherits from DCE11.0 */ +void dce60_timing_generator_construct( + struct dce110_timing_generator *tg, + struct dc_context *ctx, + uint32_t instance, + const struct dce110_timing_generator_offsets *offsets); + +#endif /* __DC_TIMING_GENERATOR_DCE60_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile index 62ad1a11bff9..733e6e6e43bd 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile @@ -31,4 +31,11 @@ DCN10 = dcn10_init.o dcn10_resource.o dcn10_ipp.o dcn10_hw_sequencer.o \ AMD_DAL_DCN10 = $(addprefix $(AMDDALPATH)/dc/dcn10/,$(DCN10)) +# fix: +# ...: '-mgeneral-regs-only' is incompatible with the use of floating-point types +# aarch64 does not support soft-float, so use hard-float and handle this in code +ifdef CONFIG_ARM64 +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn10/dcn10_resource.o := -mgeneral-regs-only +endif + AMD_DISPLAY_FILES += $(AMD_DAL_DCN10) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index a643927e272b..8ca94f506195 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -1450,33 +1450,42 @@ void dcn10_init_hw(struct dc *dc) void dcn10_power_down_on_boot(struct dc *dc) { int i = 0; + struct dc_link *edp_link; - if (dc->config.power_down_display_on_boot) { - struct dc_link *edp_link = get_edp_link(dc); - - if (edp_link && - edp_link->link_enc->funcs->is_dig_enabled && - edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc) && - dc->hwseq->funcs.edp_backlight_control && - dc->hwss.power_down && - dc->hwss.edp_power_control) { - dc->hwseq->funcs.edp_backlight_control(edp_link, false); - dc->hwss.power_down(dc); - dc->hwss.edp_power_control(edp_link, false); - } else { - for (i = 0; i < dc->link_count; i++) { - struct dc_link *link = dc->links[i]; - - if (link->link_enc->funcs->is_dig_enabled && - link->link_enc->funcs->is_dig_enabled(link->link_enc) && - dc->hwss.power_down) { - dc->hwss.power_down(dc); - break; - } + if (!dc->config.power_down_display_on_boot) + return; + + edp_link = get_edp_link(dc); + if (edp_link && + edp_link->link_enc->funcs->is_dig_enabled && + edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc) && + dc->hwseq->funcs.edp_backlight_control && + dc->hwss.power_down && + dc->hwss.edp_power_control) { + dc->hwseq->funcs.edp_backlight_control(edp_link, false); + dc->hwss.power_down(dc); + dc->hwss.edp_power_control(edp_link, false); + } else { + for (i = 0; i < dc->link_count; i++) { + struct dc_link *link = dc->links[i]; + if (link->link_enc->funcs->is_dig_enabled && + link->link_enc->funcs->is_dig_enabled(link->link_enc) && + dc->hwss.power_down) { + dc->hwss.power_down(dc); + break; } + } } + + /* + * Call update_clocks with empty context + * to send DISPLAY_OFF + * Otherwise DISPLAY_OFF may not be asserted + */ + if (dc->clk_mgr->funcs->set_low_power_state) + dc->clk_mgr->funcs->set_low_power_state(dc->clk_mgr); } void dcn10_reset_hw_ctx_wrap( @@ -2368,14 +2377,6 @@ void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx) &blnd_cfg.black_color); } - /* - * The way 420 is packed, 2 channels carry Y component, 1 channel - * alternate between Cb and Cr, so both channels need the pixel - * value for Y - */ - if (pipe_ctx->stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) - blnd_cfg.black_color.color_r_cr = blnd_cfg.black_color.color_g_y; - if (per_pixel_alpha) blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA; else @@ -2760,6 +2761,152 @@ static struct pipe_ctx *dcn10_find_top_pipe_for_stream( return NULL; } +bool dcn10_disconnect_pipes( + struct dc *dc, + struct dc_state *context) +{ + bool found_stream = false; + int i, j; + struct dce_hwseq *hws = dc->hwseq; + struct dc_state *old_ctx = dc->current_state; + bool mpcc_disconnected = false; + struct pipe_ctx *old_pipe; + struct pipe_ctx *new_pipe; + DC_LOGGER_INIT(dc->ctx->logger); + + /* Set pipe update flags and lock pipes */ + for (i = 0; i < dc->res_pool->pipe_count; i++) { + old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + new_pipe = &context->res_ctx.pipe_ctx[i]; + new_pipe->update_flags.raw = 0; + + if (!old_pipe->plane_state && !new_pipe->plane_state) + continue; + + if (old_pipe->plane_state && !new_pipe->plane_state) + new_pipe->update_flags.bits.disable = 1; + + /* Check for scl update */ + if (memcmp(&old_pipe->plane_res.scl_data, &new_pipe->plane_res.scl_data, sizeof(struct scaler_data))) + new_pipe->update_flags.bits.scaler = 1; + + /* Check for vp update */ + if (memcmp(&old_pipe->plane_res.scl_data.viewport, &new_pipe->plane_res.scl_data.viewport, sizeof(struct rect)) + || memcmp(&old_pipe->plane_res.scl_data.viewport_c, + &new_pipe->plane_res.scl_data.viewport_c, sizeof(struct rect))) + new_pipe->update_flags.bits.viewport = 1; + + } + + if (!IS_DIAG_DC(dc->ctx->dce_environment)) { + /* Disconnect mpcc here only if losing pipe split*/ + for (i = 0; i < dc->res_pool->pipe_count; i++) { + if (context->res_ctx.pipe_ctx[i].update_flags.bits.disable && + old_ctx->res_ctx.pipe_ctx[i].top_pipe) { + + /* Find the top pipe in the new ctx for the bottom pipe that we + * want to remove by comparing the streams. If both pipes are being + * disabled then do it in the regular pipe programming sequence + */ + for (j = 0; j < dc->res_pool->pipe_count; j++) { + if (old_ctx->res_ctx.pipe_ctx[i].top_pipe->stream == context->res_ctx.pipe_ctx[j].stream && + !context->res_ctx.pipe_ctx[j].top_pipe && + !context->res_ctx.pipe_ctx[j].update_flags.bits.disable) { + found_stream = true; + break; + } + } + + // Disconnect if the top pipe lost it's pipe split + if (found_stream && !context->res_ctx.pipe_ctx[j].bottom_pipe) { + hws->funcs.plane_atomic_disconnect(dc, &dc->current_state->res_ctx.pipe_ctx[i]); + DC_LOG_DC("Reset mpcc for pipe %d\n", dc->current_state->res_ctx.pipe_ctx[i].pipe_idx); + mpcc_disconnected = true; + } + } + found_stream = false; + } + } + + if (mpcc_disconnected) { + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; + struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + struct dc_plane_state *plane_state = pipe_ctx->plane_state; + struct hubp *hubp = pipe_ctx->plane_res.hubp; + + if (!pipe_ctx || !plane_state || !pipe_ctx->stream) + continue; + + // Only update scaler and viewport here if we lose a pipe split. + // This is to prevent half the screen from being black when we + // unlock after disconnecting MPCC. + if (!(old_pipe && !pipe_ctx->top_pipe && + !pipe_ctx->bottom_pipe && old_pipe->bottom_pipe)) + continue; + + if (pipe_ctx->update_flags.raw || pipe_ctx->plane_state->update_flags.raw || pipe_ctx->stream->update_flags.raw) { + if (pipe_ctx->update_flags.bits.scaler || + plane_state->update_flags.bits.scaling_change || + plane_state->update_flags.bits.position_change || + plane_state->update_flags.bits.per_pixel_alpha_change || + pipe_ctx->stream->update_flags.bits.scaling) { + + pipe_ctx->plane_res.scl_data.lb_params.alpha_en = pipe_ctx->plane_state->per_pixel_alpha; + ASSERT(pipe_ctx->plane_res.scl_data.lb_params.depth == LB_PIXEL_DEPTH_30BPP); + /* scaler configuration */ + pipe_ctx->plane_res.dpp->funcs->dpp_set_scaler( + pipe_ctx->plane_res.dpp, &pipe_ctx->plane_res.scl_data); + } + + if (pipe_ctx->update_flags.bits.viewport || + (context == dc->current_state && plane_state->update_flags.bits.position_change) || + (context == dc->current_state && plane_state->update_flags.bits.scaling_change) || + (context == dc->current_state && pipe_ctx->stream->update_flags.bits.scaling)) { + + hubp->funcs->mem_program_viewport( + hubp, + &pipe_ctx->plane_res.scl_data.viewport, + &pipe_ctx->plane_res.scl_data.viewport_c); + } + } + } + } + return mpcc_disconnected; +} + +void dcn10_wait_for_pending_cleared(struct dc *dc, + struct dc_state *context) +{ + struct pipe_ctx *pipe_ctx; + struct timing_generator *tg; + int i; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + pipe_ctx = &context->res_ctx.pipe_ctx[i]; + tg = pipe_ctx->stream_res.tg; + + /* + * Only wait for top pipe's tg penindg bit + * Also skip if pipe is disabled. + */ + if (pipe_ctx->top_pipe || + !pipe_ctx->stream || !pipe_ctx->plane_state || + !tg->funcs->is_tg_enabled(tg)) + continue; + + /* + * Wait for VBLANK then VACTIVE to ensure we get VUPDATE. + * For some reason waiting for OTG_UPDATE_PENDING cleared + * seems to not trigger the update right away, and if we + * lock again before VUPDATE then we don't get a separated + * operation. + */ + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + } +} + void dcn10_apply_ctx_for_surface( struct dc *dc, const struct dc_stream_state *stream, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h index 6d891166da8a..e5691e499023 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h @@ -194,6 +194,12 @@ void dcn10_get_surface_visual_confirm_color( void dcn10_get_hdr_visual_confirm_color( struct pipe_ctx *pipe_ctx, struct tg_color *color); +bool dcn10_disconnect_pipes( + struct dc *dc, + struct dc_state *context); + +void dcn10_wait_for_pending_cleared(struct dc *dc, + struct dc_state *context); void dcn10_set_hdr_multiplier(struct pipe_ctx *pipe_ctx); void dcn10_verify_allow_pstate_change_high(struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c index 5c98b71c1d47..a1d1559bb5d7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c @@ -34,6 +34,8 @@ static const struct hw_sequencer_funcs dcn10_funcs = { .apply_ctx_to_hw = dce110_apply_ctx_to_hw, .apply_ctx_for_surface = dcn10_apply_ctx_for_surface, .post_unlock_program_front_end = dcn10_post_unlock_program_front_end, + .disconnect_pipes = dcn10_disconnect_pipes, + .wait_for_pending_cleared = dcn10_wait_for_pending_cleared, .update_plane_addr = dcn10_update_plane_addr, .update_dchub = dcn10_update_dchub, .update_pending_status = dcn10_update_pending_status, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c index 17d5cb422025..1abd81e17f09 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c @@ -1213,6 +1213,7 @@ static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *cont bool video_large = false; bool desktop_large = false; bool dcc_disabled = false; + bool mpo_enabled = false; for (i = 0; i < context->stream_count; i++) { if (context->stream_status[i].plane_count == 0) @@ -1221,6 +1222,9 @@ static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *cont if (context->stream_status[i].plane_count > 2) return DC_FAIL_UNSUPPORTED_1; + if (context->stream_status[i].plane_count > 1) + mpo_enabled = true; + for (j = 0; j < context->stream_status[i].plane_count; j++) { struct dc_plane_state *plane = context->stream_status[i].plane_states[j]; @@ -1244,6 +1248,10 @@ static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *cont } } + /* Disable MPO in multi-display configurations. */ + if (context->stream_count > 1 && mpo_enabled) + return DC_FAIL_UNSUPPORTED_1; + /* * Workaround: On DCN10 there is UMC issue that causes underflow when * playing 4k video on 4k desktop with video downscaled and single channel @@ -1331,6 +1339,47 @@ static uint32_t read_pipe_fuses(struct dc_context *ctx) return value; } +/* + * Some architectures don't support soft-float (e.g. aarch64), on those + * this function has to be called with hardfloat enabled, make sure not + * to inline it so whatever fp stuff is done stays inside + */ +static noinline void dcn10_resource_construct_fp( + struct dc *dc) +{ + if (dc->ctx->dce_version == DCN_VERSION_1_01) { + struct dcn_soc_bounding_box *dcn_soc = dc->dcn_soc; + struct dcn_ip_params *dcn_ip = dc->dcn_ip; + struct display_mode_lib *dml = &dc->dml; + + dml->ip.max_num_dpp = 3; + /* TODO how to handle 23.84? */ + dcn_soc->dram_clock_change_latency = 23; + dcn_ip->max_num_dpp = 3; + } + if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { + dc->dcn_soc->urgent_latency = 3; + dc->debug.disable_dmcu = true; + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 41.60f; + } + + + dc->dcn_soc->number_of_channels = dc->ctx->asic_id.vram_width / ddr4_dram_width; + ASSERT(dc->dcn_soc->number_of_channels < 3); + if (dc->dcn_soc->number_of_channels == 0)/*old sbios bug*/ + dc->dcn_soc->number_of_channels = 2; + + if (dc->dcn_soc->number_of_channels == 1) { + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 19.2f; + dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = 17.066f; + dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = 14.933f; + dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 12.8f; + if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 20.80f; + } + } +} + static bool dcn10_resource_construct( uint8_t num_virtual_links, struct dc *dc, @@ -1482,37 +1531,15 @@ static bool dcn10_resource_construct( memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults)); memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults)); - if (dc->ctx->dce_version == DCN_VERSION_1_01) { - struct dcn_soc_bounding_box *dcn_soc = dc->dcn_soc; - struct dcn_ip_params *dcn_ip = dc->dcn_ip; - struct display_mode_lib *dml = &dc->dml; - - dml->ip.max_num_dpp = 3; - /* TODO how to handle 23.84? */ - dcn_soc->dram_clock_change_latency = 23; - dcn_ip->max_num_dpp = 3; - } - if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { - dc->dcn_soc->urgent_latency = 3; - dc->debug.disable_dmcu = true; - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 41.60f; - } - - - dc->dcn_soc->number_of_channels = dc->ctx->asic_id.vram_width / ddr4_dram_width; - ASSERT(dc->dcn_soc->number_of_channels < 3); - if (dc->dcn_soc->number_of_channels == 0)/*old sbios bug*/ - dc->dcn_soc->number_of_channels = 2; - - if (dc->dcn_soc->number_of_channels == 1) { - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 19.2f; - dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = 17.066f; - dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = 14.933f; - dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 12.8f; - if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 20.80f; - } - } +#if defined(CONFIG_ARM64) + /* Aarch64 does not support -msoft-float/-mfloat-abi=soft */ + DC_FP_START(); + dcn10_resource_construct_fp(dc); + DC_FP_END(); +#else + /* Other architectures we build for build this with soft-float */ + dcn10_resource_construct_fp(dc); +#endif pool->base.pp_smu = dcn10_pp_smu_create(ctx); diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c index 07b2f9399671..842abb4c475b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c @@ -121,35 +121,35 @@ void enc1_update_generic_info_packet( switch (packet_index) { case 0: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC0_FRAME_UPDATE, 1); + AFMT_GENERIC0_IMMEDIATE_UPDATE, 1); break; case 1: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC1_FRAME_UPDATE, 1); + AFMT_GENERIC1_IMMEDIATE_UPDATE, 1); break; case 2: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC2_FRAME_UPDATE, 1); + AFMT_GENERIC2_IMMEDIATE_UPDATE, 1); break; case 3: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC3_FRAME_UPDATE, 1); + AFMT_GENERIC3_IMMEDIATE_UPDATE, 1); break; case 4: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC4_FRAME_UPDATE, 1); + AFMT_GENERIC4_IMMEDIATE_UPDATE, 1); break; case 5: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC5_FRAME_UPDATE, 1); + AFMT_GENERIC5_IMMEDIATE_UPDATE, 1); break; case 6: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC6_FRAME_UPDATE, 1); + AFMT_GENERIC6_IMMEDIATE_UPDATE, 1); break; case 7: REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, - AFMT_GENERIC7_FRAME_UPDATE, 1); + AFMT_GENERIC7_IMMEDIATE_UPDATE, 1); break; default: break; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h index ed385b1477be..30eae7459d50 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h @@ -281,7 +281,14 @@ struct dcn10_stream_enc_registers { SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_FRAME_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_FRAME_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_FRAME_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC0_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC1_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_IMMEDIATE_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_IMMEDIATE_UPDATE, mask_sh),\ + SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_IMMEDIATE_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_FRAME_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_FRAME_UPDATE, mask_sh),\ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE, mask_sh),\ @@ -345,7 +352,14 @@ struct dcn10_stream_enc_registers { type AFMT_GENERIC2_FRAME_UPDATE;\ type AFMT_GENERIC3_FRAME_UPDATE;\ type AFMT_GENERIC4_FRAME_UPDATE;\ + type AFMT_GENERIC0_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC1_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC2_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC3_IMMEDIATE_UPDATE;\ type AFMT_GENERIC4_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC5_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC6_IMMEDIATE_UPDATE;\ + type AFMT_GENERIC7_IMMEDIATE_UPDATE;\ type AFMT_GENERIC5_FRAME_UPDATE;\ type AFMT_GENERIC6_FRAME_UPDATE;\ type AFMT_GENERIC7_FRAME_UPDATE;\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile index 5fcaf78334ff..624cb1341ef1 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile @@ -17,6 +17,10 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/dcn20/dcn20_resource.o := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn20/dcn20_resource.o := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h index 667640c4b288..1118e33aaa2c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h @@ -94,6 +94,7 @@ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_CLOCK_EN, mask_sh), \ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DISPCLK_R_GATE_DIS, mask_sh), \ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DSCCLK_R_GATE_DIS, mask_sh), \ + DSC_SF(DSC_TOP0_DSC_DEBUG_CONTROL, DSC_DBG_EN, mask_sh), \ DSC_SF(DSCC0_DSCC_CONFIG0, ICH_RESET_AT_END_OF_LINE, mask_sh), \ DSC_SF(DSCC0_DSCC_CONFIG0, NUMBER_OF_SLICES_PER_LINE, mask_sh), \ DSC_SF(DSCC0_DSCC_CONFIG0, ALTERNATE_ICH_ENCODING_EN, mask_sh), \ diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 66180b4332f1..c8cfd3ba1c15 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1457,8 +1457,8 @@ static void dcn20_update_dchubp_dpp( /* Any updates are handled in dc interface, just need to apply existing for plane enable */ if ((pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed || - pipe_ctx->update_flags.bits.scaler || pipe_ctx->update_flags.bits.viewport) - && pipe_ctx->stream->cursor_attributes.address.quad_part != 0) { + pipe_ctx->update_flags.bits.scaler || viewport_changed == true) && + pipe_ctx->stream->cursor_attributes.address.quad_part != 0) { dc->hwss.set_cursor_position(pipe_ctx); dc->hwss.set_cursor_attribute(pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c index 3dde6f26de47..966e1790b9bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c @@ -34,6 +34,8 @@ static const struct hw_sequencer_funcs dcn20_funcs = { .apply_ctx_to_hw = dce110_apply_ctx_to_hw, .apply_ctx_for_surface = NULL, .program_front_end_for_ctx = dcn20_program_front_end_for_ctx, + .disconnect_pipes = dcn10_disconnect_pipes, + .wait_for_pending_cleared = dcn10_wait_for_pending_cleared, .post_unlock_program_front_end = dcn20_post_unlock_program_front_end, .update_plane_addr = dcn20_update_plane_addr, .update_dchub = dcn10_update_dchub, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h index bf0044f7417e..864acd695cbb 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h @@ -167,7 +167,9 @@ LE_SF(DCIO_SOFT_RESET, UNIPHYB_SOFT_RESET, mask_sh),\ LE_SF(DCIO_SOFT_RESET, UNIPHYC_SOFT_RESET, mask_sh),\ LE_SF(DCIO_SOFT_RESET, UNIPHYD_SOFT_RESET, mask_sh),\ - LE_SF(DCIO_SOFT_RESET, UNIPHYE_SOFT_RESET, mask_sh) + LE_SF(DCIO_SOFT_RESET, UNIPHYE_SOFT_RESET, mask_sh),\ + LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL6, RDPCS_PHY_DPALT_DP4, mask_sh),\ + LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL6, RDPCS_PHY_DPALT_DISABLE, mask_sh) #define LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh)\ LINK_ENCODER_MASK_SH_LIST_DCN10(mask_sh),\ @@ -229,8 +231,6 @@ SRI(RDPCSTX_PHY_FUSE3, RDPCSTX, id), \ SRI(DPCSTX_TX_CLOCK_CNTL, DPCSTX, id), \ SRI(DPCSTX_TX_CNTL, DPCSTX, id), \ - SRI(DPCSTX_DEBUG_CONFIG, DPCSTX, id), \ - SRI(RDPCSTX_DEBUG_CONFIG, RDPCSTX, id), \ SR(RDPCSTX0_RDPCSTX_SCRATCH) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index 790baf552695..1b9874445134 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -3141,7 +3141,7 @@ static bool dcn20_validate_bandwidth_internal(struct dc *dc, struct dc_state *co int vlevel = 0; int pipe_split_from[MAX_PIPES]; int pipe_cnt = 0; - display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL); + display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_ATOMIC); DC_LOGGER_INIT(dc->ctx->logger); BW_VAL_TRACE_COUNT(); @@ -3209,6 +3209,9 @@ static noinline bool dcn20_validate_bandwidth_fp(struct dc *dc, context->bw_ctx.dml.soc.allow_dram_clock_one_display_vactive = dc->debug.enable_dram_clock_change_one_display_vactive; + /*Unsafe due to current pipe merge and split logic*/ + ASSERT(context != dc->current_state); + if (fast_validate) { return dcn20_validate_bandwidth_internal(dc, context, true); } @@ -3320,7 +3323,7 @@ enum dc_status dcn20_patch_unknown_plane_state(struct dc_plane_state *plane_stat return DC_OK; } -static struct resource_funcs dcn20_res_pool_funcs = { +static const struct resource_funcs dcn20_res_pool_funcs = { .destroy = dcn20_destroy_resource_pool, .link_enc_create = dcn20_link_encoder_create, .panel_cntl_create = dcn20_panel_cntl_create, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h index 2c1959845c29..cdd39ee9761d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h @@ -95,7 +95,6 @@ struct display_stream_compressor *dcn20_dsc_create( struct dc_context *ctx, uint32_t inst); void dcn20_dsc_destroy(struct display_stream_compressor **dsc); -void dcn20_patch_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *bb); void dcn20_cap_soc_clocks( struct _vcs_dpi_soc_bounding_box_st *bb, struct pp_smu_nv_clock_table max_clocks); diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile index 07684d3e375a..51a2f3d4c194 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile @@ -13,6 +13,10 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/dcn21/dcn21_resource.o := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn21/dcn21_resource.o := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c index b187f71afa65..2ba880c3943c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c @@ -35,6 +35,8 @@ static const struct hw_sequencer_funcs dcn21_funcs = { .apply_ctx_to_hw = dce110_apply_ctx_to_hw, .apply_ctx_for_surface = NULL, .program_front_end_for_ctx = dcn20_program_front_end_for_ctx, + .disconnect_pipes = dcn10_disconnect_pipes, + .wait_for_pending_cleared = dcn10_wait_for_pending_cleared, .post_unlock_program_front_end = dcn20_post_unlock_program_front_end, .update_plane_addr = dcn20_update_plane_addr, .update_dchub = dcn10_update_dchub, diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 88d41a385add..78743ae37851 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -1184,6 +1184,9 @@ bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, BW_VAL_TRACE_COUNT(); + /*Unsafe due to current pipe merge and split logic*/ + ASSERT(context != dc->current_state); + out = dcn20_fast_validate_bw(dc, context, pipes, &pipe_cnt, pipe_split_from, &vlevel); if (pipe_cnt == 0) @@ -1754,7 +1757,7 @@ enum dc_status dcn21_patch_unknown_plane_state(struct dc_plane_state *plane_stat return result; } -static struct resource_funcs dcn21_res_pool_funcs = { +static const struct resource_funcs dcn21_res_pool_funcs = { .destroy = dcn21_destroy_resource_pool, .link_enc_create = dcn21_link_encoder_create, .panel_cntl_create = dcn21_panel_cntl_create, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h index 8e9fd59ccde8..2fbf879cd327 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h @@ -61,7 +61,10 @@ DPCS_DCN2_MASK_SH_LIST(mask_sh),\ LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_DATA_ORDER_INVERT_18_BIT, mask_sh),\ LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL0, RDPCS_PHY_TX_VBOOST_LVL, mask_sh),\ - LE_SF(RDPCSTX0_RDPCSTX_CLOCK_CNTL, RDPCS_TX_CLK_EN, mask_sh) + LE_SF(RDPCSTX0_RDPCSTX_CLOCK_CNTL, RDPCS_TX_CLK_EN, mask_sh),\ + LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL6, RDPCS_PHY_DPALT_DP4, mask_sh),\ + LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL6, RDPCS_PHY_DPALT_DISABLE, mask_sh) + void dcn30_link_encoder_construct( struct dcn20_link_encoder *enc20, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c index 9afee7160490..19daa456e3bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c @@ -35,6 +35,8 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .apply_ctx_to_hw = dce110_apply_ctx_to_hw, .apply_ctx_for_surface = NULL, .program_front_end_for_ctx = dcn20_program_front_end_for_ctx, + .disconnect_pipes = dcn10_disconnect_pipes, + .wait_for_pending_cleared = dcn10_wait_for_pending_cleared, .post_unlock_program_front_end = dcn20_post_unlock_program_front_end, .update_plane_addr = dcn20_update_plane_addr, .update_dchub = dcn10_update_dchub, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 653a571e366d..8be4f21169d0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -340,7 +340,7 @@ static const struct dce110_clk_src_mask cs_mask = { #define abm_regs(id)\ [id] = {\ - ABM_DCN301_REG_LIST(id)\ + ABM_DCN30_REG_LIST(id)\ } static const struct dce_abm_registers abm_regs[] = { @@ -491,6 +491,8 @@ static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = { [id] = {\ LE_DCN3_REG_LIST(id), \ UNIPHY_DCN2_REG_LIST(phyid), \ + DPCS_DCN2_REG_LIST(id), \ + SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \ } static const struct dce110_aux_registers_shift aux_shift = { @@ -871,7 +873,7 @@ void dcn30_dpp_destroy(struct dpp **dpp) *dpp = NULL; } -struct dpp *dcn30_dpp_create( +static struct dpp *dcn30_dpp_create( struct dc_context *ctx, uint32_t inst) { @@ -889,7 +891,8 @@ struct dpp *dcn30_dpp_create( kfree(dpp); return NULL; } -struct output_pixel_processor *dcn30_opp_create( + +static struct output_pixel_processor *dcn30_opp_create( struct dc_context *ctx, uint32_t inst) { struct dcn20_opp *opp = @@ -905,7 +908,7 @@ struct output_pixel_processor *dcn30_opp_create( return &opp->base; } -struct dce_aux *dcn30_aux_engine_create( +static struct dce_aux *dcn30_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -924,6 +927,7 @@ struct dce_aux *dcn30_aux_engine_create( return &aux_engine->base; } + #define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) } static const struct dce_i2c_registers i2c_hw_regs[] = { @@ -943,7 +947,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCN2(_MASK) }; -struct dce_i2c_hw *dcn30_i2c_hw_create( +static struct dce_i2c_hw *dcn30_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -958,6 +962,7 @@ struct dce_i2c_hw *dcn30_i2c_hw_create( return dce_i2c_hw; } + static struct mpc *dcn30_mpc_create( struct dc_context *ctx, int num_mpcc, @@ -1008,7 +1013,7 @@ struct hubbub *dcn30_hubbub_create(struct dc_context *ctx) return &hubbub3->base; } -struct timing_generator *dcn30_timing_generator_create( +static struct timing_generator *dcn30_timing_generator_create( struct dc_context *ctx, uint32_t instance) { @@ -1042,7 +1047,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS4_CAPABLE = true }; -struct link_encoder *dcn30_link_encoder_create( +static struct link_encoder *dcn30_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dcn20_link_encoder *enc20 = @@ -1063,7 +1068,7 @@ struct link_encoder *dcn30_link_encoder_create( return &enc20->enc10.base; } -struct panel_cntl *dcn30_panel_cntl_create(const struct panel_cntl_init_data *init_data) +static struct panel_cntl *dcn30_panel_cntl_create(const struct panel_cntl_init_data *init_data) { struct dce_panel_cntl *panel_cntl = kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL); @@ -1311,7 +1316,7 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool) dcn_dccg_destroy(&pool->base.dccg); } -struct hubp *dcn30_hubp_create( +static struct hubp *dcn30_hubp_create( struct dc_context *ctx, uint32_t inst) { @@ -1330,7 +1335,7 @@ struct hubp *dcn30_hubp_create( return NULL; } -bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool) +static bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool) { int i; uint32_t pipe_count = pool->res_cap->num_dwb; @@ -1355,7 +1360,7 @@ bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool) return true; } -bool dcn30_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool) +static bool dcn30_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool) { int i; uint32_t pipe_count = pool->res_cap->num_dwb; @@ -2292,7 +2297,7 @@ static void get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, (dcn3_0_soc.return_bus_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100)); } -static void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) +void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) { unsigned int i, j; unsigned int num_states = 0; @@ -2412,7 +2417,7 @@ static void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); } -static struct resource_funcs dcn30_res_pool_funcs = { +static const struct resource_funcs dcn30_res_pool_funcs = { .destroy = dcn30_destroy_resource_pool, .link_enc_create = dcn30_link_encoder_create, .panel_cntl_create = dcn30_panel_cntl_create, @@ -2420,6 +2425,7 @@ static struct resource_funcs dcn30_res_pool_funcs = { .populate_dml_pipes = dcn30_populate_dml_pipes_from_context, .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer, .add_stream_to_ctx = dcn30_add_stream_to_ctx, + .add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource, .remove_stream_from_ctx = dcn20_remove_stream_from_ctx, .populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context, .set_mcif_arb_params = dcn30_set_mcif_arb_params, @@ -2683,7 +2689,7 @@ static bool dcn30_resource_construct( if (!resource_construct(num_virtual_links, dc, &pool->base, (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment) ? &res_create_funcs : &res_create_maximus_funcs))) - goto create_fail; + goto create_fail; /* HW Sequencer and Plane caps */ dcn30_hw_sequencer_construct(dc); diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h index 4b4a4d81c1e3..c9d5f94092a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h @@ -79,4 +79,7 @@ enum dc_status dcn30_add_stream_to_ctx( struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream); + +void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params); + #endif /* _DCN30_RESOURCE_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 417331438c30..dbc7e2abe379 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -33,6 +33,10 @@ ifdef CONFIG_PPC64 dml_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +dml_rcflags := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -60,6 +64,13 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20v2.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn21/display_mode_vba_21.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn21/display_rq_dlg_calc_21.o := $(dml_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20v2.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn21/display_mode_vba_21.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn21/display_rq_dlg_calc_21.o := $(dml_rcflags) endif ifdef CONFIG_DRM_AMD_DC_DCN3_0 CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_mode_vba_30.o := $(dml_ccflags) -Wframe-larger-than=2048 @@ -67,6 +78,8 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_ccflags) endif CFLAGS_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/display_rq_dlg_helpers.o := $(dml_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_rq_dlg_helpers.o := $(dml_rcflags) DML = display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o \ diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index b54814f11b74..2beb284f89b0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -63,6 +63,7 @@ typedef struct { #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff +#define DCN30_MAX_DSC_IMAGE_WIDTH 5184 static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib); static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation( @@ -3984,6 +3985,9 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l } else if (v->PlaneRequiredDISPCLKWithoutODMCombine > v->MaxDispclkRoundedDownToDFSGranularity) { v->ODMCombineEnablePerState[i][k] = dm_odm_combine_mode_2to1; v->PlaneRequiredDISPCLK = v->PlaneRequiredDISPCLKWithODMCombine2To1; + } else if (v->DSCEnabled[k] && (v->HActive[k] > DCN30_MAX_DSC_IMAGE_WIDTH)) { + v->ODMCombineEnablePerState[i][k] = dm_odm_combine_mode_2to1; + v->PlaneRequiredDISPCLK = v->PlaneRequiredDISPCLKWithODMCombine2To1; } else { v->ODMCombineEnablePerState[i][k] = dm_odm_combine_mode_disabled; v->PlaneRequiredDISPCLK = v->PlaneRequiredDISPCLKWithoutODMCombine; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index afdd4f0d9d71..b32093136089 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -467,7 +467,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) mode_lib->vba.AudioSampleLayout[mode_lib->vba.NumberOfActivePlanes] = 1; mode_lib->vba.DRAMClockChangeLatencyOverride = 0.0; - mode_lib->vba.DSCEnabled[mode_lib->vba.NumberOfActivePlanes] = dout->dsc_enable;; + mode_lib->vba.DSCEnabled[mode_lib->vba.NumberOfActivePlanes] = dout->dsc_enable; mode_lib->vba.DSCEnable[mode_lib->vba.NumberOfActivePlanes] = dout->dsc_enable; mode_lib->vba.NumberOfDSCSlices[mode_lib->vba.NumberOfActivePlanes] = dout->dsc_slices; diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile index ea29cf95d470..f2624a1156e5 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile @@ -10,6 +10,10 @@ ifdef CONFIG_PPC64 dsc_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +dsc_rcflags := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -28,6 +32,7 @@ endif endif CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_rcflags) DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c index 8cdaa6eef5d3..4c844cfaa956 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c +++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c @@ -34,6 +34,9 @@ /* default DSC policy target bitrate limit is 16bpp */ static uint32_t dsc_policy_max_target_bpp_limit = 16; +/* default DSC policy enables DSC only when needed */ +static bool dsc_policy_enable_dsc_when_not_needed; + static uint32_t dc_dsc_bandwidth_in_kbps_from_timing( const struct dc_crtc_timing *timing) { @@ -189,8 +192,10 @@ static bool dsc_throughput_from_dpcd(int dpcd_throughput, int *throughput) } -static bool dsc_bpp_increment_div_from_dpcd(int bpp_increment_dpcd, uint32_t *bpp_increment_div) +static bool dsc_bpp_increment_div_from_dpcd(uint8_t bpp_increment_dpcd, uint32_t *bpp_increment_div) { + // Mask bpp increment dpcd field to avoid reading other fields + bpp_increment_dpcd &= 0x7; switch (bpp_increment_dpcd) { case 0: @@ -360,7 +365,7 @@ static bool decide_dsc_target_bpp_x16( get_dsc_bandwidth_range(policy->min_target_bpp, policy->max_target_bpp, dsc_common_caps, timing, &range); - if (target_bandwidth_kbps >= range.stream_kbps) { + if (!policy->enable_dsc_when_not_needed && target_bandwidth_kbps >= range.stream_kbps) { /* enough bandwidth without dsc */ *target_bpp_x16 = 0; should_use_dsc = false; @@ -961,9 +966,20 @@ void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing *timing, struct dc /* internal upper limit, default 16 bpp */ if (policy->max_target_bpp > dsc_policy_max_target_bpp_limit) policy->max_target_bpp = dsc_policy_max_target_bpp_limit; + + /* enable DSC when not needed, default false */ + if (dsc_policy_enable_dsc_when_not_needed) + policy->enable_dsc_when_not_needed = dsc_policy_enable_dsc_when_not_needed; + else + policy->enable_dsc_when_not_needed = false; } void dc_dsc_policy_set_max_target_bpp_limit(uint32_t limit) { dsc_policy_max_target_bpp_limit = limit; } + +void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable) +{ + dsc_policy_enable_dsc_when_not_needed = enable; +} diff --git a/drivers/gpu/drm/amd/display/dc/gpio/Makefile b/drivers/gpu/drm/amd/display/dc/gpio/Makefile index 0f2f4508e564..74c0943ed644 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/Makefile +++ b/drivers/gpu/drm/amd/display/dc/gpio/Makefile @@ -31,6 +31,18 @@ AMD_DAL_GPIO = $(addprefix $(AMDDALPATH)/dc/gpio/,$(GPIO)) AMD_DISPLAY_FILES += $(AMD_DAL_GPIO) ############################################################################### +# DCE 6x +############################################################################### +# all DCE6.x are derived from DCE6.0 +ifdef CONFIG_DRM_AMD_DC_SI +GPIO_DCE60 = hw_translate_dce60.o hw_factory_dce60.o + +AMD_DAL_GPIO_DCE60 = $(addprefix $(AMDDALPATH)/dc/gpio/dce60/,$(GPIO_DCE60)) + +AMD_DISPLAY_FILES += $(AMD_DAL_GPIO_DCE60) +endif + +############################################################################### # DCE 8x ############################################################################### # all DCE8.x are derived from DCE8.0 diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.c b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.c new file mode 100644 index 000000000000..cc69acd8ada7 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.c @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dm_services.h" +#include "include/gpio_types.h" +#include "../hw_factory.h" + +#include "hw_factory_dce60.h" + +#include "../hw_gpio.h" +#include "../hw_ddc.h" +#include "../hw_hpd.h" +#include "../hw_generic.h" + +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + + +#define REG(reg_name)\ + mm ## reg_name + +#include "reg_helper.h" +#include "../hpd_regs.h" + +#define HPD_REG_LIST_DCE6(id) \ + HPD_GPIO_REG_LIST(id), \ + .int_status = mmDC_HPD ## id ## _INT_STATUS,\ + .toggle_filt_cntl = mmDC_HPD ## id ## _TOGGLE_FILT_CNTL + +#define HPD_MASK_SH_LIST_DCE6(mask_sh) \ + .DC_HPD_SENSE_DELAYED = DC_HPD1_INT_STATUS__DC_HPD1_SENSE_DELAYED ## mask_sh,\ + .DC_HPD_SENSE = DC_HPD1_INT_STATUS__DC_HPD1_SENSE ## mask_sh,\ + .DC_HPD_CONNECT_INT_DELAY = DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_CONNECT_INT_DELAY ## mask_sh,\ + .DC_HPD_DISCONNECT_INT_DELAY = DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_DISCONNECT_INT_DELAY ## mask_sh + +#define hpd_regs(id) \ +{\ + HPD_REG_LIST_DCE6(id)\ +} + +static const struct hpd_registers hpd_regs[] = { + hpd_regs(1), + hpd_regs(2), + hpd_regs(3), + hpd_regs(4), + hpd_regs(5), + hpd_regs(6) +}; + +static const struct hpd_sh_mask hpd_shift = { + HPD_MASK_SH_LIST_DCE6(__SHIFT) +}; + +static const struct hpd_sh_mask hpd_mask = { + HPD_MASK_SH_LIST_DCE6(_MASK) +}; + +#include "../ddc_regs.h" + + /* set field name */ +#define SF_DDC(reg_name, field_name, post_fix)\ + .field_name = reg_name ## __ ## field_name ## post_fix + +static const struct ddc_registers ddc_data_regs[] = { + ddc_data_regs(1), + ddc_data_regs(2), + ddc_data_regs(3), + ddc_data_regs(4), + ddc_data_regs(5), + ddc_data_regs(6), + ddc_vga_data_regs, + ddc_i2c_data_regs +}; + +static const struct ddc_registers ddc_clk_regs[] = { + ddc_clk_regs(1), + ddc_clk_regs(2), + ddc_clk_regs(3), + ddc_clk_regs(4), + ddc_clk_regs(5), + ddc_clk_regs(6), + ddc_vga_clk_regs, + ddc_i2c_clk_regs +}; + +static const struct ddc_sh_mask ddc_shift = { + DDC_MASK_SH_LIST(__SHIFT) +}; + +static const struct ddc_sh_mask ddc_mask = { + DDC_MASK_SH_LIST(_MASK) +}; + +static void define_ddc_registers( + struct hw_gpio_pin *pin, + uint32_t en) +{ + struct hw_ddc *ddc = HW_DDC_FROM_BASE(pin); + + switch (pin->id) { + case GPIO_ID_DDC_DATA: + ddc->regs = &ddc_data_regs[en]; + ddc->base.regs = &ddc_data_regs[en].gpio; + break; + case GPIO_ID_DDC_CLOCK: + ddc->regs = &ddc_clk_regs[en]; + ddc->base.regs = &ddc_clk_regs[en].gpio; + break; + default: + ASSERT_CRITICAL(false); + return; + } + + ddc->shifts = &ddc_shift; + ddc->masks = &ddc_mask; + +} + +static void define_hpd_registers(struct hw_gpio_pin *pin, uint32_t en) +{ + struct hw_hpd *hpd = HW_HPD_FROM_BASE(pin); + + hpd->regs = &hpd_regs[en]; + hpd->shifts = &hpd_shift; + hpd->masks = &hpd_mask; + hpd->base.regs = &hpd_regs[en].gpio; +} + +static const struct hw_factory_funcs funcs = { + .init_ddc_data = dal_hw_ddc_init, + .init_generic = NULL, + .init_hpd = dal_hw_hpd_init, + .get_ddc_pin = dal_hw_ddc_get_pin, + .get_hpd_pin = dal_hw_hpd_get_pin, + .get_generic_pin = NULL, + .define_hpd_registers = define_hpd_registers, + .define_ddc_registers = define_ddc_registers +}; + +void dal_hw_factory_dce60_init( + struct hw_factory *factory) +{ + factory->number_of_pins[GPIO_ID_DDC_DATA] = 8; + factory->number_of_pins[GPIO_ID_DDC_CLOCK] = 8; + factory->number_of_pins[GPIO_ID_GENERIC] = 7; + factory->number_of_pins[GPIO_ID_HPD] = 6; + factory->number_of_pins[GPIO_ID_GPIO_PAD] = 31; + factory->number_of_pins[GPIO_ID_VIP_PAD] = 0; + factory->number_of_pins[GPIO_ID_SYNC] = 2; + factory->number_of_pins[GPIO_ID_GSL] = 4; + + factory->funcs = &funcs; +} diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.h b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.h new file mode 100644 index 000000000000..1fd54ff8979c --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_factory_dce60.h @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DAL_HW_FACTORY_DCE60_H__ +#define __DAL_HW_FACTORY_DCE60_H__ + +void dal_hw_factory_dce60_init( + struct hw_factory *factory); + +#endif diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.c b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.c new file mode 100644 index 000000000000..255df31ec577 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.c @@ -0,0 +1,411 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dm_services.h" + +/* + * Pre-requisites: headers required by header of this unit + */ +#include "include/gpio_types.h" +#include "../hw_translate.h" + +#include "hw_translate_dce60.h" + +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" +#include "smu/smu_6_0_d.h" + +/* + * @brief + * Returns index of first bit (starting with LSB) which is set + */ +static uint32_t index_from_vector( + uint32_t vector) +{ + uint32_t result = 0; + uint32_t mask = 1; + + do { + if (vector == mask) + return result; + + ++result; + mask <<= 1; + } while (mask); + + BREAK_TO_DEBUGGER(); + + return GPIO_ENUM_UNKNOWN; +} + +static bool offset_to_id( + uint32_t offset, + uint32_t mask, + enum gpio_id *id, + uint32_t *en) +{ + switch (offset) { + /* GENERIC */ + case mmDC_GPIO_GENERIC_A: + *id = GPIO_ID_GENERIC; + switch (mask) { + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK: + *en = GPIO_GENERIC_A; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK: + *en = GPIO_GENERIC_B; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK: + *en = GPIO_GENERIC_C; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK: + *en = GPIO_GENERIC_D; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK: + *en = GPIO_GENERIC_E; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK: + *en = GPIO_GENERIC_F; + return true; + case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK: + *en = GPIO_GENERIC_G; + return true; + default: + BREAK_TO_DEBUGGER(); + return false; + } + break; + /* HPD */ + case mmDC_GPIO_HPD_A: + *id = GPIO_ID_HPD; + switch (mask) { + case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK: + *en = GPIO_HPD_1; + return true; + case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK: + *en = GPIO_HPD_2; + return true; + case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK: + *en = GPIO_HPD_3; + return true; + case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK: + *en = GPIO_HPD_4; + return true; + case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK: + *en = GPIO_HPD_5; + return true; + case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK: + *en = GPIO_HPD_6; + return true; + default: + BREAK_TO_DEBUGGER(); + return false; + } + break; + /* SYNCA */ + case mmDC_GPIO_SYNCA_A: + *id = GPIO_ID_SYNC; + switch (mask) { + case DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK: + *en = GPIO_SYNC_HSYNC_A; + return true; + case DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK: + *en = GPIO_SYNC_VSYNC_A; + return true; + default: + BREAK_TO_DEBUGGER(); + return false; + } + break; + /* mmDC_GPIO_GENLK_MASK */ + case mmDC_GPIO_GENLK_A: + *id = GPIO_ID_GSL; + switch (mask) { + case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK: + *en = GPIO_GSL_GENLOCK_CLOCK; + return true; + case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK: + *en = GPIO_GSL_GENLOCK_VSYNC; + return true; + case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK: + *en = GPIO_GSL_SWAPLOCK_A; + return true; + case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK: + *en = GPIO_GSL_SWAPLOCK_B; + return true; + default: + BREAK_TO_DEBUGGER(); + return false; + } + break; + /* GPIOPAD */ + case mmGPIOPAD_A: + *id = GPIO_ID_GPIO_PAD; + *en = index_from_vector(mask); + return (*en <= GPIO_GPIO_PAD_MAX); + /* DDC */ + /* we don't care about the GPIO_ID for DDC + * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK + * directly in the create method */ + case mmDC_GPIO_DDC1_A: + *en = GPIO_DDC_LINE_DDC1; + return true; + case mmDC_GPIO_DDC2_A: + *en = GPIO_DDC_LINE_DDC2; + return true; + case mmDC_GPIO_DDC3_A: + *en = GPIO_DDC_LINE_DDC3; + return true; + case mmDC_GPIO_DDC4_A: + *en = GPIO_DDC_LINE_DDC4; + return true; + case mmDC_GPIO_DDC5_A: + *en = GPIO_DDC_LINE_DDC5; + return true; + case mmDC_GPIO_DDC6_A: + *en = GPIO_DDC_LINE_DDC6; + return true; + case mmDC_GPIO_DDCVGA_A: + *en = GPIO_DDC_LINE_DDC_VGA; + return true; + /* GPIO_I2CPAD */ + case mmDC_GPIO_I2CPAD_A: + *en = GPIO_DDC_LINE_I2C_PAD; + return true; + /* Not implemented */ + case mmDC_GPIO_PWRSEQ_A: + case mmDC_GPIO_PAD_STRENGTH_1: + case mmDC_GPIO_PAD_STRENGTH_2: + case mmDC_GPIO_DEBUG: + return false; + /* UNEXPECTED */ + default: + BREAK_TO_DEBUGGER(); + return false; + } +} + +static bool id_to_offset( + enum gpio_id id, + uint32_t en, + struct gpio_pin_info *info) +{ + bool result = true; + + switch (id) { + case GPIO_ID_DDC_DATA: + info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK; + switch (en) { + case GPIO_DDC_LINE_DDC1: + info->offset = mmDC_GPIO_DDC1_A; + break; + case GPIO_DDC_LINE_DDC2: + info->offset = mmDC_GPIO_DDC2_A; + break; + case GPIO_DDC_LINE_DDC3: + info->offset = mmDC_GPIO_DDC3_A; + break; + case GPIO_DDC_LINE_DDC4: + info->offset = mmDC_GPIO_DDC4_A; + break; + case GPIO_DDC_LINE_DDC5: + info->offset = mmDC_GPIO_DDC5_A; + break; + case GPIO_DDC_LINE_DDC6: + info->offset = mmDC_GPIO_DDC6_A; + break; + case GPIO_DDC_LINE_DDC_VGA: + info->offset = mmDC_GPIO_DDCVGA_A; + break; + case GPIO_DDC_LINE_I2C_PAD: + info->offset = mmDC_GPIO_I2CPAD_A; + break; + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_DDC_CLOCK: + info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK; + switch (en) { + case GPIO_DDC_LINE_DDC1: + info->offset = mmDC_GPIO_DDC1_A; + break; + case GPIO_DDC_LINE_DDC2: + info->offset = mmDC_GPIO_DDC2_A; + break; + case GPIO_DDC_LINE_DDC3: + info->offset = mmDC_GPIO_DDC3_A; + break; + case GPIO_DDC_LINE_DDC4: + info->offset = mmDC_GPIO_DDC4_A; + break; + case GPIO_DDC_LINE_DDC5: + info->offset = mmDC_GPIO_DDC5_A; + break; + case GPIO_DDC_LINE_DDC6: + info->offset = mmDC_GPIO_DDC6_A; + break; + case GPIO_DDC_LINE_DDC_VGA: + info->offset = mmDC_GPIO_DDCVGA_A; + break; + case GPIO_DDC_LINE_I2C_PAD: + info->offset = mmDC_GPIO_I2CPAD_A; + break; + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_GENERIC: + info->offset = mmDC_GPIO_GENERIC_A; + switch (en) { + case GPIO_GENERIC_A: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK; + break; + case GPIO_GENERIC_B: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK; + break; + case GPIO_GENERIC_C: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK; + break; + case GPIO_GENERIC_D: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK; + break; + case GPIO_GENERIC_E: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK; + break; + case GPIO_GENERIC_F: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK; + break; + case GPIO_GENERIC_G: + info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK; + break; + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_HPD: + info->offset = mmDC_GPIO_HPD_A; + switch (en) { + case GPIO_HPD_1: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK; + break; + case GPIO_HPD_2: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK; + break; + case GPIO_HPD_3: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK; + break; + case GPIO_HPD_4: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK; + break; + case GPIO_HPD_5: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK; + break; + case GPIO_HPD_6: + info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK; + break; + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_SYNC: + switch (en) { + case GPIO_SYNC_HSYNC_A: + info->offset = mmDC_GPIO_SYNCA_A; + info->mask = DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK; + break; + case GPIO_SYNC_VSYNC_A: + info->offset = mmDC_GPIO_SYNCA_A; + info->mask = DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK; + break; + case GPIO_SYNC_HSYNC_B: + case GPIO_SYNC_VSYNC_B: + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_GSL: + switch (en) { + case GPIO_GSL_GENLOCK_CLOCK: + info->offset = mmDC_GPIO_GENLK_A; + info->mask = DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK; + break; + case GPIO_GSL_GENLOCK_VSYNC: + info->offset = mmDC_GPIO_GENLK_A; + info->mask = + DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK; + break; + case GPIO_GSL_SWAPLOCK_A: + info->offset = mmDC_GPIO_GENLK_A; + info->mask = DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK; + break; + case GPIO_GSL_SWAPLOCK_B: + info->offset = mmDC_GPIO_GENLK_A; + info->mask = DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK; + break; + default: + BREAK_TO_DEBUGGER(); + result = false; + } + break; + case GPIO_ID_GPIO_PAD: + info->offset = mmGPIOPAD_A; + info->mask = (1 << en); + result = (info->mask <= GPIO_GPIO_PAD_MAX); + break; + case GPIO_ID_VIP_PAD: + default: + BREAK_TO_DEBUGGER(); + result = false; + } + + if (result) { + info->offset_y = info->offset + 2; + info->offset_en = info->offset + 1; + info->offset_mask = info->offset - 1; + + info->mask_y = info->mask; + info->mask_en = info->mask; + info->mask_mask = info->mask; + } + + return result; +} + +static const struct hw_translate_funcs funcs = { + .offset_to_id = offset_to_id, + .id_to_offset = id_to_offset, +}; + +void dal_hw_translate_dce60_init( + struct hw_translate *translate) +{ + translate->funcs = &funcs; +} diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.h b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.h new file mode 100644 index 000000000000..1e811f35cec7 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/gpio/dce60/hw_translate_dce60.h @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DAL_HW_TRANSLATE_DCE60_H__ +#define __DAL_HW_TRANSLATE_DCE60_H__ + +void dal_hw_translate_dce60_init( + struct hw_translate *tr); + +#endif diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c index e5cfe28bc7bf..6fc8a6e9dc15 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c @@ -42,6 +42,9 @@ * Post-requisites: headers required by this unit */ +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/hw_factory_dce60.h" +#endif #include "dce80/hw_factory_dce80.h" #include "dce110/hw_factory_dce110.h" #include "dce120/hw_factory_dce120.h" @@ -71,6 +74,13 @@ bool dal_hw_factory_init( } switch (dce_version) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case DCE_VERSION_6_0: + case DCE_VERSION_6_1: + case DCE_VERSION_6_4: + dal_hw_factory_dce60_init(factory); + return true; +#endif case DCE_VERSION_8_0: case DCE_VERSION_8_1: case DCE_VERSION_8_3: diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c index efea7cb0f17c..3a93c945e57d 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c @@ -40,6 +40,9 @@ * Post-requisites: headers required by this unit */ +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/hw_translate_dce60.h" +#endif #include "dce80/hw_translate_dce80.h" #include "dce110/hw_translate_dce110.h" #include "dce120/hw_translate_dce120.h" @@ -69,6 +72,13 @@ bool dal_hw_translate_init( } switch (dce_version) { +#if defined(CONFIG_DRM_AMD_DC_SI) + case DCE_VERSION_6_0: + case DCE_VERSION_6_1: + case DCE_VERSION_6_4: + dal_hw_translate_dce60_init(translate); + return true; +#endif case DCE_VERSION_8_0: case DCE_VERSION_8_1: case DCE_VERSION_8_3: diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h index 5994d2a33c40..947d6106f341 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h @@ -230,6 +230,8 @@ struct clk_mgr_funcs { int (*get_dp_ref_clk_frequency)(struct clk_mgr *clk_mgr); + void (*set_low_power_state)(struct clk_mgr *clk_mgr); + void (*init_clocks)(struct clk_mgr *clk_mgr); void (*enable_pme_wa) (struct clk_mgr *clk_mgr); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h index 72743058836d..949b61351ede 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h @@ -89,6 +89,11 @@ enum dentist_divider_range { .DPREFCLK_CNTL = mmDPREFCLK_CNTL, \ .DENTIST_DISPCLK_CNTL = mmDENTIST_DISPCLK_CNTL +#if defined(CONFIG_DRM_AMD_DC_SI) +#define CLK_COMMON_REG_LIST_DCE60_BASE() \ + SR(DENTIST_DISPCLK_CNTL) +#endif + #define CLK_COMMON_REG_LIST_DCN_BASE() \ SR(DENTIST_DISPCLK_CNTL) @@ -115,6 +120,12 @@ enum dentist_divider_range { CLK_SF(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, mask_sh), \ CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, mask_sh) +#if defined(CONFIG_DRM_AMD_DC_SI) +#define CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(mask_sh) \ + CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\ + CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh) +#endif + #define CLK_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh) \ CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\ CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh) diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/panel_cntl.h b/drivers/gpu/drm/amd/display/dc/inc/hw/panel_cntl.h index f9ab5abb6462..48eac622c6a0 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/panel_cntl.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/panel_cntl.h @@ -49,6 +49,7 @@ struct panel_cntl_funcs { void (*store_backlight_level)(struct panel_cntl *panel_cntl); void (*driver_set_backlight)(struct panel_cntl *panel_cntl, uint32_t backlight_pwm_u16_16); + uint32_t (*get_current_backlight)(struct panel_cntl *panel_cntl); }; struct panel_cntl_init_data { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 3c986717dcd5..64c1be818b0e 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -67,6 +67,10 @@ struct hw_sequencer_funcs { int num_planes, struct dc_state *context); void (*program_front_end_for_ctx)(struct dc *dc, struct dc_state *context); + bool (*disconnect_pipes)(struct dc *dc, + struct dc_state *context); + void (*wait_for_pending_cleared)(struct dc *dc, + struct dc_state *context); void (*post_unlock_program_front_end)(struct dc *dc, struct dc_state *context); void (*update_plane_addr)(const struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/irq/Makefile b/drivers/gpu/drm/amd/display/dc/irq/Makefile index 3352b79fb1cb..405c25322607 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/Makefile +++ b/drivers/gpu/drm/amd/display/dc/irq/Makefile @@ -31,6 +31,17 @@ AMD_DAL_IRQ = $(addprefix $(AMDDALPATH)/dc/irq/,$(IRQ)) AMD_DISPLAY_FILES += $(AMD_DAL_IRQ) ############################################################################### +# DCE 6x +############################################################################### +ifdef CONFIG_DRM_AMD_DC_SI +IRQ_DCE60 = irq_service_dce60.o + +AMD_DAL_IRQ_DCE60 = $(addprefix $(AMDDALPATH)/dc/irq/dce60/,$(IRQ_DCE60)) + +AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCE60) +endif + +############################################################################### # DCE 8x ############################################################################### IRQ_DCE80 = irq_service_dce80.o diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.c b/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.c new file mode 100644 index 000000000000..524481885fd0 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.c @@ -0,0 +1,395 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include <linux/slab.h> + +#include "dm_services.h" + +#include "include/logger_interface.h" + +#include "irq_service_dce60.h" +#include "../dce110/irq_service_dce110.h" + +#include "dce/dce_6_0_d.h" +#include "dce/dce_6_0_sh_mask.h" + +#include "ivsrcid/ivsrcid_vislands30.h" + +#define VISLANDS30_IV_SRCID_D1_VBLANK 1 +#define VISLANDS30_IV_SRCID_D2_VBLANK 2 +#define VISLANDS30_IV_SRCID_D3_VBLANK 3 +#define VISLANDS30_IV_SRCID_D4_VBLANK 4 +#define VISLANDS30_IV_SRCID_D5_VBLANK 5 +#define VISLANDS30_IV_SRCID_D6_VBLANK 6 + +#include "dc_types.h" + +static bool hpd_ack( + struct irq_service *irq_service, + const struct irq_source_info *info) +{ + uint32_t addr = info->status_reg; + uint32_t value = dm_read_reg(irq_service->ctx, addr); + uint32_t current_status = + get_reg_field_value( + value, + DC_HPD1_INT_STATUS, + DC_HPD1_SENSE_DELAYED); + + dal_irq_service_ack_generic(irq_service, info); + + value = dm_read_reg(irq_service->ctx, info->enable_reg); + + set_reg_field_value( + value, + current_status ? 0 : 1, + DC_HPD1_INT_CONTROL, + DC_HPD1_INT_POLARITY); + + dm_write_reg(irq_service->ctx, info->enable_reg, value); + + return true; +} + +static const struct irq_source_info_funcs hpd_irq_info_funcs = { + .set = NULL, + .ack = hpd_ack +}; + +static const struct irq_source_info_funcs hpd_rx_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + +static const struct irq_source_info_funcs pflip_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + +static const struct irq_source_info_funcs vblank_irq_info_funcs = { + .set = dce110_vblank_set, + .ack = NULL +}; + +static const struct irq_source_info_funcs vblank_irq_info_funcs_dce60 = { + .set = NULL, + .ack = NULL +}; + +#define hpd_int_entry(reg_num)\ + [DC_IRQ_SOURCE_INVALID + reg_num] = {\ + .enable_reg = mmDC_HPD ## reg_num ## _INT_CONTROL,\ + .enable_mask = DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK,\ + .enable_value = {\ + DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK,\ + ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK\ + },\ + .ack_reg = mmDC_HPD ## reg_num ## _INT_CONTROL,\ + .ack_mask = DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK,\ + .ack_value = DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK,\ + .status_reg = mmDC_HPD ## reg_num ## _INT_STATUS,\ + .funcs = &hpd_irq_info_funcs\ + } + +#define hpd_rx_int_entry(reg_num)\ + [DC_IRQ_SOURCE_HPD6 + reg_num] = {\ + .enable_reg = mmDC_HPD ## reg_num ## _INT_CONTROL,\ + .enable_mask = DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_EN_MASK,\ + .enable_value = {\ + DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_EN_MASK,\ + ~DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_EN_MASK },\ + .ack_reg = mmDC_HPD ## reg_num ## _INT_CONTROL,\ + .ack_mask = DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_ACK_MASK,\ + .ack_value = DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_ACK_MASK,\ + .status_reg = mmDC_HPD ## reg_num ## _INT_STATUS,\ + .funcs = &hpd_rx_irq_info_funcs\ + } + +#define pflip_int_entry(reg_num)\ + [DC_IRQ_SOURCE_PFLIP1 + reg_num] = {\ + .enable_reg = mmDCP ## reg_num ## _GRPH_INTERRUPT_CONTROL,\ + .enable_mask =\ + GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK,\ + .enable_value = {\ + GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK,\ + ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK},\ + .ack_reg = mmDCP ## reg_num ## _GRPH_INTERRUPT_STATUS,\ + .ack_mask = GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK,\ + .ack_value = GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK,\ + .status_reg = mmDCP ## reg_num ##_GRPH_INTERRUPT_STATUS,\ + .funcs = &pflip_irq_info_funcs\ + } + +#define vupdate_int_entry(reg_num)\ + [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\ + .enable_reg = mmCRTC ## reg_num ## _CRTC_INTERRUPT_CONTROL,\ + .enable_mask =\ + CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_MSK_MASK,\ + .enable_value = {\ + CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_MSK_MASK,\ + ~CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_MSK_MASK},\ + .ack_reg = mmCRTC ## reg_num ## _CRTC_V_UPDATE_INT_STATUS,\ + .ack_mask =\ + CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\ + .ack_value =\ + CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\ + .funcs = &vblank_irq_info_funcs\ + } + +#define vblank_int_entry(reg_num)\ + [DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\ + .enable_reg = mmLB ## reg_num ## _INT_MASK,\ + .enable_mask =\ + INT_MASK__VBLANK_INT_MASK,\ + .enable_value = {\ + INT_MASK__VBLANK_INT_MASK,\ + ~INT_MASK__VBLANK_INT_MASK},\ + .ack_reg = mmLB ## reg_num ## _VBLANK_STATUS,\ + .ack_mask =\ + VBLANK_STATUS__VBLANK_ACK_MASK,\ + .ack_value =\ + VBLANK_STATUS__VBLANK_ACK_MASK,\ + .funcs = &vblank_irq_info_funcs_dce60\ + } + +#define dummy_irq_entry() \ + {\ + .funcs = &dummy_irq_info_funcs\ + } + +#define i2c_int_entry(reg_num) \ + [DC_IRQ_SOURCE_I2C_DDC ## reg_num] = dummy_irq_entry() + +#define dp_sink_int_entry(reg_num) \ + [DC_IRQ_SOURCE_DPSINK ## reg_num] = dummy_irq_entry() + +#define gpio_pad_int_entry(reg_num) \ + [DC_IRQ_SOURCE_GPIOPAD ## reg_num] = dummy_irq_entry() + +#define dc_underflow_int_entry(reg_num) \ + [DC_IRQ_SOURCE_DC ## reg_num ## UNDERFLOW] = dummy_irq_entry() + + +static const struct irq_source_info_funcs dummy_irq_info_funcs = { + .set = dal_irq_service_dummy_set, + .ack = dal_irq_service_dummy_ack +}; + +static const struct irq_source_info +irq_source_info_dce60[DAL_IRQ_SOURCES_NUMBER] = { + [DC_IRQ_SOURCE_INVALID] = dummy_irq_entry(), + hpd_int_entry(1), + hpd_int_entry(2), + hpd_int_entry(3), + hpd_int_entry(4), + hpd_int_entry(5), + hpd_int_entry(6), + hpd_rx_int_entry(1), + hpd_rx_int_entry(2), + hpd_rx_int_entry(3), + hpd_rx_int_entry(4), + hpd_rx_int_entry(5), + hpd_rx_int_entry(6), + i2c_int_entry(1), + i2c_int_entry(2), + i2c_int_entry(3), + i2c_int_entry(4), + i2c_int_entry(5), + i2c_int_entry(6), + dp_sink_int_entry(1), + dp_sink_int_entry(2), + dp_sink_int_entry(3), + dp_sink_int_entry(4), + dp_sink_int_entry(5), + dp_sink_int_entry(6), + [DC_IRQ_SOURCE_TIMER] = dummy_irq_entry(), + pflip_int_entry(0), + pflip_int_entry(1), + pflip_int_entry(2), + pflip_int_entry(3), + pflip_int_entry(4), + pflip_int_entry(5), + [DC_IRQ_SOURCE_PFLIP_UNDERLAY0] = dummy_irq_entry(), + gpio_pad_int_entry(0), + gpio_pad_int_entry(1), + gpio_pad_int_entry(2), + gpio_pad_int_entry(3), + gpio_pad_int_entry(4), + gpio_pad_int_entry(5), + gpio_pad_int_entry(6), + gpio_pad_int_entry(7), + gpio_pad_int_entry(8), + gpio_pad_int_entry(9), + gpio_pad_int_entry(10), + gpio_pad_int_entry(11), + gpio_pad_int_entry(12), + gpio_pad_int_entry(13), + gpio_pad_int_entry(14), + gpio_pad_int_entry(15), + gpio_pad_int_entry(16), + gpio_pad_int_entry(17), + gpio_pad_int_entry(18), + gpio_pad_int_entry(19), + gpio_pad_int_entry(20), + gpio_pad_int_entry(21), + gpio_pad_int_entry(22), + gpio_pad_int_entry(23), + gpio_pad_int_entry(24), + gpio_pad_int_entry(25), + gpio_pad_int_entry(26), + gpio_pad_int_entry(27), + gpio_pad_int_entry(28), + gpio_pad_int_entry(29), + gpio_pad_int_entry(30), + dc_underflow_int_entry(1), + dc_underflow_int_entry(2), + dc_underflow_int_entry(3), + dc_underflow_int_entry(4), + dc_underflow_int_entry(5), + dc_underflow_int_entry(6), + [DC_IRQ_SOURCE_DMCU_SCP] = dummy_irq_entry(), + [DC_IRQ_SOURCE_VBIOS_SW] = dummy_irq_entry(), + vupdate_int_entry(0), + vupdate_int_entry(1), + vupdate_int_entry(2), + vupdate_int_entry(3), + vupdate_int_entry(4), + vupdate_int_entry(5), + vblank_int_entry(0), + vblank_int_entry(1), + vblank_int_entry(2), + vblank_int_entry(3), + vblank_int_entry(4), + vblank_int_entry(5), +}; + +enum dc_irq_source to_dal_irq_source_dce60( + struct irq_service *irq_service, + uint32_t src_id, + uint32_t ext_id) +{ + switch (src_id) { + case VISLANDS30_IV_SRCID_D1_VBLANK: + return DC_IRQ_SOURCE_VBLANK1; + case VISLANDS30_IV_SRCID_D2_VBLANK: + return DC_IRQ_SOURCE_VBLANK2; + case VISLANDS30_IV_SRCID_D3_VBLANK: + return DC_IRQ_SOURCE_VBLANK3; + case VISLANDS30_IV_SRCID_D4_VBLANK: + return DC_IRQ_SOURCE_VBLANK4; + case VISLANDS30_IV_SRCID_D5_VBLANK: + return DC_IRQ_SOURCE_VBLANK5; + case VISLANDS30_IV_SRCID_D6_VBLANK: + return DC_IRQ_SOURCE_VBLANK6; + case VISLANDS30_IV_SRCID_D1_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE1; + case VISLANDS30_IV_SRCID_D2_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE2; + case VISLANDS30_IV_SRCID_D3_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE3; + case VISLANDS30_IV_SRCID_D4_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE4; + case VISLANDS30_IV_SRCID_D5_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE5; + case VISLANDS30_IV_SRCID_D6_V_UPDATE_INT: + return DC_IRQ_SOURCE_VUPDATE6; + case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP1; + case VISLANDS30_IV_SRCID_D2_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP2; + case VISLANDS30_IV_SRCID_D3_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP3; + case VISLANDS30_IV_SRCID_D4_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP4; + case VISLANDS30_IV_SRCID_D5_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP5; + case VISLANDS30_IV_SRCID_D6_GRPH_PFLIP: + return DC_IRQ_SOURCE_PFLIP6; + + case VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A: + /* generic src_id for all HPD and HPDRX interrupts */ + switch (ext_id) { + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_A: + return DC_IRQ_SOURCE_HPD1; + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_B: + return DC_IRQ_SOURCE_HPD2; + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_C: + return DC_IRQ_SOURCE_HPD3; + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_D: + return DC_IRQ_SOURCE_HPD4; + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_E: + return DC_IRQ_SOURCE_HPD5; + case VISLANDS30_IV_EXTID_HOTPLUG_DETECT_F: + return DC_IRQ_SOURCE_HPD6; + case VISLANDS30_IV_EXTID_HPD_RX_A: + return DC_IRQ_SOURCE_HPD1RX; + case VISLANDS30_IV_EXTID_HPD_RX_B: + return DC_IRQ_SOURCE_HPD2RX; + case VISLANDS30_IV_EXTID_HPD_RX_C: + return DC_IRQ_SOURCE_HPD3RX; + case VISLANDS30_IV_EXTID_HPD_RX_D: + return DC_IRQ_SOURCE_HPD4RX; + case VISLANDS30_IV_EXTID_HPD_RX_E: + return DC_IRQ_SOURCE_HPD5RX; + case VISLANDS30_IV_EXTID_HPD_RX_F: + return DC_IRQ_SOURCE_HPD6RX; + default: + return DC_IRQ_SOURCE_INVALID; + } + break; + + default: + return DC_IRQ_SOURCE_INVALID; + } +} + +static const struct irq_service_funcs irq_service_funcs_dce60 = { + .to_dal_irq_source = to_dal_irq_source_dce60 +}; + +static void dce60_irq_construct( + struct irq_service *irq_service, + struct irq_service_init_data *init_data) +{ + dal_irq_service_construct(irq_service, init_data); + + irq_service->info = irq_source_info_dce60; + irq_service->funcs = &irq_service_funcs_dce60; +} + +struct irq_service *dal_irq_service_dce60_create( + struct irq_service_init_data *init_data) +{ + struct irq_service *irq_service = kzalloc(sizeof(*irq_service), + GFP_KERNEL); + + if (!irq_service) + return NULL; + + dce60_irq_construct(irq_service, init_data); + return irq_service; +} + + diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.h b/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.h new file mode 100644 index 000000000000..294db29e8115 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/irq/dce60/irq_service_dce60.h @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DAL_IRQ_SERVICE_DCE60_H__ +#define __DAL_IRQ_SERVICE_DCE60_H__ + +#include "../irq_service.h" + +enum dc_irq_source to_dal_irq_source_dce60( + struct irq_service *irq_service, + uint32_t src_id, + uint32_t ext_id); + +struct irq_service *dal_irq_service_dce60_create( + struct irq_service_init_data *init_data); + +#endif + diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c index 33053b9fe6bd..6bf27bde8724 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c +++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c @@ -32,6 +32,9 @@ #include "dce110/irq_service_dce110.h" +#if defined(CONFIG_DRM_AMD_DC_SI) +#include "dce60/irq_service_dce60.h" +#endif #include "dce80/irq_service_dce80.h" diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h index c3bbfe397e8d..330acaaed79a 100644 --- a/drivers/gpu/drm/amd/display/dc/os_types.h +++ b/drivers/gpu/drm/amd/display/dc/os_types.h @@ -55,6 +55,10 @@ #include <asm/fpu/api.h> #define DC_FP_START() kernel_fpu_begin() #define DC_FP_END() kernel_fpu_end() +#elif defined(CONFIG_ARM64) +#include <asm/neon.h> +#define DC_FP_START() kernel_neon_begin() +#define DC_FP_END() kernel_neon_end() #elif defined(CONFIG_PPC64) #include <asm/switch_to.h> #include <asm/cputable.h> diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c index b8040da94b9d..944c0327763c 100644 --- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c @@ -87,6 +87,17 @@ static void virtual_enc_dp_set_odm_combine( bool odm_combine) {} +static void virtual_dig_connect_to_otg( + struct stream_encoder *enc, + int tg_inst) +{} + +static void virtual_setup_stereo_sync( + struct stream_encoder *enc, + int tg_inst, + bool enable) +{} + static const struct stream_encoder_funcs virtual_str_enc_funcs = { .dp_set_odm_combine = virtual_enc_dp_set_odm_combine, @@ -114,6 +125,8 @@ static const struct stream_encoder_funcs virtual_str_enc_funcs = { .audio_mute_control = virtual_audio_mute_control, .set_avmute = virtual_stream_encoder_set_avmute, .hdmi_reset_stream_attribute = virtual_stream_encoder_reset_hdmi_stream_attribute, + .dig_connect_to_otg = virtual_dig_connect_to_otg, + .setup_stereo_sync = virtual_setup_stereo_sync, }; bool virtual_stream_encoder_construct( diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index e013875b89ed..d7e7f2eda92f 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -36,11 +36,20 @@ /* Firmware versioning. */ #ifdef DMUB_EXPOSE_VERSION -#define DMUB_FW_VERSION_GIT_HASH 0xe6d590b09 +#define DMUB_FW_VERSION_GIT_HASH 0x4e5b2f46f #define DMUB_FW_VERSION_MAJOR 0 #define DMUB_FW_VERSION_MINOR 0 -#define DMUB_FW_VERSION_REVISION 25 -#define DMUB_FW_VERSION_UCODE ((DMUB_FW_VERSION_MAJOR << 24) | (DMUB_FW_VERSION_MINOR << 16) | DMUB_FW_VERSION_REVISION) +#define DMUB_FW_VERSION_REVISION 29 +#define DMUB_FW_VERSION_TEST 0 +#define DMUB_FW_VERSION_VBIOS 0 +#define DMUB_FW_VERSION_HOTFIX 0 +#define DMUB_FW_VERSION_UCODE (((DMUB_FW_VERSION_MAJOR & 0xFF) << 24) | \ + ((DMUB_FW_VERSION_MINOR & 0xFF) << 16) | \ + ((DMUB_FW_VERSION_REVISION & 0xFF) << 8) | \ + ((DMUB_FW_VERSION_TEST & 0x1) << 7) | \ + ((DMUB_FW_VERSION_VBIOS & 0x1) << 6) | \ + (DMUB_FW_VERSION_HOTFIX & 0x3F)) + #endif //<DMUB_TYPES>================================================================== @@ -204,6 +213,7 @@ enum dmub_cmd_vbios_type { DMUB_CMD__VBIOS_DIG1_TRANSMITTER_CONTROL = 1, DMUB_CMD__VBIOS_SET_PIXEL_CLOCK = 2, DMUB_CMD__VBIOS_ENABLE_DISP_POWER_GATING = 3, + DMUB_CMD__VBIOS_LVTMA_CONTROL = 15, }; //============================================================================== diff --git a/drivers/gpu/drm/amd/display/include/bios_parser_types.h b/drivers/gpu/drm/amd/display/include/bios_parser_types.h index c30437ae8395..21011edea337 100644 --- a/drivers/gpu/drm/amd/display/include/bios_parser_types.h +++ b/drivers/gpu/drm/amd/display/include/bios_parser_types.h @@ -101,6 +101,13 @@ enum bp_pipe_control_action { ASIC_PIPE_INIT }; +enum bp_lvtma_control_action { + LVTMA_CONTROL_LCD_BLOFF = 2, + LVTMA_CONTROL_LCD_BLON = 3, + LVTMA_CONTROL_POWER_ON = 12, + LVTMA_CONTROL_POWER_OFF = 13 +}; + struct bp_encoder_control { enum bp_encoder_control_action action; enum engine_id engine_id; diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h index abeb58d544b1..b267987aed06 100644 --- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h +++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h @@ -30,6 +30,34 @@ * ASIC internal revision ID */ +/* DCE60 (based on si_id.h in GPUOpen-Tools CodeXL) */ +#define SI_TAHITI_P_A0 0x01 +#define SI_TAHITI_P_B0 0x05 +#define SI_TAHITI_P_B1 0x06 +#define SI_PITCAIRN_PM_A0 0x14 +#define SI_PITCAIRN_PM_A1 0x15 +#define SI_CAPEVERDE_M_A0 0x28 +#define SI_CAPEVERDE_M_A1 0x29 +#define SI_OLAND_M_A0 0x3C +#define SI_HAINAN_V_A0 0x46 + +#define SI_UNKNOWN 0xFF + +#define ASIC_REV_IS_TAHITI_P(rev) \ + ((rev >= SI_TAHITI_P_A0) && (rev < SI_PITCAIRN_PM_A0)) + +#define ASIC_REV_IS_PITCAIRN_PM(rev) \ + ((rev >= SI_PITCAIRN_PM_A0) && (rev < SI_CAPEVERDE_M_A0)) + +#define ASIC_REV_IS_CAPEVERDE_M(rev) \ + ((rev >= SI_CAPEVERDE_M_A0) && (rev < SI_OLAND_M_A0)) + +#define ASIC_REV_IS_OLAND_M(rev) \ + ((rev >= SI_OLAND_M_A0) && (rev < SI_HAINAN_V_A0)) + +#define ASIC_REV_IS_HAINAN_V(rev) \ + ((rev >= SI_HAINAN_V_A0) && (rev < SI_UNKNOWN)) + /* DCE80 (based on ci_id.h in Perforce) */ #define CI_BONAIRE_M_A0 0x14 #define CI_BONAIRE_M_A1 0x15 @@ -181,6 +209,17 @@ enum { /* * ASIC chip ID */ + +/* DCE60 */ +#define DEVICE_ID_SI_TAHITI_P_6780 0x6780 +#define DEVICE_ID_SI_PITCAIRN_PM_6800 0x6800 +#define DEVICE_ID_SI_PITCAIRN_PM_6808 0x6808 +#define DEVICE_ID_SI_CAPEVERDE_M_6820 0x6820 +#define DEVICE_ID_SI_CAPEVERDE_M_6828 0x6828 +#define DEVICE_ID_SI_OLAND_M_6600 0x6600 +#define DEVICE_ID_SI_OLAND_M_6608 0x6608 +#define DEVICE_ID_SI_HAINAN_V_6660 0x6660 + /* DCE80 */ #define DEVICE_ID_KALINDI_9834 0x9834 #define DEVICE_ID_TEMASH_9839 0x9839 @@ -190,6 +229,7 @@ enum { #define DEVICE_ID_RENOIR_1636 0x1636 /* Asic Family IDs for different asic family. */ +#define FAMILY_SI 110 /* Southern Islands: Tahiti (P), Pitcairn (PM), Cape Verde (M), Oland (M), Hainan (V) */ #define FAMILY_CI 120 /* Sea Islands: Hawaii (P), Bonaire (M) */ #define FAMILY_KV 125 /* Fusion => Kaveri: Spectre, Spooky; Kabini: Kalindi */ #define FAMILY_VI 130 /* Volcanic Islands: Iceland (V), Tonga (M) */ diff --git a/drivers/gpu/drm/amd/display/include/dal_types.h b/drivers/gpu/drm/amd/display/include/dal_types.h index b67c9fa6b9cd..8aaa3af69202 100644 --- a/drivers/gpu/drm/amd/display/include/dal_types.h +++ b/drivers/gpu/drm/amd/display/include/dal_types.h @@ -34,6 +34,9 @@ struct dc_bios; enum dce_version { DCE_VERSION_UNKNOWN = (-1), + DCE_VERSION_6_0, + DCE_VERSION_6_1, + DCE_VERSION_6_4, DCE_VERSION_8_0, DCE_VERSION_8_1, DCE_VERSION_8_3, diff --git a/drivers/gpu/drm/amd/display/include/fixed31_32.h b/drivers/gpu/drm/amd/display/include/fixed31_32.h index 89ef9f6860e5..16df2a485dd0 100644 --- a/drivers/gpu/drm/amd/display/include/fixed31_32.h +++ b/drivers/gpu/drm/amd/display/include/fixed31_32.h @@ -431,6 +431,9 @@ struct fixed31_32 dc_fixpt_log(struct fixed31_32 arg); */ static inline struct fixed31_32 dc_fixpt_pow(struct fixed31_32 arg1, struct fixed31_32 arg2) { + if (arg1.value == 0) + return arg2.value == 0 ? dc_fixpt_one : dc_fixpt_zero; + return dc_fixpt_exp( dc_fixpt_mul( dc_fixpt_log(arg1), diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 81820f3d6b3b..d988533d4af5 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -324,22 +324,44 @@ static void apply_below_the_range(struct core_freesync *core_freesync, /* Choose number of frames to insert based on how close it * can get to the mid point of the variable range. + * - Delta for CEIL: delta_from_mid_point_in_us_1 + * - Delta for FLOOR: delta_from_mid_point_in_us_2 */ - if ((frame_time_in_us / mid_point_frames_ceil) > in_out_vrr->min_duration_in_us && - (delta_from_mid_point_in_us_1 < delta_from_mid_point_in_us_2 || - mid_point_frames_floor < 2)) { + if ((last_render_time_in_us / mid_point_frames_ceil) < in_out_vrr->min_duration_in_us) { + /* Check for out of range. + * If using CEIL produces a value that is out of range, + * then we are forced to use FLOOR. + */ + frames_to_insert = mid_point_frames_floor; + } else if (mid_point_frames_floor < 2) { + /* Check if FLOOR would result in non-LFC. In this case + * choose to use CEIL + */ + frames_to_insert = mid_point_frames_ceil; + } else if (delta_from_mid_point_in_us_1 < delta_from_mid_point_in_us_2) { + /* If choosing CEIL results in a frame duration that is + * closer to the mid point of the range. + * Choose CEIL + */ frames_to_insert = mid_point_frames_ceil; - delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_2 - - delta_from_mid_point_in_us_1; } else { + /* If choosing FLOOR results in a frame duration that is + * closer to the mid point of the range. + * Choose FLOOR + */ frames_to_insert = mid_point_frames_floor; - delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_1 - - delta_from_mid_point_in_us_2; } /* Prefer current frame multiplier when BTR is enabled unless it drifts * too far from the midpoint */ + if (delta_from_mid_point_in_us_1 < delta_from_mid_point_in_us_2) { + delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_2 - + delta_from_mid_point_in_us_1; + } else { + delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_1 - + delta_from_mid_point_in_us_2; + } if (in_out_vrr->btr.frames_to_insert != 0 && delta_from_mid_point_delta_in_us < BTR_DRIFT_MARGIN) { if (((last_render_time_in_us / in_out_vrr->btr.frames_to_insert) < diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h index 13c57ff2abdc..1ab813b4fd14 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h @@ -37,6 +37,6 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet); void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, - struct dc_info_packet *info_packet, int ALLMEnabled, int ALLMValue); + struct dc_info_packet *info_packet); #endif diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index 7cd8a43d1889..0fdf7a3e96de 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -421,15 +421,13 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, ***************************************************************************** */ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, - struct dc_info_packet *info_packet, int ALLMEnabled, int ALLMValue) + struct dc_info_packet *info_packet) { unsigned int length = 5; bool hdmi_vic_mode = false; uint8_t checksum = 0; uint32_t i = 0; enum dc_timing_3d_format format; - bool bALLM = (bool)ALLMEnabled; - bool bALLMVal = (bool)ALLMValue; info_packet->valid = false; format = stream->timing.timing_3d_format; @@ -442,20 +440,13 @@ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, && format == TIMING_3D_FORMAT_NONE) hdmi_vic_mode = true; - if ((format == TIMING_3D_FORMAT_NONE) && !hdmi_vic_mode && !bALLM) + if ((format == TIMING_3D_FORMAT_NONE) && !hdmi_vic_mode) return; info_packet->sb[1] = 0x03; info_packet->sb[2] = 0x0C; info_packet->sb[3] = 0x00; - if (bALLM) { - info_packet->sb[1] = 0xD8; - info_packet->sb[2] = 0x5D; - info_packet->sb[3] = 0xC4; - info_packet->sb[4] = HF_VSIF_VERSION; - } - if (format != TIMING_3D_FORMAT_NONE) info_packet->sb[4] = (2 << 5); @@ -490,9 +481,6 @@ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, info_packet->hb1 = 0x01; info_packet->hb2 = (uint8_t) (length); - if (bALLM) - info_packet->sb[5] = (info_packet->sb[5] & ~0x02) | (bALLMVal << 1); - checksum += info_packet->hb0; checksum += info_packet->hb1; checksum += info_packet->hb2; diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c index 859724771a75..61497954e67e 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c @@ -657,7 +657,7 @@ void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parame params, ram_table, big_endian); } -bool dmub_init_abm_config(struct abm *abm, +bool dmub_init_abm_config(struct resource_pool *res_pool, struct dmcu_iram_parameters params) { struct iram_table_v_2_2 ram_table; @@ -665,8 +665,13 @@ bool dmub_init_abm_config(struct abm *abm, bool result = false; uint32_t i, j = 0; - if (abm == NULL) +#if defined(CONFIG_DRM_AMD_DC_DCN3_0) + if (res_pool->abm == NULL && res_pool->multiple_abms[0] == NULL) return false; +#else + if (res_pool->abm == NULL) + return false; +#endif memset(&ram_table, 0, sizeof(ram_table)); memset(&config, 0, sizeof(config)); @@ -707,8 +712,14 @@ bool dmub_init_abm_config(struct abm *abm, config.min_abm_backlight = ram_table.min_abm_backlight; - result = abm->funcs->init_abm_config( - abm, (char *)(&config), sizeof(struct abm_config_table)); +#if defined(CONFIG_DRM_AMD_DC_DCN3_0) + if (res_pool->multiple_abms[0]) { + result = res_pool->multiple_abms[0]->funcs->init_abm_config( + res_pool->multiple_abms[0], (char *)(&config), sizeof(struct abm_config_table)); + } else +#endif + result = res_pool->abm->funcs->init_abm_config( + res_pool->abm, (char *)(&config), sizeof(struct abm_config_table)); return result; } diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h index 46fbca2e2cd1..fa4728d88092 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h @@ -28,6 +28,8 @@ #include "dc/inc/hw/dmcu.h" #include "dc/inc/hw/abm.h" +struct resource_pool; + enum abm_defines { abm_defines_max_level = 4, @@ -45,7 +47,7 @@ struct dmcu_iram_parameters { bool dmcu_load_iram(struct dmcu *dmcu, struct dmcu_iram_parameters params); -bool dmub_init_abm_config(struct abm *abm, +bool dmub_init_abm_config(struct resource_pool *res_pool, struct dmcu_iram_parameters params); #endif /* MODULES_POWER_POWER_HELPERS_H_ */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h index 27bb8c1ab858..b6f74bf4af02 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h @@ -7376,8 +7376,6 @@ #define mmCRTC4_CRTC_DRR_CONTROL 0x0f3e #define mmCRTC4_CRTC_DRR_CONTROL_BASE_IDX 2 -#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x395d -#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 // addressBlock: dce_dc_fmt4_dispdec // base address: 0x2000 diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h index ae798f768853..9de01ae574c0 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h @@ -4444,14 +4444,90 @@ /* Registers that spilled out of sid.h */ #define mmDATA_FORMAT 0x1AC0 +#define mmLB0_DATA_FORMAT 0x1AC0 +#define mmLB1_DATA_FORMAT 0x1DC0 +#define mmLB2_DATA_FORMAT 0x40C0 +#define mmLB3_DATA_FORMAT 0x43C0 +#define mmLB4_DATA_FORMAT 0x46C0 +#define mmLB5_DATA_FORMAT 0x49C0 #define mmDESKTOP_HEIGHT 0x1AC1 +#define mmLB0_DESKTOP_HEIGHT 0x1AC1 +#define mmLB1_DESKTOP_HEIGHT 0x1DC1 +#define mmLB2_DESKTOP_HEIGHT 0x40C1 +#define mmLB3_DESKTOP_HEIGHT 0x43C1 +#define mmLB4_DESKTOP_HEIGHT 0x46C1 +#define mmLB5_DESKTOP_HEIGHT 0x49C1 #define mmDC_LB_MEMORY_SPLIT 0x1AC3 +#define mmLB0_DC_LB_MEMORY_SPLIT 0x1AC3 +#define mmLB1_DC_LB_MEMORY_SPLIT 0x1DC3 +#define mmLB2_DC_LB_MEMORY_SPLIT 0x40C3 +#define mmLB3_DC_LB_MEMORY_SPLIT 0x43C3 +#define mmLB4_DC_LB_MEMORY_SPLIT 0x46C3 +#define mmLB5_DC_LB_MEMORY_SPLIT 0x49C3 +#define mmDC_LB_MEM_SIZE 0x1AC4 +#define mmLB0_DC_LB_MEM_SIZE 0x1AC4 +#define mmLB1_DC_LB_MEM_SIZE 0x1DC4 +#define mmLB2_DC_LB_MEM_SIZE 0x40C4 +#define mmLB3_DC_LB_MEM_SIZE 0x43C4 +#define mmLB4_DC_LB_MEM_SIZE 0x46C4 +#define mmLB5_DC_LB_MEM_SIZE 0x49C4 #define mmPRIORITY_A_CNT 0x1AC6 +#define mmLB0_PRIORITY_A_CNT 0x1AC6 +#define mmLB1_PRIORITY_A_CNT 0x1DC6 +#define mmLB2_PRIORITY_A_CNT 0x40C6 +#define mmLB3_PRIORITY_A_CNT 0x43C6 +#define mmLB4_PRIORITY_A_CNT 0x46C6 +#define mmLB5_PRIORITY_A_CNT 0x49C6 #define mmPRIORITY_B_CNT 0x1AC7 +#define mmLB0_PRIORITY_B_CNT 0x1AC7 +#define mmLB1_PRIORITY_B_CNT 0x1DC7 +#define mmLB2_PRIORITY_B_CNT 0x40C7 +#define mmLB3_PRIORITY_B_CNT 0x43C7 +#define mmLB4_PRIORITY_B_CNT 0x46C7 +#define mmLB5_PRIORITY_B_CNT 0x49C7 #define mmDPG_PIPE_ARBITRATION_CONTROL3 0x1B32 +#define mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL3 0x1B32 +#define mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL3 0x1E32 +#define mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL3 0x4132 +#define mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL3 0x4432 +#define mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL3 0x4732 +#define mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL3 0x4A32 #define mmINT_MASK 0x1AD0 +#define mmLB0_INT_MASK 0x1AD0 +#define mmLB1_INT_MASK 0x1DD0 +#define mmLB2_INT_MASK 0x40D0 +#define mmLB3_INT_MASK 0x43D0 +#define mmLB4_INT_MASK 0x46D0 +#define mmLB5_INT_MASK 0x49D0 #define mmVLINE_STATUS 0x1AEE +#define mmLB0_VLINE_STATUS 0x1AEE +#define mmLB1_VLINE_STATUS 0x1DEE +#define mmLB2_VLINE_STATUS 0x40EE +#define mmLB3_VLINE_STATUS 0x43EE +#define mmLB4_VLINE_STATUS 0x46EE +#define mmLB5_VLINE_STATUS 0x49EE #define mmVBLANK_STATUS 0x1AEF +#define mmLB0_VBLANK_STATUS 0x1AEF +#define mmLB1_VBLANK_STATUS 0x1DEF +#define mmLB2_VBLANK_STATUS 0x40EF +#define mmLB3_VBLANK_STATUS 0x43EF +#define mmLB4_VBLANK_STATUS 0x46EF +#define mmLB5_VBLANK_STATUS 0x49EF +#define mmSCL_HORZ_FILTER_INIT_RGB_LUMA 0x1B4C +#define mmSCL0_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x1B4C +#define mmSCL1_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x1E4C +#define mmSCL2_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x414C +#define mmSCL3_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x444C +#define mmSCL4_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x474C +#define mmSCL5_SCL_HORZ_FILTER_INIT_RGB_LUMA 0x4A4C + +#define mmSCL_HORZ_FILTER_INIT_CHROMA 0x1B4D +#define mmSCL0_SCL_HORZ_FILTER_INIT_CHROMA 0x1B4D +#define mmSCL1_SCL_HORZ_FILTER_INIT_CHROMA 0x1E4D +#define mmSCL2_SCL_HORZ_FILTER_INIT_CHROMA 0x414D +#define mmSCL3_SCL_HORZ_FILTER_INIT_CHROMA 0x444D +#define mmSCL4_SCL_HORZ_FILTER_INIT_CHROMA 0x474D +#define mmSCL5_SCL_HORZ_FILTER_INIT_CHROMA 0x4A4D #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h index abe05bc80752..41c4a46ce357 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h @@ -2076,6 +2076,8 @@ #define CRTC_CONTROL__CRTC_START_POINT_CNTL__SHIFT 0x0000000c #define CRTC_CONTROL__CRTC_SYNC_RESET_SEL_MASK 0x00000010L #define CRTC_CONTROL__CRTC_SYNC_RESET_SEL__SHIFT 0x00000004 +#define CRTC_CONTROL__CRTC_PREFETCH_EN_MASK 0x10000000L +#define CRTC_CONTROL__CRTC_PREFETCH_EN__SHIFT 0x0000001c #define CRTC_COUNT_CONTROL__CRTC_HORZ_COUNT_BY2_EN_MASK 0x00000001L #define CRTC_COUNT_CONTROL__CRTC_HORZ_COUNT_BY2_EN__SHIFT 0x00000000 #define CRTC_COUNT_CONTROL__CRTC_HORZ_REPETITION_COUNT_MASK 0x0000001eL @@ -6364,6 +6366,8 @@ #define DPG_PIPE_ARBITRATION_CONTROL2__TIME_WEIGHT__SHIFT 0x00000000 #define DPG_PIPE_ARBITRATION_CONTROL2__URGENCY_WEIGHT_MASK 0xffff0000L #define DPG_PIPE_ARBITRATION_CONTROL2__URGENCY_WEIGHT__SHIFT 0x00000010 +#define DPG_PIPE_ARBITRATION_CONTROL3__URGENCY_WATERMARK_MASK_MASK 0x00030000L +#define DPG_PIPE_ARBITRATION_CONTROL3__URGENCY_WATERMARK_MASK__SHIFT 0x00000010 #define DPG_PIPE_DPM_CONTROL__DPM_ENABLE_MASK 0x00000001L #define DPG_PIPE_DPM_CONTROL__DPM_ENABLE__SHIFT 0x00000000 #define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_ENABLE_MASK 0x00000010L @@ -6384,6 +6388,8 @@ #define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST__SHIFT 0x00000008 #define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_URGENT_DURING_REQUEST_MASK 0x00000010L #define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_URGENT_DURING_REQUEST__SHIFT 0x00000004 +#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK_MASK_MASK 0x00003000L +#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK_MASK__SHIFT 0x0000000c #define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK_MASK 0xffff0000L #define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK__SHIFT 0x00000010 #define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_ENABLE_NONLPTCH_MASK 0x00000001L @@ -6406,6 +6412,8 @@ #define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_WM_HIGH_FORCE_ON_NONLPTCH__SHIFT 0x00000008 #define DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK 0x00000001L #define DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE__SHIFT 0x00000000 +#define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK_MASK 0x00003000L +#define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK__SHIFT 0x0000000c #define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK 0xffff0000L #define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK__SHIFT 0x00000010 #define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_CURSOR_MASK 0x00000010L @@ -7256,6 +7264,8 @@ #define GRPH_CONTROL__GRPH_FORMAT__SHIFT 0x00000008 #define GRPH_CONTROL__GRPH_MACRO_TILE_ASPECT_MASK 0x000c0000L #define GRPH_CONTROL__GRPH_MACRO_TILE_ASPECT__SHIFT 0x00000012 +#define GRPH_CONTROL__GRPH_ARRAY_MODE_MASK 0x00f00000L +#define GRPH_CONTROL__GRPH_ARRAY_MODE__SHIFT 0x00000014 #define GRPH_CONTROL__GRPH_NUM_BANKS_MASK 0x0000000cL #define GRPH_CONTROL__GRPH_NUM_BANKS__SHIFT 0x00000002 #define GRPH_CONTROL__GRPH_PIPE_CONFIG_MASK 0x1f000000L @@ -9835,4 +9845,98 @@ #define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_WRITE_EN_MASK 0x00000100L #define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008 +// DATA_FORMAT +#define DATA_FORMAT__INTERLEAVE_EN_MASK 0x00000001L +#define DATA_FORMAT__INTERLEAVE_EN__SHIFT 0x00000000 +#define DATA_FORMAT__RESET_REQ_AT_EOL_MASK 0x00000010L +#define DATA_FORMAT__RESET_REQ_AT_EOL__SHIFT 0x00000004 +#define DATA_FORMAT__PREFETCH_MASK 0x00001000L +#define DATA_FORMAT__PREFETCH__SHIFT 0x0000000c +#define DATA_FORMAT__SOF_READ_PT_MASK 0x001f0000L +#define DATA_FORMAT__SOF_READ_PT__SHIFT 0x00000010 +#define DATA_FORMAT__REQUEST_MODE_MASK 0x03000000L +#define DATA_FORMAT__REQUEST_MODE__SHIFT 0x00000018 +#define DATA_FORMAT__ALLOW_REQ_MODE_1_2_MASK 0x10000000L +#define DATA_FORMAT__ALLOW_REQ_MODE_1_2__SHIFT 0x0000001c + + +// DC_LB_MEMORY_SPLIT +#define DC_LB_MEMORY_SPLIT__LB_NUM_PARTITIONS_MASK 0x000f0000L +#define DC_LB_MEMORY_SPLIT__LB_NUM_PARTITIONS__SHIFT 0x00000010 +#define DC_LB_MEMORY_SPLIT__DC_LB_MEMORY_CONFIG_MASK 0x00300000L +#define DC_LB_MEMORY_SPLIT__DC_LB_MEMORY_CONFIG__SHIFT 0x00000014 + +// DC_LB_MEM_SIZE +#define DC_LB_MEM_SIZE__DC_LB_MEM_SIZE_MASK 0x000007ffL +#define DC_LB_MEM_SIZE__DC_LB_MEM_SIZE__SHIFT 0x00000000 + +// SCL_TAP_CONTROL +#define SCL_TAP_CONTROL__SCL_V_NUM_OF_TAPS_MASK 0x00000007L +#define SCL_TAP_CONTROL__SCL_V_NUM_OF_TAPS__SHIFT 0x00000000 +#define SCL_TAP_CONTROL__SCL_H_NUM_OF_TAPS_MASK 0x00000f00L +#define SCL_TAP_CONTROL__SCL_H_NUM_OF_TAPS__SHIFT 0x00000008 + +// INT_MASK +#define INT_MASK__VBLANK_INT_MASK 0x00000001L +#define INT_MASK__VBLANK_INT__SHIFT 0x00000000 +#define INT_MASK__VLINE_INT_MASK 0x00000010L +#define INT_MASK__VLINE_INT__SHIFT 0x00000004 + +// PRIORITY_A_CNT +#define PRIORITY_A_CNT__PRIORITY_MARK_A_MASK 0x00007fffL +#define PRIORITY_A_CNT__PRIORITY_MARK_A__SHIFT 0x00000000 +#define PRIORITY_A_CNT__PRIORITY_A_OFF_MASK 0x00010000L +#define PRIORITY_A_CNT__PRIORITY_A_OFF__SHIFT 0x00000010 +#define PRIORITY_A_CNT__PRIORITY_A_ALWAYS_ON_MASK 0x00100000L +#define PRIORITY_A_CNT__PRIORITY_A_ALWAYS_ON__SHIFT 0x00000014 +#define PRIORITY_A_CNT__PRIORITY_A_FORCE_MASK_MASK 0x01000000L +#define PRIORITY_A_CNT__PRIORITY_A_FORCE_MASK__SHIFT 0x00000018 + +// PRIORITY_B_CNT +#define PRIORITY_B_CNT__PRIORITY_MARK_B_MASK 0x00007fffL +#define PRIORITY_B_CNT__PRIORITY_MARK_B__SHIFT 0x00000000 +#define PRIORITY_B_CNT__PRIORITY_B_OFF_MASK 0x00010000L +#define PRIORITY_B_CNT__PRIORITY_B_OFF__SHIFT 0x00000010 +#define PRIORITY_B_CNT__PRIORITY_B_ALWAYS_ON_MASK 0x00100000L +#define PRIORITY_B_CNT__PRIORITY_B_ALWAYS_ON__SHIFT 0x00000014 +#define PRIORITY_B_CNT__PRIORITY_B_FORCE_MASK_MASK 0x01000000L +#define PRIORITY_B_CNT__PRIORITY_B_FORCE_MASK__SHIFT 0x00000018 + +// VLINE_STATUS +#define VLINE_STATUS__VLINE_OCCURRED_MASK 0x00000001L +#define VLINE_STATUS__VLINE_OCCURRED__SHIFT 0x00000000 +#define VLINE_STATUS__VLINE_ACK_MASK 0x00000010L +#define VLINE_STATUS__VLINE_ACK__SHIFT 0x00000004 +#define VLINE_STATUS__VLINE_STAT_MASK 0x00001000L +#define VLINE_STATUS__VLINE_STAT__SHIFT 0x0000000c +#define VLINE_STATUS__VLINE_INTERRUPT_MASK 0x00010000L +#define VLINE_STATUS__VLINE_INTERRUPT__SHIFT 0x00000010 +#define VLINE_STATUS__VLINE_INTERRUPT_TYPE_MASK 0x00020000L +#define VLINE_STATUS__VLINE_INTERRUPT_TYPE__SHIFT 0x00000011 + +// VBLANK_STATUS +#define VBLANK_STATUS__VBLANK_OCCURRED_MASK 0x00000001L +#define VBLANK_STATUS__VBLANK_OCCURRED__SHIFT 0x00000000 +#define VBLANK_STATUS__VBLANK_ACK_MASK 0x00000010L +#define VBLANK_STATUS__VBLANK_ACK__SHIFT 0x00000004 +#define VBLANK_STATUS__VBLANK_STAT_MASK 0x00001000L +#define VBLANK_STATUS__VBLANK_STAT__SHIFT 0x0000000c +#define VBLANK_STATUS__VBLANK_INTERRUPT_MASK 0x00010000L +#define VBLANK_STATUS__VBLANK_INTERRUPT__SHIFT 0x00000010 +#define VBLANK_STATUS__VBLANK_INTERRUPT_TYPE_MASK 0x00020000L +#define VBLANK_STATUS__VBLANK_INTERRUPT_TYPE__SHIFT 0x00000011 + +// SCL_HORZ_FILTER_INIT_RGB_LUMA +#define SCL_HORZ_FILTER_INIT_RGB_LUMA__SCL_H_INIT_FRAC_RGB_Y_MASK 0x0000ffffL +#define SCL_HORZ_FILTER_INIT_RGB_LUMA__SCL_H_INIT_FRAC_RGB_Y__SHIFT 0x00000000 +#define SCL_HORZ_FILTER_INIT_RGB_LUMA__SCL_H_INIT_INT_RGB_Y_MASK 0x000f0000L +#define SCL_HORZ_FILTER_INIT_RGB_LUMA__SCL_H_INIT_INT_RGB_Y__SHIFT 0x00000010 + +// SCL_HORZ_FILTER_INIT_CHROMA +#define SCL_HORZ_FILTER_INIT_CHROMA__SCL_H_INIT_FRAC_CBCR_MASK 0x0000ffffL +#define SCL_HORZ_FILTER_INIT_CHROMA__SCL_H_INIT_FRAC_CBCR__SHIFT 0x00000000 +#define SCL_HORZ_FILTER_INIT_CHROMA__SCL_H_INIT_INT_CBCR_MASK 0x00070000L +#define SCL_HORZ_FILTER_INIT_CHROMA__SCL_H_INIT_INT_CBCR__SHIFT 0x00000010 + + #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h index cf166b591bc5..cf166b591bc5 100755..100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h index 0e0319e98c07..ea683f452bb3 100755..100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h @@ -50271,6 +50271,10 @@ #define DSC_TOP0_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP0_DSC_DEBUG_CONTROL +#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L + // addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec //DSCCIF0_DSCCIF_CONFIG0 @@ -50789,6 +50793,9 @@ #define DSC_TOP1_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP1_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP1_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP1_DSC_DEBUG_CONTROL +#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L // addressBlock: dce_dc_dsc1_dispdec_dsccif_dispdec @@ -51308,6 +51315,10 @@ #define DSC_TOP2_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP2_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP2_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP2_DSC_DEBUG_CONTROL +#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L + // addressBlock: dce_dc_dsc2_dispdec_dsccif_dispdec //DSCCIF2_DSCCIF_CONFIG0 @@ -51826,6 +51837,9 @@ #define DSC_TOP3_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP3_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP3_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP3_DSC_DEBUG_CONTROL +#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L // addressBlock: dce_dc_dsc3_dispdec_dsccif_dispdec @@ -52346,6 +52360,10 @@ #define DSC_TOP4_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP4_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP4_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP4_DSC_DEBUG_CONTROL +#define DSC_TOP4_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP4_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L + // addressBlock: dce_dc_dsc4_dispdec_dsccif_dispdec //DSCCIF4_DSCCIF_CONFIG0 @@ -52864,6 +52882,10 @@ #define DSC_TOP5_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK 0x00000001L #define DSC_TOP5_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK 0x00000010L #define DSC_TOP5_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK 0x00000100L +//DSC_TOP5_DSC_DEBUG_CONTROL +#define DSC_TOP5_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT 0x0 +#define DSC_TOP5_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK 0x00000001L + // addressBlock: dce_dc_dsc5_dispdec_dsccif_dispdec //DSCCIF5_DSCCIF_CONFIG0 diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_offset.h index 67faaf68e9d7..67faaf68e9d7 100755..100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_offset.h diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_sh_mask.h index b4ef50a72868..b4ef50a72868 100755..100644 --- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dpcs_3_0_0_sh_mask.h diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h index 05d1b0a5f6d2..cbaad7d83194 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h @@ -9180,6 +9180,8 @@ #define mmRLC_GPM_THREAD_ENABLE_BASE_IDX 1 #define mmRLC_RLCG_DOORBELL_RANGE 0x4c47 #define mmRLC_RLCG_DOORBELL_RANGE_BASE_IDX 1 +#define mmRLC_CGTT_MGCG_OVERRIDE 0x4c48 +#define mmRLC_CGTT_MGCG_OVERRIDE_BASE_IDX 1 #define mmRLC_CGCG_CGLS_CTRL 0x4c49 #define mmRLC_CGCG_CGLS_CTRL_BASE_IDX 1 #define mmRLC_CGCG_RAMP_CTRL 0x4c4a diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h index aac57f714cf1..c2d035ef3e94 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h @@ -32315,6 +32315,31 @@ #define RLC_RLCG_DOORBELL_CNTL__DOORBELL_3_MODE_MASK 0x000000C0L #define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_MASK 0x001F0000L #define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_EN_MASK 0x00200000L +//RLC_CGTT_MGCG_OVERRIDE +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_0__SHIFT 0x0 +#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE__SHIFT 0x1 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE__SHIFT 0x2 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE__SHIFT 0x3 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE__SHIFT 0x4 +#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE__SHIFT 0x5 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE__SHIFT 0x6 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE__SHIFT 0x7 +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE__SHIFT 0x8 +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_15_9__SHIFT 0x9 +#define RLC_CGTT_MGCG_OVERRIDE__ENABLE_CGTS_LEGACY__SHIFT 0x10 +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_17__SHIFT 0x11 +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_0_MASK 0x00000001L +#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK 0x00000002L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK 0x00000004L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK 0x00000008L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK 0x00000010L +#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK 0x00000020L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK 0x00000040L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK 0x00000080L +#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK 0x00000100L +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_15_9_MASK 0x0000FE00L +#define RLC_CGTT_MGCG_OVERRIDE__ENABLE_CGTS_LEGACY_MASK 0x00010000L +#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_17_MASK 0xFFFE0000L //RLC_RLCG_DOORBELL_STAT #define RLC_RLCG_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT 0x0 #define RLC_RLCG_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT 0x1 diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_offset.h new file mode 100644 index 000000000000..3685766c4d56 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_offset.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _umc_8_7_0_OFFSET_HEADER +#define _umc_8_7_0_OFFSET_HEADER + +#define mmUMCCH0_0_GeccErrCntSel 0x0328 +#define mmUMCCH0_0_GeccErrCntSel_BASE_IDX 0 +#define mmUMCCH0_0_GeccErrCnt 0x0329 +#define mmUMCCH0_0_GeccErrCnt_BASE_IDX 0 +#define mmMCA_UMC_UMC0_MCUMC_STATUST0 0x03c2 +#define mmMCA_UMC_UMC0_MCUMC_STATUST0_BASE_IDX 0 +#define mmMCA_UMC_UMC0_MCUMC_ADDRT0 0x03c4 +#define mmMCA_UMC_UMC0_MCUMC_ADDRT0_BASE_IDX 0 + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_sh_mask.h new file mode 100644 index 000000000000..4c5097fa0c09 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_7_0_sh_mask.h @@ -0,0 +1,79 @@ +#ifndef _umc_8_7_0_SH_MASK_HEADER
+#define _umc_8_7_0_SH_MASK_HEADER
+
+//UMCCH0_0_GeccErrCntSel
+#define UMCCH0_0_GeccErrCntSel__GeccErrCntCsSel__SHIFT 0x0
+#define UMCCH0_0_GeccErrCntSel__GeccErrInt__SHIFT 0xc
+#define UMCCH0_0_GeccErrCntSel__GeccErrCntEn__SHIFT 0xf
+#define UMCCH0_0_GeccErrCntSel__PoisonCntEn__SHIFT 0x10
+#define UMCCH0_0_GeccErrCntSel__GeccErrCntCsSel_MASK 0x0000000FL
+#define UMCCH0_0_GeccErrCntSel__GeccErrInt_MASK 0x00003000L
+#define UMCCH0_0_GeccErrCntSel__GeccErrCntEn_MASK 0x00008000L
+#define UMCCH0_0_GeccErrCntSel__PoisonCntEn_MASK 0x00030000L
+//UMCCH0_0_GeccErrCnt
+#define UMCCH0_0_GeccErrCnt__GeccErrCnt__SHIFT 0x0
+#define UMCCH0_0_GeccErrCnt__GeccUnCorrErrCnt__SHIFT 0x10
+#define UMCCH0_0_GeccErrCnt__GeccErrCnt_MASK 0x0000FFFFL
+#define UMCCH0_0_GeccErrCnt__GeccUnCorrErrCnt_MASK 0xFFFF0000L
+//MCA_UMC_UMC0_MCUMC_STATUST0
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrorCode__SHIFT 0x0
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrorCodeExt__SHIFT 0x10
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV22__SHIFT 0x16
+#define MCA_UMC_UMC0_MCUMC_STATUST0__AddrLsb__SHIFT 0x18
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV30__SHIFT 0x1e
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrCoreId__SHIFT 0x20
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV38__SHIFT 0x26
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Scrub__SHIFT 0x28
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV41__SHIFT 0x29
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Poison__SHIFT 0x2b
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Deferred__SHIFT 0x2c
+#define MCA_UMC_UMC0_MCUMC_STATUST0__UECC__SHIFT 0x2d
+#define MCA_UMC_UMC0_MCUMC_STATUST0__CECC__SHIFT 0x2e
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV47__SHIFT 0x2f
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Transparent__SHIFT 0x34
+#define MCA_UMC_UMC0_MCUMC_STATUST0__SyndV__SHIFT 0x35
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV54__SHIFT 0x36
+#define MCA_UMC_UMC0_MCUMC_STATUST0__TCC__SHIFT 0x37
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrCoreIdVal__SHIFT 0x38
+#define MCA_UMC_UMC0_MCUMC_STATUST0__PCC__SHIFT 0x39
+#define MCA_UMC_UMC0_MCUMC_STATUST0__AddrV__SHIFT 0x3a
+#define MCA_UMC_UMC0_MCUMC_STATUST0__MiscV__SHIFT 0x3b
+#define MCA_UMC_UMC0_MCUMC_STATUST0__En__SHIFT 0x3c
+#define MCA_UMC_UMC0_MCUMC_STATUST0__UC__SHIFT 0x3d
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Overflow__SHIFT 0x3e
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Val__SHIFT 0x3f
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrorCode_MASK 0x000000000000FFFFL
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrorCodeExt_MASK 0x00000000003F0000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV22_MASK 0x0000000000C00000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__AddrLsb_MASK 0x000000003F000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV30_MASK 0x00000000C0000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrCoreId_MASK 0x0000003F00000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV38_MASK 0x000000C000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Scrub_MASK 0x0000010000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV41_MASK 0x0000060000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Poison_MASK 0x0000080000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Deferred_MASK 0x0000100000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__UECC_MASK 0x0000200000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__CECC_MASK 0x0000400000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV47_MASK 0x000F800000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Transparent_MASK 0x0010000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__SyndV_MASK 0x0020000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__RESERV54_MASK 0x0040000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__TCC_MASK 0x0080000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__ErrCoreIdVal_MASK 0x0100000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__PCC_MASK 0x0200000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__AddrV_MASK 0x0400000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__MiscV_MASK 0x0800000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__En_MASK 0x1000000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__UC_MASK 0x2000000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Overflow_MASK 0x4000000000000000L
+#define MCA_UMC_UMC0_MCUMC_STATUST0__Val_MASK 0x8000000000000000L
+//MCA_UMC_UMC0_MCUMC_ADDRT0
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr__SHIFT 0x0
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__LSB__SHIFT 0x38
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved__SHIFT 0x3e
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr_MASK 0x00FFFFFFFFFFFFFFL
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__LSB_MASK 0x3F00000000000000L
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved_MASK 0xC000000000000000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index a3c238c39ef5..fc592f60e6a0 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -212,10 +212,6 @@ struct tile_config { * IH ring entry. This function allows the KFD ISR to get the VMID * from the fault status register as early as possible. * - * @get_hive_id: Returns hive id of current device, 0 if xgmi is not enabled - * - * @get_unique_id: Returns uuid id of current device - * * This structure contains function pointers to services that the kgd driver * provides to amdkfd driver. * @@ -290,9 +286,6 @@ struct kfd2kgd_calls { void (*set_vm_context_page_table_base)(struct kgd_dev *kgd, uint32_t vmid, uint64_t page_table_base); uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd); - uint64_t (*get_hive_id)(struct kgd_dev *kgd); - uint64_t (*get_unique_id)(struct kgd_dev *kgd); - }; #endif /* KGD_KFD_INTERFACE_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index a7f92d0b3a90..0aec28fda058 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -322,6 +322,115 @@ struct amd_pm_funcs { int (*asic_reset_mode_2)(void *handle); int (*set_df_cstate)(void *handle, enum pp_df_cstate state); int (*set_xgmi_pstate)(void *handle, uint32_t pstate); + ssize_t (*get_gpu_metrics)(void *handle, void **table); +}; + +struct metrics_table_header { + uint16_t structure_size; + uint8_t format_revision; + uint8_t content_revision; +}; + +struct gpu_metrics_v1_0 { + struct metrics_table_header common_header; + + /* Driver attached timestamp (in ns) */ + uint64_t system_clock_counter; + + /* Temperature */ + uint16_t temperature_edge; + uint16_t temperature_hotspot; + uint16_t temperature_mem; + uint16_t temperature_vrgfx; + uint16_t temperature_vrsoc; + uint16_t temperature_vrmem; + + /* Utilization */ + uint16_t average_gfx_activity; + uint16_t average_umc_activity; // memory controller + uint16_t average_mm_activity; // UVD or VCN + + /* Power/Energy */ + uint16_t average_socket_power; + uint32_t energy_accumulator; + + /* Average clocks */ + uint16_t average_gfxclk_frequency; + uint16_t average_socclk_frequency; + uint16_t average_uclk_frequency; + uint16_t average_vclk0_frequency; + uint16_t average_dclk0_frequency; + uint16_t average_vclk1_frequency; + uint16_t average_dclk1_frequency; + + /* Current clocks */ + uint16_t current_gfxclk; + uint16_t current_socclk; + uint16_t current_uclk; + uint16_t current_vclk0; + uint16_t current_dclk0; + uint16_t current_vclk1; + uint16_t current_dclk1; + + /* Throttle status */ + uint32_t throttle_status; + + /* Fans */ + uint16_t current_fan_speed; + + /* Link width/speed */ + uint8_t pcie_link_width; + uint8_t pcie_link_speed; // in 0.1 GT/s +}; + +struct gpu_metrics_v2_0 { + struct metrics_table_header common_header; + + /* Driver attached timestamp (in ns) */ + uint64_t system_clock_counter; + + /* Temperature */ + uint16_t temperature_gfx; // gfx temperature on APUs + uint16_t temperature_soc; // soc temperature on APUs + uint16_t temperature_core[8]; // CPU core temperature on APUs + uint16_t temperature_l3[2]; + + /* Utilization */ + uint16_t average_gfx_activity; + uint16_t average_mm_activity; // UVD or VCN + + /* Power/Energy */ + uint16_t average_socket_power; // dGPU + APU power on A + A platform + uint16_t average_cpu_power; + uint16_t average_soc_power; + uint16_t average_gfx_power; + uint16_t average_core_power[8]; // CPU core power on APUs + + /* Average clocks */ + uint16_t average_gfxclk_frequency; + uint16_t average_socclk_frequency; + uint16_t average_uclk_frequency; + uint16_t average_fclk_frequency; + uint16_t average_vclk_frequency; + uint16_t average_dclk_frequency; + + /* Current clocks */ + uint16_t current_gfxclk; + uint16_t current_socclk; + uint16_t current_uclk; + uint16_t current_fclk; + uint16_t current_vclk; + uint16_t current_dclk; + uint16_t current_coreclk[8]; // CPU core clocks + uint16_t current_l3clk[2]; + + /* Throttle status */ + uint32_t throttle_status; + + /* Fans */ + uint16_t fan_pwm; + + uint16_t padding; }; #endif diff --git a/drivers/gpu/drm/amd/pm/Makefile b/drivers/gpu/drm/amd/pm/Makefile new file mode 100644 index 000000000000..f01e86030cd1 --- /dev/null +++ b/drivers/gpu/drm/amd/pm/Makefile @@ -0,0 +1,46 @@ +# +# Copyright 2017 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +subdir-ccflags-y += \ + -I$(FULL_AMD_PATH)/pm/inc/ \ + -I$(FULL_AMD_PATH)/include/asic_reg \ + -I$(FULL_AMD_PATH)/include \ + -I$(FULL_AMD_PATH)/pm/swsmu \ + -I$(FULL_AMD_PATH)/pm/swsmu/smu11 \ + -I$(FULL_AMD_PATH)/pm/swsmu/smu12 \ + -I$(FULL_AMD_PATH)/pm/powerplay \ + -I$(FULL_AMD_PATH)/pm/powerplay/smumgr\ + -I$(FULL_AMD_PATH)/pm/powerplay/hwmgr + +AMD_PM_PATH = ../pm + +PM_LIBS = swsmu powerplay + +AMD_PM = $(addsuffix /Makefile,$(addprefix $(FULL_AMD_PATH)/pm/,$(PM_LIBS))) + +include $(AMD_PM) + +PM_MGR = amdgpu_dpm.o amdgpu_pm.o + +AMD_PM_POWER = $(addprefix $(AMD_PM_PATH)/,$(PM_MGR)) + +AMD_POWERPLAY_FILES += $(AMD_PM_POWER) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index 2082c0acd216..17a45baff638 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -28,6 +28,11 @@ #include "amdgpu_dpm.h" #include "atom.h" #include "amd_pcie.h" +#include "amdgpu_display.h" +#include "hwmgr.h" +#include <linux/power_supply.h> + +#define WIDTH_4K 3840 void amdgpu_dpm_print_class_info(u32 class, u32 class2) { @@ -117,7 +122,7 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev) { - struct drm_device *ddev = adev->ddev; + struct drm_device *ddev = adev_to_drm(adev); struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; @@ -138,7 +143,7 @@ void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev) u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; u32 vblank_in_pixels; @@ -165,7 +170,7 @@ u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev) u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev) { - struct drm_device *dev = adev->ddev; + struct drm_device *dev = adev_to_drm(adev); struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; u32 vrefresh = 0; @@ -1110,8 +1115,6 @@ int amdgpu_dpm_baco_reset(struct amdgpu_device *adev) struct smu_context *smu = &adev->smu; int ret = 0; - dev_info(adev->dev, "GPU BACO reset\n"); - if (is_support_sw_smu(adev)) { ret = smu_baco_enter(smu); if (ret) @@ -1216,3 +1219,469 @@ int amdgpu_dpm_allow_xgmi_power_down(struct amdgpu_device *adev, bool en) return 0; } + +int amdgpu_dpm_enable_mgpu_fan_boost(struct amdgpu_device *adev) +{ + void *pp_handle = adev->powerplay.pp_handle; + const struct amd_pm_funcs *pp_funcs = + adev->powerplay.pp_funcs; + struct smu_context *smu = &adev->smu; + int ret = 0; + + if (is_support_sw_smu(adev)) + ret = smu_enable_mgpu_fan_boost(smu); + else if (pp_funcs && pp_funcs->enable_mgpu_fan_boost) + ret = pp_funcs->enable_mgpu_fan_boost(pp_handle); + + return ret; +} + +int amdgpu_dpm_set_clockgating_by_smu(struct amdgpu_device *adev, + uint32_t msg_id) +{ + void *pp_handle = adev->powerplay.pp_handle; + const struct amd_pm_funcs *pp_funcs = + adev->powerplay.pp_funcs; + int ret = 0; + + if (pp_funcs && pp_funcs->set_clockgating_by_smu) + ret = pp_funcs->set_clockgating_by_smu(pp_handle, + msg_id); + + return ret; +} + +int amdgpu_dpm_smu_i2c_bus_access(struct amdgpu_device *adev, + bool acquire) +{ + void *pp_handle = adev->powerplay.pp_handle; + const struct amd_pm_funcs *pp_funcs = + adev->powerplay.pp_funcs; + int ret = -EOPNOTSUPP; + + if (pp_funcs && pp_funcs->smu_i2c_bus_access) + ret = pp_funcs->smu_i2c_bus_access(pp_handle, + acquire); + + return ret; +} + +void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev) +{ + if (adev->pm.dpm_enabled) { + mutex_lock(&adev->pm.mutex); + if (power_supply_is_system_supplied() > 0) + adev->pm.ac_power = true; + else + adev->pm.ac_power = false; + if (adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->enable_bapm) + amdgpu_dpm_enable_bapm(adev, adev->pm.ac_power); + mutex_unlock(&adev->pm.mutex); + + if (is_support_sw_smu(adev)) + smu_set_ac_dc(&adev->smu); + } +} + +int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor, + void *data, uint32_t *size) +{ + int ret = 0; + + if (!data || !size) + return -EINVAL; + + if (is_support_sw_smu(adev)) + ret = smu_read_sensor(&adev->smu, sensor, data, size); + else { + if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor) + ret = adev->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, + sensor, data, size); + else + ret = -EINVAL; + } + + return ret; +} + +void amdgpu_dpm_thermal_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, + pm.dpm.thermal.work); + /* switch to the thermal state */ + enum amd_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL; + int temp, size = sizeof(temp); + + if (!adev->pm.dpm_enabled) + return; + + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP, + (void *)&temp, &size)) { + if (temp < adev->pm.dpm.thermal.min_temp) + /* switch back the user state */ + dpm_state = adev->pm.dpm.user_state; + } else { + if (adev->pm.dpm.thermal.high_to_low) + /* switch back the user state */ + dpm_state = adev->pm.dpm.user_state; + } + mutex_lock(&adev->pm.mutex); + if (dpm_state == POWER_STATE_TYPE_INTERNAL_THERMAL) + adev->pm.dpm.thermal_active = true; + else + adev->pm.dpm.thermal_active = false; + adev->pm.dpm.state = dpm_state; + mutex_unlock(&adev->pm.mutex); + + amdgpu_pm_compute_clocks(adev); +} + +static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev, + enum amd_pm_state_type dpm_state) +{ + int i; + struct amdgpu_ps *ps; + u32 ui_class; + bool single_display = (adev->pm.dpm.new_active_crtc_count < 2) ? + true : false; + + /* check if the vblank period is too short to adjust the mclk */ + if (single_display && adev->powerplay.pp_funcs->vblank_too_short) { + if (amdgpu_dpm_vblank_too_short(adev)) + single_display = false; + } + + /* certain older asics have a separare 3D performance state, + * so try that first if the user selected performance + */ + if (dpm_state == POWER_STATE_TYPE_PERFORMANCE) + dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF; + /* balanced states don't exist at the moment */ + if (dpm_state == POWER_STATE_TYPE_BALANCED) + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + +restart_search: + /* Pick the best power state based on current conditions */ + for (i = 0; i < adev->pm.dpm.num_ps; i++) { + ps = &adev->pm.dpm.ps[i]; + ui_class = ps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK; + switch (dpm_state) { + /* user states */ + case POWER_STATE_TYPE_BATTERY: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_BALANCED: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BALANCED) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_PERFORMANCE: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + /* internal states */ + case POWER_STATE_TYPE_INTERNAL_UVD: + if (adev->pm.dpm.uvd_ps) + return adev->pm.dpm.uvd_ps; + else + break; + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_BOOT: + return adev->pm.dpm.boot_ps; + case POWER_STATE_TYPE_INTERNAL_THERMAL: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ACPI: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ULV: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_3DPERF: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) + return ps; + break; + default: + break; + } + } + /* use a fallback state if we didn't match */ + switch (dpm_state) { + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (adev->pm.dpm.uvd_ps) { + return adev->pm.dpm.uvd_ps; + } else { + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + } + case POWER_STATE_TYPE_INTERNAL_THERMAL: + dpm_state = POWER_STATE_TYPE_INTERNAL_ACPI; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_ACPI: + dpm_state = POWER_STATE_TYPE_BATTERY; + goto restart_search; + case POWER_STATE_TYPE_BATTERY: + case POWER_STATE_TYPE_BALANCED: + case POWER_STATE_TYPE_INTERNAL_3DPERF: + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + default: + break; + } + + return NULL; +} + +static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) +{ + struct amdgpu_ps *ps; + enum amd_pm_state_type dpm_state; + int ret; + bool equal = false; + + /* if dpm init failed */ + if (!adev->pm.dpm_enabled) + return; + + if (adev->pm.dpm.user_state != adev->pm.dpm.state) { + /* add other state override checks here */ + if ((!adev->pm.dpm.thermal_active) && + (!adev->pm.dpm.uvd_active)) + adev->pm.dpm.state = adev->pm.dpm.user_state; + } + dpm_state = adev->pm.dpm.state; + + ps = amdgpu_dpm_pick_power_state(adev, dpm_state); + if (ps) + adev->pm.dpm.requested_ps = ps; + else + return; + + if (amdgpu_dpm == 1 && adev->powerplay.pp_funcs->print_power_state) { + printk("switching from power state:\n"); + amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps); + printk("switching to power state:\n"); + amdgpu_dpm_print_power_state(adev, adev->pm.dpm.requested_ps); + } + + /* update whether vce is active */ + ps->vce_active = adev->pm.dpm.vce_active; + if (adev->powerplay.pp_funcs->display_configuration_changed) + amdgpu_dpm_display_configuration_changed(adev); + + ret = amdgpu_dpm_pre_set_power_state(adev); + if (ret) + return; + + if (adev->powerplay.pp_funcs->check_state_equal) { + if (0 != amdgpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)) + equal = false; + } + + if (equal) + return; + + amdgpu_dpm_set_power_state(adev); + amdgpu_dpm_post_set_power_state(adev); + + adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs; + adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count; + + if (adev->powerplay.pp_funcs->force_performance_level) { + if (adev->pm.dpm.thermal_active) { + enum amd_dpm_forced_level level = adev->pm.dpm.forced_level; + /* force low perf level for thermal */ + amdgpu_dpm_force_performance_level(adev, AMD_DPM_FORCED_LEVEL_LOW); + /* save the user's level */ + adev->pm.dpm.forced_level = level; + } else { + /* otherwise, user selected level */ + amdgpu_dpm_force_performance_level(adev, adev->pm.dpm.forced_level); + } + } +} + +void amdgpu_pm_compute_clocks(struct amdgpu_device *adev) +{ + int i = 0; + + if (!adev->pm.dpm_enabled) + return; + + if (adev->mode_info.num_crtc) + amdgpu_display_bandwidth_update(adev); + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (ring && ring->sched.ready) + amdgpu_fence_wait_empty(ring); + } + + if (is_support_sw_smu(adev)) { + struct smu_dpm_context *smu_dpm = &adev->smu.smu_dpm; + smu_handle_task(&adev->smu, + smu_dpm->dpm_level, + AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, + true); + } else { + if (adev->powerplay.pp_funcs->dispatch_tasks) { + if (!amdgpu_device_has_dc_support(adev)) { + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_get_active_displays(adev); + adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count; + adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev); + adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev); + /* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */ + if (adev->pm.pm_display_cfg.vrefresh > 120) + adev->pm.pm_display_cfg.min_vblank_time = 0; + if (adev->powerplay.pp_funcs->display_configuration_change) + adev->powerplay.pp_funcs->display_configuration_change( + adev->powerplay.pp_handle, + &adev->pm.pm_display_cfg); + mutex_unlock(&adev->pm.mutex); + } + amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL); + } else { + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_get_active_displays(adev); + amdgpu_dpm_change_power_state_locked(adev); + mutex_unlock(&adev->pm.mutex); + } + } +} + +void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable) +{ + int ret = 0; + + if (adev->family == AMDGPU_FAMILY_SI) { + mutex_lock(&adev->pm.mutex); + if (enable) { + adev->pm.dpm.uvd_active = true; + adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD; + } else { + adev->pm.dpm.uvd_active = false; + } + mutex_unlock(&adev->pm.mutex); + + amdgpu_pm_compute_clocks(adev); + } else { + ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_UVD, !enable); + if (ret) + DRM_ERROR("Dpm %s uvd failed, ret = %d. \n", + enable ? "enable" : "disable", ret); + + /* enable/disable Low Memory PState for UVD (4k videos) */ + if (adev->asic_type == CHIP_STONEY && + adev->uvd.decode_image_width >= WIDTH_4K) { + struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle; + + if (hwmgr && hwmgr->hwmgr_func && + hwmgr->hwmgr_func->update_nbdpm_pstate) + hwmgr->hwmgr_func->update_nbdpm_pstate(hwmgr, + !enable, + true); + } + } +} + +void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable) +{ + int ret = 0; + + if (adev->family == AMDGPU_FAMILY_SI) { + mutex_lock(&adev->pm.mutex); + if (enable) { + adev->pm.dpm.vce_active = true; + /* XXX select vce level based on ring/task */ + adev->pm.dpm.vce_level = AMD_VCE_LEVEL_AC_ALL; + } else { + adev->pm.dpm.vce_active = false; + } + mutex_unlock(&adev->pm.mutex); + + amdgpu_pm_compute_clocks(adev); + } else { + ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_VCE, !enable); + if (ret) + DRM_ERROR("Dpm %s vce failed, ret = %d. \n", + enable ? "enable" : "disable", ret); + } +} + +void amdgpu_pm_print_power_states(struct amdgpu_device *adev) +{ + int i; + + if (adev->powerplay.pp_funcs->print_power_state == NULL) + return; + + for (i = 0; i < adev->pm.dpm.num_ps; i++) + amdgpu_dpm_print_power_state(adev, &adev->pm.dpm.ps[i]); + +} + +void amdgpu_dpm_enable_jpeg(struct amdgpu_device *adev, bool enable) +{ + int ret = 0; + + ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_JPEG, !enable); + if (ret) + DRM_ERROR("Dpm %s jpeg failed, ret = %d. \n", + enable ? "enable" : "disable", ret); +} + +int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version) +{ + int r; + + if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->load_firmware) { + r = adev->powerplay.pp_funcs->load_firmware(adev->powerplay.pp_handle); + if (r) { + pr_err("smu firmware loading failed\n"); + return r; + } + *smu_version = adev->pm.fw_version; + } + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index e4dbf14320b6..2d924e88a215 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -29,17 +29,14 @@ #include "amdgpu_drv.h" #include "amdgpu_pm.h" #include "amdgpu_dpm.h" -#include "amdgpu_display.h" #include "amdgpu_smu.h" #include "atom.h" -#include <linux/power_supply.h> #include <linux/pci.h> #include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> #include <linux/nospec.h> #include <linux/pm_runtime.h> #include "hwmgr.h" -#define WIDTH_4K 3840 static const struct cg_flag_name clocks[] = { {AMD_CG_SUPPORT_GFX_MGCG, "Graphics Medium Grain Clock Gating"}, @@ -81,45 +78,6 @@ static const struct hwmon_temp_label { {PP_TEMP_MEM, "mem"}, }; -void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev) -{ - if (adev->pm.dpm_enabled) { - mutex_lock(&adev->pm.mutex); - if (power_supply_is_system_supplied() > 0) - adev->pm.ac_power = true; - else - adev->pm.ac_power = false; - if (adev->powerplay.pp_funcs && - adev->powerplay.pp_funcs->enable_bapm) - amdgpu_dpm_enable_bapm(adev, adev->pm.ac_power); - mutex_unlock(&adev->pm.mutex); - - if (is_support_sw_smu(adev)) - smu_set_ac_dc(&adev->smu); - } -} - -int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor, - void *data, uint32_t *size) -{ - int ret = 0; - - if (!data || !size) - return -EINVAL; - - if (is_support_sw_smu(adev)) - ret = smu_read_sensor(&adev->smu, sensor, data, size); - else { - if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor) - ret = adev->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, - sensor, data, size); - else - ret = -EINVAL; - } - - return ret; -} - /** * DOC: power_dpm_state * @@ -159,11 +117,11 @@ static ssize_t amdgpu_get_power_dpm_state(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); enum amd_pm_state_type pm; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -197,11 +155,11 @@ static ssize_t amdgpu_set_power_dpm_state(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); enum amd_pm_state_type state; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (strncmp("battery", buf, strlen("battery")) == 0) @@ -303,11 +261,11 @@ static ssize_t amdgpu_get_power_dpm_force_performance_level(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); enum amd_dpm_forced_level level = 0xff; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -344,12 +302,12 @@ static ssize_t amdgpu_set_power_dpm_force_performance_level(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); enum amd_dpm_forced_level level; enum amd_dpm_forced_level current_level = 0xff; int ret = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (strncmp("low", buf, strlen("low")) == 0) { @@ -449,11 +407,11 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct pp_states_info data; int i, buf_len, ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -491,13 +449,13 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); struct pp_states_info data; struct smu_context *smu = &adev->smu; enum amd_pm_state_type pm = 0; int i = 0, ret = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -536,9 +494,9 @@ static ssize_t amdgpu_get_pp_force_state(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (adev->pp_force_state_enabled) @@ -553,12 +511,12 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); enum amd_pm_state_type state = 0; unsigned long idx; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (strlen(buf) == 1) @@ -614,11 +572,11 @@ static ssize_t amdgpu_get_pp_table(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); char *table = NULL; int size, ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -659,10 +617,10 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -694,6 +652,52 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * in each power level within a power state. The pp_od_clk_voltage is used for * this. * + * Note that the actual memory controller clock rate are exposed, not + * the effective memory clock of the DRAMs. To translate it, use the + * following formula: + * + * Clock conversion (Mhz): + * + * HBM: effective_memory_clock = memory_controller_clock * 1 + * + * G5: effective_memory_clock = memory_controller_clock * 1 + * + * G6: effective_memory_clock = memory_controller_clock * 2 + * + * DRAM data rate (MT/s): + * + * HBM: effective_memory_clock * 2 = data_rate + * + * G5: effective_memory_clock * 4 = data_rate + * + * G6: effective_memory_clock * 8 = data_rate + * + * Bandwidth (MB/s): + * + * data_rate * vram_bit_width / 8 = memory_bandwidth + * + * Some examples: + * + * G5 on RX460: + * + * memory_controller_clock = 1750 Mhz + * + * effective_memory_clock = 1750 Mhz * 1 = 1750 Mhz + * + * data rate = 1750 * 4 = 7000 MT/s + * + * memory_bandwidth = 7000 * 128 bits / 8 = 112000 MB/s + * + * G6 on RX5700: + * + * memory_controller_clock = 875 Mhz + * + * effective_memory_clock = 875 Mhz * 2 = 1750 Mhz + * + * data rate = 1750 * 8 = 14000 MT/s + * + * memory_bandwidth = 14000 * 256 bits / 8 = 448000 MB/s + * * < For Vega10 and previous ASICs > * * Reading the file will display: @@ -759,7 +763,7 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t parameter_size = 0; long parameter[64]; @@ -769,7 +773,7 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, const char delimiter[3] = {' ', '\n', '\0'}; uint32_t type; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (count > 127) @@ -796,7 +800,8 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, tmp_str++; while (isspace(*++tmp_str)); - while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + while (tmp_str[0]) { + sub_str = strsep(&tmp_str, delimiter); ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -858,11 +863,11 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -912,11 +917,11 @@ static ssize_t amdgpu_set_pp_features(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint64_t featuremask; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = kstrtou64(buf, 0, &featuremask); @@ -957,11 +962,11 @@ static ssize_t amdgpu_get_pp_features(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1018,11 +1023,11 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1066,7 +1071,8 @@ static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) memcpy(buf_cpy, buf, bytes); buf_cpy[bytes] = '\0'; tmp = buf_cpy; - while ((sub_str = strsep(&tmp, delimiter)) != NULL) { + while (tmp[0]) { + sub_str = strsep(&tmp, delimiter); if (strlen(sub_str)) { ret = kstrtol(sub_str, 0, &level); if (ret) @@ -1085,11 +1091,11 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t mask = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1121,11 +1127,11 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1153,11 +1159,11 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t mask = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1189,11 +1195,11 @@ static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1221,11 +1227,11 @@ static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t mask = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1259,11 +1265,11 @@ static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1291,11 +1297,11 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t mask = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1329,11 +1335,11 @@ static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1361,11 +1367,11 @@ static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t mask = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1399,11 +1405,11 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1431,11 +1437,11 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; uint32_t mask = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = amdgpu_read_mask(buf, count, &mask); @@ -1469,11 +1475,11 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t value = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1499,11 +1505,11 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; long int value; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = kstrtol(buf, 0, &value); @@ -1542,11 +1548,11 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t value = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1572,11 +1578,11 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int ret; long int value; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = kstrtol(buf, 0, &value); @@ -1635,11 +1641,11 @@ static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); ssize_t size; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; ret = pm_runtime_get_sync(ddev->dev); @@ -1669,7 +1675,7 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, { int ret; struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint32_t parameter_size = 0; long parameter[64]; char *sub_str, buf_cpy[128]; @@ -1679,7 +1685,7 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, long int profile_mode = 0; const char delimiter[3] = {' ', '\n', '\0'}; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; tmp[0] = *(buf); @@ -1695,7 +1701,8 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, i++; memcpy(buf_cpy, buf, count-i); tmp_str = buf_cpy; - while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + while (tmp_str[0]) { + sub_str = strsep(&tmp_str, delimiter); ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -1739,10 +1746,10 @@ static ssize_t amdgpu_get_gpu_busy_percent(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int r, value, size = sizeof(value); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; r = pm_runtime_get_sync(ddev->dev); @@ -1777,10 +1784,10 @@ static ssize_t amdgpu_get_mem_busy_percent(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); int r, value, size = sizeof(value); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; r = pm_runtime_get_sync(ddev->dev); @@ -1819,11 +1826,11 @@ static ssize_t amdgpu_get_pcie_bw(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); uint64_t count0 = 0, count1 = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (adev->flags & AMD_IS_APU) @@ -1862,9 +1869,9 @@ static ssize_t amdgpu_get_unique_id(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (adev->unique_id) @@ -1893,10 +1900,10 @@ static ssize_t amdgpu_get_thermal_throttling_logging(struct device *dev, char *buf) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); return snprintf(buf, PAGE_SIZE, "%s: thermal throttling logging %s, with interval %d seconds\n", - adev->ddev->unique, + adev_to_drm(adev)->unique, atomic_read(&adev->throttling_logging_enabled) ? "enabled" : "disabled", adev->throttling_logging_rs.interval / HZ + 1); } @@ -1907,7 +1914,7 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, size_t count) { struct drm_device *ddev = dev_get_drvdata(dev); - struct amdgpu_device *adev = ddev->dev_private; + struct amdgpu_device *adev = drm_to_adev(ddev); long throttling_logging_interval; unsigned long flags; int ret = 0; @@ -1940,6 +1947,57 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, return count; } +/** + * DOC: gpu_metrics + * + * The amdgpu driver provides a sysfs API for retrieving current gpu + * metrics data. The file gpu_metrics is used for this. Reading the + * file will dump all the current gpu metrics data. + * + * These data include temperature, frequency, engines utilization, + * power consume, throttler status, fan speed and cpu core statistics( + * available for APU only). That's it will give a snapshot of all sensors + * at the same time. + */ +static ssize_t amdgpu_get_gpu_metrics(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + void *gpu_metrics; + ssize_t size = 0; + int ret; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + size = smu_sys_get_gpu_metrics(&adev->smu, &gpu_metrics); + else if (adev->powerplay.pp_funcs->get_gpu_metrics) + size = amdgpu_dpm_get_gpu_metrics(adev, &gpu_metrics); + + if (size <= 0) + goto out; + + if (size >= PAGE_SIZE) + size = PAGE_SIZE - 1; + + memcpy(buf, gpu_metrics, size); + +out: + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + return size; +} + static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RW(power_dpm_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(power_dpm_force_performance_level, ATTR_FLAG_BASIC), @@ -1963,6 +2021,7 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RW(pp_features, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RO(unique_id, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(thermal_throttling_logging, ATTR_FLAG_BASIC), + AMDGPU_DEVICE_ATTR_RO(gpu_metrics, ATTR_FLAG_BASIC), }; static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_attr *attr, @@ -2012,6 +2071,9 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ } else if (DEVICE_ATTR_IS(pp_features)) { if (adev->flags & AMD_IS_APU || asic_type < CHIP_VEGA10) *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(gpu_metrics)) { + if (asic_type < CHIP_VEGA12) + *states = ATTR_STATE_UNSUPPORTED; } if (asic_type == CHIP_ARCTURUS) { @@ -2131,15 +2193,15 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev, int channel = to_sensor_dev_attr(attr)->index; int r, temp = 0, size = sizeof(temp); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (channel >= PP_TEMP_MAX) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2164,8 +2226,8 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev, break; } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2267,12 +2329,12 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev, u32 pwm_mode = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - ret = pm_runtime_get_sync(adev->ddev->dev); + ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (ret < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return ret; } @@ -2280,16 +2342,16 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev, pwm_mode = smu_get_fan_control_mode(&adev->smu); } else { if (!adev->powerplay.pp_funcs->get_fan_control_mode) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -EINVAL; } pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return sprintf(buf, "%i\n", pwm_mode); } @@ -2303,16 +2365,16 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, int err, ret; int value; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; err = kstrtoint(buf, 10, &value); if (err) return err; - ret = pm_runtime_get_sync(adev->ddev->dev); + ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (ret < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return ret; } @@ -2320,16 +2382,16 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, smu_set_fan_control_mode(&adev->smu, value); } else { if (!adev->powerplay.pp_funcs->set_fan_control_mode) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -EINVAL; } amdgpu_dpm_set_fan_control_mode(adev, value); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return count; } @@ -2357,12 +2419,12 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, u32 value; u32 pwm_mode; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2373,15 +2435,15 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, if (pwm_mode != AMD_FAN_CTRL_MANUAL) { pr_info("manual fan speed control should be enabled first\n"); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -EINVAL; } err = kstrtou32(buf, 10, &value); if (err) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2394,8 +2456,8 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2411,12 +2473,12 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, int err; u32 speed = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2427,8 +2489,8 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2446,12 +2508,12 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, int err; u32 speed = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2462,8 +2524,8 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2480,20 +2542,20 @@ static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev, u32 size = sizeof(min_rpm); int r; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MIN_FAN_RPM, (void *)&min_rpm, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2510,20 +2572,20 @@ static ssize_t amdgpu_hwmon_get_fan1_max(struct device *dev, u32 size = sizeof(max_rpm); int r; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MAX_FAN_RPM, (void *)&max_rpm, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2539,12 +2601,12 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev, int err; u32 rpm = 0; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2555,8 +2617,8 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2573,12 +2635,12 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, u32 value; u32 pwm_mode; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2588,15 +2650,15 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); if (pwm_mode != AMD_FAN_CTRL_MANUAL) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -ENODATA; } err = kstrtou32(buf, 10, &value); if (err) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2607,8 +2669,8 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2624,12 +2686,12 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev, u32 pwm_mode = 0; int ret; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - ret = pm_runtime_get_sync(adev->ddev->dev); + ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (ret < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return ret; } @@ -2637,16 +2699,16 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev, pwm_mode = smu_get_fan_control_mode(&adev->smu); } else { if (!adev->powerplay.pp_funcs->get_fan_control_mode) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -EINVAL; } pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return sprintf(buf, "%i\n", pwm_mode == AMD_FAN_CTRL_AUTO ? 0 : 1); } @@ -2661,7 +2723,7 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, int value; u32 pwm_mode; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; err = kstrtoint(buf, 10, &value); @@ -2675,9 +2737,9 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, else return -EINVAL; - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2685,15 +2747,15 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, smu_set_fan_control_mode(&adev->smu, pwm_mode); } else { if (!adev->powerplay.pp_funcs->set_fan_control_mode) { - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return -EINVAL; } amdgpu_dpm_set_fan_control_mode(adev, pwm_mode); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return count; } @@ -2706,12 +2768,12 @@ static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev, u32 vddgfx; int r, size = sizeof(vddgfx); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2719,8 +2781,8 @@ static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev, r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDGFX, (void *)&vddgfx, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2743,16 +2805,16 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev, u32 vddnb; int r, size = sizeof(vddnb); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; /* only APUs have vddnb */ if (!(adev->flags & AMD_IS_APU)) return -EINVAL; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2760,8 +2822,8 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev, r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB, (void *)&vddnb, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2785,12 +2847,12 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, int r, size = sizeof(u32); unsigned uw; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2798,8 +2860,8 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER, (void *)&query, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2826,12 +2888,12 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, ssize_t size; int r; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2845,8 +2907,8 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, size = snprintf(buf, PAGE_SIZE, "\n"); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return size; } @@ -2860,12 +2922,12 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, ssize_t size; int r; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2879,8 +2941,8 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, size = snprintf(buf, PAGE_SIZE, "\n"); } - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return size; } @@ -2895,7 +2957,7 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, int err; u32 value; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; if (amdgpu_sriov_vf(adev)) @@ -2908,9 +2970,9 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, value = value / 1000000; /* convert to Watt */ - err = pm_runtime_get_sync(adev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return err; } @@ -2921,8 +2983,8 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, else err = -EINVAL; - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (err) return err; @@ -2938,12 +3000,12 @@ static ssize_t amdgpu_hwmon_show_sclk(struct device *dev, uint32_t sclk; int r, size = sizeof(sclk); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2951,8 +3013,8 @@ static ssize_t amdgpu_hwmon_show_sclk(struct device *dev, r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, (void *)&sclk, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -2975,12 +3037,12 @@ static ssize_t amdgpu_hwmon_show_mclk(struct device *dev, uint32_t mclk; int r, size = sizeof(mclk); - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; - r = pm_runtime_get_sync(adev->ddev->dev); + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); return r; } @@ -2988,8 +3050,8 @@ static ssize_t amdgpu_hwmon_show_mclk(struct device *dev, r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, (void *)&mclk, &size); - pm_runtime_mark_last_busy(adev->ddev->dev); - pm_runtime_put_autosuspend(adev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); if (r) return r; @@ -3249,14 +3311,18 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, } if (((adev->flags & AMD_IS_APU) || - adev->family == AMDGPU_FAMILY_SI || /* not implemented yet */ - adev->family == AMDGPU_FAMILY_KV) && /* not implemented yet */ - (attr == &sensor_dev_attr_power1_average.dev_attr.attr || - attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr || + adev->family == AMDGPU_FAMILY_SI) && /* not implemented yet */ + (attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr || attr == &sensor_dev_attr_power1_cap_min.dev_attr.attr|| attr == &sensor_dev_attr_power1_cap.dev_attr.attr)) return 0; + if (((adev->family == AMDGPU_FAMILY_SI) || + ((adev->flags & AMD_IS_APU) && + (adev->asic_type < CHIP_RENOIR))) && /* not implemented yet */ + (attr == &sensor_dev_attr_power1_average.dev_attr.attr)) + return 0; + if (!is_support_sw_smu(adev)) { /* hide max/min values if we can't both query and manage the fan */ if ((!adev->powerplay.pp_funcs->set_fan_speed_percent && @@ -3321,338 +3387,6 @@ static const struct attribute_group *hwmon_groups[] = { NULL }; -void amdgpu_dpm_thermal_work_handler(struct work_struct *work) -{ - struct amdgpu_device *adev = - container_of(work, struct amdgpu_device, - pm.dpm.thermal.work); - /* switch to the thermal state */ - enum amd_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL; - int temp, size = sizeof(temp); - - if (!adev->pm.dpm_enabled) - return; - - if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP, - (void *)&temp, &size)) { - if (temp < adev->pm.dpm.thermal.min_temp) - /* switch back the user state */ - dpm_state = adev->pm.dpm.user_state; - } else { - if (adev->pm.dpm.thermal.high_to_low) - /* switch back the user state */ - dpm_state = adev->pm.dpm.user_state; - } - mutex_lock(&adev->pm.mutex); - if (dpm_state == POWER_STATE_TYPE_INTERNAL_THERMAL) - adev->pm.dpm.thermal_active = true; - else - adev->pm.dpm.thermal_active = false; - adev->pm.dpm.state = dpm_state; - mutex_unlock(&adev->pm.mutex); - - amdgpu_pm_compute_clocks(adev); -} - -static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev, - enum amd_pm_state_type dpm_state) -{ - int i; - struct amdgpu_ps *ps; - u32 ui_class; - bool single_display = (adev->pm.dpm.new_active_crtc_count < 2) ? - true : false; - - /* check if the vblank period is too short to adjust the mclk */ - if (single_display && adev->powerplay.pp_funcs->vblank_too_short) { - if (amdgpu_dpm_vblank_too_short(adev)) - single_display = false; - } - - /* certain older asics have a separare 3D performance state, - * so try that first if the user selected performance - */ - if (dpm_state == POWER_STATE_TYPE_PERFORMANCE) - dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF; - /* balanced states don't exist at the moment */ - if (dpm_state == POWER_STATE_TYPE_BALANCED) - dpm_state = POWER_STATE_TYPE_PERFORMANCE; - -restart_search: - /* Pick the best power state based on current conditions */ - for (i = 0; i < adev->pm.dpm.num_ps; i++) { - ps = &adev->pm.dpm.ps[i]; - ui_class = ps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK; - switch (dpm_state) { - /* user states */ - case POWER_STATE_TYPE_BATTERY: - if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) { - if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { - if (single_display) - return ps; - } else - return ps; - } - break; - case POWER_STATE_TYPE_BALANCED: - if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BALANCED) { - if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { - if (single_display) - return ps; - } else - return ps; - } - break; - case POWER_STATE_TYPE_PERFORMANCE: - if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { - if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { - if (single_display) - return ps; - } else - return ps; - } - break; - /* internal states */ - case POWER_STATE_TYPE_INTERNAL_UVD: - if (adev->pm.dpm.uvd_ps) - return adev->pm.dpm.uvd_ps; - else - break; - case POWER_STATE_TYPE_INTERNAL_UVD_SD: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_UVD_HD: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_UVD_HD2: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_UVD_MVC: - if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_BOOT: - return adev->pm.dpm.boot_ps; - case POWER_STATE_TYPE_INTERNAL_THERMAL: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_ACPI: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_ULV: - if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) - return ps; - break; - case POWER_STATE_TYPE_INTERNAL_3DPERF: - if (ps->class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) - return ps; - break; - default: - break; - } - } - /* use a fallback state if we didn't match */ - switch (dpm_state) { - case POWER_STATE_TYPE_INTERNAL_UVD_SD: - dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; - goto restart_search; - case POWER_STATE_TYPE_INTERNAL_UVD_HD: - case POWER_STATE_TYPE_INTERNAL_UVD_HD2: - case POWER_STATE_TYPE_INTERNAL_UVD_MVC: - if (adev->pm.dpm.uvd_ps) { - return adev->pm.dpm.uvd_ps; - } else { - dpm_state = POWER_STATE_TYPE_PERFORMANCE; - goto restart_search; - } - case POWER_STATE_TYPE_INTERNAL_THERMAL: - dpm_state = POWER_STATE_TYPE_INTERNAL_ACPI; - goto restart_search; - case POWER_STATE_TYPE_INTERNAL_ACPI: - dpm_state = POWER_STATE_TYPE_BATTERY; - goto restart_search; - case POWER_STATE_TYPE_BATTERY: - case POWER_STATE_TYPE_BALANCED: - case POWER_STATE_TYPE_INTERNAL_3DPERF: - dpm_state = POWER_STATE_TYPE_PERFORMANCE; - goto restart_search; - default: - break; - } - - return NULL; -} - -static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) -{ - struct amdgpu_ps *ps; - enum amd_pm_state_type dpm_state; - int ret; - bool equal = false; - - /* if dpm init failed */ - if (!adev->pm.dpm_enabled) - return; - - if (adev->pm.dpm.user_state != adev->pm.dpm.state) { - /* add other state override checks here */ - if ((!adev->pm.dpm.thermal_active) && - (!adev->pm.dpm.uvd_active)) - adev->pm.dpm.state = adev->pm.dpm.user_state; - } - dpm_state = adev->pm.dpm.state; - - ps = amdgpu_dpm_pick_power_state(adev, dpm_state); - if (ps) - adev->pm.dpm.requested_ps = ps; - else - return; - - if (amdgpu_dpm == 1 && adev->powerplay.pp_funcs->print_power_state) { - printk("switching from power state:\n"); - amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps); - printk("switching to power state:\n"); - amdgpu_dpm_print_power_state(adev, adev->pm.dpm.requested_ps); - } - - /* update whether vce is active */ - ps->vce_active = adev->pm.dpm.vce_active; - if (adev->powerplay.pp_funcs->display_configuration_changed) - amdgpu_dpm_display_configuration_changed(adev); - - ret = amdgpu_dpm_pre_set_power_state(adev); - if (ret) - return; - - if (adev->powerplay.pp_funcs->check_state_equal) { - if (0 != amdgpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)) - equal = false; - } - - if (equal) - return; - - amdgpu_dpm_set_power_state(adev); - amdgpu_dpm_post_set_power_state(adev); - - adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs; - adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count; - - if (adev->powerplay.pp_funcs->force_performance_level) { - if (adev->pm.dpm.thermal_active) { - enum amd_dpm_forced_level level = adev->pm.dpm.forced_level; - /* force low perf level for thermal */ - amdgpu_dpm_force_performance_level(adev, AMD_DPM_FORCED_LEVEL_LOW); - /* save the user's level */ - adev->pm.dpm.forced_level = level; - } else { - /* otherwise, user selected level */ - amdgpu_dpm_force_performance_level(adev, adev->pm.dpm.forced_level); - } - } -} - -void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable) -{ - int ret = 0; - - if (adev->family == AMDGPU_FAMILY_SI) { - mutex_lock(&adev->pm.mutex); - if (enable) { - adev->pm.dpm.uvd_active = true; - adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD; - } else { - adev->pm.dpm.uvd_active = false; - } - mutex_unlock(&adev->pm.mutex); - - amdgpu_pm_compute_clocks(adev); - } else { - ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_UVD, !enable); - if (ret) - DRM_ERROR("Dpm %s uvd failed, ret = %d. \n", - enable ? "enable" : "disable", ret); - - /* enable/disable Low Memory PState for UVD (4k videos) */ - if (adev->asic_type == CHIP_STONEY && - adev->uvd.decode_image_width >= WIDTH_4K) { - struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle; - - if (hwmgr && hwmgr->hwmgr_func && - hwmgr->hwmgr_func->update_nbdpm_pstate) - hwmgr->hwmgr_func->update_nbdpm_pstate(hwmgr, - !enable, - true); - } - } -} - -void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable) -{ - int ret = 0; - - if (adev->family == AMDGPU_FAMILY_SI) { - mutex_lock(&adev->pm.mutex); - if (enable) { - adev->pm.dpm.vce_active = true; - /* XXX select vce level based on ring/task */ - adev->pm.dpm.vce_level = AMD_VCE_LEVEL_AC_ALL; - } else { - adev->pm.dpm.vce_active = false; - } - mutex_unlock(&adev->pm.mutex); - - amdgpu_pm_compute_clocks(adev); - } else { - ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_VCE, !enable); - if (ret) - DRM_ERROR("Dpm %s vce failed, ret = %d. \n", - enable ? "enable" : "disable", ret); - } -} - -void amdgpu_pm_print_power_states(struct amdgpu_device *adev) -{ - int i; - - if (adev->powerplay.pp_funcs->print_power_state == NULL) - return; - - for (i = 0; i < adev->pm.dpm.num_ps; i++) - amdgpu_dpm_print_power_state(adev, &adev->pm.dpm.ps[i]); - -} - -void amdgpu_dpm_enable_jpeg(struct amdgpu_device *adev, bool enable) -{ - int ret = 0; - - ret = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_JPEG, !enable); - if (ret) - DRM_ERROR("Dpm %s jpeg failed, ret = %d. \n", - enable ? "enable" : "disable", ret); -} - -int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version) -{ - int r; - - if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->load_firmware) { - r = adev->powerplay.pp_funcs->load_firmware(adev->powerplay.pp_handle); - if (r) { - pr_err("smu firmware loading failed\n"); - return r; - } - *smu_version = adev->pm.fw_version; - } - return 0; -} - int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) { int ret; @@ -3713,55 +3447,6 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) amdgpu_device_attr_remove_groups(adev, &adev->pm.pm_attr_list); } -void amdgpu_pm_compute_clocks(struct amdgpu_device *adev) -{ - int i = 0; - - if (!adev->pm.dpm_enabled) - return; - - if (adev->mode_info.num_crtc) - amdgpu_display_bandwidth_update(adev); - - for (i = 0; i < AMDGPU_MAX_RINGS; i++) { - struct amdgpu_ring *ring = adev->rings[i]; - if (ring && ring->sched.ready) - amdgpu_fence_wait_empty(ring); - } - - if (is_support_sw_smu(adev)) { - struct smu_dpm_context *smu_dpm = &adev->smu.smu_dpm; - smu_handle_task(&adev->smu, - smu_dpm->dpm_level, - AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, - true); - } else { - if (adev->powerplay.pp_funcs->dispatch_tasks) { - if (!amdgpu_device_has_dc_support(adev)) { - mutex_lock(&adev->pm.mutex); - amdgpu_dpm_get_active_displays(adev); - adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count; - adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev); - adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev); - /* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */ - if (adev->pm.pm_display_cfg.vrefresh > 120) - adev->pm.pm_display_cfg.min_vblank_time = 0; - if (adev->powerplay.pp_funcs->display_configuration_change) - adev->powerplay.pp_funcs->display_configuration_change( - adev->powerplay.pp_handle, - &adev->pm.pm_display_cfg); - mutex_unlock(&adev->pm.mutex); - } - amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL); - } else { - mutex_lock(&adev->pm.mutex); - amdgpu_dpm_get_active_displays(adev); - amdgpu_dpm_change_power_state_locked(adev); - mutex_unlock(&adev->pm.mutex); - } - } -} - /* * Debugfs info */ @@ -3869,11 +3554,11 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_device *adev = drm_to_adev(dev); u32 flags = 0; int r; - if (adev->in_gpu_reset) + if (amdgpu_in_reset(adev)) return -EPERM; r = pm_runtime_get_sync(dev->dev); @@ -3882,11 +3567,6 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data) return r; } - amdgpu_device_ip_get_clockgating_state(adev, &flags); - seq_printf(m, "Clock Gating Flags Mask: 0x%x\n", flags); - amdgpu_parse_cg_state(m, flags); - seq_printf(m, "\n"); - if (!adev->pm.dpm_enabled) { seq_printf(m, "dpm not enabled\n"); pm_runtime_mark_last_busy(dev->dev); @@ -3906,7 +3586,16 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data) } else { r = amdgpu_debugfs_pm_info_pp(m, adev); } + if (r) + goto out; + + amdgpu_device_ip_get_clockgating_state(adev, &flags); + + seq_printf(m, "Clock Gating Flags Mask: 0x%x\n", flags); + amdgpu_parse_cg_state(m, flags); + seq_printf(m, "\n"); +out: pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/pm/inc/amd_powerplay.h index fe3665965416..fe3665965416 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h +++ b/drivers/gpu/drm/amd/pm/inc/amd_powerplay.h diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h index aa27fe65cdfa..dff4a5f99bb0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h @@ -341,10 +341,6 @@ enum amdgpu_pcie_gen { ((adev)->powerplay.pp_funcs->reset_power_profile_state(\ (adev)->powerplay.pp_handle, request)) -#define amdgpu_dpm_set_clockgating_by_smu(adev, msg_id) \ - ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\ - (adev)->powerplay.pp_handle, msg_id)) - #define amdgpu_dpm_get_power_profile_mode(adev, buf) \ ((adev)->powerplay.pp_funcs->get_power_profile_mode(\ (adev)->powerplay.pp_handle, buf)) @@ -357,10 +353,6 @@ enum amdgpu_pcie_gen { ((adev)->powerplay.pp_funcs->odn_edit_dpm_table(\ (adev)->powerplay.pp_handle, type, parameter, size)) -#define amdgpu_dpm_enable_mgpu_fan_boost(adev) \ - ((adev)->powerplay.pp_funcs->enable_mgpu_fan_boost(\ - (adev)->powerplay.pp_handle)) - #define amdgpu_dpm_get_ppfeature_status(adev, buf) \ ((adev)->powerplay.pp_funcs->get_ppfeature_status(\ (adev)->powerplay.pp_handle, (buf))) @@ -369,6 +361,9 @@ enum amdgpu_pcie_gen { ((adev)->powerplay.pp_funcs->set_ppfeature_status(\ (adev)->powerplay.pp_handle, (ppfeatures))) +#define amdgpu_dpm_get_gpu_metrics(adev, table) \ + ((adev)->powerplay.pp_funcs->get_gpu_metrics((adev)->powerplay.pp_handle, table)) + struct amdgpu_dpm { struct amdgpu_ps *ps; /* number of valid power states */ @@ -545,4 +540,26 @@ int amdgpu_dpm_set_df_cstate(struct amdgpu_device *adev, int amdgpu_dpm_allow_xgmi_power_down(struct amdgpu_device *adev, bool en); +int amdgpu_dpm_enable_mgpu_fan_boost(struct amdgpu_device *adev); + +int amdgpu_dpm_set_clockgating_by_smu(struct amdgpu_device *adev, + uint32_t msg_id); + +int amdgpu_dpm_smu_i2c_bus_access(struct amdgpu_device *adev, + bool acquire); + +void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev); + +int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor, + void *data, uint32_t *size); + +void amdgpu_dpm_thermal_work_handler(struct work_struct *work); + +void amdgpu_pm_compute_clocks(struct amdgpu_device *adev); +void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable); +void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable); +void amdgpu_dpm_enable_jpeg(struct amdgpu_device *adev, bool enable); +void amdgpu_pm_print_power_states(struct amdgpu_device *adev); +int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h index d9ae2b49a402..45a22e101d15 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h @@ -79,18 +79,10 @@ struct amdgpu_device_attr_entry { amdgpu_get_##_name, NULL, \ _flags, ##__VA_ARGS__) -void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev); int amdgpu_pm_sysfs_init(struct amdgpu_device *adev); int amdgpu_pm_virt_sysfs_init(struct amdgpu_device *adev); void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev); void amdgpu_pm_virt_sysfs_fini(struct amdgpu_device *adev); -void amdgpu_pm_print_power_states(struct amdgpu_device *adev); -int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version); -void amdgpu_pm_compute_clocks(struct amdgpu_device *adev); -void amdgpu_dpm_thermal_work_handler(struct work_struct *work); -void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable); -void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable); -void amdgpu_dpm_enable_jpeg(struct amdgpu_device *adev, bool enable); int amdgpu_debugfs_pm_init(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h index 074458eb5407..d22a759b6b43 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h @@ -274,6 +274,9 @@ struct smu_table_context void *overdrive_table; void *boot_overdrive_table; + + uint32_t gpu_metrics_table_size; + void *gpu_metrics_table; }; struct smu_dpm_context { @@ -448,6 +451,10 @@ struct smu_context bool dc_controlled_by_gpio; struct work_struct throttling_logging_work; + atomic64_t throttle_int_counter; + + unsigned fan_max_rpm; + unsigned manual_fan_speed_rpm; }; struct i2c_adapter; @@ -491,7 +498,6 @@ struct pptable_funcs { int (*notify_smc_display_config)(struct smu_context *smu); int (*set_cpu_power_state)(struct smu_context *smu); bool (*is_dpm_running)(struct smu_context *smu); - int (*get_fan_speed_percent)(struct smu_context *smu, uint32_t *speed); int (*get_fan_speed_rpm)(struct smu_context *smu, uint32_t *speed); int (*set_watermarks_table)(struct smu_context *smu, struct dm_pp_wm_sets_with_clock_ranges_soc15 *clock_ranges); @@ -567,7 +573,6 @@ struct pptable_funcs { int (*conv_power_profile_to_pplib_workload)(int power_profile); uint32_t (*get_fan_control_mode)(struct smu_context *smu); int (*set_fan_control_mode)(struct smu_context *smu, uint32_t mode); - int (*set_fan_speed_percent)(struct smu_context *smu, uint32_t speed); int (*set_fan_speed_rpm)(struct smu_context *smu, uint32_t speed); int (*set_xgmi_pstate)(struct smu_context *smu, uint32_t pstate); int (*gfx_off_control)(struct smu_context *smu, bool enable); @@ -590,6 +595,11 @@ struct pptable_funcs { void (*log_thermal_throttling_event)(struct smu_context *smu); size_t (*get_pp_feature_mask)(struct smu_context *smu, char *buf); int (*set_pp_feature_mask)(struct smu_context *smu, uint64_t new_mask); + ssize_t (*get_gpu_metrics)(struct smu_context *smu, void **table); + int (*enable_mgpu_fan_boost)(struct smu_context *smu); + int (*gfx_ulv_control)(struct smu_context *smu, bool enablement); + int (*deep_sleep_control)(struct smu_context *smu, bool enablement); + int (*get_fan_parameters)(struct smu_context *smu); }; typedef enum { @@ -792,5 +802,9 @@ int smu_get_dpm_clock_table(struct smu_context *smu, int smu_get_status_gfxoff(struct amdgpu_device *adev, uint32_t *value); +ssize_t smu_sys_get_gpu_metrics(struct smu_context *smu, void **table); + +int smu_enable_mgpu_fan_boost(struct smu_context *smu); + #endif #endif diff --git a/drivers/gpu/drm/amd/powerplay/inc/arcturus_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/arcturus_ppsmc.h index 79afb132164e..79afb132164e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/arcturus_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/arcturus_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/cz_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/cz_ppsmc.h index 9b698780aed8..9b698780aed8 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/cz_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/cz_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/fiji_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/fiji_ppsmc.h index 7ae494569a60..7ae494569a60 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/fiji_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/fiji_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/pm/inc/hardwaremanager.h index 6e0be6027705..6e0be6027705 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h +++ b/drivers/gpu/drm/amd/pm/inc/hardwaremanager.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/inc/hwmgr.h index 15ed6cbdf366..1b3529efc91e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h +++ b/drivers/gpu/drm/amd/pm/inc/hwmgr.h @@ -359,6 +359,7 @@ struct pp_hwmgr_func { int (*set_xgmi_pstate)(struct pp_hwmgr *hwmgr, uint32_t pstate); int (*disable_power_features_for_compute_performance)(struct pp_hwmgr *hwmgr, bool disable); + ssize_t (*get_gpu_metrics)(struct pp_hwmgr *hwmgr, void **table); }; struct pp_table_func { diff --git a/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h b/drivers/gpu/drm/amd/pm/inc/polaris10_pwrvirus.h index 6a53b7e74ccd..6a53b7e74ccd 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h +++ b/drivers/gpu/drm/amd/pm/inc/polaris10_pwrvirus.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/power_state.h b/drivers/gpu/drm/amd/pm/inc/power_state.h index a5f2227a3971..a5f2227a3971 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/power_state.h +++ b/drivers/gpu/drm/amd/pm/inc/power_state.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h b/drivers/gpu/drm/amd/pm/inc/pp_debug.h index cea65093b6ad..cea65093b6ad 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h +++ b/drivers/gpu/drm/amd/pm/inc/pp_debug.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_endian.h b/drivers/gpu/drm/amd/pm/inc/pp_endian.h index f49d1963fe85..f49d1963fe85 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/pp_endian.h +++ b/drivers/gpu/drm/amd/pm/inc/pp_endian.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_thermal.h b/drivers/gpu/drm/amd/pm/inc/pp_thermal.h index 3e30768f9e1c..3e30768f9e1c 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/pp_thermal.h +++ b/drivers/gpu/drm/amd/pm/inc/pp_thermal.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/ppinterrupt.h b/drivers/gpu/drm/amd/pm/inc/ppinterrupt.h index c067e0925b6b..c067e0925b6b 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/ppinterrupt.h +++ b/drivers/gpu/drm/amd/pm/inc/ppinterrupt.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/rv_ppsmc.h index df4677da736c..df4677da736c 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/rv_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu10.h b/drivers/gpu/drm/amd/pm/inc/smu10.h index b96520528240..b96520528240 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu10.h +++ b/drivers/gpu/drm/amd/pm/inc/smu10.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu10_driver_if.h b/drivers/gpu/drm/amd/pm/inc/smu10_driver_if.h index dea8fe93da63..dea8fe93da63 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu10_driver_if.h +++ b/drivers/gpu/drm/amd/pm/inc/smu10_driver_if.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if.h index fdc6b7a57bc9..fdc6b7a57bc9 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_arcturus.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_arcturus.h index 43d43d6addc0..43d43d6addc0 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_arcturus.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_arcturus.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_navi10.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_navi10.h index 4b2da98afcd2..246d3951a78a 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_navi10.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_navi10.h @@ -885,6 +885,45 @@ typedef struct { } SmuMetrics_t; typedef struct { + uint16_t CurrClock[PPCLK_COUNT]; + uint16_t AverageGfxclkFrequency; + uint16_t AverageSocclkFrequency; + uint16_t AverageUclkFrequency ; + uint16_t AverageGfxActivity ; + uint16_t AverageUclkActivity ; + uint8_t CurrSocVoltageOffset ; + uint8_t CurrGfxVoltageOffset ; + uint8_t CurrMemVidOffset ; + uint8_t Padding8 ; + uint16_t AverageSocketPower ; + uint16_t TemperatureEdge ; + uint16_t TemperatureHotspot ; + uint16_t TemperatureMem ; + uint16_t TemperatureVrGfx ; + uint16_t TemperatureVrMem0 ; + uint16_t TemperatureVrMem1 ; + uint16_t TemperatureVrSoc ; + uint16_t TemperatureLiquid0 ; + uint16_t TemperatureLiquid1 ; + uint16_t TemperaturePlx ; + uint16_t Padding16 ; + uint32_t ThrottlerStatus ; + + uint8_t LinkDpmLevel; + uint8_t Padding8_2; + uint16_t CurrFanSpeed; + + uint32_t EnergyAccumulator; + uint16_t AverageVclkFrequency ; + uint16_t AverageDclkFrequency ; + uint16_t VcnActivityPercentage ; + uint16_t padding16_2; + + // Padding - ignore + uint32_t MmHubPadding[8]; // SMU internal use +} SmuMetrics_NV12_t; + +typedef struct { uint16_t MinClock; // This is either DCEFCLK or SOCCLK (in MHz) uint16_t MaxClock; // This is either DCEFCLK or SOCCLK (in MHz) uint16_t MinUclk; diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_sienna_cichlid.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_sienna_cichlid.h index aa2708fccb6d..5ef9c92f57c4 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_sienna_cichlid.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_sienna_cichlid.h @@ -27,7 +27,7 @@ // *** IMPORTANT *** // SMU TEAM: Always increment the interface version if // any structure is changed in this file -#define SMU11_DRIVER_IF_VERSION 0x34 +#define SMU11_DRIVER_IF_VERSION 0x35 #define PPTABLE_Sienna_Cichlid_SMU_VERSION 5 @@ -127,7 +127,7 @@ #define FEATURE_DF_CSTATE_BIT 45 #define FEATURE_2_STEP_PSTATE_BIT 46 #define FEATURE_SMNCLK_DPM_BIT 47 -#define FEATURE_SPARE_48_BIT 48 +#define FEATURE_PERLINK_GMIDOWN_BIT 48 #define FEATURE_GFX_EDC_BIT 49 #define FEATURE_SPARE_50_BIT 50 #define FEATURE_SPARE_51_BIT 51 @@ -169,7 +169,7 @@ typedef enum { #define DPM_OVERRIDE_DISABLE_DFLL_PLL_SHUTDOWN 0x00000200 #define DPM_OVERRIDE_DISABLE_MEMORY_TEMPERATURE_READ 0x00000400 #define DPM_OVERRIDE_DISABLE_VOLT_LINK_VCN_DCEFCLK 0x00000800 -#define DPM_OVERRIDE_DISABLE_FAST_FCLK_TIMER 0x00001000 +#define DPM_OVERRIDE_ENABLE_FAST_FCLK_TIMER 0x00001000 #define DPM_OVERRIDE_DISABLE_VCN_PG 0x00002000 #define DPM_OVERRIDE_DISABLE_FMAX_VMAX 0x00004000 diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu12_driver_if.h b/drivers/gpu/drm/amd/pm/inc/smu12_driver_if.h index e9315eb5b48e..e9315eb5b48e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu12_driver_if.h +++ b/drivers/gpu/drm/amd/pm/inc/smu12_driver_if.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7.h b/drivers/gpu/drm/amd/pm/inc/smu7.h index e14072d45918..e14072d45918 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu7.h +++ b/drivers/gpu/drm/amd/pm/inc/smu7.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu71.h b/drivers/gpu/drm/amd/pm/inc/smu71.h index 71c9b2d28640..71c9b2d28640 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu71.h +++ b/drivers/gpu/drm/amd/pm/inc/smu71.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu71_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu71_discrete.h index c0e3936d5c2e..c0e3936d5c2e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu71_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu71_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu72.h b/drivers/gpu/drm/amd/pm/inc/smu72.h index 9ad1cefff79f..9ad1cefff79f 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu72.h +++ b/drivers/gpu/drm/amd/pm/inc/smu72.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu72_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu72_discrete.h index 2aefbb85f620..2aefbb85f620 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu72_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu72_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu73.h b/drivers/gpu/drm/amd/pm/inc/smu73.h index c6b12a4c00db..c6b12a4c00db 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu73.h +++ b/drivers/gpu/drm/amd/pm/inc/smu73.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu73_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu73_discrete.h index 5916be08a7fe..5916be08a7fe 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu73_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu73_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu74.h b/drivers/gpu/drm/amd/pm/inc/smu74.h index fd10a9fa843d..fd10a9fa843d 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu74.h +++ b/drivers/gpu/drm/amd/pm/inc/smu74.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu74_discrete.h index 899d6d8108c2..899d6d8108c2 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu74_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu75.h b/drivers/gpu/drm/amd/pm/inc/smu75.h index 771523001533..771523001533 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu75.h +++ b/drivers/gpu/drm/amd/pm/inc/smu75.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu75_discrete.h index b64e58a22ddf..b64e58a22ddf 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu75_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_common.h b/drivers/gpu/drm/amd/pm/inc/smu7_common.h index 94bf7b649c20..94bf7b649c20 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu7_common.h +++ b/drivers/gpu/drm/amd/pm/inc/smu7_common.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_discrete.h b/drivers/gpu/drm/amd/pm/inc/smu7_discrete.h index ee876745dd12..ee876745dd12 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu7_discrete.h +++ b/drivers/gpu/drm/amd/pm/inc/smu7_discrete.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_fusion.h b/drivers/gpu/drm/amd/pm/inc/smu7_fusion.h index 78ada9ffd508..78ada9ffd508 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu7_fusion.h +++ b/drivers/gpu/drm/amd/pm/inc/smu7_fusion.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu7_ppsmc.h index 6e19f4c7cf8f..6e19f4c7cf8f 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu7_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu8.h b/drivers/gpu/drm/amd/pm/inc/smu8.h index d758d07b6a31..d758d07b6a31 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu8.h +++ b/drivers/gpu/drm/amd/pm/inc/smu8.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu8_fusion.h b/drivers/gpu/drm/amd/pm/inc/smu8_fusion.h index 0c37c94e9414..0c37c94e9414 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu8_fusion.h +++ b/drivers/gpu/drm/amd/pm/inc/smu8_fusion.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu9.h b/drivers/gpu/drm/amd/pm/inc/smu9.h index 70ac4d477be2..70ac4d477be2 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu9.h +++ b/drivers/gpu/drm/amd/pm/inc/smu9.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h b/drivers/gpu/drm/amd/pm/inc/smu9_driver_if.h index 2818c98ff5ca..2818c98ff5ca 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h +++ b/drivers/gpu/drm/amd/pm/inc/smu9_driver_if.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 7b585e205a5a..7b585e205a5a 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_cz.h b/drivers/gpu/drm/amd/pm/inc/smu_ucode_xfer_cz.h index eb0f79f9c876..eb0f79f9c876 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_cz.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_ucode_xfer_cz.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_vi.h b/drivers/gpu/drm/amd/pm/inc/smu_ucode_xfer_vi.h index 880152c0f775..880152c0f775 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_ucode_xfer_vi.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_ucode_xfer_vi.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h index 6a42331aba8a..1f9575a4dfe7 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h @@ -28,10 +28,10 @@ #define SMU11_DRIVER_IF_VERSION_INV 0xFFFFFFFF #define SMU11_DRIVER_IF_VERSION_ARCT 0x17 #define SMU11_DRIVER_IF_VERSION_NV10 0x36 -#define SMU11_DRIVER_IF_VERSION_NV12 0x33 +#define SMU11_DRIVER_IF_VERSION_NV12 0x36 #define SMU11_DRIVER_IF_VERSION_NV14 0x36 -#define SMU11_DRIVER_IF_VERSION_Sienna_Cichlid 0x34 -#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0x3 +#define SMU11_DRIVER_IF_VERSION_Sienna_Cichlid 0x35 +#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0x4 /* MP Apertures */ #define MP0_Public 0x03800000 @@ -200,12 +200,12 @@ int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode); -int -smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed); - int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed); +int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed); + int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate); @@ -264,5 +264,21 @@ int smu_v11_0_get_dpm_level_range(struct smu_context *smu, uint32_t *min_value, uint32_t *max_value); +int smu_v11_0_get_current_pcie_link_width_level(struct smu_context *smu); + +int smu_v11_0_get_current_pcie_link_width(struct smu_context *smu); + +int smu_v11_0_get_current_pcie_link_speed_level(struct smu_context *smu); + +int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu); + +void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics); + +int smu_v11_0_gfx_ulv_control(struct smu_context *smu, + bool enablement); + +int smu_v11_0_deep_sleep_control(struct smu_context *smu, + bool enablement); + #endif #endif diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_7_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_7_ppsmc.h index 35dd6072cc45..35dd6072cc45 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_7_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_7_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_7_pptable.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_7_pptable.h index 247c6e9632ba..247c6e9632ba 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_7_pptable.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_7_pptable.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_ppsmc.h index 406bfd187ce8..fa0174dc7e0e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_ppsmc.h @@ -123,7 +123,9 @@ #define PPSMC_MSG_DALDisableDummyPstateChange 0x49 #define PPSMC_MSG_DALEnableDummyPstateChange 0x4A -#define PPSMC_Message_Count 0x4B +#define PPSMC_MSG_SetMGpuFanBoostLimitRpm 0x4C + +#define PPSMC_Message_Count 0x4D typedef uint32_t PPSMC_Result; typedef uint32_t PPSMC_Msg; diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_pptable.h index 7a63cf8e85ed..7a63cf8e85ed 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0_pptable.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v12_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h index 02de3b6199e5..fa2e8cb07967 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v12_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h @@ -60,5 +60,7 @@ int smu_v12_0_set_soft_freq_limited_range(struct smu_context *smu, enum smu_clk_ int smu_v12_0_set_driver_table_location(struct smu_context *smu); +void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics); + #endif #endif diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v12_0_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v12_0_ppsmc.h index 9ac9f3bd3664..9ac9f3bd3664 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v12_0_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v12_0_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/pm/inc/smumgr.h index ad100b533d04..ad100b533d04 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h +++ b/drivers/gpu/drm/amd/pm/inc/smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/tonga_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/tonga_ppsmc.h index 63631296d751..63631296d751 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/tonga_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/tonga_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/vega10_ppsmc.h index 715b5a168831..715b5a168831 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/vega10_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h b/drivers/gpu/drm/amd/pm/inc/vega12/smu9_driver_if.h index b6ffd08784e7..b6ffd08784e7 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h +++ b/drivers/gpu/drm/amd/pm/inc/vega12/smu9_driver_if.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega12_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/vega12_ppsmc.h index f985c78d746a..f985c78d746a 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/vega12_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/vega12_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/vega20_ppsmc.h index 0c66f0fe1aaf..0c66f0fe1aaf 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/vega20_ppsmc.h diff --git a/drivers/gpu/drm/amd/powerplay/Makefile b/drivers/gpu/drm/amd/pm/powerplay/Makefile index e9c48f99f71b..0fb114adc79f 100644 --- a/drivers/gpu/drm/amd/powerplay/Makefile +++ b/drivers/gpu/drm/amd/pm/powerplay/Makefile @@ -1,5 +1,5 @@ # -# Copyright 2017 Advanced Micro Devices, Inc. +# Copyright 2020 Advanced Micro Devices, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -20,25 +20,20 @@ # OTHER DEALINGS IN THE SOFTWARE. # -subdir-ccflags-y += \ - -I$(FULL_AMD_PATH)/powerplay/inc/ \ - -I$(FULL_AMD_PATH)/include/asic_reg \ - -I$(FULL_AMD_PATH)/include \ - -I$(FULL_AMD_PATH)/powerplay/smumgr\ - -I$(FULL_AMD_PATH)/powerplay/hwmgr - -AMD_PP_PATH = ../powerplay +AMD_PP_PATH = ../pm/powerplay PP_LIBS = smumgr hwmgr -AMD_POWERPLAY = $(addsuffix /Makefile,$(addprefix $(FULL_AMD_PATH)/powerplay/,$(PP_LIBS))) +AMD_POWERPLAY = $(addsuffix /Makefile,$(addprefix $(FULL_AMD_PATH)/pm/powerplay/,$(PP_LIBS))) include $(AMD_POWERPLAY) -POWER_MGR = amd_powerplay.o amdgpu_smu.o smu_v11_0.o \ - smu_v12_0.o arcturus_ppt.o navi10_ppt.o \ - renoir_ppt.o sienna_cichlid_ppt.o smu_cmn.o +POWER_MGR-y = amd_powerplay.o + +POWER_MGR-$(CONFIG_DRM_AMDGPU_CIK)+= kv_dpm.o kv_smc.o + +POWER_MGR-$(CONFIG_DRM_AMDGPU_SI)+= si_dpm.o si_smc.o -AMD_PP_POWER = $(addprefix $(AMD_PP_PATH)/,$(POWER_MGR)) +AMD_PP_POWER = $(addprefix $(AMD_PP_PATH)/,$(POWER_MGR-y)) AMD_POWERPLAY_FILES += $(AMD_PP_POWER) diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index 7e6dcdf7df73..a6321f2063c1 100644 --- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -1598,6 +1598,24 @@ static int pp_set_xgmi_pstate(void *handle, uint32_t pstate) return 0; } +static ssize_t pp_get_gpu_metrics(void *handle, void **table) +{ + struct pp_hwmgr *hwmgr = handle; + ssize_t size; + + if (!hwmgr) + return -EINVAL; + + if (!hwmgr->pm_en || !hwmgr->hwmgr_func->get_gpu_metrics) + return -EOPNOTSUPP; + + mutex_lock(&hwmgr->smu_lock); + size = hwmgr->hwmgr_func->get_gpu_metrics(hwmgr, table); + mutex_unlock(&hwmgr->smu_lock); + + return size; +} + static const struct amd_pm_funcs pp_dpm_funcs = { .load_firmware = pp_dpm_load_fw, .wait_for_fw_loading_complete = pp_dpm_fw_loading_complete, @@ -1658,4 +1676,5 @@ static const struct amd_pm_funcs pp_dpm_funcs = { .smu_i2c_bus_access = pp_smu_i2c_bus_access, .set_df_cstate = pp_set_df_cstate, .set_xgmi_pstate = pp_set_xgmi_pstate, + .get_gpu_metrics = pp_get_gpu_metrics, }; diff --git a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h b/drivers/gpu/drm/amd/pm/powerplay/cik_dpm.h index 2fcc4b60153c..2fcc4b60153c 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h +++ b/drivers/gpu/drm/amd/pm/powerplay/cik_dpm.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/Makefile index 2773966ae434..2773966ae434 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/Makefile diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ci_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ci_baco.c index 3be40114e63d..3be40114e63d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ci_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ci_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ci_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ci_baco.h index 17041f187020..17041f187020 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ci_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ci_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/common_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/common_baco.c index 1c73776bd606..1c73776bd606 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/common_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/common_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/common_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/common_baco.h index 8393eb62706d..8393eb62706d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/common_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/common_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/fiji_baco.c index c0368f2dfb21..c0368f2dfb21 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/fiji_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/fiji_baco.h index 47f402900bdb..47f402900bdb 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/fiji_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hardwaremanager.c index 9454ab50f9a1..1f9b9facdf1f 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hardwaremanager.c @@ -271,7 +271,10 @@ int phm_start_thermal_controller(struct pp_hwmgr *hwmgr) bool phm_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr) { - PHM_FUNC_CHECK(hwmgr); + if (hwmgr == NULL || + hwmgr->hwmgr_func == NULL) + return false; + if (hwmgr->pp_one_vf) return false; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index f48fdc7f0382..f48fdc7f0382 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr_ppt.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr_ppt.h index c0193e09d58a..c0193e09d58a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr_ppt.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr_ppt.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/polaris_baco.c index 8f8e296f2fe9..8f8e296f2fe9 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/polaris_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/polaris_baco.h index 87a5fa0a157a..87a5fa0a157a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/polaris_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_overdriver.c index 8de384bf9a8f..8de384bf9a8f 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_overdriver.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_overdriver.h index 4112a9398163..4112a9398163 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_overdriver.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c index 31a32a79cfc2..31a32a79cfc2 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.h index b62d55f1f289..b62d55f1f289 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index 01dc46dc9c8a..01dc46dc9c8a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.h index 3ee54f182943..76ed2e413594 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.h @@ -26,15 +26,6 @@ #include "hwmgr.h" -#define MEM_TYPE_GDDR5 0x50 -#define MEM_TYPE_GDDR4 0x40 -#define MEM_TYPE_GDDR3 0x30 -#define MEM_TYPE_DDR2 0x20 -#define MEM_TYPE_GDDR1 0x10 -#define MEM_TYPE_DDR3 0xb0 -#define MEM_TYPE_MASK 0xF0 - - /* As returned from PowerConnectorDetectionTable. */ #define PP_ATOM_POWER_BUDGET_DISABLE_OVERDRIVE 0x80 #define PP_ATOM_POWER_BUDGET_SHOW_WARNING 0x40 diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c index 615cf2c09e54..615cf2c09e54 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.h index b7e2651b570b..b7e2651b570b 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h index 8f50a038396c..8f50a038396c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppevvmath.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pppcielanes.c index 186496a34cbe..186496a34cbe 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pppcielanes.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pppcielanes.h index 70b163b35570..70b163b35570 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pppcielanes.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pppcielanes.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pptable_v1_0.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pptable_v1_0.h index 1e870f58dd12..1e870f58dd12 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pptable_v1_0.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pptable_v1_0.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c index b760f95e7fa7..b760f95e7fa7 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.h index b9710abdff01..b9710abdff01 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c index 719597c5d27d..719597c5d27d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.h index baddaa75693b..baddaa75693b 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c index c9cfe90a2947..9ee8cf8267c8 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c @@ -204,8 +204,7 @@ static int smu10_set_min_deep_sleep_dcefclk(struct pp_hwmgr *hwmgr, uint32_t clo { struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend); - if (smu10_data->need_min_deep_sleep_dcefclk && - smu10_data->deep_sleep_dcefclk != clock) { + if (clock && smu10_data->deep_sleep_dcefclk != clock) { smu10_data->deep_sleep_dcefclk = clock; smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetMinDeepSleepDcefclk, @@ -219,8 +218,7 @@ static int smu10_set_hard_min_dcefclk_by_freq(struct pp_hwmgr *hwmgr, uint32_t c { struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend); - if (smu10_data->dcf_actual_hard_min_freq && - smu10_data->dcf_actual_hard_min_freq != clock) { + if (clock && smu10_data->dcf_actual_hard_min_freq != clock) { smu10_data->dcf_actual_hard_min_freq = clock; smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetHardMinDcefclkByFreq, @@ -234,8 +232,7 @@ static int smu10_set_hard_min_fclk_by_freq(struct pp_hwmgr *hwmgr, uint32_t cloc { struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend); - if (smu10_data->f_actual_hard_min_freq && - smu10_data->f_actual_hard_min_freq != clock) { + if (clock && smu10_data->f_actual_hard_min_freq != clock) { smu10_data->f_actual_hard_min_freq = clock; smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetHardMinFclkByFreq, diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.h index 0f969de10fab..0f969de10fab 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_inc.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_inc.h index edb68e302f6f..edb68e302f6f 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_inc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_inc.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c index 044cda005aed..044cda005aed 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h index be0d98abb536..be0d98abb536 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_clockpowergating.c index f2bda3bcbbde..f2bda3bcbbde 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_clockpowergating.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_clockpowergating.h index fc8f8a6acc72..fc8f8a6acc72 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_clockpowergating.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_dyn_defaults.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_dyn_defaults.h index 3477d4dfff70..3477d4dfff70 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_dyn_defaults.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_dyn_defaults.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index ffe05b7cc1f0..4a3b64aa21ce 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -3581,7 +3581,8 @@ static int smu7_read_sensor(struct pp_hwmgr *hwmgr, int idx, case AMDGPU_PP_SENSOR_GPU_POWER: return smu7_get_gpu_power(hwmgr, (uint32_t *)value); case AMDGPU_PP_SENSOR_VDDGFX: - if ((data->vr_config & 0xff) == 0x2) + if ((data->vr_config & VRCONF_VDDGFX_MASK) == + (VR_SVI2_PLANE_2 << VRCONF_VDDGFX_SHIFT)) val_vid = PHM_READ_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_SVI2_STATUS, PLANE2_VID); else diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h index 69d361f8dfca..69d361f8dfca 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_powertune.c index 5d4971576111..5d4971576111 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_powertune.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_powertune.h index 22f86b6bf1be..22f86b6bf1be 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_powertune.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c index 0b30f73649a8..0b30f73649a8 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h index 42c1ba0fad78..42c1ba0fad78 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c index 35ed47ebaf09..35ed47ebaf09 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.h index 05a06083e1b8..05a06083e1b8 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c index de0a37f7c632..de0a37f7c632 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h index 84e90f801ac3..84e90f801ac3 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c index 60b5ca974356..60b5ca974356 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h index ad33983a8064..ad33983a8064 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/tonga_baco.c index ea743bea8e29..ea743bea8e29 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/tonga_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/tonga_baco.h index 5dc16cc8a295..5dc16cc8a295 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/tonga_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_baco.c index 46bb16c29cf6..46bb16c29cf6 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_baco.h index 96d793f026a5..96d793f026a5 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index c378a000c934..c378a000c934 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h index f752b4ad0c8a..f752b4ad0c8a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_inc.h index faf7ac044348..faf7ac044348 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_inc.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_powertune.c index 9757d47dd6b8..9757d47dd6b8 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_powertune.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_powertune.h index b95771ab89cd..b95771ab89cd 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_powertune.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_pptable.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_pptable.h index c934e9612c1b..c934e9612c1b 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_pptable.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_pptable.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c index f29af5ca0aa0..f29af5ca0aa0 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.h index da5fbec9b0cd..da5fbec9b0cd 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index 468bdd6f6697..952cd3d7240e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -363,17 +363,29 @@ int vega10_thermal_get_temperature(struct pp_hwmgr *hwmgr) static int vega10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, struct PP_TemperatureRange *range) { + struct phm_ppt_v2_information *pp_table_info = + (struct phm_ppt_v2_information *)(hwmgr->pptable); + struct phm_tdp_table *tdp_table = pp_table_info->tdp_table; struct amdgpu_device *adev = hwmgr->adev; - int low = VEGA10_THERMAL_MINIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; - int high = VEGA10_THERMAL_MAXIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + int low = VEGA10_THERMAL_MINIMUM_ALERT_TEMP; + int high = VEGA10_THERMAL_MAXIMUM_ALERT_TEMP; uint32_t val; - if (low < range->min) - low = range->min; - if (high > range->max) - high = range->max; + /* compare them in unit celsius degree */ + if (low < range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES) + low = range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + + /* + * As a common sense, usSoftwareShutdownTemp should be bigger + * than ThotspotLimit. For any invalid usSoftwareShutdownTemp, + * we will just use the max possible setting VEGA10_THERMAL_MAXIMUM_ALERT_TEMP + * to avoid false alarms. + */ + if ((tdp_table->usSoftwareShutdownTemp > + range->hotspot_crit_max / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)) { + if (high > tdp_table->usSoftwareShutdownTemp) + high = tdp_table->usSoftwareShutdownTemp; + } if (low > high) return -EINVAL; @@ -382,8 +394,8 @@ static int vega10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5); val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1); - val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); - val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); + val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, high); + val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, low); val &= (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK) & (~THM_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK) & (~THM_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h index 4a0ede7c1f07..4a0ede7c1f07 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_baco.c index bc53cce4f32d..bc53cce4f32d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_baco.h index 57b72e5a95ae..57b72e5a95ae 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c index a678a67f1c0d..f0680dd58508 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c @@ -47,6 +47,13 @@ #include "pp_thermal.h" #include "vega12_baco.h" +#define smnPCIE_LC_SPEED_CNTL 0x11140290 +#define smnPCIE_LC_LINK_WIDTH_CNTL 0x11140288 + +#define LINK_WIDTH_MAX 6 +#define LINK_SPEED_MAX 3 +static int link_width[] = {0, 1, 2, 4, 8, 12, 16}; +static int link_speed[] = {25, 50, 80, 160}; static int vega12_force_clock_level(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask); @@ -1255,22 +1262,29 @@ static uint32_t vega12_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low) return (mem_clk * 100); } -static int vega12_get_metrics_table(struct pp_hwmgr *hwmgr, SmuMetrics_t *metrics_table) +static int vega12_get_metrics_table(struct pp_hwmgr *hwmgr, + SmuMetrics_t *metrics_table, + bool bypass_cache) { struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend); int ret = 0; - if (!data->metrics_time || time_after(jiffies, data->metrics_time + HZ / 2)) { - ret = smum_smc_table_manager(hwmgr, (uint8_t *)metrics_table, - TABLE_SMU_METRICS, true); + if (bypass_cache || + !data->metrics_time || + time_after(jiffies, data->metrics_time + msecs_to_jiffies(1))) { + ret = smum_smc_table_manager(hwmgr, + (uint8_t *)(&data->metrics_table), + TABLE_SMU_METRICS, + true); if (ret) { pr_info("Failed to export SMU metrics table!\n"); return ret; } - memcpy(&data->metrics_table, metrics_table, sizeof(SmuMetrics_t)); data->metrics_time = jiffies; - } else + } + + if (metrics_table) memcpy(metrics_table, &data->metrics_table, sizeof(SmuMetrics_t)); return ret; @@ -1281,7 +1295,7 @@ static int vega12_get_gpu_power(struct pp_hwmgr *hwmgr, uint32_t *query) SmuMetrics_t metrics_table; int ret = 0; - ret = vega12_get_metrics_table(hwmgr, &metrics_table); + ret = vega12_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -1332,7 +1346,7 @@ static int vega12_get_current_activity_percent( SmuMetrics_t metrics_table; int ret = 0; - ret = vega12_get_metrics_table(hwmgr, &metrics_table); + ret = vega12_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -1380,7 +1394,7 @@ static int vega12_read_sensor(struct pp_hwmgr *hwmgr, int idx, *size = 4; break; case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: - ret = vega12_get_metrics_table(hwmgr, &metrics_table); + ret = vega12_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -1389,7 +1403,7 @@ static int vega12_read_sensor(struct pp_hwmgr *hwmgr, int idx, *size = 4; break; case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = vega12_get_metrics_table(hwmgr, &metrics_table); + ret = vega12_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -2095,6 +2109,46 @@ static int vega12_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return 0; } +static int vega12_get_current_pcie_link_width_level(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + + return (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & + PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) + >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; +} + +static int vega12_get_current_pcie_link_width(struct pp_hwmgr *hwmgr) +{ + uint32_t width_level; + + width_level = vega12_get_current_pcie_link_width_level(hwmgr); + if (width_level > LINK_WIDTH_MAX) + width_level = 0; + + return link_width[width_level]; +} + +static int vega12_get_current_pcie_link_speed_level(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + + return (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & + PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) + >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; +} + +static int vega12_get_current_pcie_link_speed(struct pp_hwmgr *hwmgr) +{ + uint32_t speed_level; + + speed_level = vega12_get_current_pcie_link_speed_level(hwmgr); + if (speed_level > LINK_SPEED_MAX) + speed_level = 0; + + return link_speed[speed_level]; +} + static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf) { @@ -2682,6 +2736,69 @@ static int vega12_set_mp1_state(struct pp_hwmgr *hwmgr, return 0; } +static void vega12_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics) +{ + memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v1_0)); + + gpu_metrics->common_header.structure_size = + sizeof(struct gpu_metrics_v1_0); + gpu_metrics->common_header.format_revision = 1; + gpu_metrics->common_header.content_revision = 0; + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); +} + +static ssize_t vega12_get_gpu_metrics(struct pp_hwmgr *hwmgr, + void **table) +{ + struct vega12_hwmgr *data = + (struct vega12_hwmgr *)(hwmgr->backend); + struct gpu_metrics_v1_0 *gpu_metrics = + &data->gpu_metrics_table; + SmuMetrics_t metrics; + uint32_t fan_speed_rpm; + int ret; + + ret = vega12_get_metrics_table(hwmgr, &metrics, true); + if (ret) + return ret; + + vega12_init_gpu_metrics_v1_0(gpu_metrics); + + gpu_metrics->temperature_edge = metrics.TemperatureEdge; + gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; + gpu_metrics->temperature_mem = metrics.TemperatureHBM; + gpu_metrics->temperature_vrgfx = metrics.TemperatureVrGfx; + gpu_metrics->temperature_vrmem = metrics.TemperatureVrMem; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_umc_activity = metrics.AverageUclkActivity; + + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.AverageUclkFrequency; + + gpu_metrics->current_gfxclk = metrics.CurrClock[PPCLK_GFXCLK]; + gpu_metrics->current_socclk = metrics.CurrClock[PPCLK_SOCCLK]; + gpu_metrics->current_uclk = metrics.CurrClock[PPCLK_UCLK]; + gpu_metrics->current_vclk0 = metrics.CurrClock[PPCLK_VCLK]; + gpu_metrics->current_dclk0 = metrics.CurrClock[PPCLK_DCLK]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + vega12_fan_ctrl_get_fan_speed_rpm(hwmgr, &fan_speed_rpm); + gpu_metrics->current_fan_speed = (uint16_t)fan_speed_rpm; + + gpu_metrics->pcie_link_width = + vega12_get_current_pcie_link_width(hwmgr); + gpu_metrics->pcie_link_speed = + vega12_get_current_pcie_link_speed(hwmgr); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v1_0); +} + static const struct pp_hwmgr_func vega12_hwmgr_funcs = { .backend_init = vega12_hwmgr_backend_init, .backend_fini = vega12_hwmgr_backend_fini, @@ -2739,6 +2856,7 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = { .get_ppfeature_status = vega12_get_ppfeature_status, .set_ppfeature_status = vega12_set_ppfeature_status, .set_mp1_state = vega12_set_mp1_state, + .get_gpu_metrics = vega12_get_gpu_metrics, }; int vega12_hwmgr_init(struct pp_hwmgr *hwmgr) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.h index 73875399666a..aa63ae41942d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.h @@ -399,6 +399,7 @@ struct vega12_hwmgr { unsigned long metrics_time; SmuMetrics_t metrics_table; + struct gpu_metrics_v1_0 gpu_metrics_table; }; #define VEGA12_DPM2_NEAR_TDP_DEC 10 diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_inc.h index e6d9e84059e1..0d08c57d3bca 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_inc.h @@ -35,7 +35,6 @@ #include "asic_reg/gc/gc_9_2_1_sh_mask.h" #include "asic_reg/nbio/nbio_6_1_offset.h" -#include "asic_reg/nbio/nbio_6_1_offset.h" #include "asic_reg/nbio/nbio_6_1_sh_mask.h" #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_pptable.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_pptable.h index bf4f5095b80d..bf4f5095b80d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_pptable.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_pptable.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c index 195d8539fbb4..740e2fc7a034 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c @@ -252,7 +252,7 @@ static int init_powerplay_table_information( phm_copy_clock_limits_array(hwmgr, &pptable_information->power_saving_clock_max, powerplay_table->PowerSavingClockMax, ATOM_VEGA12_PPCLOCK_COUNT); phm_copy_clock_limits_array(hwmgr, &pptable_information->power_saving_clock_min, powerplay_table->PowerSavingClockMin, ATOM_VEGA12_PPCLOCK_COUNT); - pptable_information->smc_pptable = (PPTable_t *)kmalloc(sizeof(PPTable_t), GFP_KERNEL); + pptable_information->smc_pptable = kmalloc(sizeof(PPTable_t), GFP_KERNEL); if (pptable_information->smc_pptable == NULL) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.h index 65652ae65929..65652ae65929 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_thermal.c index c15b9756025d..7ace439dcde7 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_thermal.c @@ -170,17 +170,18 @@ int vega12_thermal_get_temperature(struct pp_hwmgr *hwmgr) static int vega12_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, struct PP_TemperatureRange *range) { + struct phm_ppt_v3_information *pptable_information = + (struct phm_ppt_v3_information *)hwmgr->pptable; struct amdgpu_device *adev = hwmgr->adev; - int low = VEGA12_THERMAL_MINIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; - int high = VEGA12_THERMAL_MAXIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + int low = VEGA12_THERMAL_MINIMUM_ALERT_TEMP; + int high = VEGA12_THERMAL_MAXIMUM_ALERT_TEMP; uint32_t val; - if (low < range->min) - low = range->min; - if (high > range->max) - high = range->max; + /* compare them in unit celsius degree */ + if (low < range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES) + low = range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + if (high > pptable_information->us_software_shutdown_temp) + high = pptable_information->us_software_shutdown_temp; if (low > high) return -EINVAL; @@ -189,8 +190,8 @@ static int vega12_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5); val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1); - val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); - val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); + val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, high); + val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, low); val = val & (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK); WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL, val); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_thermal.h index 0d8ed039ab12..0d8ed039ab12 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_thermal.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c index 2a28c9df15a0..2a28c9df15a0 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h index f06471e712dc..f06471e712dc 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c index 3b8839641770..da84012b7fd5 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c @@ -55,6 +55,11 @@ #define smnPCIE_LC_SPEED_CNTL 0x11140290 #define smnPCIE_LC_LINK_WIDTH_CNTL 0x11140288 +#define LINK_WIDTH_MAX 6 +#define LINK_SPEED_MAX 3 +static int link_width[] = {0, 1, 2, 4, 8, 12, 16}; +static int link_speed[] = {25, 50, 80, 160}; + static void vega20_set_default_registry_data(struct pp_hwmgr *hwmgr) { struct vega20_hwmgr *data = @@ -484,7 +489,7 @@ static int vega20_setup_asic_task(struct pp_hwmgr *hwmgr) { struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev); int ret = 0; - bool use_baco = (adev->in_gpu_reset && + bool use_baco = (amdgpu_in_reset(adev) && (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO)) || (adev->in_runpm && amdgpu_asic_supports_baco(adev)); @@ -979,10 +984,7 @@ static int vega20_disable_all_smu_features(struct pp_hwmgr *hwmgr) { struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); - uint64_t features_enabled; - int i; - bool enabled; - int ret = 0; + int i, ret = 0; PP_ASSERT_WITH_CODE((ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_DisableAllSmuFeatures, @@ -990,17 +992,8 @@ static int vega20_disable_all_smu_features(struct pp_hwmgr *hwmgr) "[DisableAllSMUFeatures] Failed to disable all smu features!", return ret); - ret = vega20_get_enabled_smc_features(hwmgr, &features_enabled); - PP_ASSERT_WITH_CODE(!ret, - "[DisableAllSMUFeatures] Failed to get enabled smc features!", - return ret); - - for (i = 0; i < GNLD_FEATURES_MAX; i++) { - enabled = (features_enabled & data->smu_features[i].smu_feature_bitmap) ? - true : false; - data->smu_features[i].enabled = enabled; - data->smu_features[i].supported = enabled; - } + for (i = 0; i < GNLD_FEATURES_MAX; i++) + data->smu_features[i].enabled = 0; return 0; } @@ -1652,12 +1645,6 @@ static void vega20_init_powergate_state(struct pp_hwmgr *hwmgr) data->uvd_power_gated = true; data->vce_power_gated = true; - - if (data->smu_features[GNLD_DPM_UVD].enabled) - data->uvd_power_gated = false; - - if (data->smu_features[GNLD_DPM_VCE].enabled) - data->vce_power_gated = false; } static int vega20_enable_dpm_tasks(struct pp_hwmgr *hwmgr) @@ -2085,22 +2072,29 @@ static uint32_t vega20_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low) return (mem_clk * 100); } -static int vega20_get_metrics_table(struct pp_hwmgr *hwmgr, SmuMetrics_t *metrics_table) +static int vega20_get_metrics_table(struct pp_hwmgr *hwmgr, + SmuMetrics_t *metrics_table, + bool bypass_cache) { struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); int ret = 0; - if (!data->metrics_time || time_after(jiffies, data->metrics_time + HZ / 2)) { - ret = smum_smc_table_manager(hwmgr, (uint8_t *)metrics_table, - TABLE_SMU_METRICS, true); + if (bypass_cache || + !data->metrics_time || + time_after(jiffies, data->metrics_time + msecs_to_jiffies(1))) { + ret = smum_smc_table_manager(hwmgr, + (uint8_t *)(&data->metrics_table), + TABLE_SMU_METRICS, + true); if (ret) { pr_info("Failed to export SMU metrics table!\n"); return ret; } - memcpy(&data->metrics_table, metrics_table, sizeof(SmuMetrics_t)); data->metrics_time = jiffies; - } else + } + + if (metrics_table) memcpy(metrics_table, &data->metrics_table, sizeof(SmuMetrics_t)); return ret; @@ -2112,7 +2106,7 @@ static int vega20_get_gpu_power(struct pp_hwmgr *hwmgr, int ret = 0; SmuMetrics_t metrics_table; - ret = vega20_get_metrics_table(hwmgr, &metrics_table); + ret = vega20_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -2150,7 +2144,7 @@ static int vega20_get_current_activity_percent(struct pp_hwmgr *hwmgr, int ret = 0; SmuMetrics_t metrics_table; - ret = vega20_get_metrics_table(hwmgr, &metrics_table); + ret = vega20_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -2180,7 +2174,7 @@ static int vega20_read_sensor(struct pp_hwmgr *hwmgr, int idx, switch (idx) { case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = vega20_get_metrics_table(hwmgr, &metrics_table); + ret = vega20_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -2205,7 +2199,7 @@ static int vega20_read_sensor(struct pp_hwmgr *hwmgr, int idx, *size = 4; break; case AMDGPU_PP_SENSOR_EDGE_TEMP: - ret = vega20_get_metrics_table(hwmgr, &metrics_table); + ret = vega20_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -2214,7 +2208,7 @@ static int vega20_read_sensor(struct pp_hwmgr *hwmgr, int idx, *size = 4; break; case AMDGPU_PP_SENSOR_MEM_TEMP: - ret = vega20_get_metrics_table(hwmgr, &metrics_table); + ret = vega20_get_metrics_table(hwmgr, &metrics_table, false); if (ret) return ret; @@ -3230,10 +3224,11 @@ static int vega20_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf) static int vega20_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfeature_masks) { - uint64_t features_enabled; - uint64_t features_to_enable; - uint64_t features_to_disable; - int ret = 0; + struct vega20_hwmgr *data = + (struct vega20_hwmgr *)(hwmgr->backend); + uint64_t features_enabled, features_to_enable, features_to_disable; + int i, ret = 0; + bool enabled; if (new_ppfeature_masks >= (1ULL << GNLD_FEATURES_MAX)) return -EINVAL; @@ -3262,9 +3257,60 @@ static int vega20_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; } + /* Update the cached feature enablement state */ + ret = vega20_get_enabled_smc_features(hwmgr, &features_enabled); + if (ret) + return ret; + + for (i = 0; i < GNLD_FEATURES_MAX; i++) { + enabled = (features_enabled & data->smu_features[i].smu_feature_bitmap) ? + true : false; + data->smu_features[i].enabled = enabled; + } + return 0; } +static int vega20_get_current_pcie_link_width_level(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + + return (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & + PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) + >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; +} + +static int vega20_get_current_pcie_link_width(struct pp_hwmgr *hwmgr) +{ + uint32_t width_level; + + width_level = vega20_get_current_pcie_link_width_level(hwmgr); + if (width_level > LINK_WIDTH_MAX) + width_level = 0; + + return link_width[width_level]; +} + +static int vega20_get_current_pcie_link_speed_level(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + + return (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & + PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) + >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; +} + +static int vega20_get_current_pcie_link_speed(struct pp_hwmgr *hwmgr) +{ + uint32_t speed_level; + + speed_level = vega20_get_current_pcie_link_speed_level(hwmgr); + if (speed_level > LINK_SPEED_MAX) + speed_level = 0; + + return link_speed[speed_level]; +} + static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf) { @@ -3277,7 +3323,6 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, struct phm_ppt_v3_information *pptable_information = (struct phm_ppt_v3_information *)hwmgr->pptable; PPTable_t *pptable = (PPTable_t *)pptable_information->smc_pptable; - struct amdgpu_device *adev = hwmgr->adev; struct pp_clock_levels_with_latency clocks; struct vega20_single_dpm_table *fclk_dpm_table = &(data->dpm_table.fclk_table); @@ -3371,12 +3416,10 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, break; case PP_PCIE: - current_gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & - PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) - >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - current_lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & - PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) - >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; + current_gen_speed = + vega20_get_current_pcie_link_speed_level(hwmgr); + current_lane_width = + vega20_get_current_pcie_link_width_level(hwmgr); for (i = 0; i < NUM_LINK_LEVELS; i++) { if (i == 1 && data->pcie_parameters_override) { gen_speed = data->pcie_gen_level1; @@ -4218,6 +4261,72 @@ static int vega20_set_xgmi_pstate(struct pp_hwmgr *hwmgr, return ret; } +static void vega20_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics) +{ + memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v1_0)); + + gpu_metrics->common_header.structure_size = + sizeof(struct gpu_metrics_v1_0); + gpu_metrics->common_header.format_revision = 1; + gpu_metrics->common_header.content_revision = 0; + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); +} + +static ssize_t vega20_get_gpu_metrics(struct pp_hwmgr *hwmgr, + void **table) +{ + struct vega20_hwmgr *data = + (struct vega20_hwmgr *)(hwmgr->backend); + struct gpu_metrics_v1_0 *gpu_metrics = + &data->gpu_metrics_table; + SmuMetrics_t metrics; + uint32_t fan_speed_rpm; + int ret; + + ret = vega20_get_metrics_table(hwmgr, &metrics, true); + if (ret) + return ret; + + vega20_init_gpu_metrics_v1_0(gpu_metrics); + + gpu_metrics->temperature_edge = metrics.TemperatureEdge; + gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; + gpu_metrics->temperature_mem = metrics.TemperatureHBM; + gpu_metrics->temperature_vrgfx = metrics.TemperatureVrGfx; + gpu_metrics->temperature_vrsoc = metrics.TemperatureVrSoc; + gpu_metrics->temperature_vrmem = metrics.TemperatureVrMem0; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_umc_activity = metrics.AverageUclkActivity; + + gpu_metrics->average_socket_power = metrics.AverageSocketPower; + + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.AverageUclkFrequency; + + gpu_metrics->current_gfxclk = metrics.CurrClock[PPCLK_GFXCLK]; + gpu_metrics->current_socclk = metrics.CurrClock[PPCLK_SOCCLK]; + gpu_metrics->current_uclk = metrics.CurrClock[PPCLK_UCLK]; + gpu_metrics->current_vclk0 = metrics.CurrClock[PPCLK_VCLK]; + gpu_metrics->current_dclk0 = metrics.CurrClock[PPCLK_DCLK]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + vega20_fan_ctrl_get_fan_speed_rpm(hwmgr, &fan_speed_rpm); + gpu_metrics->current_fan_speed = (uint16_t)fan_speed_rpm; + + gpu_metrics->pcie_link_width = + vega20_get_current_pcie_link_width(hwmgr); + gpu_metrics->pcie_link_speed = + vega20_get_current_pcie_link_speed(hwmgr); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v1_0); +} + static const struct pp_hwmgr_func vega20_hwmgr_funcs = { /* init/fini related */ .backend_init = vega20_hwmgr_backend_init, @@ -4288,6 +4397,7 @@ static const struct pp_hwmgr_func vega20_hwmgr_funcs = { .smu_i2c_bus_access = vega20_smu_i2c_bus_access, .set_df_cstate = vega20_set_df_cstate, .set_xgmi_pstate = vega20_set_xgmi_pstate, + .get_gpu_metrics = vega20_get_gpu_metrics, }; int vega20_hwmgr_init(struct pp_hwmgr *hwmgr) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.h index 2c3125f82b24..075c0094da9c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.h @@ -527,6 +527,7 @@ struct vega20_hwmgr { unsigned long metrics_time; SmuMetrics_t metrics_table; + struct gpu_metrics_v1_0 gpu_metrics_table; bool pcie_parameters_override; uint32_t pcie_gen_level1; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_inc.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_inc.h index 613cb1989b3d..613cb1989b3d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_inc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_inc.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_powertune.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_powertune.c index d7cc3d2d9e17..d7cc3d2d9e17 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_powertune.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_powertune.c diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_powertune.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_powertune.h index d68c734c0f4e..d68c734c0f4e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_powertune.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_powertune.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_pptable.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_pptable.h index 2222e29405c6..2222e29405c6 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_pptable.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_pptable.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c index 7a7f15d0c53a..1f9082539457 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c @@ -890,14 +890,12 @@ static int init_powerplay_table_information( power_saving_clock_count); } - pptable_information->smc_pptable = (PPTable_t *)kmalloc(sizeof(PPTable_t), GFP_KERNEL); + pptable_information->smc_pptable = kmemdup(&(powerplay_table->smcPPTable), + sizeof(PPTable_t), + GFP_KERNEL); if (pptable_information->smc_pptable == NULL) return -ENOMEM; - memcpy(pptable_information->smc_pptable, - &(powerplay_table->smcPPTable), - sizeof(PPTable_t)); - result = append_vbios_pptable(hwmgr, (pptable_information->smc_pptable)); if (result) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.h index 846c2cb40b35..846c2cb40b35 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.h diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c index 7add2f60f49c..364162ddaa9c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c @@ -240,17 +240,18 @@ int vega20_thermal_get_temperature(struct pp_hwmgr *hwmgr) static int vega20_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, struct PP_TemperatureRange *range) { + struct phm_ppt_v3_information *pptable_information = + (struct phm_ppt_v3_information *)hwmgr->pptable; struct amdgpu_device *adev = hwmgr->adev; - int low = VEGA20_THERMAL_MINIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; - int high = VEGA20_THERMAL_MAXIMUM_ALERT_TEMP * - PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + int low = VEGA20_THERMAL_MINIMUM_ALERT_TEMP; + int high = VEGA20_THERMAL_MAXIMUM_ALERT_TEMP; uint32_t val; - if (low < range->min) - low = range->min; - if (high > range->max) - high = range->max; + /* compare them in unit celsius degree */ + if (low < range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES) + low = range->min / PP_TEMPERATURE_UNITS_PER_CENTIGRADES; + if (high > pptable_information->us_software_shutdown_temp) + high = pptable_information->us_software_shutdown_temp; if (low > high) return -EINVAL; @@ -259,8 +260,8 @@ static int vega20_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5); val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1); - val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); - val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES)); + val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, high); + val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, low); val = val & (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK); WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL, val); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h index 2d1769bbd24e..2d1769bbd24e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c index 4b3faaccecb9..4b3faaccecb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.h b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.h index 6df0ed41317c..6df0ed41317c 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.h +++ b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.h diff --git a/drivers/gpu/drm/amd/amdgpu/kv_smc.c b/drivers/gpu/drm/amd/pm/powerplay/kv_smc.c index 2d9ab6b8be66..2d9ab6b8be66 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_smc.c +++ b/drivers/gpu/drm/amd/pm/powerplay/kv_smc.c diff --git a/drivers/gpu/drm/amd/amdgpu/ppsmc.h b/drivers/gpu/drm/amd/pm/powerplay/ppsmc.h index 8463245f424f..8463245f424f 100644 --- a/drivers/gpu/drm/amd/amdgpu/ppsmc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/ppsmc.h diff --git a/drivers/gpu/drm/amd/amdgpu/r600_dpm.h b/drivers/gpu/drm/amd/pm/powerplay/r600_dpm.h index 055321f61ca7..055321f61ca7 100644 --- a/drivers/gpu/drm/amd/amdgpu/r600_dpm.h +++ b/drivers/gpu/drm/amd/pm/powerplay/r600_dpm.h diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c index ea914b256ebd..b5986d19dc08 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c @@ -6196,7 +6196,7 @@ static void si_request_link_speed_change_before_state_change(struct amdgpu_devic si_pi->force_pcie_gen = AMDGPU_PCIE_GEN2; if (current_link_speed == AMDGPU_PCIE_GEN2) break; - /* fall through */ + fallthrough; case AMDGPU_PCIE_GEN2: if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.h b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.h index bc0be6818e21..bc0be6818e21 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.h +++ b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.h diff --git a/drivers/gpu/drm/amd/amdgpu/si_smc.c b/drivers/gpu/drm/amd/pm/powerplay/si_smc.c index 8f994ffa9cd1..8f994ffa9cd1 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_smc.c +++ b/drivers/gpu/drm/amd/pm/powerplay/si_smc.c diff --git a/drivers/gpu/drm/amd/amdgpu/sislands_smc.h b/drivers/gpu/drm/amd/pm/powerplay/sislands_smc.h index d2930eceaf3c..d2930eceaf3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sislands_smc.h +++ b/drivers/gpu/drm/amd/pm/powerplay/sislands_smc.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile b/drivers/gpu/drm/amd/pm/powerplay/smumgr/Makefile index 6c59c61a0d81..6c59c61a0d81 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/Makefile diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c index c18169aa59ce..e4d1f3d66ef4 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c @@ -37,6 +37,7 @@ #include "cgs_common.h" #include "atombios.h" #include "pppcielanes.h" +#include "smu7_smumgr.h" #include "smu/smu_7_0_1_d.h" #include "smu/smu_7_0_1_sh_mask.h" @@ -2948,6 +2949,7 @@ const struct pp_smumgr_func ci_smu_funcs = { .request_smu_load_specific_fw = NULL, .send_msg_to_smc = ci_send_msg_to_smc, .send_msg_to_smc_with_parameter = ci_send_msg_to_smc_with_parameter, + .get_argument = smu7_get_argument, .download_pptable_settings = NULL, .upload_pptable_settings = NULL, .get_offsetof = ci_get_offsetof, diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.h index a8282705c569..a8282705c569 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c index ecb9ee46d6b3..ecb9ee46d6b3 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.h index 6d3746268ccf..6d3746268ccf 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/fiji_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c index 431ad2fd38df..431ad2fd38df 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.h index f32c506779c9..f32c506779c9 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/iceland_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c index c3d2e6dcf62a..c3d2e6dcf62a 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.h index 1ec425df9eda..1ec425df9eda 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu10_smumgr.c index ea2279bb8cbf..ea2279bb8cbf 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu10_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu10_smumgr.h index 9c2be74a2b2f..9c2be74a2b2f 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu10_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c index aae25243eb10..aae25243eb10 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.h index e7303dc8c260..e7303dc8c260 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu8_smumgr.c index 76d4f12ceedf..76d4f12ceedf 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu8_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu8_smumgr.h index c7b61222d258..c7b61222d258 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu8_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu9_smumgr.c index adfbcbe5d113..adfbcbe5d113 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu9_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu9_smumgr.h index 1462279ca128..1462279ca128 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu9_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smumgr.c index b6fb48066841..b6fb48066841 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/tonga_smumgr.c index 4bfadb49521b..4bfadb49521b 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/tonga_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/tonga_smumgr.h index d664fedd3d85..d664fedd3d85 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/tonga_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega10_smumgr.c index 1e222c5d91a4..1e222c5d91a4 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega10_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega10_smumgr.h index bad760f22624..bad760f22624 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega10_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega12_smumgr.c index f54df76537e4..f54df76537e4 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega12_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega12_smumgr.h index aeec965ce81f..aeec965ce81f 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega12_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega20_smumgr.c index cf43629d29d2..cf43629d29d2 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega20_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega20_smumgr.h index 62ebbfd6068f..62ebbfd6068f 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vega20_smumgr.h diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vegam_smumgr.c index 0ecc18b55ffb..0ecc18b55ffb 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vegam_smumgr.c diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vegam_smumgr.h index 2b6558238500..2b6558238500 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/vegam_smumgr.h diff --git a/drivers/gpu/drm/amd/pm/swsmu/Makefile b/drivers/gpu/drm/amd/pm/swsmu/Makefile new file mode 100644 index 000000000000..6f281990b7b4 --- /dev/null +++ b/drivers/gpu/drm/amd/pm/swsmu/Makefile @@ -0,0 +1,36 @@ +# +# Copyright 2020 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +AMD_SWSMU_PATH = ../pm/swsmu + +SWSMU_LIBS = smu11 smu12 + +AMD_SWSMU = $(addsuffix /Makefile,$(addprefix $(FULL_AMD_PATH)/pm/swsmu/,$(SWSMU_LIBS))) + +include $(AMD_SWSMU) + +SWSMU_MGR = amdgpu_smu.o \ + smu_cmn.o \ + +AMD_SWSMU_POWER = $(addprefix $(AMD_SWSMU_PATH)/,$(SWSMU_MGR)) + +AMD_POWERPLAY_FILES += $(AMD_SWSMU_POWER) diff --git a/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 0826625573dc..7a55ece1f124 100644 --- a/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -504,6 +504,8 @@ static int smu_late_init(void *handle) smu_get_unique_id(smu); + smu_get_fan_parameters(smu); + smu_handle_task(&adev->smu, smu->smu_dpm.dpm_level, AMD_PP_TASK_COMPLETE_INIT, @@ -756,6 +758,7 @@ static int smu_sw_init(void *handle) mutex_init(&smu->message_lock); INIT_WORK(&smu->throttling_logging_work, smu_throttling_logging_work_fn); + atomic64_set(&smu->throttle_int_counter, 0); smu->watermarks_bitmap = 0; smu->power_profile_mode = PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT; smu->default_power_profile_mode = PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT; @@ -1109,7 +1112,7 @@ static int smu_disable_dpms(struct smu_context *smu) struct amdgpu_device *adev = smu->adev; int ret = 0; bool use_baco = !smu->is_apu && - ((adev->in_gpu_reset && + ((amdgpu_in_reset(adev) && (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO)) || ((adev->in_runpm || adev->in_hibernate) && amdgpu_asic_supports_baco(adev))); @@ -1439,6 +1442,8 @@ static int smu_enable_umd_pstate(void *handle, amdgpu_device_ip_set_clockgating_state(smu->adev, AMD_IP_BLOCK_TYPE_GFX, AMD_CG_STATE_UNGATE); + smu_gfx_ulv_control(smu, false); + smu_deep_sleep_control(smu, false); } } else { /* exit umd pstate, restore level, enable gfx cg*/ @@ -1446,6 +1451,8 @@ static int smu_enable_umd_pstate(void *handle, if (*level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT) *level = smu_dpm_ctx->saved_dpm_level; smu_dpm_ctx->enable_umd_pstate = false; + smu_deep_sleep_control(smu, true); + smu_gfx_ulv_control(smu, true); amdgpu_device_ip_set_clockgating_state(smu->adev, AMD_IP_BLOCK_TYPE_GFX, AMD_CG_STATE_GATE); @@ -2185,31 +2192,44 @@ int smu_set_fan_control_mode(struct smu_context *smu, int value) int smu_get_fan_speed_percent(struct smu_context *smu, uint32_t *speed) { int ret = 0; + uint32_t percent; + uint32_t current_rpm; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_percent) - ret = smu->ppt_funcs->get_fan_speed_percent(smu, speed); + if (smu->ppt_funcs->get_fan_speed_rpm) { + ret = smu->ppt_funcs->get_fan_speed_rpm(smu, ¤t_rpm); + if (!ret) { + percent = current_rpm * 100 / smu->fan_max_rpm; + *speed = percent > 100 ? 100 : percent; + } + } mutex_unlock(&smu->mutex); + return ret; } int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) { int ret = 0; + uint32_t rpm; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_percent) - ret = smu->ppt_funcs->set_fan_speed_percent(smu, speed); + if (smu->ppt_funcs->set_fan_speed_rpm) { + if (speed > 100) + speed = 100; + rpm = speed * smu->fan_max_rpm / 100; + ret = smu->ppt_funcs->set_fan_speed_rpm(smu, rpm); + } mutex_unlock(&smu->mutex); @@ -2631,3 +2651,40 @@ int smu_get_dpm_clock_table(struct smu_context *smu, return ret; } + +ssize_t smu_sys_get_gpu_metrics(struct smu_context *smu, + void **table) +{ + ssize_t size; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return -EOPNOTSUPP; + + if (!smu->ppt_funcs->get_gpu_metrics) + return -EOPNOTSUPP; + + mutex_lock(&smu->mutex); + + size = smu->ppt_funcs->get_gpu_metrics(smu, table); + + mutex_unlock(&smu->mutex); + + return size; +} + +int smu_enable_mgpu_fan_boost(struct smu_context *smu) +{ + int ret = 0; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return -EOPNOTSUPP; + + mutex_lock(&smu->mutex); + + if (smu->ppt_funcs->enable_mgpu_fan_boost) + ret = smu->ppt_funcs->enable_mgpu_fan_boost(smu); + + mutex_unlock(&smu->mutex); + + return ret; +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/Makefile b/drivers/gpu/drm/amd/pm/swsmu/smu11/Makefile new file mode 100644 index 000000000000..f98d97192635 --- /dev/null +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/Makefile @@ -0,0 +1,33 @@ +# +# Copyright 2020 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +# Makefile for the 'smu manager' sub-component of powerplay. +# It provides the smu management services for the driver. + +SMU11_MGR = arcturus_ppt.o \ + navi10_ppt.o \ + sienna_cichlid_ppt.o \ + smu_v11_0.o + +AMD_SWSMU_SMU11MGR = $(addprefix $(AMD_SWSMU_PATH)/smu11/,$(SMU11_MGR)) + +AMD_POWERPLAY_FILES += $(AMD_SWSMU_SMU11MGR) diff --git a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 6c991de8f371..2ce6ad9c6609 100644 --- a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -79,6 +79,8 @@ /* possible frequency drift (1Mhz) */ #define EPSILON 1 +#define smnPCIE_ESM_CTRL 0x111003D0 + static const struct cmn2asic_msg_mapping arcturus_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 0), MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), @@ -234,6 +236,13 @@ static int arcturus_tables_init(struct smu_context *smu) return -ENOMEM; smu_table->metrics_time = 0; + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_0); + smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); + if (!smu_table->gpu_metrics_table) { + kfree(smu_table->metrics_table); + return -ENOMEM; + } + return 0; } @@ -542,19 +551,12 @@ static int arcturus_get_smu_metrics_data(struct smu_context *smu, mutex_lock(&smu->metrics_lock); - if (!smu_table->metrics_time || - time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(1))) { - ret = smu_cmn_update_table(smu, - SMU_TABLE_SMU_METRICS, - 0, - smu_table->metrics_table, - false); - if (ret) { - dev_info(smu->adev->dev, "Failed to export SMU metrics table!\n"); - mutex_unlock(&smu->metrics_lock); - return ret; - } - smu_table->metrics_time = jiffies; + ret = smu_cmn_get_metrics_table_locked(smu, + NULL, + false); + if (ret) { + mutex_unlock(&smu->metrics_lock); + return ret; } switch (member) { @@ -897,9 +899,10 @@ static int arcturus_force_clk_levels(struct smu_context *smu, return ret; } - if (smu_version >= 0x361200) { + if ((smu_version >= 0x361200) && + (smu_version <= 0x361a00)) { dev_err(smu->adev->dev, "Forcing clock level is not supported with " - "54.18 and onwards SMU firmwares\n"); + "54.18 - 54.26(included) SMU firmwares\n"); return -EOPNOTSUPP; } @@ -1120,29 +1123,23 @@ static int arcturus_get_fan_speed_rpm(struct smu_context *smu, if (!speed) return -EINVAL; - return arcturus_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + return arcturus_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + speed); + default: + return smu_v11_0_get_fan_speed_rpm(smu, speed); + } } -static int arcturus_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int arcturus_get_fan_parameters(struct smu_context *smu) { PPTable_t *pptable = smu->smu_table.driver_pptable; - uint32_t percent, current_rpm; - int ret = 0; - if (!speed) - return -EINVAL; + smu->fan_max_rpm = pptable->FanMaximumRpm; - ret = arcturus_get_fan_speed_rpm(smu, ¤t_rpm); - if (ret) - return ret; - - percent = current_rpm * 100 / pptable->FanMaximumRpm; - *speed = percent > 100 ? 100 : percent; - - return ret; + return 0; } static int arcturus_get_power_limit(struct smu_context *smu) @@ -1392,9 +1389,10 @@ static int arcturus_set_performance_level(struct smu_context *smu, case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: - if (smu_version >= 0x361200) { + if ((smu_version >= 0x361200) && + (smu_version <= 0x361a00)) { dev_err(smu->adev->dev, "Forcing clock level is not supported with " - "54.18 and onwards SMU firmwares\n"); + "54.18 - 54.26(included) SMU firmwares\n"); return -EOPNOTSUPP; } break; @@ -1840,10 +1838,14 @@ static bool arcturus_is_dpm_running(struct smu_context *smu) { int ret = 0; uint32_t feature_mask[2]; - unsigned long feature_enabled; + uint64_t feature_enabled; + ret = smu_cmn_get_enabled_mask(smu, feature_mask, 2); - feature_enabled = (unsigned long)((uint64_t)feature_mask[0] | - ((uint64_t)feature_mask[1] << 32)); + if (ret) + return false; + + feature_enabled = (uint64_t)feature_mask[1] << 32 | feature_mask[0]; + return !!(feature_enabled & SMC_DPM_FEATURE); } @@ -2204,14 +2206,17 @@ static const struct throttling_logging_label { }; static void arcturus_log_thermal_throttling_event(struct smu_context *smu) { + int ret; int throttler_idx, throtting_events = 0, buf_idx = 0; struct amdgpu_device *adev = smu->adev; uint32_t throttler_status; char log_buf[256]; - arcturus_get_smu_metrics_data(smu, - METRICS_THROTTLER_STATUS, - &throttler_status); + ret = arcturus_get_smu_metrics_data(smu, + METRICS_THROTTLER_STATUS, + &throttler_status); + if (ret) + return; memset(log_buf, 0, sizeof(log_buf)); for (throttler_idx = 0; throttler_idx < ARRAY_SIZE(logging_label); @@ -2233,6 +2238,77 @@ static void arcturus_log_thermal_throttling_event(struct smu_context *smu) dev_warn(adev->dev, "WARN: GPU thermal throttling temperature reached, expect performance decrease. %s.\n", log_buf); + kgd2kfd_smi_event_throttle(smu->adev->kfd.dev, throttler_status); +} + +static int arcturus_get_current_pcie_link_speed(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t esm_ctrl; + + /* TODO: confirm this on real target */ + esm_ctrl = RREG32_PCIE(smnPCIE_ESM_CTRL); + if ((esm_ctrl >> 15) & 0x1FFFF) + return (((esm_ctrl >> 8) & 0x3F) + 128); + + return smu_v11_0_get_current_pcie_link_speed(smu); +} + +static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v1_0 *gpu_metrics = + (struct gpu_metrics_v1_0 *)smu_table->gpu_metrics_table; + SmuMetrics_t metrics; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, + &metrics, + true); + if (ret) + return ret; + + smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + + gpu_metrics->temperature_edge = metrics.TemperatureEdge; + gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; + gpu_metrics->temperature_mem = metrics.TemperatureHBM; + gpu_metrics->temperature_vrgfx = metrics.TemperatureVrGfx; + gpu_metrics->temperature_vrsoc = metrics.TemperatureVrSoc; + gpu_metrics->temperature_vrmem = metrics.TemperatureVrMem; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_umc_activity = metrics.AverageUclkActivity; + gpu_metrics->average_mm_activity = metrics.VcnActivityPercentage; + + gpu_metrics->average_socket_power = metrics.AverageSocketPower; + gpu_metrics->energy_accumulator = metrics.EnergyAccumulator; + + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.AverageUclkFrequency; + gpu_metrics->average_vclk0_frequency = metrics.AverageVclkFrequency; + gpu_metrics->average_dclk0_frequency = metrics.AverageDclkFrequency; + + gpu_metrics->current_gfxclk = metrics.CurrClock[PPCLK_GFXCLK]; + gpu_metrics->current_socclk = metrics.CurrClock[PPCLK_SOCCLK]; + gpu_metrics->current_uclk = metrics.CurrClock[PPCLK_UCLK]; + gpu_metrics->current_vclk0 = metrics.CurrClock[PPCLK_VCLK]; + gpu_metrics->current_dclk0 = metrics.CurrClock[PPCLK_DCLK]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + gpu_metrics->current_fan_speed = metrics.CurrFanSpeed; + + gpu_metrics->pcie_link_width = + smu_v11_0_get_current_pcie_link_width(smu); + gpu_metrics->pcie_link_speed = + arcturus_get_current_pcie_link_speed(smu); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v1_0); } static const struct pptable_funcs arcturus_ppt_funcs = { @@ -2247,7 +2323,6 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .print_clk_levels = arcturus_print_clk_levels, .force_clk_levels = arcturus_force_clk_levels, .read_sensor = arcturus_read_sensor, - .get_fan_speed_percent = arcturus_get_fan_speed_percent, .get_fan_speed_rpm = arcturus_get_fan_speed_rpm, .get_power_profile_mode = arcturus_get_power_profile_mode, .set_power_profile_mode = arcturus_set_power_profile_mode, @@ -2293,7 +2368,6 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, @@ -2312,6 +2386,10 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .log_thermal_throttling_event = arcturus_log_thermal_throttling_event, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, + .get_gpu_metrics = arcturus_get_gpu_metrics, + .gfx_ulv_control = smu_v11_0_gfx_ulv_control, + .deep_sleep_control = smu_v11_0_deep_sleep_control, + .get_fan_parameters = arcturus_get_fan_parameters, }; void arcturus_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.h index d756b16924b8..d756b16924b8 100644 --- a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.h diff --git a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index 9f62af9abd23..42d53cca7360 100644 --- a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -138,6 +138,7 @@ static struct cmn2asic_msg_mapping navi10_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(DAL_ENABLE_DUMMY_PSTATE_CHANGE, PPSMC_MSG_DALEnableDummyPstateChange, 0), MSG_MAP(GetVoltageByDpm, PPSMC_MSG_GetVoltageByDpm, 0), MSG_MAP(GetVoltageByDpmOverdrive, PPSMC_MSG_GetVoltageByDpmOverdrive, 0), + MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), }; static struct cmn2asic_mapping navi10_clk_map[SMU_CLK_COUNT] = { @@ -456,13 +457,18 @@ static int navi10_tables_init(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; struct smu_table *tables = smu_table->tables; + struct amdgpu_device *adev = smu->adev; SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t), PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetrics_t), - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); + if (adev->asic_type == CHIP_NAVI12) + SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetrics_NV12_t), + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); + else + SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetrics_t), + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t), PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t), @@ -473,16 +479,30 @@ static int navi10_tables_init(struct smu_context *smu) sizeof(DpmActivityMonitorCoeffInt_t), PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM); - smu_table->metrics_table = kzalloc(sizeof(SmuMetrics_t), GFP_KERNEL); + smu_table->metrics_table = kzalloc(adev->asic_type == CHIP_NAVI12 ? + sizeof(SmuMetrics_NV12_t) : + sizeof(SmuMetrics_t), GFP_KERNEL); if (!smu_table->metrics_table) - return -ENOMEM; + goto err0_out; smu_table->metrics_time = 0; + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_0); + smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); + if (!smu_table->gpu_metrics_table) + goto err1_out; + smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); if (!smu_table->watermarks_table) - return -ENOMEM; + goto err2_out; return 0; + +err2_out: + kfree(smu_table->gpu_metrics_table); +err1_out: + kfree(smu_table->metrics_table); +err0_out: + return -ENOMEM; } static int navi10_get_smu_metrics_data(struct smu_context *smu, @@ -490,23 +510,22 @@ static int navi10_get_smu_metrics_data(struct smu_context *smu, uint32_t *value) { struct smu_table_context *smu_table= &smu->smu_table; + /* + * This works for NV12 also. As although NV12 uses a different + * SmuMetrics structure from other NV1X ASICs, they share the + * same offsets for the heading parts(those members used here). + */ SmuMetrics_t *metrics = (SmuMetrics_t *)smu_table->metrics_table; int ret = 0; mutex_lock(&smu->metrics_lock); - if (!smu_table->metrics_time || - time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(1))) { - ret = smu_cmn_update_table(smu, - SMU_TABLE_SMU_METRICS, - 0, - smu_table->metrics_table, - false); - if (ret) { - dev_info(smu->adev->dev, "Failed to export SMU metrics table!\n"); - mutex_unlock(&smu->metrics_lock); - return ret; - } - smu_table->metrics_time = jiffies; + + ret = smu_cmn_get_metrics_table_locked(smu, + NULL, + false); + if (ret) { + mutex_unlock(&smu->metrics_lock); + return ret; } switch (member) { @@ -909,7 +928,6 @@ static int navi10_print_clk_levels(struct smu_context *smu, uint32_t gen_speed, lane_width; struct smu_dpm_context *smu_dpm = &smu->smu_dpm; struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context; - struct amdgpu_device *adev = smu->adev; PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable; OverDriveTable_t *od_table = (OverDriveTable_t *)table_context->overdrive_table; @@ -963,12 +981,8 @@ static int navi10_print_clk_levels(struct smu_context *smu, } break; case SMU_PCIE: - gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & - PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) - >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & - PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) - >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; + gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu); + lane_width = smu_v11_0_get_current_pcie_link_width_level(smu); for (i = 0; i < NUM_LINK_LEVELS; i++) size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," : @@ -1331,10 +1345,14 @@ static bool navi10_is_dpm_running(struct smu_context *smu) { int ret = 0; uint32_t feature_mask[2]; - unsigned long feature_enabled; + uint64_t feature_enabled; + ret = smu_cmn_get_enabled_mask(smu, feature_mask, 2); - feature_enabled = (unsigned long)((uint64_t)feature_mask[0] | - ((uint64_t)feature_mask[1] << 32)); + if (ret) + return false; + + feature_enabled = (uint64_t)feature_mask[1] << 32 | feature_mask[0]; + return !!(feature_enabled & SMC_DPM_FEATURE); } @@ -1344,27 +1362,23 @@ static int navi10_get_fan_speed_rpm(struct smu_context *smu, if (!speed) return -EINVAL; - return navi10_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + return navi10_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + speed); + default: + return smu_v11_0_get_fan_speed_rpm(smu, speed); + } } -static int navi10_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int navi10_get_fan_parameters(struct smu_context *smu) { - int ret = 0; - uint32_t percent = 0; - uint32_t current_rpm; PPTable_t *pptable = smu->smu_table.driver_pptable; - ret = navi10_get_fan_speed_rpm(smu, ¤t_rpm); - if (ret) - return ret; - - percent = current_rpm * 100 / pptable->FanMaximumRpm; - *speed = percent > 100 ? 100 : percent; + smu->fan_max_rpm = pptable->FanMaximumRpm; - return ret; + return 0; } static int navi10_get_power_profile_mode(struct smu_context *smu, char *buf) @@ -2472,6 +2486,97 @@ static void navi10_i2c_control_fini(struct smu_context *smu, struct i2c_adapter i2c_del_adapter(control); } +static ssize_t navi10_get_gpu_metrics(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v1_0 *gpu_metrics = + (struct gpu_metrics_v1_0 *)smu_table->gpu_metrics_table; + struct amdgpu_device *adev = smu->adev; + SmuMetrics_NV12_t nv12_metrics = { 0 }; + SmuMetrics_t metrics; + int ret = 0; + + mutex_lock(&smu->metrics_lock); + + ret = smu_cmn_get_metrics_table_locked(smu, + NULL, + true); + if (ret) { + mutex_unlock(&smu->metrics_lock); + return ret; + } + + memcpy(&metrics, smu_table->metrics_table, sizeof(SmuMetrics_t)); + if (adev->asic_type == CHIP_NAVI12) + memcpy(&nv12_metrics, smu_table->metrics_table, sizeof(SmuMetrics_NV12_t)); + + mutex_unlock(&smu->metrics_lock); + + smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + + gpu_metrics->temperature_edge = metrics.TemperatureEdge; + gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; + gpu_metrics->temperature_mem = metrics.TemperatureMem; + gpu_metrics->temperature_vrgfx = metrics.TemperatureVrGfx; + gpu_metrics->temperature_vrsoc = metrics.TemperatureVrSoc; + gpu_metrics->temperature_vrmem = metrics.TemperatureVrMem0; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_umc_activity = metrics.AverageUclkActivity; + + gpu_metrics->average_socket_power = metrics.AverageSocketPower; + + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.AverageUclkFrequency; + + if (adev->asic_type == CHIP_NAVI12) { + gpu_metrics->energy_accumulator = nv12_metrics.EnergyAccumulator; + gpu_metrics->average_vclk0_frequency = nv12_metrics.AverageVclkFrequency; + gpu_metrics->average_dclk0_frequency = nv12_metrics.AverageDclkFrequency; + gpu_metrics->average_mm_activity = nv12_metrics.VcnActivityPercentage; + } + + gpu_metrics->current_gfxclk = metrics.CurrClock[PPCLK_GFXCLK]; + gpu_metrics->current_socclk = metrics.CurrClock[PPCLK_SOCCLK]; + gpu_metrics->current_uclk = metrics.CurrClock[PPCLK_UCLK]; + gpu_metrics->current_vclk0 = metrics.CurrClock[PPCLK_VCLK]; + gpu_metrics->current_dclk0 = metrics.CurrClock[PPCLK_DCLK]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + gpu_metrics->current_fan_speed = metrics.CurrFanSpeed; + + gpu_metrics->pcie_link_width = + smu_v11_0_get_current_pcie_link_width(smu); + gpu_metrics->pcie_link_speed = + smu_v11_0_get_current_pcie_link_speed(smu); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v1_0); +} + +static int navi10_enable_mgpu_fan_boost(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t param = 0; + + /* Navi12 does not support this */ + if (adev->asic_type == CHIP_NAVI12) + return 0; + + /* Workaround for WS SKU */ + if (adev->pdev->device == 0x7312 && + adev->pdev->revision == 0) + param = 0xD188; + + return smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetMGpuFanBoostLimitRpm, + param, + NULL); +} static const struct pptable_funcs navi10_ppt_funcs = { .get_allowed_feature_mask = navi10_get_allowed_feature_mask, @@ -2488,7 +2593,6 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_config_changed = navi10_display_config_changed, .notify_smc_display_config = navi10_notify_smc_display_config, .is_dpm_running = navi10_is_dpm_running, - .get_fan_speed_percent = navi10_get_fan_speed_percent, .get_fan_speed_rpm = navi10_get_fan_speed_rpm, .get_power_profile_mode = navi10_get_power_profile_mode, .set_power_profile_mode = navi10_set_power_profile_mode, @@ -2532,7 +2636,6 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, @@ -2553,6 +2656,11 @@ static const struct pptable_funcs navi10_ppt_funcs = { .set_power_source = smu_v11_0_set_power_source, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, + .get_gpu_metrics = navi10_get_gpu_metrics, + .enable_mgpu_fan_boost = navi10_enable_mgpu_fan_boost, + .gfx_ulv_control = smu_v11_0_gfx_ulv_control, + .deep_sleep_control = smu_v11_0_deep_sleep_control, + .get_fan_parameters = navi10_get_fan_parameters, }; void navi10_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/navi10_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.h index 2abb4ba01db1..84dc5a1b6830 100644 --- a/drivers/gpu/drm/amd/powerplay/navi10_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.h @@ -49,9 +49,6 @@ #define NAVI10_VOLTAGE_SCALE (4) -#define smnPCIE_LC_SPEED_CNTL 0x11140290 -#define smnPCIE_LC_LINK_WIDTH_CNTL 0x11140288 - extern void navi10_set_ppt_funcs(struct smu_context *smu); #endif diff --git a/drivers/gpu/drm/amd/powerplay/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 3865dbed5f93..5c22611d5878 100644 --- a/drivers/gpu/drm/amd/powerplay/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -68,7 +68,8 @@ FEATURE_MASK(FEATURE_DPM_LINK_BIT) | \ FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT) | \ FEATURE_MASK(FEATURE_DPM_FCLK_BIT) | \ - FEATURE_MASK(FEATURE_DPM_DCEFCLK_BIT)) + FEATURE_MASK(FEATURE_DPM_DCEFCLK_BIT) | \ + FEATURE_MASK(FEATURE_DPM_MP0CLK_BIT)) #define SMU_11_0_7_GFX_BUSY_THRESHOLD 15 @@ -95,6 +96,7 @@ static struct cmn2asic_msg_mapping sienna_cichlid_message_map[SMU_MSG_MAX_COUNT] MSG_MAP(TransferTableSmu2Dram, PPSMC_MSG_TransferTableSmu2Dram, 0), MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), MSG_MAP(UseDefaultPPTable, PPSMC_MSG_UseDefaultPPTable, 0), + MSG_MAP(RunDcBtc, PPSMC_MSG_RunDcBtc, 0), MSG_MAP(EnterBaco, PPSMC_MSG_EnterBaco, 0), MSG_MAP(SetSoftMinByFreq, PPSMC_MSG_SetSoftMinByFreq, 0), MSG_MAP(SetSoftMaxByFreq, PPSMC_MSG_SetSoftMaxByFreq, 0), @@ -124,6 +126,7 @@ static struct cmn2asic_msg_mapping sienna_cichlid_message_map[SMU_MSG_MAX_COUNT] MSG_MAP(BacoAudioD3PME, PPSMC_MSG_BacoAudioD3PME, 0), MSG_MAP(ArmD3, PPSMC_MSG_ArmD3, 0), MSG_MAP(Mode1Reset, PPSMC_MSG_Mode1Reset, 0), + MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), }; static struct cmn2asic_mapping sienna_cichlid_clk_map[SMU_CLK_COUNT] = { @@ -228,6 +231,7 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_PREFETCHER_BIT) | FEATURE_MASK(FEATURE_DPM_FCLK_BIT) + | FEATURE_MASK(FEATURE_DPM_MP0CLK_BIT) | FEATURE_MASK(FEATURE_DS_SOCCLK_BIT) | FEATURE_MASK(FEATURE_DS_DCEFCLK_BIT) | FEATURE_MASK(FEATURE_DS_FCLK_BIT) @@ -385,14 +389,26 @@ static int sienna_cichlid_tables_init(struct smu_context *smu) smu_table->metrics_table = kzalloc(sizeof(SmuMetrics_t), GFP_KERNEL); if (!smu_table->metrics_table) - return -ENOMEM; + goto err0_out; smu_table->metrics_time = 0; + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_0); + smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); + if (!smu_table->gpu_metrics_table) + goto err1_out; + smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); if (!smu_table->watermarks_table) - return -ENOMEM; + goto err2_out; return 0; + +err2_out: + kfree(smu_table->gpu_metrics_table); +err1_out: + kfree(smu_table->metrics_table); +err0_out: + return -ENOMEM; } static int sienna_cichlid_get_smu_metrics_data(struct smu_context *smu, @@ -404,19 +420,13 @@ static int sienna_cichlid_get_smu_metrics_data(struct smu_context *smu, int ret = 0; mutex_lock(&smu->metrics_lock); - if (!smu_table->metrics_time || - time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(1))) { - ret = smu_cmn_update_table(smu, - SMU_TABLE_SMU_METRICS, - 0, - smu_table->metrics_table, - false); - if (ret) { - dev_info(smu->adev->dev, "Failed to export SMU metrics table!\n"); - mutex_unlock(&smu->metrics_lock); - return ret; - } - smu_table->metrics_time = jiffies; + + ret = smu_cmn_get_metrics_table_locked(smu, + NULL, + false); + if (ret) { + mutex_unlock(&smu->metrics_lock); + return ret; } switch (member) { @@ -775,7 +785,7 @@ static int sienna_cichlid_dpm_set_vcn_enable(struct smu_context *smu, bool enabl ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0, NULL); if (ret) return ret; - if (adev->asic_type == CHIP_SIENNA_CICHLID) { + if (adev->vcn.num_vcn_inst > 1) { ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0x10000, NULL); if (ret) @@ -787,7 +797,7 @@ static int sienna_cichlid_dpm_set_vcn_enable(struct smu_context *smu, bool enabl ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0, NULL); if (ret) return ret; - if (adev->asic_type == CHIP_SIENNA_CICHLID) { + if (adev->vcn.num_vcn_inst > 1) { ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0x10000, NULL); if (ret) @@ -951,12 +961,8 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, } break; case SMU_PCIE: - gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & - PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) - >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & - PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) - >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; + gen_speed = smu_v11_0_get_current_pcie_link_speed(smu); + lane_width = smu_v11_0_get_current_pcie_link_width(smu); for (i = 0; i < NUM_LINK_LEVELS; i++) size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," : @@ -1146,10 +1152,14 @@ static bool sienna_cichlid_is_dpm_running(struct smu_context *smu) { int ret = 0; uint32_t feature_mask[2]; - unsigned long feature_enabled; + uint64_t feature_enabled; + ret = smu_cmn_get_enabled_mask(smu, feature_mask, 2); - feature_enabled = (unsigned long)((uint64_t)feature_mask[0] | - ((uint64_t)feature_mask[1] << 32)); + if (ret) + return false; + + feature_enabled = (uint64_t)feature_mask[1] << 32 | feature_mask[0]; + return !!(feature_enabled & SMC_DPM_FEATURE); } @@ -1159,27 +1169,23 @@ static int sienna_cichlid_get_fan_speed_rpm(struct smu_context *smu, if (!speed) return -EINVAL; - return sienna_cichlid_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + return sienna_cichlid_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + speed); + default: + return smu_v11_0_get_fan_speed_rpm(smu, speed); + } } -static int sienna_cichlid_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int sienna_cichlid_get_fan_parameters(struct smu_context *smu) { - int ret = 0; - uint32_t percent = 0; - uint32_t current_rpm; PPTable_t *pptable = smu->smu_table.driver_pptable; - ret = sienna_cichlid_get_fan_speed_rpm(smu, ¤t_rpm); - if (ret) - return ret; - - percent = current_rpm * 100 / pptable->FanMaximumRpm; - *speed = percent > 100 ? 100 : percent; + smu->fan_max_rpm = pptable->FanMaximumRpm; - return ret; + return 0; } static int sienna_cichlid_get_power_profile_mode(struct smu_context *smu, char *buf) @@ -1732,6 +1738,11 @@ static int sienna_cichlid_get_dpm_ultimate_freq(struct smu_context *smu, return ret; } +static int sienna_cichlid_run_btc(struct smu_context *smu) +{ + return smu_cmn_send_smc_msg(smu, SMU_MSG_RunDcBtc, NULL); +} + static bool sienna_cichlid_is_baco_supported(struct smu_context *smu) { struct amdgpu_device *adev = smu->adev; @@ -2644,6 +2655,76 @@ static void sienna_cichlid_i2c_control_fini(struct smu_context *smu, struct i2c_ i2c_del_adapter(control); } +static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v1_0 *gpu_metrics = + (struct gpu_metrics_v1_0 *)smu_table->gpu_metrics_table; + SmuMetrics_t metrics; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, + &metrics, + true); + if (ret) + return ret; + + smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + + gpu_metrics->temperature_edge = metrics.TemperatureEdge; + gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; + gpu_metrics->temperature_mem = metrics.TemperatureMem; + gpu_metrics->temperature_vrgfx = metrics.TemperatureVrGfx; + gpu_metrics->temperature_vrsoc = metrics.TemperatureVrSoc; + gpu_metrics->temperature_vrmem = metrics.TemperatureVrMem0; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_umc_activity = metrics.AverageUclkActivity; + gpu_metrics->average_mm_activity = metrics.VcnActivityPercentage; + + gpu_metrics->average_socket_power = metrics.AverageSocketPower; + gpu_metrics->energy_accumulator = metrics.EnergyAccumulator; + + if (metrics.AverageGfxActivity <= SMU_11_0_7_GFX_BUSY_THRESHOLD) + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequencyPostDs; + else + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequencyPreDs; + gpu_metrics->average_uclk_frequency = metrics.AverageUclkFrequencyPostDs; + gpu_metrics->average_vclk0_frequency = metrics.AverageVclk0Frequency; + gpu_metrics->average_dclk0_frequency = metrics.AverageDclk0Frequency; + gpu_metrics->average_vclk1_frequency = metrics.AverageVclk1Frequency; + gpu_metrics->average_dclk1_frequency = metrics.AverageDclk1Frequency; + + gpu_metrics->current_gfxclk = metrics.CurrClock[PPCLK_GFXCLK]; + gpu_metrics->current_socclk = metrics.CurrClock[PPCLK_SOCCLK]; + gpu_metrics->current_uclk = metrics.CurrClock[PPCLK_UCLK]; + gpu_metrics->current_vclk0 = metrics.CurrClock[PPCLK_VCLK_0]; + gpu_metrics->current_dclk0 = metrics.CurrClock[PPCLK_DCLK_0]; + gpu_metrics->current_vclk1 = metrics.CurrClock[PPCLK_VCLK_1]; + gpu_metrics->current_dclk1 = metrics.CurrClock[PPCLK_DCLK_1]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + gpu_metrics->current_fan_speed = metrics.CurrFanSpeed; + + gpu_metrics->pcie_link_width = + smu_v11_0_get_current_pcie_link_width(smu); + gpu_metrics->pcie_link_speed = + smu_v11_0_get_current_pcie_link_speed(smu); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v1_0); +} + +static int sienna_cichlid_enable_mgpu_fan_boost(struct smu_context *smu) +{ + return smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetMGpuFanBoostLimitRpm, + 0, + NULL); +} static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_allowed_feature_mask = sienna_cichlid_get_allowed_feature_mask, @@ -2659,7 +2740,6 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_config_changed = sienna_cichlid_display_config_changed, .notify_smc_display_config = sienna_cichlid_notify_smc_display_config, .is_dpm_running = sienna_cichlid_is_dpm_running, - .get_fan_speed_percent = sienna_cichlid_get_fan_speed_percent, .get_fan_speed_rpm = sienna_cichlid_get_fan_speed_rpm, .get_power_profile_mode = sienna_cichlid_get_power_profile_mode, .set_power_profile_mode = sienna_cichlid_set_power_profile_mode, @@ -2703,7 +2783,6 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, @@ -2719,8 +2798,14 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .mode1_reset = smu_v11_0_mode1_reset, .get_dpm_ultimate_freq = sienna_cichlid_get_dpm_ultimate_freq, .set_soft_freq_limited_range = smu_v11_0_set_soft_freq_limited_range, + .run_btc = sienna_cichlid_run_btc, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, + .get_gpu_metrics = sienna_cichlid_get_gpu_metrics, + .enable_mgpu_fan_boost = sienna_cichlid_enable_mgpu_fan_boost, + .gfx_ulv_control = smu_v11_0_gfx_ulv_control, + .deep_sleep_control = smu_v11_0_deep_sleep_control, + .get_fan_parameters = sienna_cichlid_get_fan_parameters, }; void sienna_cichlid_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/sienna_cichlid_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h index 8078886e4cbc..57e120c440ea 100644 --- a/drivers/gpu/drm/amd/powerplay/sienna_cichlid_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h @@ -31,7 +31,4 @@ typedef enum { extern void sienna_cichlid_set_ppt_funcs(struct smu_context *smu); -#define smnPCIE_LC_SPEED_CNTL 0x11140290 -#define smnPCIE_LC_LINK_WIDTH_CNTL 0x11140288 - #endif diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 7b950a582a28..f5aeb0b5cf97 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -67,6 +67,19 @@ MODULE_FIRMWARE("amdgpu/navy_flounder_smc.bin"); #define SMU11_MODE1_RESET_WAIT_TIME_IN_MS 500 //500ms +#define LINK_WIDTH_MAX 6 +#define LINK_SPEED_MAX 3 + +#define smnPCIE_LC_LINK_WIDTH_CNTL 0x11140288 +#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK 0x00000070L +#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT 0x4 +#define smnPCIE_LC_SPEED_CNTL 0x11140290 +#define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK 0xC000 +#define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT 0xE + +static int link_width[] = {0, 1, 2, 4, 8, 12, 16}; +static int link_speed[] = {25, 50, 80, 160}; + int smu_v11_0_init_microcode(struct smu_context *smu) { struct amdgpu_device *adev = smu->adev; @@ -404,10 +417,12 @@ int smu_v11_0_fini_smc_tables(struct smu_context *smu) struct smu_table_context *smu_table = &smu->smu_table; struct smu_dpm_context *smu_dpm = &smu->smu_dpm; + kfree(smu_table->gpu_metrics_table); kfree(smu_table->boot_overdrive_table); kfree(smu_table->overdrive_table); kfree(smu_table->max_sustainable_clocks); kfree(smu_table->driver_pptable); + smu_table->gpu_metrics_table = NULL; smu_table->boot_overdrive_table = NULL; smu_table->overdrive_table = NULL; smu_table->max_sustainable_clocks = NULL; @@ -1085,35 +1100,6 @@ smu_v11_0_set_fan_static_mode(struct smu_context *smu, uint32_t mode) } int -smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) -{ - struct amdgpu_device *adev = smu->adev; - uint32_t duty100, duty; - uint64_t tmp64; - - if (speed > 100) - speed = 100; - - if (smu_v11_0_auto_fan_control(smu, 0)) - return -EINVAL; - - duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), - CG_FDO_CTRL1, FMAX_DUTY100); - if (!duty100) - return -EINVAL; - - tmp64 = (uint64_t)speed * duty100; - do_div(tmp64, 100); - duty = (uint32_t)tmp64; - - WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0, - REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL0), - CG_FDO_CTRL0, FDO_STATIC_DUTY, duty)); - - return smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC); -} - -int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode) { @@ -1121,7 +1107,7 @@ smu_v11_0_set_fan_control_mode(struct smu_context *smu, switch (mode) { case AMD_FAN_CTRL_NONE: - ret = smu_v11_0_set_fan_speed_percent(smu, 100); + ret = smu_v11_0_set_fan_speed_rpm(smu, smu->fan_max_rpm); break; case AMD_FAN_CTRL_MANUAL: ret = smu_v11_0_auto_fan_control(smu, 0); @@ -1167,6 +1153,27 @@ int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, return ret; } +int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t tach_period, crystal_clock_freq; + uint64_t tmp64; + + tach_period = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), + CG_TACH_CTRL, TARGET_PERIOD); + if (!tach_period) + return -EINVAL; + + crystal_clock_freq = amdgpu_asic_get_xclk(adev); + + tmp64 = (uint64_t)crystal_clock_freq * 60 * 10000; + do_div(tmp64, (tach_period * 8)); + *speed = (uint32_t)tmp64; + + return 0; +} + int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate) { @@ -1312,6 +1319,11 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev, smu_v11_0_ack_ac_dc_interrupt(&adev->smu); break; case 0x7: + /* + * Increment the throttle interrupt counter + */ + atomic64_inc(&smu->throttle_int_counter); + if (!atomic_read(&adev->throttling_logging_enabled)) return 0; @@ -1913,3 +1925,99 @@ int smu_v11_0_get_dpm_level_range(struct smu_context *smu, return ret; } + +int smu_v11_0_get_current_pcie_link_width_level(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + + return (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & + PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) + >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; +} + +int smu_v11_0_get_current_pcie_link_width(struct smu_context *smu) +{ + uint32_t width_level; + + width_level = smu_v11_0_get_current_pcie_link_width_level(smu); + if (width_level > LINK_WIDTH_MAX) + width_level = 0; + + return link_width[width_level]; +} + +int smu_v11_0_get_current_pcie_link_speed_level(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + + return (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & + PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) + >> PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; +} + +int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu) +{ + uint32_t speed_level; + + speed_level = smu_v11_0_get_current_pcie_link_speed_level(smu); + if (speed_level > LINK_SPEED_MAX) + speed_level = 0; + + return link_speed[speed_level]; +} + +void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics) +{ + memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v1_0)); + + gpu_metrics->common_header.structure_size = + sizeof(struct gpu_metrics_v1_0); + gpu_metrics->common_header.format_revision = 1; + gpu_metrics->common_header.content_revision = 0; + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); +} + +int smu_v11_0_gfx_ulv_control(struct smu_context *smu, + bool enablement) +{ + int ret = 0; + + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_GFX_ULV_BIT)) + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_GFX_ULV_BIT, enablement); + + return ret; +} + +int smu_v11_0_deep_sleep_control(struct smu_context *smu, + bool enablement) +{ + struct amdgpu_device *adev = smu->adev; + int ret = 0; + + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_DS_GFXCLK_BIT)) { + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_DS_GFXCLK_BIT, enablement); + if (ret) { + dev_err(adev->dev, "Failed to %s GFXCLK DS!\n", enablement ? "enable" : "disable"); + return ret; + } + } + + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_DS_SOCCLK_BIT)) { + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_DS_SOCCLK_BIT, enablement); + if (ret) { + dev_err(adev->dev, "Failed to %s SOCCLK DS!\n", enablement ? "enable" : "disable"); + return ret; + } + } + + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_DS_LCLK_BIT)) { + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_DS_LCLK_BIT, enablement); + if (ret) { + dev_err(adev->dev, "Failed to %s LCLK DS!\n", enablement ? "enable" : "disable"); + return ret; + } + } + + return ret; +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/Makefile b/drivers/gpu/drm/amd/pm/swsmu/smu12/Makefile new file mode 100644 index 000000000000..67e53f7da3ce --- /dev/null +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/Makefile @@ -0,0 +1,31 @@ +# +# Copyright 2020 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +# Makefile for the 'smu manager' sub-component of powerplay. +# It provides the smu management services for the driver. + +SMU12_MGR = renoir_ppt.o \ + smu_v12_0.o + +AMD_SWSMU_SMU12MGR = $(addprefix $(AMD_SWSMU_PATH)/smu12/,$(SMU12_MGR)) + +AMD_POWERPLAY_FILES += $(AMD_SWSMU_SMU12MGR) diff --git a/drivers/gpu/drm/amd/powerplay/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index dbb676c482fd..3b9ac72c7571 100644 --- a/drivers/gpu/drm/amd/powerplay/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -128,30 +128,6 @@ static struct cmn2asic_mapping renoir_workload_map[PP_SMC_POWER_PROFILE_COUNT] = WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), }; -static int renoir_get_metrics_table(struct smu_context *smu, - SmuMetrics_t *metrics_table) -{ - struct smu_table_context *smu_table= &smu->smu_table; - int ret = 0; - - mutex_lock(&smu->metrics_lock); - if (!smu_table->metrics_time || time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(100))) { - ret = smu_cmn_update_table(smu, SMU_TABLE_SMU_METRICS, 0, - (void *)smu_table->metrics_table, false); - if (ret) { - dev_info(smu->adev->dev, "Failed to export SMU metrics table!\n"); - mutex_unlock(&smu->metrics_lock); - return ret; - } - smu_table->metrics_time = jiffies; - } - - memcpy(metrics_table, smu_table->metrics_table, sizeof(SmuMetrics_t)); - mutex_unlock(&smu->metrics_lock); - - return ret; -} - static int renoir_init_smc_tables(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; @@ -166,18 +142,32 @@ static int renoir_init_smc_tables(struct smu_context *smu) smu_table->clocks_table = kzalloc(sizeof(DpmClocks_t), GFP_KERNEL); if (!smu_table->clocks_table) - return -ENOMEM; + goto err0_out; smu_table->metrics_table = kzalloc(sizeof(SmuMetrics_t), GFP_KERNEL); if (!smu_table->metrics_table) - return -ENOMEM; + goto err1_out; smu_table->metrics_time = 0; smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL); if (!smu_table->watermarks_table) - return -ENOMEM; + goto err2_out; + + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_0); + smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); + if (!smu_table->gpu_metrics_table) + goto err3_out; return 0; + +err3_out: + kfree(smu_table->watermarks_table); +err2_out: + kfree(smu_table->metrics_table); +err1_out: + kfree(smu_table->clocks_table); +err0_out: + return -ENOMEM; } /** @@ -361,7 +351,7 @@ static int renoir_print_clk_levels(struct smu_context *smu, memset(&metrics, 0, sizeof(metrics)); - ret = renoir_get_metrics_table(smu, &metrics); + ret = smu_cmn_get_metrics_table(smu, &metrics, false); if (ret) return ret; @@ -507,7 +497,7 @@ static int renoir_get_current_clk_freq_by_table(struct smu_context *smu, int ret = 0, clk_id = 0; SmuMetrics_t metrics; - ret = renoir_get_metrics_table(smu, &metrics); + ret = smu_cmn_get_metrics_table(smu, &metrics, false); if (ret) return ret; @@ -590,7 +580,7 @@ static int renoir_get_gpu_temperature(struct smu_context *smu, uint32_t *value) if (!value) return -EINVAL; - ret = renoir_get_metrics_table(smu, &metrics); + ret = smu_cmn_get_metrics_table(smu, &metrics, false); if (ret) return ret; @@ -610,7 +600,7 @@ static int renoir_get_current_activity_percent(struct smu_context *smu, if (!value) return -EINVAL; - ret = renoir_get_metrics_table(smu, &metrics); + ret = smu_cmn_get_metrics_table(smu, &metrics, false); if (ret) return ret; @@ -626,6 +616,44 @@ static int renoir_get_current_activity_percent(struct smu_context *smu, return 0; } +static int renoir_get_vddc(struct smu_context *smu, uint32_t *value, + unsigned int index) +{ + int ret = 0; + SmuMetrics_t metrics; + + if (index >= 2) + return -EINVAL; + + if (!value) + return -EINVAL; + + ret = smu_cmn_get_metrics_table(smu, &metrics, false); + if (ret) + return ret; + + *value = metrics.Voltage[index]; + + return 0; +} + +static int renoir_get_power(struct smu_context *smu, uint32_t *value) +{ + int ret = 0; + SmuMetrics_t metrics; + + if (!value) + return -EINVAL; + + ret = smu_cmn_get_metrics_table(smu, &metrics, false); + if (ret) + return ret; + + *value = metrics.CurrentSocketPower << 8; + + return 0; +} + /** * This interface get dpm clock table for dc */ @@ -962,6 +990,18 @@ static int renoir_read_sensor(struct smu_context *smu, *(uint32_t *)data *= 100; *size = 4; break; + case AMDGPU_PP_SENSOR_VDDGFX: + ret = renoir_get_vddc(smu, (uint32_t *)data, 0); + *size = 4; + break; + case AMDGPU_PP_SENSOR_VDDNB: + ret = renoir_get_vddc(smu, (uint32_t *)data, 1); + *size = 4; + break; + case AMDGPU_PP_SENSOR_GPU_POWER: + ret = renoir_get_power(smu, (uint32_t *)data); + *size = 4; + break; default: ret = -EOPNOTSUPP; break; @@ -987,6 +1027,65 @@ static bool renoir_is_dpm_running(struct smu_context *smu) } +static ssize_t renoir_get_gpu_metrics(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v2_0 *gpu_metrics = + (struct gpu_metrics_v2_0 *)smu_table->gpu_metrics_table; + SmuMetrics_t metrics; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, &metrics, true); + if (ret) + return ret; + + smu_v12_0_init_gpu_metrics_v2_0(gpu_metrics); + + gpu_metrics->temperature_gfx = metrics.GfxTemperature; + gpu_metrics->temperature_soc = metrics.SocTemperature; + memcpy(&gpu_metrics->temperature_core[0], + &metrics.CoreTemperature[0], + sizeof(uint16_t) * 8); + gpu_metrics->temperature_l3[0] = metrics.L3Temperature[0]; + gpu_metrics->temperature_l3[1] = metrics.L3Temperature[1]; + + gpu_metrics->average_gfx_activity = metrics.AverageGfxActivity; + gpu_metrics->average_mm_activity = metrics.AverageUvdActivity; + + gpu_metrics->average_socket_power = metrics.CurrentSocketPower; + gpu_metrics->average_cpu_power = metrics.Power[0]; + gpu_metrics->average_soc_power = metrics.Power[1]; + memcpy(&gpu_metrics->average_core_power[0], + &metrics.CorePower[0], + sizeof(uint16_t) * 8); + + gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency; + gpu_metrics->average_fclk_frequency = metrics.AverageFclkFrequency; + gpu_metrics->average_vclk_frequency = metrics.AverageVclkFrequency; + + gpu_metrics->current_gfxclk = metrics.ClockFrequency[CLOCK_GFXCLK]; + gpu_metrics->current_socclk = metrics.ClockFrequency[CLOCK_SOCCLK]; + gpu_metrics->current_uclk = metrics.ClockFrequency[CLOCK_UMCCLK]; + gpu_metrics->current_fclk = metrics.ClockFrequency[CLOCK_FCLK]; + gpu_metrics->current_vclk = metrics.ClockFrequency[CLOCK_VCLK]; + gpu_metrics->current_dclk = metrics.ClockFrequency[CLOCK_DCLK]; + memcpy(&gpu_metrics->current_coreclk[0], + &metrics.CoreFrequency[0], + sizeof(uint16_t) * 8); + gpu_metrics->current_l3clk[0] = metrics.L3Frequency[0]; + gpu_metrics->current_l3clk[1] = metrics.L3Frequency[1]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + + gpu_metrics->fan_pwm = metrics.FanPwm; + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v2_0); +} + static const struct pptable_funcs renoir_ppt_funcs = { .set_power_state = NULL, .print_clk_levels = renoir_print_clk_levels, @@ -1021,6 +1120,7 @@ static const struct pptable_funcs renoir_ppt_funcs = { .is_dpm_running = renoir_is_dpm_running, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, + .get_gpu_metrics = renoir_get_gpu_metrics, }; void renoir_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/renoir_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.h index 8c3f004cdf8d..8c3f004cdf8d 100644 --- a/drivers/gpu/drm/amd/powerplay/renoir_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.h diff --git a/drivers/gpu/drm/amd/powerplay/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c index 31456437bb18..660f403d5770 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v12_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c @@ -274,3 +274,15 @@ int smu_v12_0_set_driver_table_location(struct smu_context *smu) return ret; } + +void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics) +{ + memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v2_0)); + + gpu_metrics->common_header.structure_size = + sizeof(struct gpu_metrics_v2_0); + gpu_metrics->common_header.format_revision = 2; + gpu_metrics->common_header.content_revision = 0; + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); +} diff --git a/drivers/gpu/drm/amd/powerplay/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 5c23c44c33bd..a58ea08cd115 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -604,7 +604,7 @@ int smu_cmn_update_table(struct smu_context *smu, memcpy(table_data, table->cpu_addr, table_size); } - return ret; + return 0; } int smu_cmn_write_watermarks_table(struct smu_context *smu) @@ -631,3 +631,48 @@ int smu_cmn_write_pptable(struct smu_context *smu) pptable, true); } + +int smu_cmn_get_metrics_table_locked(struct smu_context *smu, + void *metrics_table, + bool bypass_cache) +{ + struct smu_table_context *smu_table= &smu->smu_table; + uint32_t table_size = + smu_table->tables[SMU_TABLE_SMU_METRICS].size; + int ret = 0; + + if (bypass_cache || + !smu_table->metrics_time || + time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(1))) { + ret = smu_cmn_update_table(smu, + SMU_TABLE_SMU_METRICS, + 0, + smu_table->metrics_table, + false); + if (ret) { + dev_info(smu->adev->dev, "Failed to export SMU metrics table!\n"); + return ret; + } + smu_table->metrics_time = jiffies; + } + + if (metrics_table) + memcpy(metrics_table, smu_table->metrics_table, table_size); + + return 0; +} + +int smu_cmn_get_metrics_table(struct smu_context *smu, + void *metrics_table, + bool bypass_cache) +{ + int ret = 0; + + mutex_lock(&smu->metrics_lock); + ret = smu_cmn_get_metrics_table_locked(smu, + metrics_table, + bypass_cache); + mutex_unlock(&smu->metrics_lock); + + return ret; +} diff --git a/drivers/gpu/drm/amd/powerplay/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index 98face8c5fd6..6d00ad740c27 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -79,5 +79,13 @@ int smu_cmn_write_watermarks_table(struct smu_context *smu); int smu_cmn_write_pptable(struct smu_context *smu); +int smu_cmn_get_metrics_table_locked(struct smu_context *smu, + void *metrics_table, + bool bypass_cache); + +int smu_cmn_get_metrics_table(struct smu_context *smu, + void *metrics_table, + bool bypass_cache); + #endif #endif diff --git a/drivers/gpu/drm/amd/powerplay/smu_internal.h b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h index 264073d4e263..38c10177ed21 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_internal.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h @@ -92,6 +92,9 @@ #define smu_get_asic_power_limits(smu) smu_ppt_funcs(get_power_limit, 0, smu) #define smu_get_pp_feature_mask(smu, buf) smu_ppt_funcs(get_pp_feature_mask, 0, smu, buf) #define smu_set_pp_feature_mask(smu, new_mask) smu_ppt_funcs(set_pp_feature_mask, 0, smu, new_mask) +#define smu_gfx_ulv_control(smu, enablement) smu_ppt_funcs(gfx_ulv_control, 0, smu, enablement) +#define smu_deep_sleep_control(smu, enablement) smu_ppt_funcs(deep_sleep_control, 0, smu, enablement) +#define smu_get_fan_parameters(smu) smu_ppt_funcs(get_fan_parameters, 0, smu) #endif #endif diff --git a/drivers/gpu/drm/arm/malidp_hw.c b/drivers/gpu/drm/arm/malidp_hw.c index ca570b135478..e9de542f9b7c 100644 --- a/drivers/gpu/drm/arm/malidp_hw.c +++ b/drivers/gpu/drm/arm/malidp_hw.c @@ -532,7 +532,7 @@ static int malidp500_enable_memwrite(struct malidp_hw_device *hwdev, malidp_hw_write(hwdev, lower_32_bits(addrs[1]), base + MALIDP_MW_P2_PTR_LOW); malidp_hw_write(hwdev, upper_32_bits(addrs[1]), base + MALIDP_MW_P2_PTR_HIGH); malidp_hw_write(hwdev, pitches[1], base + MALIDP_MW_P2_STRIDE); - /* fall through */ + fallthrough; case 1: malidp_hw_write(hwdev, lower_32_bits(addrs[0]), base + MALIDP_MW_P1_PTR_LOW); malidp_hw_write(hwdev, upper_32_bits(addrs[0]), base + MALIDP_MW_P1_PTR_HIGH); @@ -869,7 +869,7 @@ static int malidp550_enable_memwrite(struct malidp_hw_device *hwdev, malidp_hw_write(hwdev, lower_32_bits(addrs[1]), base + MALIDP_MW_P2_PTR_LOW); malidp_hw_write(hwdev, upper_32_bits(addrs[1]), base + MALIDP_MW_P2_PTR_HIGH); malidp_hw_write(hwdev, pitches[1], base + MALIDP_MW_P2_STRIDE); - /* fall through */ + fallthrough; case 1: malidp_hw_write(hwdev, lower_32_bits(addrs[0]), base + MALIDP_MW_P1_PTR_LOW); malidp_hw_write(hwdev, upper_32_bits(addrs[0]), base + MALIDP_MW_P1_PTR_HIGH); @@ -1324,7 +1324,7 @@ static irqreturn_t malidp_se_irq(int irq, void *arg) break; case MW_RESTART: drm_writeback_signal_completion(&malidp->mw_connector, 0); - /* fall through - to a new start */ + fallthrough; /* to a new start */ case MW_START: /* writeback started, need to emulate one-shot mode */ hw->disable_memwrite(hwdev); diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c index d62749a10cdd..77066bca8793 100644 --- a/drivers/gpu/drm/ast/ast_main.c +++ b/drivers/gpu/drm/ast/ast_main.c @@ -240,7 +240,7 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post) ast->dp501_fw_addr = NULL; } } - /* fallthrough */ + fallthrough; case 0x0c: ast->tx_chip_type = AST_TX_DP501; } diff --git a/drivers/gpu/drm/bridge/nwl-dsi.c b/drivers/gpu/drm/bridge/nwl-dsi.c index ce94f797d090..66b67402f1ac 100644 --- a/drivers/gpu/drm/bridge/nwl-dsi.c +++ b/drivers/gpu/drm/bridge/nwl-dsi.c @@ -409,7 +409,6 @@ static bool nwl_dsi_read_packet(struct nwl_dsi *dsi, u32 status) switch (data_type) { case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_2BYTE: - fallthrough; case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE: if (xfer->msg->rx_len > 1) { /* read second byte */ @@ -418,7 +417,6 @@ static bool nwl_dsi_read_packet(struct nwl_dsi *dsi, u32 status) } fallthrough; case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE: - fallthrough; case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE: if (xfer->msg->rx_len > 0) { /* read first byte */ diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c index d7e65c869415..9fef6413741d 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c @@ -61,10 +61,10 @@ static int dw_hdmi_i2s_hw_params(struct device *dev, void *data, switch (hparms->channels) { case 7 ... 8: conf0 |= HDMI_AUD_CONF0_I2S_EN3; - /* Fall-thru */ + fallthrough; case 5 ... 6: conf0 |= HDMI_AUD_CONF0_I2S_EN2; - /* Fall-thru */ + fallthrough; case 3 ... 4: conf0 |= HDMI_AUD_CONF0_I2S_EN1; /* Fall-thru */ diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 454544e0da7d..ecdf9b01340f 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -601,13 +601,13 @@ static void ti_sn_bridge_read_valid_rates(struct ti_sn_bridge *pdata, DRM_DEV_ERROR(pdata->dev, "Unexpected max rate (%#x); assuming 5.4 GHz\n", (int)dpcd_val); - /* fall through */ + fallthrough; case DP_LINK_BW_5_4: rate_valid[7] = 1; - /* fall through */ + fallthrough; case DP_LINK_BW_2_7: rate_valid[4] = 1; - /* fall through */ + fallthrough; case DP_LINK_BW_1_62: rate_valid[1] = 1; break; diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index f68c69a45752..9e1ad493e689 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -34,6 +34,7 @@ #include <drm/drm_bridge.h> #include <drm/drm_damage_helper.h> #include <drm/drm_device.h> +#include <drm/drm_drv.h> #include <drm/drm_plane_helper.h> #include <drm/drm_print.h> #include <drm/drm_self_refresh_helper.h> @@ -3106,7 +3107,7 @@ void drm_atomic_helper_shutdown(struct drm_device *dev) if (ret) DRM_ERROR("Disabling all crtc's during unload failed with %i\n", ret); - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); } EXPORT_SYMBOL(drm_atomic_helper_shutdown); @@ -3246,7 +3247,7 @@ struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev) } unlock: - DRM_MODESET_LOCK_ALL_END(ctx, err); + DRM_MODESET_LOCK_ALL_END(dev, ctx, err); if (err) return ERR_PTR(err); @@ -3327,7 +3328,7 @@ int drm_atomic_helper_resume(struct drm_device *dev, err = drm_atomic_helper_commit_duplicated_state(state, &ctx); - DRM_MODESET_LOCK_ALL_END(ctx, err); + DRM_MODESET_LOCK_ALL_END(dev, ctx, err); drm_atomic_state_put(state); return err; diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c index a0735fbc144b..7a01d0918861 100644 --- a/drivers/gpu/drm/drm_bufs.c +++ b/drivers/gpu/drm/drm_bufs.c @@ -537,7 +537,7 @@ int drm_legacy_rmmap_locked(struct drm_device *dev, struct drm_local_map *map) switch (map->type) { case _DRM_REGISTERS: iounmap(map->handle); - /* FALLTHROUGH */ + fallthrough; case _DRM_FRAME_BUFFER: arch_phys_wc_del(map->mtrr); break; diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index c93123ff7c21..138ff34b31db 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -294,7 +294,7 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev, crtc->gamma_size, &ctx); out: - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); return ret; } diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c index 283bcc4362ca..aecdd7ea26dc 100644 --- a/drivers/gpu/drm/drm_crtc.c +++ b/drivers/gpu/drm/drm_crtc.c @@ -588,7 +588,6 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data, if (crtc_req->mode_valid && !drm_lease_held(file_priv, plane->base.id)) return -EACCES; - mutex_lock(&crtc->dev->mode_config.mutex); DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE, ret); @@ -756,8 +755,7 @@ out: fb = NULL; mode = NULL; - DRM_MODESET_LOCK_ALL_END(ctx, ret); - mutex_unlock(&crtc->dev->mode_config.mutex); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); return ret; } diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c index 4c21cf69dad5..1e7c638873c8 100644 --- a/drivers/gpu/drm/drm_dp_helper.c +++ b/drivers/gpu/drm/drm_dp_helper.c @@ -423,6 +423,133 @@ bool drm_dp_send_real_edid_checksum(struct drm_dp_aux *aux, } EXPORT_SYMBOL(drm_dp_send_real_edid_checksum); +static u8 drm_dp_downstream_port_count(const u8 dpcd[DP_RECEIVER_CAP_SIZE]) +{ + u8 port_count = dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_PORT_COUNT_MASK; + + if (dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DETAILED_CAP_INFO_AVAILABLE && port_count > 4) + port_count = 4; + + return port_count; +} + +static int drm_dp_read_extended_dpcd_caps(struct drm_dp_aux *aux, + u8 dpcd[DP_RECEIVER_CAP_SIZE]) +{ + u8 dpcd_ext[6]; + int ret; + + /* + * Prior to DP1.3 the bit represented by + * DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT was reserved. + * If it is set DP_DPCD_REV at 0000h could be at a value less than + * the true capability of the panel. The only way to check is to + * then compare 0000h and 2200h. + */ + if (!(dpcd[DP_TRAINING_AUX_RD_INTERVAL] & + DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)) + return 0; + + ret = drm_dp_dpcd_read(aux, DP_DP13_DPCD_REV, &dpcd_ext, + sizeof(dpcd_ext)); + if (ret < 0) + return ret; + if (ret != sizeof(dpcd_ext)) + return -EIO; + + if (dpcd[DP_DPCD_REV] > dpcd_ext[DP_DPCD_REV]) { + DRM_DEBUG_KMS("%s: Extended DPCD rev less than base DPCD rev (%d > %d)\n", + aux->name, dpcd[DP_DPCD_REV], + dpcd_ext[DP_DPCD_REV]); + return 0; + } + + if (!memcmp(dpcd, dpcd_ext, sizeof(dpcd_ext))) + return 0; + + DRM_DEBUG_KMS("%s: Base DPCD: %*ph\n", + aux->name, DP_RECEIVER_CAP_SIZE, dpcd); + + memcpy(dpcd, dpcd_ext, sizeof(dpcd_ext)); + + return 0; +} + +/** + * drm_dp_read_dpcd_caps() - read DPCD caps and extended DPCD caps if + * available + * @aux: DisplayPort AUX channel + * @dpcd: Buffer to store the resulting DPCD in + * + * Attempts to read the base DPCD caps for @aux. Additionally, this function + * checks for and reads the extended DPRX caps (%DP_DP13_DPCD_REV) if + * present. + * + * Returns: %0 if the DPCD was read successfully, negative error code + * otherwise. + */ +int drm_dp_read_dpcd_caps(struct drm_dp_aux *aux, + u8 dpcd[DP_RECEIVER_CAP_SIZE]) +{ + int ret; + + ret = drm_dp_dpcd_read(aux, DP_DPCD_REV, dpcd, DP_RECEIVER_CAP_SIZE); + if (ret < 0) + return ret; + if (ret != DP_RECEIVER_CAP_SIZE || dpcd[DP_DPCD_REV] == 0) + return -EIO; + + ret = drm_dp_read_extended_dpcd_caps(aux, dpcd); + if (ret < 0) + return ret; + + DRM_DEBUG_KMS("%s: DPCD: %*ph\n", + aux->name, DP_RECEIVER_CAP_SIZE, dpcd); + + return ret; +} +EXPORT_SYMBOL(drm_dp_read_dpcd_caps); + +/** + * drm_dp_read_downstream_info() - read DPCD downstream port info if available + * @aux: DisplayPort AUX channel + * @dpcd: A cached copy of the port's DPCD + * @downstream_ports: buffer to store the downstream port info in + * + * See also: + * drm_dp_downstream_max_clock() + * drm_dp_downstream_max_bpc() + * + * Returns: 0 if either the downstream port info was read successfully or + * there was no downstream info to read, or a negative error code otherwise. + */ +int drm_dp_read_downstream_info(struct drm_dp_aux *aux, + const u8 dpcd[DP_RECEIVER_CAP_SIZE], + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]) +{ + int ret; + u8 len; + + memset(downstream_ports, 0, DP_MAX_DOWNSTREAM_PORTS); + + /* No downstream info to read */ + if (!drm_dp_is_branch(dpcd) || + dpcd[DP_DPCD_REV] < DP_DPCD_REV_10 || + !(dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT)) + return 0; + + len = drm_dp_downstream_port_count(dpcd); + if (dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DETAILED_CAP_INFO_AVAILABLE) + len *= 4; + + ret = drm_dp_dpcd_read(aux, DP_DOWNSTREAM_PORT_0, downstream_ports, len); + if (ret < 0) + return ret; + + return ret == len ? 0 : -EIO; +} +EXPORT_SYMBOL(drm_dp_read_downstream_info); + /** * drm_dp_downstream_max_clock() - extract branch device max * pixel rate for legacy VGA @@ -431,7 +558,11 @@ EXPORT_SYMBOL(drm_dp_send_real_edid_checksum); * @dpcd: DisplayPort configuration data * @port_cap: port capabilities * - * Returns max clock in kHz on success or 0 if max clock not defined + * See also: + * drm_dp_read_downstream_info() + * drm_dp_downstream_max_bpc() + * + * Returns: Max clock in kHz on success or 0 if max clock not defined */ int drm_dp_downstream_max_clock(const u8 dpcd[DP_RECEIVER_CAP_SIZE], const u8 port_cap[4]) @@ -462,7 +593,11 @@ EXPORT_SYMBOL(drm_dp_downstream_max_clock); * @dpcd: DisplayPort configuration data * @port_cap: port capabilities * - * Returns max bpc on success or 0 if max bpc not defined + * See also: + * drm_dp_read_downstream_info() + * drm_dp_downstream_max_clock() + * + * Returns: Max bpc on success or 0 if max bpc not defined */ int drm_dp_downstream_max_bpc(const u8 dpcd[DP_RECEIVER_CAP_SIZE], const u8 port_cap[4]) @@ -492,7 +627,7 @@ int drm_dp_downstream_max_bpc(const u8 dpcd[DP_RECEIVER_CAP_SIZE], case DP_DS_16BPC: return 16; } - /* fall through */ + fallthrough; default: return 0; } @@ -668,6 +803,54 @@ void drm_dp_set_subconnector_property(struct drm_connector *connector, } EXPORT_SYMBOL(drm_dp_set_subconnector_property); +/** + * drm_dp_read_sink_count_cap() - Check whether a given connector has a valid sink + * count + * @connector: The DRM connector to check + * @dpcd: A cached copy of the connector's DPCD RX capabilities + * @desc: A cached copy of the connector's DP descriptor + * + * See also: drm_dp_read_sink_count() + * + * Returns: %True if the (e)DP connector has a valid sink count that should + * be probed, %false otherwise. + */ +bool drm_dp_read_sink_count_cap(struct drm_connector *connector, + const u8 dpcd[DP_RECEIVER_CAP_SIZE], + const struct drm_dp_desc *desc) +{ + /* Some eDP panels don't set a valid value for the sink count */ + return connector->connector_type != DRM_MODE_CONNECTOR_eDP && + dpcd[DP_DPCD_REV] >= DP_DPCD_REV_11 && + dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT && + !drm_dp_has_quirk(desc, 0, DP_DPCD_QUIRK_NO_SINK_COUNT); +} +EXPORT_SYMBOL(drm_dp_read_sink_count_cap); + +/** + * drm_dp_read_sink_count() - Retrieve the sink count for a given sink + * @aux: The DP AUX channel to use + * + * See also: drm_dp_read_sink_count_cap() + * + * Returns: The current sink count reported by @aux, or a negative error code + * otherwise. + */ +int drm_dp_read_sink_count(struct drm_dp_aux *aux) +{ + u8 count; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_SINK_COUNT, &count); + if (ret < 0) + return ret; + if (ret != 1) + return -EIO; + + return DP_GET_SINK_COUNT(count); +} +EXPORT_SYMBOL(drm_dp_read_sink_count); + /* * I2C-over-AUX implementation */ diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index 166405d4c536..b9c5a98c5c9c 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -3489,6 +3489,28 @@ static int drm_dp_get_vc_payload_bw(u8 dp_link_bw, u8 dp_link_count) } /** + * drm_dp_read_mst_cap() - check whether or not a sink supports MST + * @aux: The DP AUX channel to use + * @dpcd: A cached copy of the DPCD capabilities for this sink + * + * Returns: %True if the sink supports MST, %false otherwise + */ +bool drm_dp_read_mst_cap(struct drm_dp_aux *aux, + const u8 dpcd[DP_RECEIVER_CAP_SIZE]) +{ + u8 mstm_cap; + + if (dpcd[DP_DPCD_REV] < DP_DPCD_REV_12) + return false; + + if (drm_dp_dpcd_readb(aux, DP_MSTM_CAP, &mstm_cap) != 1) + return false; + + return mstm_cap & DP_MST_CAP; +} +EXPORT_SYMBOL(drm_dp_read_mst_cap); + +/** * drm_dp_mst_topology_mgr_set_mst() - Set the MST state for a topology manager * @mgr: manager to set state for * @mst_state: true to enable MST on this connector - false to disable. @@ -5042,8 +5064,8 @@ int drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state, struct drm crtc = conn_state->crtc; - if (WARN_ON(!crtc)) - return -EINVAL; + if (!crtc) + continue; if (!drm_dp_mst_dsc_aux_for_port(pos->port)) continue; diff --git a/drivers/gpu/drm/drm_mode_object.c b/drivers/gpu/drm/drm_mode_object.c index 901b078abf40..db05f386a709 100644 --- a/drivers/gpu/drm/drm_mode_object.c +++ b/drivers/gpu/drm/drm_mode_object.c @@ -428,7 +428,7 @@ int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data, out_unref: drm_mode_object_put(obj); out: - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); return ret; } @@ -470,7 +470,7 @@ static int set_property_legacy(struct drm_mode_object *obj, break; } drm_property_change_valid_put(prop, ref); - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); return ret; } diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index 14b6f7638728..501b4fe55a3d 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -1930,7 +1930,7 @@ void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out, default: WARN(1, "Invalid aspect ratio (0%x) on mode\n", in->picture_aspect_ratio); - /* fall through */ + fallthrough; case HDMI_PICTURE_ASPECT_NONE: out->flags |= DRM_MODE_FLAG_PIC_AR_NONE; break; diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index b7b90b3a2e38..affe1cfed009 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -792,7 +792,7 @@ static int setplane_internal(struct drm_plane *plane, crtc_x, crtc_y, crtc_w, crtc_h, src_x, src_y, src_w, src_h, &ctx); - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(plane->dev, ctx, ret); return ret; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index d5a4cd85a0f6..c6404b8d067f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -337,9 +337,16 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL); gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV); - gpu->identity.product_id = gpu_read(gpu, VIVS_HI_CHIP_PRODUCT_ID); gpu->identity.customer_id = gpu_read(gpu, VIVS_HI_CHIP_CUSTOMER_ID); - gpu->identity.eco_id = gpu_read(gpu, VIVS_HI_CHIP_ECO_ID); + + /* + * Reading these two registers on GC600 rev 0x19 result in a + * unhandled fault: external abort on non-linefetch + */ + if (!etnaviv_is_model_rev(gpu, GC600, 0x19)) { + gpu->identity.product_id = gpu_read(gpu, VIVS_HI_CHIP_PRODUCT_ID); + gpu->identity.eco_id = gpu_read(gpu, VIVS_HI_CHIP_ECO_ID); + } /* * !!!! HACK ALERT !!!! diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 4e3e95dce6d8..cd46c882269c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -89,12 +89,15 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) u32 dma_addr; int change; + /* block scheduler */ + drm_sched_stop(&gpu->sched, sched_job); + /* * If the GPU managed to complete this jobs fence, the timout is * spurious. Bail out. */ if (dma_fence_is_signaled(submit->out_fence)) - return; + goto out_no_timeout; /* * If the GPU is still making forward progress on the front-end (which @@ -105,12 +108,9 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) change = dma_addr - gpu->hangcheck_dma_addr; if (change < 0 || change > 16) { gpu->hangcheck_dma_addr = dma_addr; - return; + goto out_no_timeout; } - /* block scheduler */ - drm_sched_stop(&gpu->sched, sched_job); - if(sched_job) drm_sched_increase_karma(sched_job); @@ -120,6 +120,7 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) drm_sched_resubmit_jobs(&gpu->sched); +out_no_timeout: /* restart scheduler after GPU is usable again */ drm_sched_start(&gpu->sched, true); } diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c index 843dfcefc46a..1a1a2853a842 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c @@ -987,10 +987,10 @@ static void exynos_dsi_send_to_fifo(struct exynos_dsi *dsi, switch (length) { case 3: reg |= payload[2] << 16; - /* Fall through */ + fallthrough; case 2: reg |= payload[1] << 8; - /* Fall through */ + fallthrough; case 1: reg |= payload[0]; exynos_dsi_write(dsi, DSIM_PAYLOAD_REG, reg); @@ -1038,7 +1038,7 @@ static void exynos_dsi_read_from_fifo(struct exynos_dsi *dsi, payload[1] = reg >> 16; ++xfer->rx_done; } - /* Fall through */ + fallthrough; case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE: case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE: payload[0] = reg >> 8; @@ -1082,10 +1082,10 @@ static void exynos_dsi_read_from_fifo(struct exynos_dsi *dsi, switch (length) { case 3: payload[2] = (reg >> 16) & 0xff; - /* Fall through */ + fallthrough; case 2: payload[1] = (reg >> 8) & 0xff; - /* Fall through */ + fallthrough; case 1: payload[0] = reg & 0xff; } diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c index 56a2b47e1af7..5147f5929be7 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c @@ -92,7 +92,7 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper, offset = fbi->var.xoffset * fb->format->cpp[0]; offset += fbi->var.yoffset * fb->pitches[0]; - fbi->screen_base = exynos_gem->kvaddr + offset; + fbi->screen_buffer = exynos_gem->kvaddr + offset; fbi->screen_size = size; fbi->fix.smem_len = size; diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h index 7445748288da..74e926abeff0 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gem.h +++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h @@ -40,7 +40,7 @@ struct exynos_drm_gem { unsigned int flags; unsigned long size; void *cookie; - void __iomem *kvaddr; + void *kvaddr; dma_addr_t dma_addr; unsigned long dma_attrs; struct sg_table *sgt; diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c index 86fac677fe69..3c6d9f3913d5 100644 --- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c +++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c @@ -101,19 +101,19 @@ static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane, break; case DRM_FORMAT_ARGB8888: alpha = DCU_LAYER_AB_WHOLE_FRAME; - /* fall-through */ + fallthrough; case DRM_FORMAT_XRGB8888: bpp = FSL_DCU_ARGB8888; break; case DRM_FORMAT_ARGB4444: alpha = DCU_LAYER_AB_WHOLE_FRAME; - /* fall-through */ + fallthrough; case DRM_FORMAT_XRGB4444: bpp = FSL_DCU_ARGB4444; break; case DRM_FORMAT_ARGB1555: alpha = DCU_LAYER_AB_WHOLE_FRAME; - /* fall-through */ + fallthrough; case DRM_FORMAT_XRGB1555: bpp = FSL_DCU_ARGB1555; break; diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 8c55f5bee9ab..f4053dd6bde9 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -712,7 +712,7 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder, switch (intel_dsi->pixel_format) { default: MISSING_CASE(intel_dsi->pixel_format); - /* fallthrough */ + fallthrough; case MIPI_DSI_FMT_RGB565: tmp |= PIX_FMT_RGB565; break; @@ -739,7 +739,7 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder, switch (intel_dsi->video_mode_format) { default: MISSING_CASE(intel_dsi->video_mode_format); - /* fallthrough */ + fallthrough; case VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS: tmp |= VIDEO_MODE_SYNC_EVENT; break; @@ -792,7 +792,7 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder, switch (pipe) { default: MISSING_CASE(pipe); - /* fallthrough */ + fallthrough; case PIPE_A: tmp |= TRANS_DDI_EDP_INPUT_A_ON; break; diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index c53c85d38fa5..a0a41ec5c341 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -905,7 +905,7 @@ parse_psr(struct drm_i915_private *dev_priv, const struct bdb_header *bdb) drm_dbg_kms(&dev_priv->drm, "VBT tp1 wakeup time value %d is outside range[0-3], defaulting to max value 2500us\n", psr_table->tp1_wakeup_time); - /* fallthrough */ + fallthrough; case 2: dev_priv->vbt.psr.tp1_wakeup_time_us = 2500; break; @@ -925,7 +925,7 @@ parse_psr(struct drm_i915_private *dev_priv, const struct bdb_header *bdb) drm_dbg_kms(&dev_priv->drm, "VBT tp2_tp3 wakeup time value %d is outside range[0-3], defaulting to max value 2500us\n", psr_table->tp2_tp3_wakeup_time); - /* fallthrough */ + fallthrough; case 2: dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = 2500; break; @@ -1775,7 +1775,7 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, switch (child->hdmi_max_data_rate) { default: MISSING_CASE(child->hdmi_max_data_rate); - /* fall through */ + fallthrough; case HDMI_MAX_DATA_RATE_PLATFORM: max_tmds_clock = 0; break; diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index bb91dace304a..577c4441f32d 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -326,7 +326,7 @@ static void pnv_get_cdclk(struct drm_i915_private *dev_priv, default: drm_err(&dev_priv->drm, "Unknown pnv display core clock 0x%04x\n", gcfgc); - /* fall through */ + fallthrough; case GC_DISPLAY_CLOCK_133_MHZ_PNV: cdclk_config->cdclk = 133333; break; @@ -766,7 +766,7 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv, switch (cdclk) { default: MISSING_CASE(cdclk); - /* fall through */ + fallthrough; case 337500: val |= LCPLL_CLK_FREQ_337_5_BDW; break; @@ -1042,7 +1042,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, drm_WARN_ON(&dev_priv->drm, cdclk != dev_priv->cdclk.hw.bypass); drm_WARN_ON(&dev_priv->drm, vco != 0); - /* fall through */ + fallthrough; case 308571: case 337500: freq_select = CDCLK_FREQ_337_308; @@ -1333,7 +1333,7 @@ static void icl_readout_refclk(struct drm_i915_private *dev_priv, switch (dssm) { default: MISSING_CASE(dssm); - /* fall through */ + fallthrough; case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz: cdclk_config->ref = 24000; break; @@ -1561,7 +1561,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, drm_WARN_ON(&dev_priv->drm, cdclk != dev_priv->cdclk.hw.bypass); drm_WARN_ON(&dev_priv->drm, vco != 0); - /* fall through */ + fallthrough; case 2: divider = BXT_CDCLK_CD2X_DIV_SEL_1; break; @@ -2677,7 +2677,7 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv) */ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) intel_de_write(dev_priv, GMBUSFREQ_VLV, - DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000)); + DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000)); } static int cnp_rawclk(struct drm_i915_private *dev_priv) @@ -2903,9 +2903,10 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) dev_priv->display.get_cdclk = i85x_get_cdclk; else if (IS_I845G(dev_priv)) dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk; - else { /* 830 */ - drm_WARN(&dev_priv->drm, !IS_I830(dev_priv), - "Unknown platform. Assuming 133 MHz CDCLK\n"); + else if (IS_I830(dev_priv)) + dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; + + if (drm_WARN(&dev_priv->drm, !dev_priv->display.get_cdclk, + "Unknown platform. Assuming 133 MHz CDCLK\n")) dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; - } } diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h index 5731806e4cee..6b31fde4be16 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.h +++ b/drivers/gpu/drm/i915/display/intel_cdclk.h @@ -17,8 +17,8 @@ struct intel_atomic_state; struct intel_crtc_state; struct intel_cdclk_vals { - u16 refclk; u32 cdclk; + u16 refclk; u8 divider; /* CD2X divider * 2 */ u8 ratio; }; diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c index eccaa79cb4a9..157d8c8c605a 100644 --- a/drivers/gpu/drm/i915/display/intel_combo_phy.c +++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c @@ -52,7 +52,7 @@ cnl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) { default: MISSING_CASE(val); - /* fall through */ + fallthrough; case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0: procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0]; break; @@ -258,7 +258,7 @@ static bool phy_is_master(struct drm_i915_private *dev_priv, enum phy phy) static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, enum phy phy) { - bool ret; + bool ret = true; u32 expected_val = 0; if (!icl_combo_phy_enabled(dev_priv, phy)) @@ -276,7 +276,7 @@ static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, DCC_MODE_SELECT_CONTINUOSLY); } - ret = cnl_verify_procmon_ref_values(dev_priv, phy); + ret &= cnl_verify_procmon_ref_values(dev_priv, phy); if (phy_is_master(dev_priv, phy)) { ret &= check_phy_reg(dev_priv, phy, ICL_PORT_COMP_DW8(phy), @@ -320,7 +320,7 @@ void intel_combo_phy_power_up_lanes(struct drm_i915_private *dev_priv, break; default: MISSING_CASE(lane_count); - /* fall-through */ + fallthrough; case 4: lane_mask = PWR_UP_ALL_LANES; break; @@ -337,7 +337,7 @@ void intel_combo_phy_power_up_lanes(struct drm_i915_private *dev_priv, break; default: MISSING_CASE(lane_count); - /* fall-through */ + fallthrough; case 4: lane_mask = PWR_UP_ALL_LANES; break; diff --git a/drivers/gpu/drm/i915/display/intel_csr.c b/drivers/gpu/drm/i915/display/intel_csr.c index f22a7645c249..d5db16764619 100644 --- a/drivers/gpu/drm/i915/display/intel_csr.c +++ b/drivers/gpu/drm/i915/display/intel_csr.c @@ -40,12 +40,12 @@ #define GEN12_CSR_MAX_FW_SIZE ICL_CSR_MAX_FW_SIZE -#define RKL_CSR_PATH "i915/rkl_dmc_ver2_01.bin" -#define RKL_CSR_VERSION_REQUIRED CSR_VERSION(2, 1) +#define RKL_CSR_PATH "i915/rkl_dmc_ver2_02.bin" +#define RKL_CSR_VERSION_REQUIRED CSR_VERSION(2, 2) MODULE_FIRMWARE(RKL_CSR_PATH); -#define TGL_CSR_PATH "i915/tgl_dmc_ver2_06.bin" -#define TGL_CSR_VERSION_REQUIRED CSR_VERSION(2, 6) +#define TGL_CSR_PATH "i915/tgl_dmc_ver2_08.bin" +#define TGL_CSR_VERSION_REQUIRED CSR_VERSION(2, 8) #define TGL_CSR_MAX_FW_SIZE 0x6000 MODULE_FIRMWARE(TGL_CSR_PATH); diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index 2c484b55bcdf..19ac6b2a664c 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -706,6 +706,42 @@ static const struct cnl_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr2[] = { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ }; +static const struct cnl_ddi_buf_trans tgl_uy_combo_phy_ddi_translations_dp_hbr2[] = { + /* NT mV Trans mV db */ + { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ + { 0xA, 0x4F, 0x36, 0x00, 0x09 }, /* 350 500 3.1 */ + { 0xC, 0x60, 0x32, 0x00, 0x0D }, /* 350 700 6.0 */ + { 0xC, 0x7F, 0x2D, 0x00, 0x12 }, /* 350 900 8.2 */ + { 0xC, 0x47, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ + { 0xC, 0x6F, 0x36, 0x00, 0x09 }, /* 500 700 2.9 */ + { 0x6, 0x7D, 0x32, 0x00, 0x0D }, /* 500 900 5.1 */ + { 0x6, 0x60, 0x3C, 0x00, 0x03 }, /* 650 700 0.6 */ + { 0x6, 0x7F, 0x34, 0x00, 0x0B }, /* 600 900 3.5 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ +}; + +/* + * Cloned the HOBL entry to comply with the voltage and pre-emphasis entries + * that DisplayPort specification requires + */ +static const struct cnl_ddi_buf_trans tgl_combo_phy_ddi_translations_edp_hbr2_hobl[] = { + /* VS pre-emp */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 0 0 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 0 1 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 0 2 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 0 3 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1 0 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1 1 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1 2 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 2 0 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 2 1 */ +}; + +static bool is_hobl_buf_trans(const struct cnl_ddi_buf_trans *table) +{ + return table == tgl_combo_phy_ddi_translations_edp_hbr2_hobl; +} + static const struct ddi_buf_trans * bdw_get_buf_trans_edp(struct intel_encoder *encoder, int *n_entries) { @@ -1050,9 +1086,26 @@ static const struct cnl_ddi_buf_trans * tgl_get_combo_buf_trans(struct intel_encoder *encoder, int type, int rate, int *n_entries) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + + if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.hobl) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + if (!intel_dp->hobl_failed && rate <= 540000) { + /* Same table applies to TGL, RKL and DG1 */ + *n_entries = ARRAY_SIZE(tgl_combo_phy_ddi_translations_edp_hbr2_hobl); + return tgl_combo_phy_ddi_translations_edp_hbr2_hobl; + } + } + if (type == INTEL_OUTPUT_HDMI || type == INTEL_OUTPUT_EDP) { return icl_get_combo_buf_trans(encoder, type, rate, n_entries); } else if (rate > 270000) { + if (IS_TGL_U(dev_priv) || IS_TGL_Y(dev_priv)) { + *n_entries = ARRAY_SIZE(tgl_uy_combo_phy_ddi_translations_dp_hbr2); + return tgl_uy_combo_phy_ddi_translations_dp_hbr2; + } + *n_entries = ARRAY_SIZE(tgl_combo_phy_ddi_translations_dp_hbr2); return tgl_combo_phy_ddi_translations_dp_hbr2; } @@ -1888,7 +1941,7 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { default: MISSING_CASE(tmp & TRANS_DDI_EDP_INPUT_MASK); - /* fallthrough */ + fallthrough; case TRANS_DDI_EDP_INPUT_A_ON: case TRANS_DDI_EDP_INPUT_A_ONOFF: *pipe_mask = BIT(PIPE_A); @@ -2392,6 +2445,15 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, level = n_entries - 1; } + if (type == INTEL_OUTPUT_EDP) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + val = EDP4K2K_MODE_OVRD_EN | EDP4K2K_MODE_OVRD_OPTIMIZED; + intel_dp->hobl_active = is_hobl_buf_trans(ddi_translations); + intel_de_rmw(dev_priv, ICL_PORT_CL_DW10(phy), val, + intel_dp->hobl_active ? val : 0); + } + /* Set PORT_TX_DW5 */ val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); val &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK | @@ -2802,7 +2864,9 @@ hsw_set_signal_levels(struct intel_dp *intel_dp) static u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, enum phy phy) { - if (intel_phy_is_combo(dev_priv, phy)) { + if (IS_ROCKETLAKE(dev_priv)) { + return RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); + } else if (intel_phy_is_combo(dev_priv, phy)) { return ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); } else if (intel_phy_is_tc(dev_priv, phy)) { enum tc_port tc_port = intel_port_to_tc(dev_priv, @@ -2829,6 +2893,16 @@ static void icl_map_plls_to_ports(struct intel_encoder *encoder, (val & icl_dpclka_cfgcr0_clk_off(dev_priv, phy)) == 0); if (intel_phy_is_combo(dev_priv, phy)) { + u32 mask, sel; + + if (IS_ROCKETLAKE(dev_priv)) { + mask = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + sel = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy); + } else { + mask = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + sel = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy); + } + /* * Even though this register references DDIs, note that we * want to pass the PHY rather than the port (DDI). For @@ -2839,8 +2913,8 @@ static void icl_map_plls_to_ports(struct intel_encoder *encoder, * Clock Select chooses the PLL for both DDIA and DDID and * drives port A in all cases." */ - val &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); - val |= ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy); + val &= ~mask; + val |= sel; intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val); intel_de_posting_read(dev_priv, ICL_DPCLKA_CFGCR0); } @@ -4037,8 +4111,7 @@ static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) intel_wait_ddi_buf_idle(dev_priv, port); } - dp_tp_ctl = DP_TP_CTL_ENABLE | - DP_TP_CTL_LINK_TRAIN_PAT1 | DP_TP_CTL_SCRAMBLE_DISABLE; + dp_tp_ctl = DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_PAT1; if (intel_dp->link_mst) dp_tp_ctl |= DP_TP_CTL_MODE_MST; else { @@ -4061,16 +4134,10 @@ static void intel_ddi_set_link_train(struct intel_dp *intel_dp, { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); - enum port port = dp_to_dig_port(intel_dp)->base.port; u32 temp; temp = intel_de_read(dev_priv, intel_dp->regs.dp_tp_ctl); - if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) - temp |= DP_TP_CTL_SCRAMBLE_DISABLE; - else - temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; - temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; switch (dp_train_pat & train_pat_mask) { case DP_TRAINING_PATTERN_DISABLE: @@ -4091,9 +4158,6 @@ static void intel_ddi_set_link_train(struct intel_dp *intel_dp, } intel_de_write(dev_priv, intel_dp->regs.dp_tp_ctl, temp); - - intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP); - intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); } static void intel_ddi_set_idle_link_train(struct intel_dp *intel_dp) @@ -4268,7 +4332,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder, pipe_config->hdmi_scrambling = true; if (temp & TRANS_DDI_HIGH_TMDS_CHAR_RATE) pipe_config->hdmi_high_tmds_clock_ratio = true; - /* fall through */ + fallthrough; case TRANS_DDI_MODE_SELECT_DVI: pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI); pipe_config->lane_count = 4; @@ -4878,6 +4942,13 @@ intel_ddi_max_lanes(struct intel_digital_port *dig_port) return max_lanes; } +static bool hti_uses_phy(struct drm_i915_private *i915, enum phy phy) +{ + return i915->hti_state & HDPORT_ENABLED && + (i915->hti_state & HDPORT_PHY_USED_DP(phy) || + i915->hti_state & HDPORT_PHY_USED_HDMI(phy)); +} + void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) { struct intel_digital_port *dig_port; @@ -4885,6 +4956,18 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) bool init_hdmi, init_dp, init_lspcon = false; enum phy phy = intel_port_to_phy(dev_priv, port); + /* + * On platforms with HTI (aka HDPORT), if it's enabled at boot it may + * have taken over some of the PHYs and made them unavailable to the + * driver. In that case we should skip initializing the corresponding + * outputs. + */ + if (hti_uses_phy(dev_priv, phy)) { + drm_dbg_kms(&dev_priv->drm, "PORT %c / PHY %c reserved by HTI\n", + port_name(port), phy_name(phy)); + return; + } + init_hdmi = intel_bios_port_supports_dvi(dev_priv, port) || intel_bios_port_supports_hdmi(dev_priv, port); init_dp = intel_bios_port_supports_dp(dev_priv, port); diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 729ec6e0d43a..ddb588f863f8 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -47,6 +47,7 @@ #include "display/intel_ddi.h" #include "display/intel_dp.h" #include "display/intel_dp_mst.h" +#include "display/intel_dpll_mgr.h" #include "display/intel_dsi.h" #include "display/intel_dvo.h" #include "display/intel_gmbus.h" @@ -2029,12 +2030,12 @@ intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) case I915_FORMAT_MOD_Y_TILED_CCS: if (is_ccs_plane(fb, color_plane)) return 128; - /* fall through */ + fallthrough; case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: if (is_ccs_plane(fb, color_plane)) return 64; - /* fall through */ + fallthrough; case I915_FORMAT_MOD_Y_TILED: if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv)) return 128; @@ -2043,7 +2044,7 @@ intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) case I915_FORMAT_MOD_Yf_TILED_CCS: if (is_ccs_plane(fb, color_plane)) return 128; - /* fall through */ + fallthrough; case I915_FORMAT_MOD_Yf_TILED: switch (cpp) { case 1: @@ -2185,7 +2186,7 @@ static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: if (is_semiplanar_uv_plane(fb, color_plane)) return intel_tile_row_size(fb, color_plane); - /* Fall-through */ + fallthrough; case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: return 16 * 1024; case I915_FORMAT_MOD_Y_TILED_CCS: @@ -2194,7 +2195,7 @@ static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, if (INTEL_GEN(dev_priv) >= 12 && is_semiplanar_uv_plane(fb, color_plane)) return intel_tile_row_size(fb, color_plane); - /* Fall-through */ + fallthrough; case I915_FORMAT_MOD_Yf_TILED: return 1 * 1024 * 1024; default: @@ -2310,7 +2311,7 @@ err: void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) { - i915_gem_object_lock(vma->obj); + i915_gem_object_lock(vma->obj, NULL); if (flags & PLANE_HAS_FENCE) i915_vma_unpin_fence(vma); i915_gem_object_unpin_from_display_plane(vma); @@ -3450,7 +3451,7 @@ initial_plane_vma(struct drm_i915_private *i915, if (IS_ERR(vma)) goto err_obj; - if (i915_ggtt_pin(vma, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base)) + if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base)) goto err_obj; if (i915_gem_object_is_tiled(obj) && @@ -3761,6 +3762,44 @@ static int glk_max_plane_width(const struct drm_framebuffer *fb, } } +static int icl_min_plane_width(const struct drm_framebuffer *fb) +{ + /* Wa_14011264657, Wa_14011050563: gen11+ */ + switch (fb->format->format) { + case DRM_FORMAT_C8: + return 18; + case DRM_FORMAT_RGB565: + return 10; + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ARGB8888: + case DRM_FORMAT_ABGR8888: + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_XBGR2101010: + case DRM_FORMAT_ARGB2101010: + case DRM_FORMAT_ABGR2101010: + case DRM_FORMAT_XVYU2101010: + case DRM_FORMAT_Y212: + case DRM_FORMAT_Y216: + return 6; + case DRM_FORMAT_NV12: + return 20; + case DRM_FORMAT_P010: + case DRM_FORMAT_P012: + case DRM_FORMAT_P016: + return 12; + case DRM_FORMAT_XRGB16161616F: + case DRM_FORMAT_XBGR16161616F: + case DRM_FORMAT_ARGB16161616F: + case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XVYU12_16161616: + case DRM_FORMAT_XVYU16161616: + return 4; + default: + return 1; + } +} + static int icl_max_plane_width(const struct drm_framebuffer *fb, int color_plane, unsigned int rotation) @@ -3843,29 +3882,31 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) int y = plane_state->uapi.src.y1 >> 16; int w = drm_rect_width(&plane_state->uapi.src) >> 16; int h = drm_rect_height(&plane_state->uapi.src) >> 16; - int max_width; - int max_height; - u32 alignment; - u32 offset; + int max_width, min_width, max_height; + u32 alignment, offset; int aux_plane = intel_main_to_aux_plane(fb, 0); u32 aux_offset = plane_state->color_plane[aux_plane].offset; - if (INTEL_GEN(dev_priv) >= 11) + if (INTEL_GEN(dev_priv) >= 11) { max_width = icl_max_plane_width(fb, 0, rotation); - else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + min_width = icl_min_plane_width(fb); + } else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { max_width = glk_max_plane_width(fb, 0, rotation); - else + min_width = 1; + } else { max_width = skl_max_plane_width(fb, 0, rotation); + min_width = 1; + } if (INTEL_GEN(dev_priv) >= 11) max_height = icl_max_plane_height(); else max_height = skl_max_plane_height(); - if (w > max_width || h > max_height) { + if (w > max_width || w < min_width || h > max_height) { drm_dbg_kms(&dev_priv->drm, - "requested Y/RGB source size %dx%d too big (limit %dx%d)\n", - w, h, max_width, max_height); + "requested Y/RGB source size %dx%d outside limits (min: %dx1 max: %dx%d)\n", + w, h, min_width, max_width, max_height); return -EINVAL; } @@ -6211,7 +6252,7 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, case DRM_FORMAT_ARGB16161616F: if (INTEL_GEN(dev_priv) >= 11) break; - /* fall through */ + fallthrough; default: drm_dbg_kms(&dev_priv->drm, "[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n", @@ -10802,9 +10843,18 @@ static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, u32 temp; if (intel_phy_is_combo(dev_priv, phy)) { - temp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0) & - ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); - id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy); + u32 mask, shift; + + if (IS_ROCKETLAKE(dev_priv)) { + mask = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + shift = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy); + } else { + mask = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + shift = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy); + } + + temp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0) & mask; + id = temp >> shift; port_dpll_id = ICL_PORT_DPLL_DEFAULT; } else if (intel_phy_is_tc(dev_priv, phy)) { u32 clk_sel = intel_de_read(dev_priv, DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK; @@ -10896,7 +10946,7 @@ static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, break; default: MISSING_CASE(ddi_pll_sel); - /* fall through */ + fallthrough; case PORT_CLK_SEL_NONE: return; } @@ -10956,10 +11006,10 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, drm_WARN(dev, 1, "unknown pipe linked to transcoder %s\n", transcoder_name(panel_transcoder)); - /* fall through */ + fallthrough; case TRANS_DDI_EDP_INPUT_A_ONOFF: force_thru = true; - /* fall through */ + fallthrough; case TRANS_DDI_EDP_INPUT_A_ON: trans_pipe = PIPE_A; break; @@ -12760,6 +12810,9 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, } + if (!mode_changed) + intel_psr2_sel_fetch_update(state, crtc); + return 0; } @@ -13183,7 +13236,7 @@ static bool check_digital_port_conflicts(struct intel_atomic_state *state) case INTEL_OUTPUT_DDI: if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev)))) break; - /* else, fall through */ + fallthrough; case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_EDP: @@ -14930,7 +14983,7 @@ static int intel_atomic_check(struct drm_device *dev, if (any_ms && !check_digital_port_conflicts(state)) { drm_dbg_kms(&dev_priv->drm, "rejecting conflicting digital port configuration\n"); - ret = EINVAL; + ret = -EINVAL; goto fail; } @@ -14956,12 +15009,6 @@ static int intel_atomic_check(struct drm_device *dev, if (dev_priv->wm.distrust_bios_wm) any_ms = true; - if (any_ms) { - ret = intel_modeset_checks(state); - if (ret) - goto fail; - } - intel_fbc_choose_crtc(dev_priv, state); ret = calc_watermark_data(state); if (ret) @@ -14976,6 +15023,10 @@ static int intel_atomic_check(struct drm_device *dev, goto fail; if (any_ms) { + ret = intel_modeset_checks(state); + if (ret) + goto fail; + ret = intel_modeset_calc_cdclk(state); if (ret) return ret; @@ -15136,6 +15187,8 @@ static void commit_pipe_config(struct intel_atomic_state *state, if (new_crtc_state->update_pipe) intel_pipe_fastset(old_crtc_state, new_crtc_state); + + intel_psr2_program_trans_man_trk_ctl(new_crtc_state); } if (dev_priv->display.atomic_update_watermarks) @@ -17139,7 +17192,7 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, if (!intel_fb->frontbuffer) return -ENOMEM; - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); tiling = i915_gem_object_get_tiling(obj); stride = i915_gem_object_get_stride(obj); i915_gem_object_unlock(obj); @@ -17894,6 +17947,13 @@ int intel_modeset_init(struct drm_i915_private *i915) if (i915->max_cdclk_freq == 0) intel_update_max_cdclk(i915); + /* + * If the platform has HTI, we need to find out whether it has reserved + * any display resources before we create our display outputs. + */ + if (INTEL_INFO(i915)->display.has_hti) + i915->hti_state = intel_de_read(i915, HDPORT_STATE); + /* Just disable it once at startup */ intel_vga_disable(i915); intel_setup_outputs(i915); diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index 3644752cc5ec..f549381048b3 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -417,6 +417,9 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame); seq_printf(m, "%d\t%d\n", frame, su_blocks); } + + seq_printf(m, "PSR2 selective fetch: %s\n", + enableddisabled(psr->psr2_sel_fetch_enabled)); } unlock: @@ -2044,9 +2047,12 @@ DEFINE_SHOW_ATTRIBUTE(i915_hdcp_sink_capability); static int i915_lpsp_capability_show(struct seq_file *m, void *data) { struct drm_connector *connector = m->private; - struct intel_encoder *encoder = - intel_attached_encoder(to_intel_connector(connector)); struct drm_i915_private *i915 = to_i915(connector->dev); + struct intel_encoder *encoder; + + encoder = intel_attached_encoder(to_intel_connector(connector)); + if (!encoder) + return -ENODEV; if (connector->status != connector_status_connected) return -ENODEV; diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 0c713e83274d..7946c6af4b1e 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -3927,12 +3927,13 @@ tgl_tc_cold_request(struct drm_i915_private *i915, bool block) int ret; while (1) { - u32 low_val = 0, high_val; + u32 low_val; + u32 high_val = 0; if (block) - high_val = TGL_PCODE_EXIT_TCCOLD_DATA_H_BLOCK_REQ; + low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ; else - high_val = TGL_PCODE_EXIT_TCCOLD_DATA_H_UNBLOCK_REQ; + low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ; /* * Spec states that we should timeout the request after 200us @@ -3951,8 +3952,7 @@ tgl_tc_cold_request(struct drm_i915_private *i915, bool block) if (++tries == 3) break; - if (ret == -EAGAIN) - msleep(1); + msleep(1); } if (ret) @@ -4147,6 +4147,12 @@ static const struct i915_power_well_desc tgl_power_wells[] = { }, }, { + .name = "TC cold off", + .domains = TGL_TC_COLD_OFF_POWER_DOMAINS, + .ops = &tgl_tc_cold_off_ops, + .id = DISP_PW_ID_NONE, + }, + { .name = "AUX A", .domains = TGL_AUX_A_IO_POWER_DOMAINS, .ops = &icl_aux_power_well_ops, @@ -4332,12 +4338,6 @@ static const struct i915_power_well_desc tgl_power_wells[] = { .hsw.irq_pipe_mask = BIT(PIPE_D), }, }, - { - .name = "TC cold off", - .domains = TGL_TC_COLD_OFF_POWER_DOMAINS, - .ops = &tgl_tc_cold_off_ops, - .id = DISP_PW_ID_NONE, - }, }; static const struct i915_power_well_desc rkl_power_wells[] = { @@ -5240,10 +5240,10 @@ struct buddy_page_mask { }; static const struct buddy_page_mask tgl_buddy_page_masks[] = { - { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0xE }, { .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0xF }, { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C }, { .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x1F }, + { .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 }, {} }; @@ -5302,6 +5302,12 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv, gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); + /* Wa_14011294188:ehl,jsl,tgl,rkl */ + if (INTEL_PCH_TYPE(dev_priv) >= PCH_JSP && + INTEL_PCH_TYPE(dev_priv) < PCH_DG1) + intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0, + PCH_DPMGUNIT_CLOCK_GATE_DISABLE); + /* 1. Enable PCH reset handshake. */ intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv)); diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index e8f809161c75..9349b15afff6 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -931,6 +931,7 @@ struct intel_crtc_state { bool has_psr; bool has_psr2; + bool enable_psr2_sel_fetch; u32 dc3co_exitline; /* @@ -1073,6 +1074,8 @@ struct intel_crtc_state { /* For DSB related info */ struct intel_dsb *dsb; + + u32 psr2_man_track_ctl; }; enum intel_pipe_crc_source { @@ -1375,6 +1378,9 @@ struct intel_dp { /* Display stream compression testing */ bool force_dsc_en; + + bool hobl_failed; + bool hobl_active; }; enum lspcon_vendor { diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 79c27f91f42c..284b15f84592 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -4449,62 +4449,6 @@ intel_dp_link_down(struct intel_encoder *encoder, } } -static void -intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) -{ - struct drm_i915_private *i915 = dp_to_i915(intel_dp); - u8 dpcd_ext[6]; - - /* - * Prior to DP1.3 the bit represented by - * DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT was reserved. - * if it is set DP_DPCD_REV at 0000h could be at a value less than - * the true capability of the panel. The only way to check is to - * then compare 0000h and 2200h. - */ - if (!(intel_dp->dpcd[DP_TRAINING_AUX_RD_INTERVAL] & - DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)) - return; - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_DP13_DPCD_REV, - &dpcd_ext, sizeof(dpcd_ext)) != sizeof(dpcd_ext)) { - drm_err(&i915->drm, - "DPCD failed read at extended capabilities\n"); - return; - } - - if (intel_dp->dpcd[DP_DPCD_REV] > dpcd_ext[DP_DPCD_REV]) { - drm_dbg_kms(&i915->drm, - "DPCD extended DPCD rev less than base DPCD rev\n"); - return; - } - - if (!memcmp(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext))) - return; - - drm_dbg_kms(&i915->drm, "Base DPCD: %*ph\n", - (int)sizeof(intel_dp->dpcd), intel_dp->dpcd); - - memcpy(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext)); -} - -bool -intel_dp_read_dpcd(struct intel_dp *intel_dp) -{ - struct drm_i915_private *i915 = dp_to_i915(intel_dp); - - if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, - sizeof(intel_dp->dpcd)) < 0) - return false; /* aux transfer failed */ - - intel_dp_extended_receiver_capabilities(intel_dp); - - drm_dbg_kms(&i915->drm, "DPCD: %*ph\n", (int)sizeof(intel_dp->dpcd), - intel_dp->dpcd); - - return intel_dp->dpcd[DP_DPCD_REV] != 0; -} - bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp) { u8 dprx = 0; @@ -4563,7 +4507,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) /* this function is meant to be called only once */ drm_WARN_ON(&dev_priv->drm, intel_dp->dpcd[DP_DPCD_REV] != 0); - if (!intel_dp_read_dpcd(intel_dp)) + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd) != 0) return false; drm_dp_read_desc(&intel_dp->aux, &intel_dp->desc, @@ -4634,11 +4578,23 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) return true; } +static bool +intel_dp_has_sink_count(struct intel_dp *intel_dp) +{ + if (!intel_dp->attached_connector) + return false; + + return drm_dp_read_sink_count_cap(&intel_dp->attached_connector->base, + intel_dp->dpcd, + &intel_dp->desc); +} static bool intel_dp_get_dpcd(struct intel_dp *intel_dp) { - if (!intel_dp_read_dpcd(intel_dp)) + int ret; + + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd)) return false; /* @@ -4653,18 +4609,9 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) intel_dp_set_common_rates(intel_dp); } - /* - * Some eDP panels do not set a valid value for sink count, that is why - * it don't care about read it here and in intel_edp_init_dpcd(). - */ - if (!intel_dp_is_edp(intel_dp) && - !drm_dp_has_quirk(&intel_dp->desc, 0, - DP_DPCD_QUIRK_NO_SINK_COUNT)) { - u8 count; - ssize_t r; - - r = drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_COUNT, &count); - if (r < 1) + if (intel_dp_has_sink_count(intel_dp)) { + ret = drm_dp_read_sink_count(&intel_dp->aux); + if (ret < 0) return false; /* @@ -4672,7 +4619,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) * a member variable in intel_dp will track any changes * between short pulse interrupts. */ - intel_dp->sink_count = DP_GET_SINK_COUNT(count); + intel_dp->sink_count = ret; /* * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that @@ -4685,32 +4632,8 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) return false; } - if (!drm_dp_is_branch(intel_dp->dpcd)) - return true; /* native DP sink */ - - if (intel_dp->dpcd[DP_DPCD_REV] == 0x10) - return true; /* no per-port downstream info */ - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_DOWNSTREAM_PORT_0, - intel_dp->downstream_ports, - DP_MAX_DOWNSTREAM_PORTS) < 0) - return false; /* downstream port status fetch failed */ - - return true; -} - -static bool -intel_dp_sink_can_mst(struct intel_dp *intel_dp) -{ - u8 mstm_cap; - - if (intel_dp->dpcd[DP_DPCD_REV] < 0x12) - return false; - - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_MSTM_CAP, &mstm_cap) != 1) - return false; - - return mstm_cap & DP_MST_CAP; + return drm_dp_read_downstream_info(&intel_dp->aux, intel_dp->dpcd, + intel_dp->downstream_ports) == 0; } static bool @@ -4720,7 +4643,7 @@ intel_dp_can_mst(struct intel_dp *intel_dp) return i915->params.enable_dp_mst && intel_dp->can_mst && - intel_dp_sink_can_mst(intel_dp); + drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd); } static void @@ -4729,7 +4652,7 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - bool sink_can_mst = intel_dp_sink_can_mst(intel_dp); + bool sink_can_mst = drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd); drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s\n", @@ -5963,9 +5886,8 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp) return connector_status_connected; /* If we're HPD-aware, SINK_COUNT changes dynamically */ - if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && + if (intel_dp_has_sink_count(intel_dp) && intel_dp->downstream_ports[0] & DP_DS_PORT_HPD) { - return intel_dp->sink_count ? connector_status_connected : connector_status_disconnected; } diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h index b901ab850cbd..0a3af3410d52 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.h +++ b/drivers/gpu/drm/i915/display/intel_dp.h @@ -99,7 +99,6 @@ bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp); bool intel_dp_get_link_status(struct intel_dp *intel_dp, u8 *link_status); -bool intel_dp_read_dpcd(struct intel_dp *intel_dp); bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp); int intel_dp_link_required(int pixel_clock, int bpp); int intel_dp_max_data_rate(int max_link_clock, int max_lanes); diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index a23ed7290843..f2c8b56be9ea 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -410,10 +410,17 @@ intel_dp_start_link_train(struct intel_dp *intel_dp) intel_connector->base.base.id, intel_connector->base.name, intel_dp->link_rate, intel_dp->lane_count); - if (!intel_dp_get_link_train_fallback_values(intel_dp, - intel_dp->link_rate, - intel_dp->lane_count)) - /* Schedule a Hotplug Uevent to userspace to start modeset */ - schedule_work(&intel_connector->modeset_retry_work); - return; + + if (intel_dp->hobl_active) { + drm_dbg_kms(&dp_to_i915(intel_dp)->drm, + "Link Training failed with HOBL active, not enabling it from now on"); + intel_dp->hobl_failed = true; + } else if (intel_dp_get_link_train_fallback_values(intel_dp, + intel_dp->link_rate, + intel_dp->lane_count)) { + return; + } + + /* Schedule a Hotplug Uevent to userspace to start modeset */ + schedule_work(&intel_connector->modeset_retry_work); } diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index aeb6ee395cce..c9013f8f766f 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -892,7 +892,7 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, refclk = dev_priv->dpll.ref_clks.nssc; break; } - /* fall through */ + fallthrough; case WRPLL_REF_PCH_SSC: /* * We could calculate spread here, but our checking @@ -2977,7 +2977,7 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, switch (dev_priv->dpll.ref_clks.nssc) { default: MISSING_CASE(dev_priv->dpll.ref_clks.nssc); - /* fall-through */ + fallthrough; case 19200: *pll_params = tgl_tbt_pll_19_2MHz_values; break; @@ -2992,7 +2992,7 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, switch (dev_priv->dpll.ref_clks.nssc) { default: MISSING_CASE(dev_priv->dpll.ref_clks.nssc); - /* fall-through */ + fallthrough; case 19200: case 38400: *pll_params = icl_tbt_pll_19_2MHz_values; @@ -3120,7 +3120,7 @@ static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, switch (div1) { default: MISSING_CASE(div1); - /* fall through */ + fallthrough; case 2: hsdiv = MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2; break; @@ -3475,6 +3475,14 @@ static void icl_update_active_dpll(struct intel_atomic_state *state, icl_set_active_port_dpll(crtc_state, port_dpll_id); } +static u32 intel_get_hti_plls(struct drm_i915_private *i915) +{ + if (!(i915->hti_state & HDPORT_ENABLED)) + return 0; + + return REG_FIELD_GET(HDPORT_DPLL_USED_MASK, i915->hti_state); +} + static bool icl_get_combo_phy_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) @@ -3504,13 +3512,22 @@ static bool icl_get_combo_phy_dpll(struct intel_atomic_state *state, icl_calc_dpll_state(dev_priv, &pll_params, &port_dpll->hw_state); - if (IS_ELKHARTLAKE(dev_priv) && port != PORT_A) + if (IS_ROCKETLAKE(dev_priv)) { dpll_mask = BIT(DPLL_ID_EHL_DPLL4) | BIT(DPLL_ID_ICL_DPLL1) | BIT(DPLL_ID_ICL_DPLL0); - else + } else if (IS_ELKHARTLAKE(dev_priv) && port != PORT_A) { + dpll_mask = + BIT(DPLL_ID_EHL_DPLL4) | + BIT(DPLL_ID_ICL_DPLL1) | + BIT(DPLL_ID_ICL_DPLL0); + } else { dpll_mask = BIT(DPLL_ID_ICL_DPLL1) | BIT(DPLL_ID_ICL_DPLL0); + } + + /* Eliminate DPLLs from consideration if reserved by HTI */ + dpll_mask &= ~intel_get_hti_plls(dev_priv); port_dpll->pll = intel_find_shared_dpll(state, crtc, &port_dpll->hw_state, @@ -3791,7 +3808,12 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, if (!(val & PLL_ENABLE)) goto out; - if (INTEL_GEN(dev_priv) >= 12) { + if (IS_ROCKETLAKE(dev_priv)) { + hw_state->cfgcr0 = intel_de_read(dev_priv, + RKL_DPLL_CFGCR0(id)); + hw_state->cfgcr1 = intel_de_read(dev_priv, + RKL_DPLL_CFGCR1(id)); + } else if (INTEL_GEN(dev_priv) >= 12) { hw_state->cfgcr0 = intel_de_read(dev_priv, TGL_DPLL_CFGCR0(id)); hw_state->cfgcr1 = intel_de_read(dev_priv, @@ -3844,7 +3866,10 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv, const enum intel_dpll_id id = pll->info->id; i915_reg_t cfgcr0_reg, cfgcr1_reg; - if (INTEL_GEN(dev_priv) >= 12) { + if (IS_ROCKETLAKE(dev_priv)) { + cfgcr0_reg = RKL_DPLL_CFGCR0(id); + cfgcr1_reg = RKL_DPLL_CFGCR1(id); + } else if (INTEL_GEN(dev_priv) >= 12) { cfgcr0_reg = TGL_DPLL_CFGCR0(id); cfgcr1_reg = TGL_DPLL_CFGCR1(id); } else { @@ -4276,6 +4301,21 @@ static const struct intel_dpll_mgr tgl_pll_mgr = { .dump_hw_state = icl_dump_hw_state, }; +static const struct dpll_info rkl_plls[] = { + { "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 }, + { "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 }, + { "DPLL 4", &combo_pll_funcs, DPLL_ID_EHL_DPLL4, 0 }, + { }, +}; + +static const struct intel_dpll_mgr rkl_pll_mgr = { + .dpll_info = rkl_plls, + .get_dplls = icl_get_dplls, + .put_dplls = icl_put_dplls, + .update_ref_clks = icl_update_dpll_ref_clks, + .dump_hw_state = icl_dump_hw_state, +}; + /** * intel_shared_dpll_init - Initialize shared DPLLs * @dev: drm device @@ -4289,7 +4329,9 @@ void intel_shared_dpll_init(struct drm_device *dev) const struct dpll_info *dpll_info; int i; - if (INTEL_GEN(dev_priv) >= 12) + if (IS_ROCKETLAKE(dev_priv)) + dpll_mgr = &rkl_pll_mgr; + else if (INTEL_GEN(dev_priv) >= 12) dpll_mgr = &tgl_pll_mgr; else if (IS_ELKHARTLAKE(dev_priv)) dpll_mgr = &ehl_pll_mgr; diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 24c3a0f212c6..135f5e8a4d70 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -424,6 +424,14 @@ static void intel_fbc_deactivate(struct drm_i915_private *dev_priv, fbc->no_fbc_reason = reason; } +static u64 intel_fbc_cfb_base_max(struct drm_i915_private *i915) +{ + if (INTEL_GEN(i915) >= 5 || IS_G4X(i915)) + return BIT_ULL(28); + else + return BIT_ULL(32); +} + static int find_compression_threshold(struct drm_i915_private *dev_priv, struct drm_mm_node *node, unsigned int size, @@ -442,6 +450,8 @@ static int find_compression_threshold(struct drm_i915_private *dev_priv, else end = U64_MAX; + end = min(end, intel_fbc_cfb_base_max(dev_priv)); + /* HACK: This code depends on what we will do in *_enable_fbc. If that * code changes, this code needs to change as well. * @@ -1416,6 +1426,13 @@ static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv) if (!HAS_FBC(dev_priv)) return 0; + /* + * Fbc is causing random underruns in CI execution on TGL platforms. + * Disabling the same while the problem is being debugged and analyzed. + */ + if (IS_TIGERLAKE(dev_priv)) + return 0; + if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) return 1; diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 2979ed2588eb..d898b370d7a4 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -232,6 +232,8 @@ static void frontbuffer_release(struct kref *ref) RCU_INIT_POINTER(obj->frontbuffer, NULL); spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock); + i915_active_fini(&front->write); + i915_gem_object_put(obj); kfree_rcu(front, rcu); } diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index 89a4d294822d..1a0d49af2a08 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -336,8 +336,10 @@ int intel_hdcp_validate_v_prime(struct intel_connector *connector, /* Fill up the empty slots in sha_text and write it out */ sha_empty = sizeof(sha_text) - sha_leftovers; - for (j = 0; j < sha_empty; j++) - sha_text |= ksv[j] << ((sizeof(sha_text) - j - 1) * 8); + for (j = 0; j < sha_empty; j++) { + u8 off = ((sizeof(sha_text) - j - 1 - sha_leftovers) * 8); + sha_text |= ksv[j] << off; + } ret = intel_write_sha_text(dev_priv, sha_text); if (ret < 0) @@ -435,7 +437,7 @@ int intel_hdcp_validate_v_prime(struct intel_connector *connector, /* Write 32 bits of text */ intel_de_write(dev_priv, HDCP_REP_CTL, rep_ctl | HDCP_SHA1_TEXT_32); - sha_text |= bstatus[0] << 24 | bstatus[1] << 16; + sha_text |= bstatus[0] << 8 | bstatus[1]; ret = intel_write_sha_text(dev_priv, sha_text); if (ret < 0) return ret; @@ -450,17 +452,29 @@ int intel_hdcp_validate_v_prime(struct intel_connector *connector, return ret; sha_idx += sizeof(sha_text); } + + /* + * Terminate the SHA-1 stream by hand. For the other leftover + * cases this is appended by the hardware. + */ + intel_de_write(dev_priv, HDCP_REP_CTL, + rep_ctl | HDCP_SHA1_TEXT_32); + sha_text = DRM_HDCP_SHA1_TERMINATOR << 24; + ret = intel_write_sha_text(dev_priv, sha_text); + if (ret < 0) + return ret; + sha_idx += sizeof(sha_text); } else if (sha_leftovers == 3) { - /* Write 32 bits of text */ + /* Write 32 bits of text (filled from LSB) */ intel_de_write(dev_priv, HDCP_REP_CTL, rep_ctl | HDCP_SHA1_TEXT_32); - sha_text |= bstatus[0] << 24; + sha_text |= bstatus[0]; ret = intel_write_sha_text(dev_priv, sha_text); if (ret < 0) return ret; sha_idx += sizeof(sha_text); - /* Write 8 bits of text, 24 bits of M0 */ + /* Write 8 bits of text (filled from LSB), 24 bits of M0 */ intel_de_write(dev_priv, HDCP_REP_CTL, rep_ctl | HDCP_SHA1_TEXT_8); ret = intel_write_sha_text(dev_priv, bstatus[1]); @@ -781,6 +795,7 @@ static int _intel_hdcp_disable(struct intel_connector *connector) struct intel_hdcp *hdcp = &connector->hdcp; enum port port = dig_port->base.port; enum transcoder cpu_transcoder = hdcp->cpu_transcoder; + u32 repeater_ctl; int ret; drm_dbg_kms(&dev_priv->drm, "[%s:%d] HDCP is being disabled...\n", @@ -796,6 +811,11 @@ static int _intel_hdcp_disable(struct intel_connector *connector) return -ETIMEDOUT; } + repeater_ctl = intel_hdcp_get_repeater_ctl(dev_priv, cpu_transcoder, + port); + intel_de_write(dev_priv, HDCP_REP_CTL, + intel_de_read(dev_priv, HDCP_REP_CTL) & ~repeater_ctl); + ret = hdcp->shim->toggle_signalling(dig_port, false); if (ret) { drm_err(&dev_priv->drm, "Failed to disable HDCP signalling\n"); diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c index b781bf469644..dc1b35559afd 100644 --- a/drivers/gpu/drm/i915/display/intel_lspcon.c +++ b/drivers/gpu/drm/i915/display/intel_lspcon.c @@ -571,7 +571,7 @@ bool lspcon_init(struct intel_digital_port *dig_port) return false; } - if (!intel_dp_read_dpcd(dp)) { + if (drm_dp_read_dpcd_caps(&dp->aux, dp->dpcd) != 0) { DRM_ERROR("LSPCON DPCD read failed\n"); return false; } diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index bbde3b12c311..4072d7062efd 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -229,7 +229,7 @@ int intel_pch_panel_fitting(struct intel_crtc_state *crtc_state, case DRM_MODE_SCALE_NONE: WARN_ON(adjusted_mode->crtc_hdisplay != crtc_state->pipe_src_w); WARN_ON(adjusted_mode->crtc_vdisplay != crtc_state->pipe_src_h); - /* fall through */ + fallthrough; case DRM_MODE_SCALE_FULLSCREEN: x = y = 0; width = adjusted_mode->crtc_hdisplay; diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index bf9e320c547d..2b004ee9619c 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -553,6 +553,22 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) val |= EDP_PSR2_FAST_WAKE(7); } + if (dev_priv->psr.psr2_sel_fetch_enabled) { + /* WA 1408330847 */ + if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0) || + IS_RKL_REVID(dev_priv, RKL_REVID_A0, RKL_REVID_A0)) + intel_de_rmw(dev_priv, CHICKEN_PAR1_1, + DIS_RAM_BYPASS_PSR2_MAN_TRACK, + DIS_RAM_BYPASS_PSR2_MAN_TRACK); + + intel_de_write(dev_priv, + PSR2_MAN_TRK_CTL(dev_priv->psr.transcoder), + PSR2_MAN_TRK_CTL_ENABLE); + } else if (HAS_PSR2_SEL_FETCH(dev_priv)) { + intel_de_write(dev_priv, + PSR2_MAN_TRK_CTL(dev_priv->psr.transcoder), 0); + } + /* * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is * recommending keep this bit unset while PSR2 is enabled. @@ -663,6 +679,38 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, crtc_state->dc3co_exitline = crtc_vdisplay - exit_scanlines; } +static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, + struct intel_crtc_state *crtc_state) +{ + struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state); + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_plane_state *plane_state; + struct intel_plane *plane; + int i; + + if (!dev_priv->params.enable_psr2_sel_fetch) { + drm_dbg_kms(&dev_priv->drm, + "PSR2 sel fetch not enabled, disabled by parameter\n"); + return false; + } + + if (crtc_state->uapi.async_flip) { + drm_dbg_kms(&dev_priv->drm, + "PSR2 sel fetch not enabled, async flip enabled\n"); + return false; + } + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane_state->uapi.rotation != DRM_MODE_ROTATE_0) { + drm_dbg_kms(&dev_priv->drm, + "PSR2 sel fetch not enabled, plane rotated\n"); + return false; + } + } + + return crtc_state->enable_psr2_sel_fetch = true; +} + static bool intel_psr2_config_valid(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state) { @@ -732,22 +780,17 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, return false; } - /* - * Some platforms lack PSR2 HW tracking and instead require manual - * tracking by software. In this case, the driver is required to track - * the areas that need updates and program hardware to send selective - * updates. - * - * So until the software tracking is implemented, PSR2 needs to be - * disabled for platforms without PSR2 HW tracking. - */ - if (!HAS_PSR_HW_TRACKING(dev_priv)) { - drm_dbg_kms(&dev_priv->drm, - "No PSR2 HW tracking in the platform\n"); - return false; + if (HAS_PSR2_SEL_FETCH(dev_priv)) { + if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) && + !HAS_PSR_HW_TRACKING(dev_priv)) { + drm_dbg_kms(&dev_priv->drm, + "PSR2 not enabled, selective fetch not valid and no HW tracking available\n"); + return false; + } } - if (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v) { + if (!crtc_state->enable_psr2_sel_fetch && + (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v)) { drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, resolution %dx%d > max supported %dx%d\n", crtc_hdisplay, crtc_vdisplay, @@ -898,6 +941,11 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, val |= EXITLINE_ENABLE; intel_de_write(dev_priv, EXITLINE(cpu_transcoder), val); } + + if (HAS_PSR_HW_TRACKING(dev_priv)) + intel_de_rmw(dev_priv, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING, + dev_priv->psr.psr2_sel_fetch_enabled ? + IGNORE_PSR2_HW_TRACKING : 0); } static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, @@ -919,6 +967,7 @@ static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, /* DC5/DC6 requires at least 6 idle frames */ val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6); dev_priv->psr.dc3co_exit_delay = val; + dev_priv->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch; /* * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR @@ -1058,6 +1107,13 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) psr_status_mask, 2000)) drm_err(&dev_priv->drm, "Timed out waiting PSR idle state\n"); + /* WA 1408330847 */ + if (dev_priv->psr.psr2_sel_fetch_enabled && + (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0) || + IS_RKL_REVID(dev_priv, RKL_REVID_A0, RKL_REVID_A0))) + intel_de_rmw(dev_priv, CHICKEN_PAR1_1, + DIS_RAM_BYPASS_PSR2_MAN_TRACK, 0); + /* Disable PSR on Sink */ drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0); @@ -1115,6 +1171,32 @@ static void psr_force_hw_tracking_exit(struct drm_i915_private *dev_priv) intel_psr_exit(dev_priv); } +void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct i915_psr *psr = &dev_priv->psr; + + if (!HAS_PSR2_SEL_FETCH(dev_priv) || + !crtc_state->enable_psr2_sel_fetch) + return; + + intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(psr->transcoder), + crtc_state->psr2_man_track_ctl); +} + +void intel_psr2_sel_fetch_update(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + + if (!crtc_state->enable_psr2_sel_fetch) + return; + + crtc_state->psr2_man_track_ctl = PSR2_MAN_TRK_CTL_ENABLE | + PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; +} + /** * intel_psr_update - Update PSR state * @intel_dp: Intel DP diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h index b4515186d5f4..6a83c8e682e6 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.h +++ b/drivers/gpu/drm/i915/display/intel_psr.h @@ -13,6 +13,8 @@ struct drm_connector_state; struct drm_i915_private; struct intel_crtc_state; struct intel_dp; +struct intel_crtc; +struct intel_atomic_state; #define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support) void intel_psr_init_dpcd(struct intel_dp *intel_dp); @@ -43,5 +45,8 @@ void intel_psr_atomic_check(struct drm_connector *connector, struct drm_connector_state *old_state, struct drm_connector_state *new_state); void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp); +void intel_psr2_sel_fetch_update(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state); #endif /* __INTEL_PSR_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 2da4388e1540..5e9fb349c829 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -1531,7 +1531,7 @@ static void intel_sdvo_pre_enable(struct intel_atomic_state *state, default: drm_WARN(&dev_priv->drm, 1, "unknown pixel multiplier specified\n"); - /* fall through */ + fallthrough; case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break; case 2: rate = SDVO_CLOCK_RATE_MULT_2X; break; case 4: rate = SDVO_CLOCK_RATE_MULT_4X; break; @@ -2549,19 +2549,19 @@ intel_sdvo_guess_ddc_bus(struct intel_sdvo *sdvo) switch (sdvo->controlled_output) { case SDVO_OUTPUT_LVDS1: mask |= SDVO_OUTPUT_LVDS1; - /* fall through */ + fallthrough; case SDVO_OUTPUT_LVDS0: mask |= SDVO_OUTPUT_LVDS0; - /* fall through */ + fallthrough; case SDVO_OUTPUT_TMDS1: mask |= SDVO_OUTPUT_TMDS1; - /* fall through */ + fallthrough; case SDVO_OUTPUT_TMDS0: mask |= SDVO_OUTPUT_TMDS0; - /* fall through */ + fallthrough; case SDVO_OUTPUT_RGB1: mask |= SDVO_OUTPUT_RGB1; - /* fall through */ + fallthrough; case SDVO_OUTPUT_RGB0: mask |= SDVO_OUTPUT_RGB0; break; diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index d03860fef2d7..6b72223981be 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -2147,7 +2147,7 @@ static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state, case DRM_FORMAT_RGB565: if (INTEL_GEN(dev_priv) >= 11) break; - /* fall through */ + fallthrough; case DRM_FORMAT_C8: case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_XBGR16161616F: @@ -2702,7 +2702,7 @@ static bool g4x_sprite_format_mod_supported(struct drm_plane *_plane, if (modifier == DRM_FORMAT_MOD_LINEAR || modifier == I915_FORMAT_MOD_X_TILED) return true; - /* fall through */ + fallthrough; default: return false; } @@ -2733,7 +2733,7 @@ static bool snb_sprite_format_mod_supported(struct drm_plane *_plane, if (modifier == DRM_FORMAT_MOD_LINEAR || modifier == I915_FORMAT_MOD_X_TILED) return true; - /* fall through */ + fallthrough; default: return false; } @@ -2768,7 +2768,7 @@ static bool vlv_sprite_format_mod_supported(struct drm_plane *_plane, if (modifier == DRM_FORMAT_MOD_LINEAR || modifier == I915_FORMAT_MOD_X_TILED) return true; - /* fall through */ + fallthrough; default: return false; } @@ -2801,7 +2801,7 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_ABGR8888: if (is_ccs_modifier(modifier)) return true; - /* fall through */ + fallthrough; case DRM_FORMAT_RGB565: case DRM_FORMAT_XRGB2101010: case DRM_FORMAT_XBGR2101010: @@ -2819,7 +2819,7 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_XVYU2101010: if (modifier == I915_FORMAT_MOD_Yf_TILED) return true; - /* fall through */ + fallthrough; case DRM_FORMAT_C8: case DRM_FORMAT_XBGR16161616F: case DRM_FORMAT_ABGR16161616F: @@ -2834,7 +2834,7 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, modifier == I915_FORMAT_MOD_X_TILED || modifier == I915_FORMAT_MOD_Y_TILED) return true; - /* fall through */ + fallthrough; default: return false; } @@ -2843,8 +2843,9 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, static bool gen12_plane_supports_mc_ccs(struct drm_i915_private *dev_priv, enum plane_id plane_id) { - /* Wa_14010477008:tgl[a0..c0] */ - if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_C0)) + /* Wa_14010477008:tgl[a0..c0],rkl[all] */ + if (IS_ROCKETLAKE(dev_priv) || + IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_C0)) return false; return plane_id < PLANE_SPRITE4; @@ -2860,7 +2861,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: if (!gen12_plane_supports_mc_ccs(dev_priv, plane->id)) return false; - /* fall through */ + fallthrough; case DRM_FORMAT_MOD_LINEAR: case I915_FORMAT_MOD_X_TILED: case I915_FORMAT_MOD_Y_TILED: @@ -2877,7 +2878,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_ABGR8888: if (is_ccs_modifier(modifier)) return true; - /* fall through */ + fallthrough; case DRM_FORMAT_YUYV: case DRM_FORMAT_YVYU: case DRM_FORMAT_UYVY: @@ -2889,7 +2890,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_P016: if (modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS) return true; - /* fall through */ + fallthrough; case DRM_FORMAT_RGB565: case DRM_FORMAT_XRGB2101010: case DRM_FORMAT_XBGR2101010: @@ -2910,7 +2911,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, modifier == I915_FORMAT_MOD_X_TILED || modifier == I915_FORMAT_MOD_Y_TILED) return true; - /* fall through */ + fallthrough; default: return false; } diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index 5b5dc86a5737..8f67aef18b2d 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -159,7 +159,7 @@ int intel_tc_port_fia_max_lane_count(struct intel_digital_port *dig_port) switch (lane_mask) { default: MISSING_CASE(lane_mask); - /* fall-through */ + fallthrough; case 0x1: case 0x2: case 0x4: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c index 278664f831e7..272cf3ea68d5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c @@ -32,12 +32,13 @@ static void vma_clear_pages(struct i915_vma *vma) vma->pages = NULL; } -static int vma_bind(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static void vma_bind(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { - return vm->vma_ops.bind_vma(vm, vma, cache_level, flags); + vm->vma_ops.bind_vma(vm, stash, vma, cache_level, flags); } static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) @@ -157,6 +158,7 @@ static void clear_pages_worker(struct work_struct *work) struct clear_pages_work *w = container_of(work, typeof(*w), work); struct drm_i915_gem_object *obj = w->sleeve->vma->obj; struct i915_vma *vma = w->sleeve->vma; + struct i915_gem_ww_ctx ww; struct i915_request *rq; struct i915_vma *batch; int err = w->dma.error; @@ -172,17 +174,20 @@ static void clear_pages_worker(struct work_struct *work) obj->read_domains = I915_GEM_GPU_DOMAINS; obj->write_domain = 0; - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (unlikely(err)) + i915_gem_ww_ctx_init(&ww, false); + intel_engine_pm_get(w->ce->engine); +retry: + err = intel_context_pin_ww(w->ce, &ww); + if (err) goto out_signal; - batch = intel_emit_vma_fill_blt(w->ce, vma, w->value); + batch = intel_emit_vma_fill_blt(w->ce, vma, &ww, w->value); if (IS_ERR(batch)) { err = PTR_ERR(batch); - goto out_unpin; + goto out_ctx; } - rq = intel_context_create_request(w->ce); + rq = i915_request_create(w->ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out_batch; @@ -224,9 +229,19 @@ out_request: i915_request_add(rq); out_batch: intel_emit_vma_release(w->ce, batch); -out_unpin: - i915_vma_unpin(vma); +out_ctx: + intel_context_unpin(w->ce); out_signal: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + + i915_vma_unpin(w->sleeve->vma); + intel_engine_pm_put(w->ce->engine); + if (unlikely(err)) { dma_fence_set_error(&w->dma, err); dma_fence_signal(&w->dma); @@ -234,6 +249,44 @@ out_signal: } } +static int pin_wait_clear_pages_work(struct clear_pages_work *w, + struct intel_context *ce) +{ + struct i915_vma *vma = w->sleeve->vma; + struct i915_gem_ww_ctx ww; + int err; + + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(vma->obj, &ww); + if (err) + goto out; + + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); + if (unlikely(err)) + goto out; + + err = i915_sw_fence_await_reservation(&w->wait, + vma->obj->base.resv, NULL, + true, 0, I915_FENCE_GFP); + if (err) + goto err_unpin_vma; + + dma_resv_add_excl_fence(vma->obj->base.resv, &w->dma); + +err_unpin_vma: + if (err) + i915_vma_unpin(vma); +out: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + return err; +} + static int __i915_sw_fence_call clear_pages_work_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) @@ -287,17 +340,9 @@ int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0); i915_sw_fence_init(&work->wait, clear_pages_work_notify); - i915_gem_object_lock(obj); - err = i915_sw_fence_await_reservation(&work->wait, - obj->base.resv, NULL, true, 0, - I915_FENCE_GFP); - if (err < 0) { + err = pin_wait_clear_pages_work(work, ce); + if (err < 0) dma_fence_set_error(&work->dma, err); - } else { - dma_resv_add_excl_fence(obj->base.resv, &work->dma); - err = 0; - } - i915_gem_object_unlock(obj); dma_fence_get(&work->dma); i915_sw_fence_commit(&work->wait); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index d0bdb6d447ed..cf5ecbde9e06 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -439,29 +439,36 @@ static bool __cancel_engine(struct intel_engine_cs *engine) return __reset_engine(engine); } -static struct intel_engine_cs *__active_engine(struct i915_request *rq) +static bool +__active_engine(struct i915_request *rq, struct intel_engine_cs **active) { struct intel_engine_cs *engine, *locked; + bool ret = false; /* * Serialise with __i915_request_submit() so that it sees * is-banned?, or we know the request is already inflight. + * + * Note that rq->engine is unstable, and so we double + * check that we have acquired the lock on the final engine. */ locked = READ_ONCE(rq->engine); spin_lock_irq(&locked->active.lock); while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { spin_unlock(&locked->active.lock); - spin_lock(&engine->active.lock); locked = engine; + spin_lock(&locked->active.lock); } - engine = NULL; - if (i915_request_is_active(rq) && rq->fence.error != -EIO) - engine = rq->engine; + if (!i915_request_completed(rq)) { + if (i915_request_is_active(rq) && rq->fence.error != -EIO) + *active = locked; + ret = true; + } spin_unlock_irq(&locked->active.lock); - return engine; + return ret; } static struct intel_engine_cs *active_engine(struct intel_context *ce) @@ -472,17 +479,16 @@ static struct intel_engine_cs *active_engine(struct intel_context *ce) if (!ce->timeline) return NULL; - mutex_lock(&ce->timeline->mutex); - list_for_each_entry_reverse(rq, &ce->timeline->requests, link) { - if (i915_request_completed(rq)) - break; + rcu_read_lock(); + list_for_each_entry_rcu(rq, &ce->timeline->requests, link) { + if (i915_request_is_active(rq) && i915_request_completed(rq)) + continue; /* Check with the backend if the request is inflight */ - engine = __active_engine(rq); - if (engine) + if (__active_engine(rq, &engine)) break; } - mutex_unlock(&ce->timeline->mutex); + rcu_read_unlock(); return engine; } @@ -713,6 +719,7 @@ __create_context(struct drm_i915_private *i915) ctx->i915 = i915; ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); mutex_init(&ctx->mutex); + INIT_LIST_HEAD(&ctx->link); spin_lock_init(&ctx->stale.lock); INIT_LIST_HEAD(&ctx->stale.engines); @@ -740,10 +747,6 @@ __create_context(struct drm_i915_private *i915) for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; - spin_lock(&i915->gem.contexts.lock); - list_add_tail(&ctx->link, &i915->gem.contexts.list); - spin_unlock(&i915->gem.contexts.lock); - return ctx; err_free: @@ -889,7 +892,7 @@ i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { struct intel_timeline *timeline; - timeline = intel_timeline_create(&i915->gt, NULL); + timeline = intel_timeline_create(&i915->gt); if (IS_ERR(timeline)) { context_close(ctx); return ERR_CAST(timeline); @@ -931,6 +934,7 @@ static int gem_context_register(struct i915_gem_context *ctx, struct drm_i915_file_private *fpriv, u32 *id) { + struct drm_i915_private *i915 = ctx->i915; struct i915_address_space *vm; int ret; @@ -949,8 +953,16 @@ static int gem_context_register(struct i915_gem_context *ctx, /* And finally expose ourselves to userspace via the idr */ ret = xa_alloc(&fpriv->context_xa, id, ctx, xa_limit_32b, GFP_KERNEL); if (ret) - put_pid(fetch_and_zero(&ctx->pid)); + goto err_pid; + + spin_lock(&i915->gem.contexts.lock); + list_add_tail(&ctx->link, &i915->gem.contexts.list); + spin_unlock(&i915->gem.contexts.lock); + return 0; + +err_pid: + put_pid(fetch_and_zero(&ctx->pid)); return ret; } @@ -1094,6 +1106,7 @@ I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); static int context_barrier_task(struct i915_gem_context *ctx, intel_engine_mask_t engines, bool (*skip)(struct intel_context *ce, void *data), + int (*pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data), int (*emit)(struct i915_request *rq, void *data), void (*task)(void *data), void *data) @@ -1101,6 +1114,7 @@ static int context_barrier_task(struct i915_gem_context *ctx, struct context_barrier_task *cb; struct i915_gem_engines_iter it; struct i915_gem_engines *e; + struct i915_gem_ww_ctx ww; struct intel_context *ce; int err = 0; @@ -1138,10 +1152,21 @@ static int context_barrier_task(struct i915_gem_context *ctx, if (skip && skip(ce, data)) continue; - rq = intel_context_create_request(ce); + i915_gem_ww_ctx_init(&ww, true); +retry: + err = intel_context_pin_ww(ce, &ww); + if (err) + goto err; + + if (pin) + err = pin(ce, &ww, data); + if (err) + goto err_unpin; + + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); - break; + goto err_unpin; } err = 0; @@ -1151,6 +1176,16 @@ static int context_barrier_task(struct i915_gem_context *ctx, err = i915_active_add_request(&cb->base, rq); i915_request_add(rq); +err_unpin: + intel_context_unpin(ce); +err: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + if (err) break; } @@ -1206,6 +1241,17 @@ static void set_ppgtt_barrier(void *data) i915_vm_close(old); } +static int pin_ppgtt_update(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data) +{ + struct i915_address_space *vm = ce->vm; + + if (!HAS_LOGICAL_RING_CONTEXTS(vm->i915)) + /* ppGTT is not part of the legacy context image */ + return gen6_ppgtt_pin(i915_vm_to_ppgtt(vm), ww); + + return 0; +} + static int emit_ppgtt_update(struct i915_request *rq, void *data) { struct i915_address_space *vm = rq->context->vm; @@ -1262,20 +1308,10 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data) static bool skip_ppgtt_update(struct intel_context *ce, void *data) { - if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) - return true; - if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) - return false; - - if (!atomic_read(&ce->pin_count)) - return true; - - /* ppGTT is not part of the legacy context image */ - if (gen6_ppgtt_pin(i915_vm_to_ppgtt(ce->vm))) - return true; - - return false; + return !ce->state; + else + return !atomic_read(&ce->pin_count); } static int set_ppgtt(struct drm_i915_file_private *file_priv, @@ -1326,6 +1362,7 @@ static int set_ppgtt(struct drm_i915_file_private *file_priv, */ err = context_barrier_task(ctx, ALL_ENGINES, skip_ppgtt_update, + pin_ppgtt_update, emit_ppgtt_update, set_ppgtt_barrier, old); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index 2679380159fc..27fddc22a7c6 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -128,7 +128,7 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire if (err) return err; - err = i915_gem_object_lock_interruptible(obj); + err = i915_gem_object_lock_interruptible(obj, NULL); if (err) goto out; @@ -149,7 +149,7 @@ static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direct if (err) return err; - err = i915_gem_object_lock_interruptible(obj); + err = i915_gem_object_lock_interruptible(obj, NULL); if (err) goto out; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c index 7f76fc68f498..7c90a63c273d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c @@ -32,11 +32,17 @@ void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) if (!i915_gem_object_is_framebuffer(obj)) return; - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); __i915_gem_object_flush_for_display(obj); i915_gem_object_unlock(obj); } +void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj) +{ + if (i915_gem_object_is_framebuffer(obj)) + __i915_gem_object_flush_for_display(obj); +} + /** * Moves a single object to the WC read, and possibly write domain. * @obj: object to act on @@ -197,18 +203,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_lock_interruptible(obj); - if (ret) - return ret; - /* Always invalidate stale cachelines */ if (obj->cache_level != cache_level) { i915_gem_object_set_cache_coherency(obj, cache_level); obj->cache_dirty = true; } - i915_gem_object_unlock(obj); - /* The cache-level will be applied when each vma is rebound. */ return i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE | @@ -293,7 +293,12 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, goto out; } + ret = i915_gem_object_lock_interruptible(obj, NULL); + if (ret) + goto out; + ret = i915_gem_object_set_cache_level(obj, level); + i915_gem_object_unlock(obj); out: i915_gem_object_put(obj); @@ -313,6 +318,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, unsigned int flags) { struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_gem_ww_ctx ww; struct i915_vma *vma; int ret; @@ -320,6 +326,11 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) return ERR_PTR(-EINVAL); + i915_gem_ww_ctx_init(&ww, true); +retry: + ret = i915_gem_object_lock(obj, &ww); + if (ret) + goto err; /* * The display engine is not coherent with the LLC cache on gen6. As * a result, we make sure that the pinning that is about to occur is @@ -334,7 +345,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE); if (ret) - return ERR_PTR(ret); + goto err; /* * As the user may map the buffer once pinned in the display plane @@ -347,18 +358,31 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, vma = ERR_PTR(-ENOSPC); if ((flags & PIN_MAPPABLE) == 0 && (!view || view->type == I915_GGTT_VIEW_NORMAL)) - vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, - flags | - PIN_MAPPABLE | - PIN_NONBLOCK); - if (IS_ERR(vma)) - vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); - if (IS_ERR(vma)) - return vma; + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment, + flags | PIN_MAPPABLE | + PIN_NONBLOCK); + if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, + alignment, flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto err; + } vma->display_alignment = max_t(u64, vma->display_alignment, alignment); - i915_gem_object_flush_if_display(obj); + i915_gem_object_flush_if_display_locked(obj); + +err: + if (ret == -EDEADLK) { + ret = i915_gem_ww_ctx_backoff(&ww); + if (!ret) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + + if (ret) + return ERR_PTR(ret); return vma; } @@ -536,7 +560,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, if (err) goto out; - err = i915_gem_object_lock_interruptible(obj); + err = i915_gem_object_lock_interruptible(obj, NULL); if (err) goto out_unpin; @@ -576,19 +600,17 @@ int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_lock_interruptible(obj); - if (ret) - return ret; + assert_object_held(obj); ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); if (ret) - goto err_unlock; + return ret; ret = i915_gem_object_pin_pages(obj); if (ret) - goto err_unlock; + return ret; if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || !static_cpu_has(X86_FEATURE_CLFLUSH)) { @@ -616,8 +638,6 @@ out: err_unpin: i915_gem_object_unpin_pages(obj); -err_unlock: - i915_gem_object_unlock(obj); return ret; } @@ -630,20 +650,18 @@ int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_lock_interruptible(obj); - if (ret) - return ret; + assert_object_held(obj); ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT); if (ret) - goto err_unlock; + return ret; ret = i915_gem_object_pin_pages(obj); if (ret) - goto err_unlock; + return ret; if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || !static_cpu_has(X86_FEATURE_CLFLUSH)) { @@ -680,7 +698,5 @@ out: err_unpin: i915_gem_object_unpin_pages(obj); -err_unlock: - i915_gem_object_unlock(obj); return ret; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 6b4ec66cb558..5509946f1a1d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -26,6 +26,7 @@ #include "i915_gem_ioctls.h" #include "i915_sw_fence_work.h" #include "i915_trace.h" +#include "i915_user_extensions.h" struct eb_vma { struct i915_vma *vma; @@ -40,9 +41,11 @@ struct eb_vma { u32 handle; }; -struct eb_vma_array { - struct kref kref; - struct eb_vma vma[]; +enum { + FORCE_CPU_RELOC = 1, + FORCE_GTT_RELOC, + FORCE_GPU_RELOC, +#define DBG_FORCE_RELOC 0 /* choose one of the above! */ }; #define __EXEC_OBJECT_HAS_PIN BIT(31) @@ -50,9 +53,11 @@ struct eb_vma_array { #define __EXEC_OBJECT_NEEDS_MAP BIT(29) #define __EXEC_OBJECT_NEEDS_BIAS BIT(28) #define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */ +#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE) #define __EXEC_HAS_RELOC BIT(31) -#define __EXEC_INTERNAL_FLAGS (~0u << 31) +#define __EXEC_ENGINE_PINNED BIT(30) +#define __EXEC_INTERNAL_FLAGS (~0u << 30) #define UPDATE PIN_OFFSET_FIXED #define BATCH_OFFSET_BIAS (256*1024) @@ -222,6 +227,13 @@ struct eb_vma_array { * the batchbuffer in trusted mode, otherwise the ioctl is rejected. */ +struct eb_fence { + struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */ + struct dma_fence *dma_fence; + u64 value; + struct dma_fence_chain *chain_fence; +}; + struct i915_execbuffer { struct drm_i915_private *i915; /** i915 backpointer */ struct drm_file *file; /** per-file lookup tables and limits */ @@ -246,6 +258,8 @@ struct i915_execbuffer { /** list of vma that have execobj.relocation_count */ struct list_head relocs; + struct i915_gem_ww_ctx ww; + /** * Track the most recently used object for relocations, as we * frequently have to perform multiple relocations within the same @@ -253,25 +267,30 @@ struct i915_execbuffer { */ struct reloc_cache { struct drm_mm_node node; /** temporary GTT binding */ + unsigned long vaddr; /** Current kmap address */ + unsigned long page; /** Currently mapped page index */ unsigned int gen; /** Cached value of INTEL_GEN */ bool use_64bit_reloc : 1; bool has_llc : 1; bool has_fence : 1; bool needs_unfenced : 1; - struct i915_vma *target; struct i915_request *rq; - struct i915_vma *rq_vma; u32 *rq_cmd; unsigned int rq_size; + struct intel_gt_buffer_pool_node *pool; } reloc_cache; + struct intel_gt_buffer_pool_node *reloc_pool; /** relocation pool for -EDEADLK handling */ + struct intel_context *reloc_context; + u64 invalid_flags; /** Set of execobj.flags that are invalid */ u32 context_flags; /** Set of execobj.flags to insert from the ctx */ u32 batch_start_offset; /** Location within object of batch */ u32 batch_len; /** Length of batch within object */ u32 batch_flags; /** Flags composed for emit_bb_start() */ + struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */ /** * Indicate either the size of the hastable used to resolve @@ -280,9 +299,16 @@ struct i915_execbuffer { */ int lut_size; struct hlist_head *buckets; /** ht for relocation handles */ - struct eb_vma_array *array; + + struct eb_fence *fences; + unsigned long num_fences; }; +static int eb_parse(struct i915_execbuffer *eb); +static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, + bool throttle); +static void eb_unpin_engine(struct i915_execbuffer *eb); + static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) { return intel_engine_requires_cmd_parser(eb->engine) || @@ -290,62 +316,8 @@ static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) eb->args->batch_len); } -static struct eb_vma_array *eb_vma_array_create(unsigned int count) -{ - struct eb_vma_array *arr; - - arr = kvmalloc(struct_size(arr, vma, count), GFP_KERNEL | __GFP_NOWARN); - if (!arr) - return NULL; - - kref_init(&arr->kref); - arr->vma[0].vma = NULL; - - return arr; -} - -static inline void eb_unreserve_vma(struct eb_vma *ev) -{ - struct i915_vma *vma = ev->vma; - - if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) - __i915_vma_unpin_fence(vma); - - if (ev->flags & __EXEC_OBJECT_HAS_PIN) - __i915_vma_unpin(vma); - - ev->flags &= ~(__EXEC_OBJECT_HAS_PIN | - __EXEC_OBJECT_HAS_FENCE); -} - -static void eb_vma_array_destroy(struct kref *kref) -{ - struct eb_vma_array *arr = container_of(kref, typeof(*arr), kref); - struct eb_vma *ev = arr->vma; - - while (ev->vma) { - eb_unreserve_vma(ev); - i915_vma_put(ev->vma); - ev++; - } - - kvfree(arr); -} - -static void eb_vma_array_put(struct eb_vma_array *arr) -{ - kref_put(&arr->kref, eb_vma_array_destroy); -} - static int eb_create(struct i915_execbuffer *eb) { - /* Allocate an extra slot for use by the command parser + sentinel */ - eb->array = eb_vma_array_create(eb->buffer_count + 2); - if (!eb->array) - return -ENOMEM; - - eb->vma = eb->array->vma; - if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { unsigned int size = 1 + ilog2(eb->buffer_count); @@ -379,10 +351,8 @@ static int eb_create(struct i915_execbuffer *eb) break; } while (--size); - if (unlikely(!size)) { - eb_vma_array_put(eb->array); + if (unlikely(!size)) return -ENOMEM; - } eb->lut_size = size; } else { @@ -466,16 +436,17 @@ eb_pin_vma(struct i915_execbuffer *eb, pin_flags |= PIN_GLOBAL; /* Attempt to reuse the current location if available */ - if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags))) { + /* TODO: Add -EDEADLK handling here */ + if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) { if (entry->flags & EXEC_OBJECT_PINNED) return false; /* Failing that pick any _free_ space if suitable */ - if (unlikely(i915_vma_pin(vma, - entry->pad_to_size, - entry->alignment, - eb_pin_flags(entry, ev->flags) | - PIN_USER | PIN_NOEVICT))) + if (unlikely(i915_vma_pin_ww(vma, &eb->ww, + entry->pad_to_size, + entry->alignment, + eb_pin_flags(entry, ev->flags) | + PIN_USER | PIN_NOEVICT))) return false; } @@ -493,6 +464,19 @@ eb_pin_vma(struct i915_execbuffer *eb, return !eb_vma_misplaced(entry, vma, ev->flags); } +static inline void +eb_unreserve_vma(struct eb_vma *ev) +{ + if (!(ev->flags & __EXEC_OBJECT_HAS_PIN)) + return; + + if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) + __i915_vma_unpin_fence(ev->vma); + + __i915_vma_unpin(ev->vma); + ev->flags &= ~__EXEC_OBJECT_RESERVED; +} + static int eb_validate_vma(struct i915_execbuffer *eb, struct drm_i915_gem_exec_object2 *entry, @@ -584,19 +568,26 @@ eb_add_vma(struct i915_execbuffer *eb, eb->batch = ev; } +} - if (eb_pin_vma(eb, entry, ev)) { - if (entry->offset != vma->node.start) { - entry->offset = vma->node.start | UPDATE; - eb->args->flags |= __EXEC_HAS_RELOC; - } - } else { - eb_unreserve_vma(ev); - list_add_tail(&ev->bind_link, &eb->unbound); - } +static inline int use_cpu_reloc(const struct reloc_cache *cache, + const struct drm_i915_gem_object *obj) +{ + if (!i915_gem_object_has_struct_page(obj)) + return false; + + if (DBG_FORCE_RELOC == FORCE_CPU_RELOC) + return true; + + if (DBG_FORCE_RELOC == FORCE_GTT_RELOC) + return false; + + return (cache->has_llc || + obj->cache_dirty || + obj->cache_level != I915_CACHE_NONE); } -static int eb_reserve_vma(const struct i915_execbuffer *eb, +static int eb_reserve_vma(struct i915_execbuffer *eb, struct eb_vma *ev, u64 pin_flags) { @@ -611,7 +602,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb, return err; } - err = i915_vma_pin(vma, + err = i915_vma_pin_ww(vma, &eb->ww, entry->pad_to_size, entry->alignment, eb_pin_flags(entry, ev->flags) | pin_flags); if (err) @@ -661,10 +652,6 @@ static int eb_reserve(struct i915_execbuffer *eb) * This avoid unnecessary unbinding of later objects in order to make * room for the earlier objects *unless* we need to defragment. */ - - if (mutex_lock_interruptible(&eb->i915->drm.struct_mutex)) - return -EINTR; - pass = 0; do { list_for_each_entry(ev, &eb->unbound, bind_link) { @@ -672,8 +659,8 @@ static int eb_reserve(struct i915_execbuffer *eb) if (err) break; } - if (!(err == -ENOSPC || err == -EAGAIN)) - break; + if (err != -ENOSPC) + return err; /* Resort *all* the objects into priority order */ INIT_LIST_HEAD(&eb->unbound); @@ -703,13 +690,6 @@ static int eb_reserve(struct i915_execbuffer *eb) } list_splice_tail(&last, &eb->unbound); - if (err == -EAGAIN) { - mutex_unlock(&eb->i915->drm.struct_mutex); - flush_workqueue(eb->i915->mm.userptr_wq); - mutex_lock(&eb->i915->drm.struct_mutex); - continue; - } - switch (pass++) { case 0: break; @@ -720,20 +700,15 @@ static int eb_reserve(struct i915_execbuffer *eb) err = i915_gem_evict_vm(eb->context->vm); mutex_unlock(&eb->context->vm->mutex); if (err) - goto unlock; + return err; break; default: - err = -ENOSPC; - goto unlock; + return -ENOSPC; } pin_flags = PIN_USER; } while (1); - -unlock: - mutex_unlock(&eb->i915->drm.struct_mutex); - return err; } static unsigned int eb_batch_index(const struct i915_execbuffer *eb) @@ -856,12 +831,12 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) static int eb_lookup_vmas(struct i915_execbuffer *eb) { + struct drm_i915_private *i915 = eb->i915; unsigned int batch = eb_batch_index(eb); unsigned int i; int err = 0; INIT_LIST_HEAD(&eb->relocs); - INIT_LIST_HEAD(&eb->unbound); for (i = 0; i < eb->buffer_count; i++) { struct i915_vma *vma; @@ -869,22 +844,83 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) vma = eb_lookup_vma(eb, eb->exec[i].handle); if (IS_ERR(vma)) { err = PTR_ERR(vma); - break; + goto err; } err = eb_validate_vma(eb, &eb->exec[i], vma); if (unlikely(err)) { i915_vma_put(vma); - break; + goto err; } eb_add_vma(eb, i, batch, vma); } + if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) { + drm_dbg(&i915->drm, + "Attempting to use self-modifying batch buffer\n"); + return -EINVAL; + } + + if (range_overflows_t(u64, + eb->batch_start_offset, eb->batch_len, + eb->batch->vma->size)) { + drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); + return -EINVAL; + } + + if (eb->batch_len == 0) + eb->batch_len = eb->batch->vma->size - eb->batch_start_offset; + + return 0; + +err: eb->vma[i].vma = NULL; return err; } +static int eb_validate_vmas(struct i915_execbuffer *eb) +{ + unsigned int i; + int err; + + INIT_LIST_HEAD(&eb->unbound); + + for (i = 0; i < eb->buffer_count; i++) { + struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; + struct eb_vma *ev = &eb->vma[i]; + struct i915_vma *vma = ev->vma; + + err = i915_gem_object_lock(vma->obj, &eb->ww); + if (err) + return err; + + if (eb_pin_vma(eb, entry, ev)) { + if (entry->offset != vma->node.start) { + entry->offset = vma->node.start | UPDATE; + eb->args->flags |= __EXEC_HAS_RELOC; + } + } else { + eb_unreserve_vma(ev); + + list_add_tail(&ev->bind_link, &eb->unbound); + if (drm_mm_node_allocated(&vma->node)) { + err = i915_vma_unbind(vma); + if (err) + return err; + } + } + + GEM_BUG_ON(drm_mm_node_allocated(&vma->node) && + eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); + } + + if (!list_empty(&eb->unbound)) + return eb_reserve(eb); + + return 0; +} + static struct eb_vma * eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) { @@ -905,13 +941,31 @@ eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) } } +static void eb_release_vmas(struct i915_execbuffer *eb, bool final) +{ + const unsigned int count = eb->buffer_count; + unsigned int i; + + for (i = 0; i < count; i++) { + struct eb_vma *ev = &eb->vma[i]; + struct i915_vma *vma = ev->vma; + + if (!vma) + break; + + eb_unreserve_vma(ev); + + if (final) + i915_vma_put(vma); + } + + eb_unpin_engine(eb); +} + static void eb_destroy(const struct i915_execbuffer *eb) { GEM_BUG_ON(eb->reloc_cache.rq); - if (eb->array) - eb_vma_array_put(eb->array); - if (eb->lut_size > 0) kfree(eb->buckets); } @@ -923,9 +977,19 @@ relocation_target(const struct drm_i915_gem_relocation_entry *reloc, return gen8_canonical_addr((int)reloc->delta + target->node.start); } +static void reloc_cache_clear(struct reloc_cache *cache) +{ + cache->rq = NULL; + cache->rq_cmd = NULL; + cache->pool = NULL; + cache->rq_size = 0; +} + static void reloc_cache_init(struct reloc_cache *cache, struct drm_i915_private *i915) { + cache->page = -1; + cache->vaddr = 0; /* Must be a variable in the struct to allow GCC to unroll. */ cache->gen = INTEL_GEN(i915); cache->has_llc = HAS_LLC(i915); @@ -933,120 +997,249 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->has_fence = cache->gen < 4; cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; cache->node.flags = 0; - cache->rq = NULL; - cache->target = NULL; + reloc_cache_clear(cache); } -#define RELOC_TAIL 4 +static inline void *unmask_page(unsigned long p) +{ + return (void *)(uintptr_t)(p & PAGE_MASK); +} -static int reloc_gpu_chain(struct reloc_cache *cache) +static inline unsigned int unmask_flags(unsigned long p) { - struct intel_gt_buffer_pool_node *pool; - struct i915_request *rq = cache->rq; - struct i915_vma *batch; - u32 *cmd; - int err; + return p & ~PAGE_MASK; +} - pool = intel_gt_get_buffer_pool(rq->engine->gt, PAGE_SIZE); - if (IS_ERR(pool)) - return PTR_ERR(pool); +#define KMAP 0x4 /* after CLFLUSH_FLAGS */ - batch = i915_vma_instance(pool->obj, rq->context->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_pool; - } +static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) +{ + struct drm_i915_private *i915 = + container_of(cache, struct i915_execbuffer, reloc_cache)->i915; + return &i915->ggtt; +} - err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); - if (err) - goto out_pool; +static void reloc_cache_put_pool(struct i915_execbuffer *eb, struct reloc_cache *cache) +{ + if (!cache->pool) + return; - GEM_BUG_ON(cache->rq_size + RELOC_TAIL > PAGE_SIZE / sizeof(u32)); - cmd = cache->rq_cmd + cache->rq_size; - *cmd++ = MI_ARB_CHECK; - if (cache->gen >= 8) - *cmd++ = MI_BATCH_BUFFER_START_GEN8; - else if (cache->gen >= 6) - *cmd++ = MI_BATCH_BUFFER_START; - else - *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; - *cmd++ = lower_32_bits(batch->node.start); - *cmd++ = upper_32_bits(batch->node.start); /* Always 0 for gen<8 */ - i915_gem_object_flush_map(cache->rq_vma->obj); - i915_gem_object_unpin_map(cache->rq_vma->obj); - cache->rq_vma = NULL; + /* + * This is a bit nasty, normally we keep objects locked until the end + * of execbuffer, but we already submit this, and have to unlock before + * dropping the reference. Fortunately we can only hold 1 pool node at + * a time, so this should be harmless. + */ + i915_gem_ww_unlock_single(cache->pool->obj); + intel_gt_buffer_pool_put(cache->pool); + cache->pool = NULL; +} - err = intel_gt_buffer_pool_mark_active(pool, rq); - if (err == 0) { - i915_vma_lock(batch); - err = i915_request_await_object(rq, batch->obj, false); - if (err == 0) - err = i915_vma_move_to_active(batch, rq, 0); - i915_vma_unlock(batch); - } - i915_vma_unpin(batch); - if (err) - goto out_pool; +static void reloc_gpu_flush(struct i915_execbuffer *eb, struct reloc_cache *cache) +{ + struct drm_i915_gem_object *obj = cache->rq->batch->obj; - cmd = i915_gem_object_pin_map(batch->obj, - cache->has_llc ? - I915_MAP_FORCE_WB : - I915_MAP_FORCE_WC); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto out_pool; - } + GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); + cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END; - /* Return with batch mapping (cmd) still pinned */ - cache->rq_cmd = cmd; - cache->rq_size = 0; - cache->rq_vma = batch; + __i915_gem_object_flush_map(obj, 0, sizeof(u32) * (cache->rq_size + 1)); + i915_gem_object_unpin_map(obj); -out_pool: - intel_gt_buffer_pool_put(pool); - return err; + intel_gt_chipset_flush(cache->rq->engine->gt); + + i915_request_add(cache->rq); + reloc_cache_put_pool(eb, cache); + reloc_cache_clear(cache); + + eb->reloc_pool = NULL; } -static unsigned int reloc_bb_flags(const struct reloc_cache *cache) +static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) { - return cache->gen > 5 ? 0 : I915_DISPATCH_SECURE; + void *vaddr; + + if (cache->rq) + reloc_gpu_flush(eb, cache); + + if (!cache->vaddr) + return; + + vaddr = unmask_page(cache->vaddr); + if (cache->vaddr & KMAP) { + struct drm_i915_gem_object *obj = + (struct drm_i915_gem_object *)cache->node.mm; + if (cache->vaddr & CLFLUSH_AFTER) + mb(); + + kunmap_atomic(vaddr); + i915_gem_object_finish_access(obj); + } else { + struct i915_ggtt *ggtt = cache_to_ggtt(cache); + + intel_gt_flush_ggtt_writes(ggtt->vm.gt); + io_mapping_unmap_atomic((void __iomem *)vaddr); + + if (drm_mm_node_allocated(&cache->node)) { + ggtt->vm.clear_range(&ggtt->vm, + cache->node.start, + cache->node.size); + mutex_lock(&ggtt->vm.mutex); + drm_mm_remove_node(&cache->node); + mutex_unlock(&ggtt->vm.mutex); + } else { + i915_vma_unpin((struct i915_vma *)cache->node.mm); + } + } + + cache->vaddr = 0; + cache->page = -1; } -static int reloc_gpu_flush(struct reloc_cache *cache) +static void *reloc_kmap(struct drm_i915_gem_object *obj, + struct reloc_cache *cache, + unsigned long pageno) { - struct i915_request *rq; - int err; + void *vaddr; + struct page *page; - rq = fetch_and_zero(&cache->rq); - if (!rq) - return 0; + if (cache->vaddr) { + kunmap_atomic(unmask_page(cache->vaddr)); + } else { + unsigned int flushes; + int err; - if (cache->rq_vma) { - struct drm_i915_gem_object *obj = cache->rq_vma->obj; + err = i915_gem_object_prepare_write(obj, &flushes); + if (err) + return ERR_PTR(err); - GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); - cache->rq_cmd[cache->rq_size++] = MI_BATCH_BUFFER_END; + BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS); + BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK); - __i915_gem_object_flush_map(obj, - 0, sizeof(u32) * cache->rq_size); - i915_gem_object_unpin_map(obj); + cache->vaddr = flushes | KMAP; + cache->node.mm = (void *)obj; + if (flushes) + mb(); } - err = 0; - if (rq->engine->emit_init_breadcrumb) - err = rq->engine->emit_init_breadcrumb(rq); - if (!err) - err = rq->engine->emit_bb_start(rq, - rq->batch->node.start, - PAGE_SIZE, - reloc_bb_flags(cache)); - if (err) - i915_request_set_error_once(rq, err); + page = i915_gem_object_get_page(obj, pageno); + if (!obj->mm.dirty) + set_page_dirty(page); - intel_gt_chipset_flush(rq->engine->gt); - i915_request_add(rq); + vaddr = kmap_atomic(page); + cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; + cache->page = pageno; - return err; + return vaddr; +} + +static void *reloc_iomap(struct drm_i915_gem_object *obj, + struct i915_execbuffer *eb, + unsigned long page) +{ + struct reloc_cache *cache = &eb->reloc_cache; + struct i915_ggtt *ggtt = cache_to_ggtt(cache); + unsigned long offset; + void *vaddr; + + if (cache->vaddr) { + intel_gt_flush_ggtt_writes(ggtt->vm.gt); + io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); + } else { + struct i915_vma *vma; + int err; + + if (i915_gem_object_is_tiled(obj)) + return ERR_PTR(-EINVAL); + + if (use_cpu_reloc(cache, obj)) + return NULL; + + err = i915_gem_object_set_to_gtt_domain(obj, true); + if (err) + return ERR_PTR(err); + + vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, + PIN_MAPPABLE | + PIN_NONBLOCK /* NOWARN */ | + PIN_NOEVICT); + if (vma == ERR_PTR(-EDEADLK)) + return vma; + + if (IS_ERR(vma)) { + memset(&cache->node, 0, sizeof(cache->node)); + mutex_lock(&ggtt->vm.mutex); + err = drm_mm_insert_node_in_range + (&ggtt->vm.mm, &cache->node, + PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, + 0, ggtt->mappable_end, + DRM_MM_INSERT_LOW); + mutex_unlock(&ggtt->vm.mutex); + if (err) /* no inactive aperture space, use cpu reloc */ + return NULL; + } else { + cache->node.start = vma->node.start; + cache->node.mm = (void *)vma; + } + } + + offset = cache->node.start; + if (drm_mm_node_allocated(&cache->node)) { + ggtt->vm.insert_page(&ggtt->vm, + i915_gem_object_get_dma_address(obj, page), + offset, I915_CACHE_NONE, 0); + } else { + offset += page << PAGE_SHIFT; + } + + vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, + offset); + cache->page = page; + cache->vaddr = (unsigned long)vaddr; + + return vaddr; +} + +static void *reloc_vaddr(struct drm_i915_gem_object *obj, + struct i915_execbuffer *eb, + unsigned long page) +{ + struct reloc_cache *cache = &eb->reloc_cache; + void *vaddr; + + if (cache->page == page) { + vaddr = unmask_page(cache->vaddr); + } else { + vaddr = NULL; + if ((cache->vaddr & KMAP) == 0) + vaddr = reloc_iomap(obj, eb, page); + if (!vaddr) + vaddr = reloc_kmap(obj, cache, page); + } + + return vaddr; +} + +static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) +{ + if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) { + if (flushes & CLFLUSH_BEFORE) { + clflushopt(addr); + mb(); + } + + *addr = value; + + /* + * Writes to the same cacheline are serialised by the CPU + * (including clflush). On the write path, we only require + * that it hits memory in an orderly fashion and place + * mb barriers at the start and end of the relocation phase + * to ensure ordering of clflush wrt to the system. + */ + if (flushes & CLFLUSH_AFTER) + clflushopt(addr); + } else + *addr = value; } static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) @@ -1054,7 +1247,7 @@ static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) struct drm_i915_gem_object *obj = vma->obj; int err; - i915_vma_lock(vma); + assert_vma_held(vma); if (obj->cache_dirty & ~obj->cache_coherent) i915_gem_clflush_object(obj, 0); @@ -1064,25 +1257,31 @@ static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); - return err; } static int __reloc_gpu_alloc(struct i915_execbuffer *eb, struct intel_engine_cs *engine, + struct i915_vma *vma, unsigned int len) { struct reloc_cache *cache = &eb->reloc_cache; - struct intel_gt_buffer_pool_node *pool; + struct intel_gt_buffer_pool_node *pool = eb->reloc_pool; struct i915_request *rq; struct i915_vma *batch; u32 *cmd; int err; - pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE); - if (IS_ERR(pool)) - return PTR_ERR(pool); + if (!pool) { + pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE); + if (IS_ERR(pool)) + return PTR_ERR(pool); + } + eb->reloc_pool = NULL; + + err = i915_gem_object_lock(pool->obj, &eb->ww); + if (err) + goto err_pool; cmd = i915_gem_object_pin_map(pool->obj, cache->has_llc ? @@ -1090,35 +1289,42 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, I915_MAP_FORCE_WC); if (IS_ERR(cmd)) { err = PTR_ERR(cmd); - goto out_pool; + goto err_pool; } - batch = i915_vma_instance(pool->obj, eb->context->vm, NULL); + batch = i915_vma_instance(pool->obj, vma->vm, NULL); if (IS_ERR(batch)) { err = PTR_ERR(batch); goto err_unmap; } - err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); + err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER | PIN_NONBLOCK); if (err) goto err_unmap; if (engine == eb->context->engine) { rq = i915_request_create(eb->context); } else { - struct intel_context *ce; + struct intel_context *ce = eb->reloc_context; - ce = intel_context_create(engine); - if (IS_ERR(ce)) { - err = PTR_ERR(ce); - goto err_unpin; + if (!ce) { + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto err_unpin; + } + + i915_vm_put(ce->vm); + ce->vm = i915_vm_get(eb->context->vm); + eb->reloc_context = ce; } - i915_vm_put(ce->vm); - ce->vm = i915_vm_get(eb->context->vm); + err = intel_context_pin_ww(ce, &eb->ww); + if (err) + goto err_unpin; - rq = intel_context_create_request(ce); - intel_context_put(ce); + rq = i915_request_create(ce); + intel_context_unpin(ce); } if (IS_ERR(rq)) { err = PTR_ERR(rq); @@ -1129,11 +1335,20 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, if (err) goto err_request; - i915_vma_lock(batch); + err = reloc_move_to_gpu(rq, vma); + if (err) + goto err_request; + + err = eb->engine->emit_bb_start(rq, + batch->node.start, PAGE_SIZE, + cache->gen > 5 ? 0 : I915_DISPATCH_SECURE); + if (err) + goto skip_request; + + assert_vma_held(batch); err = i915_request_await_object(rq, batch->obj, false); if (err == 0) err = i915_vma_move_to_active(batch, rq, 0); - i915_vma_unlock(batch); if (err) goto skip_request; @@ -1143,10 +1358,10 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, cache->rq = rq; cache->rq_cmd = cmd; cache->rq_size = 0; - cache->rq_vma = batch; + cache->pool = pool; /* Return with batch mapping (cmd) still pinned */ - goto out_pool; + return 0; skip_request: i915_request_set_error_once(rq, err); @@ -1156,8 +1371,8 @@ err_unpin: i915_vma_unpin(batch); err_unmap: i915_gem_object_unpin_map(pool->obj); -out_pool: - intel_gt_buffer_pool_put(pool); +err_pool: + eb->reloc_pool = pool; return err; } @@ -1172,9 +1387,12 @@ static u32 *reloc_gpu(struct i915_execbuffer *eb, { struct reloc_cache *cache = &eb->reloc_cache; u32 *cmd; - int err; + + if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1)) + reloc_gpu_flush(eb, cache); if (unlikely(!cache->rq)) { + int err; struct intel_engine_cs *engine = eb->engine; if (!reloc_can_use_engine(engine)) { @@ -1183,37 +1401,28 @@ static u32 *reloc_gpu(struct i915_execbuffer *eb, return ERR_PTR(-ENODEV); } - err = __reloc_gpu_alloc(eb, engine, len); + err = __reloc_gpu_alloc(eb, engine, vma, len); if (unlikely(err)) return ERR_PTR(err); } - if (vma != cache->target) { - err = reloc_move_to_gpu(cache->rq, vma); - if (unlikely(err)) { - i915_request_set_error_once(cache->rq, err); - return ERR_PTR(err); - } - - cache->target = vma; - } - - if (unlikely(cache->rq_size + len > - PAGE_SIZE / sizeof(u32) - RELOC_TAIL)) { - err = reloc_gpu_chain(cache); - if (unlikely(err)) { - i915_request_set_error_once(cache->rq, err); - return ERR_PTR(err); - } - } - - GEM_BUG_ON(cache->rq_size + len >= PAGE_SIZE / sizeof(u32)); cmd = cache->rq_cmd + cache->rq_size; cache->rq_size += len; return cmd; } +static inline bool use_reloc_gpu(struct i915_vma *vma) +{ + if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) + return true; + + if (DBG_FORCE_RELOC) + return false; + + return !dma_resv_test_signaled_rcu(vma->resv, true); +} + static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) { struct page *page; @@ -1229,9 +1438,9 @@ static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) } static int __reloc_entry_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - u64 offset, - u64 target_addr) + struct i915_vma *vma, + u64 offset, + u64 target_addr) { const unsigned int gen = eb->reloc_cache.gen; unsigned int len; @@ -1246,8 +1455,10 @@ static int __reloc_entry_gpu(struct i915_execbuffer *eb, len = 3; batch = reloc_gpu(eb, vma, len); - if (IS_ERR(batch)) - return PTR_ERR(batch); + if (batch == ERR_PTR(-EDEADLK)) + return -EDEADLK; + else if (IS_ERR(batch)) + return false; addr = gen8_canonical_addr(vma->node.start + offset); if (gen >= 8) { @@ -1296,21 +1507,58 @@ static int __reloc_entry_gpu(struct i915_execbuffer *eb, *batch++ = target_addr; } - return 0; + return true; +} + +static int reloc_entry_gpu(struct i915_execbuffer *eb, + struct i915_vma *vma, + u64 offset, + u64 target_addr) +{ + if (eb->reloc_cache.vaddr) + return false; + + if (!use_reloc_gpu(vma)) + return false; + + return __reloc_entry_gpu(eb, vma, offset, target_addr); } static u64 -relocate_entry(struct i915_execbuffer *eb, - struct i915_vma *vma, +relocate_entry(struct i915_vma *vma, const struct drm_i915_gem_relocation_entry *reloc, + struct i915_execbuffer *eb, const struct i915_vma *target) { u64 target_addr = relocation_target(reloc, target); - int err; - - err = __reloc_entry_gpu(eb, vma, reloc->offset, target_addr); - if (err) - return err; + u64 offset = reloc->offset; + int reloc_gpu = reloc_entry_gpu(eb, vma, offset, target_addr); + + if (reloc_gpu < 0) + return reloc_gpu; + + if (!reloc_gpu) { + bool wide = eb->reloc_cache.use_64bit_reloc; + void *vaddr; + +repeat: + vaddr = reloc_vaddr(vma->obj, eb, + offset >> PAGE_SHIFT); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); + clflush_write32(vaddr + offset_in_page(offset), + lower_32_bits(target_addr), + eb->reloc_cache.vaddr); + + if (wide) { + offset += sizeof(u32); + target_addr >>= 32; + wide = false; + goto repeat; + } + } return target->node.start | UPDATE; } @@ -1375,7 +1623,8 @@ eb_relocate_entry(struct i915_execbuffer *eb, * If the relocation already has the right value in it, no * more work needs to be done. */ - if (gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) + if (!DBG_FORCE_RELOC && + gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) return 0; /* Check that the relocation address is valid... */ @@ -1407,7 +1656,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, ev->flags &= ~EXEC_OBJECT_ASYNC; /* and update the user's relocation entry */ - return relocate_entry(eb, ev->vma, reloc, target->vma); + return relocate_entry(ev->vma, reloc, eb, target->vma); } static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) @@ -1444,9 +1693,13 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) * we would try to acquire the struct mutex again. Obviously * this is bad and so lockdep complains vehemently. */ - copied = __copy_from_user(r, urelocs, count * sizeof(r[0])); - if (unlikely(copied)) - return -EFAULT; + pagefault_disable(); + copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0])); + pagefault_enable(); + if (unlikely(copied)) { + remain = -EFAULT; + goto out; + } remain -= count; do { @@ -1454,7 +1707,8 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) if (likely(offset == 0)) { } else if ((s64)offset < 0) { - return (int)offset; + remain = (int)offset; + goto out; } else { /* * Note that reporting an error now @@ -1484,74 +1738,401 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) } while (r++, --count); urelocs += ARRAY_SIZE(stack); } while (remain); +out: + reloc_cache_reset(&eb->reloc_cache, eb); + return remain; +} - return 0; +static int +eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev) +{ + const struct drm_i915_gem_exec_object2 *entry = ev->exec; + struct drm_i915_gem_relocation_entry *relocs = + u64_to_ptr(typeof(*relocs), entry->relocs_ptr); + unsigned int i; + int err; + + for (i = 0; i < entry->relocation_count; i++) { + u64 offset = eb_relocate_entry(eb, ev, &relocs[i]); + + if ((s64)offset < 0) { + err = (int)offset; + goto err; + } + } + err = 0; +err: + reloc_cache_reset(&eb->reloc_cache, eb); + return err; +} + +static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) +{ + const char __user *addr, *end; + unsigned long size; + char __maybe_unused c; + + size = entry->relocation_count; + if (size == 0) + return 0; + + if (size > N_RELOC(ULONG_MAX)) + return -EINVAL; + + addr = u64_to_user_ptr(entry->relocs_ptr); + size *= sizeof(struct drm_i915_gem_relocation_entry); + if (!access_ok(addr, size)) + return -EFAULT; + + end = addr + size; + for (; addr < end; addr += PAGE_SIZE) { + int err = __get_user(c, addr); + if (err) + return err; + } + return __get_user(c, end - 1); } -static int eb_relocate(struct i915_execbuffer *eb) +static int eb_copy_relocations(const struct i915_execbuffer *eb) { + struct drm_i915_gem_relocation_entry *relocs; + const unsigned int count = eb->buffer_count; + unsigned int i; int err; - err = eb_lookup_vmas(eb); - if (err) - return err; + for (i = 0; i < count; i++) { + const unsigned int nreloc = eb->exec[i].relocation_count; + struct drm_i915_gem_relocation_entry __user *urelocs; + unsigned long size; + unsigned long copied; + + if (nreloc == 0) + continue; + + err = check_relocations(&eb->exec[i]); + if (err) + goto err; + + urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); + size = nreloc * sizeof(*relocs); + + relocs = kvmalloc_array(size, 1, GFP_KERNEL); + if (!relocs) { + err = -ENOMEM; + goto err; + } + + /* copy_from_user is limited to < 4GiB */ + copied = 0; + do { + unsigned int len = + min_t(u64, BIT_ULL(31), size - copied); + + if (__copy_from_user((char *)relocs + copied, + (char __user *)urelocs + copied, + len)) + goto end; + + copied += len; + } while (copied < size); + + /* + * As we do not update the known relocation offsets after + * relocating (due to the complexities in lock handling), + * we need to mark them as invalid now so that we force the + * relocation processing next time. Just in case the target + * object is evicted and then rebound into its old + * presumed_offset before the next execbuffer - if that + * happened we would make the mistake of assuming that the + * relocations were valid. + */ + if (!user_access_begin(urelocs, size)) + goto end; + + for (copied = 0; copied < nreloc; copied++) + unsafe_put_user(-1, + &urelocs[copied].presumed_offset, + end_user); + user_access_end(); + + eb->exec[i].relocs_ptr = (uintptr_t)relocs; + } + + return 0; + +end_user: + user_access_end(); +end: + kvfree(relocs); + err = -EFAULT; +err: + while (i--) { + relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); + if (eb->exec[i].relocation_count) + kvfree(relocs); + } + return err; +} + +static int eb_prefault_relocations(const struct i915_execbuffer *eb) +{ + const unsigned int count = eb->buffer_count; + unsigned int i; + + for (i = 0; i < count; i++) { + int err; - if (!list_empty(&eb->unbound)) { - err = eb_reserve(eb); + err = check_relocations(&eb->exec[i]); if (err) return err; } - /* The objects are in their final locations, apply the relocations. */ - if (eb->args->flags & __EXEC_HAS_RELOC) { - struct eb_vma *ev; - int flush; + return 0; +} - list_for_each_entry(ev, &eb->relocs, reloc_link) { +static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb, + struct i915_request *rq) +{ + bool have_copy = false; + struct eb_vma *ev; + int err = 0; + +repeat: + if (signal_pending(current)) { + err = -ERESTARTSYS; + goto out; + } + + /* We may process another execbuffer during the unlock... */ + eb_release_vmas(eb, false); + i915_gem_ww_ctx_fini(&eb->ww); + + if (rq) { + /* nonblocking is always false */ + if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT) < 0) { + i915_request_put(rq); + rq = NULL; + + err = -EINTR; + goto err_relock; + } + + i915_request_put(rq); + rq = NULL; + } + + /* + * We take 3 passes through the slowpatch. + * + * 1 - we try to just prefault all the user relocation entries and + * then attempt to reuse the atomic pagefault disabled fast path again. + * + * 2 - we copy the user entries to a local buffer here outside of the + * local and allow ourselves to wait upon any rendering before + * relocations + * + * 3 - we already have a local copy of the relocation entries, but + * were interrupted (EAGAIN) whilst waiting for the objects, try again. + */ + if (!err) { + err = eb_prefault_relocations(eb); + } else if (!have_copy) { + err = eb_copy_relocations(eb); + have_copy = err == 0; + } else { + cond_resched(); + err = 0; + } + + if (!err) + flush_workqueue(eb->i915->mm.userptr_wq); + +err_relock: + i915_gem_ww_ctx_init(&eb->ww, true); + if (err) + goto out; + + /* reacquire the objects */ +repeat_validate: + rq = eb_pin_engine(eb, false); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + rq = NULL; + goto err; + } + + /* We didn't throttle, should be NULL */ + GEM_WARN_ON(rq); + + err = eb_validate_vmas(eb); + if (err) + goto err; + + GEM_BUG_ON(!eb->batch); + + list_for_each_entry(ev, &eb->relocs, reloc_link) { + if (!have_copy) { + pagefault_disable(); err = eb_relocate_vma(eb, ev); + pagefault_enable(); + if (err) + break; + } else { + err = eb_relocate_vma_slow(eb, ev); if (err) break; } + } + + if (err == -EDEADLK) + goto err; + + if (err && !have_copy) + goto repeat; + + if (err) + goto err; - flush = reloc_gpu_flush(&eb->reloc_cache); + /* as last step, parse the command buffer */ + err = eb_parse(eb); + if (err) + goto err; + + /* + * Leave the user relocations as are, this is the painfully slow path, + * and we want to avoid the complication of dropping the lock whilst + * having buffers reserved in the aperture and so causing spurious + * ENOSPC for random operations. + */ + +err: + if (err == -EDEADLK) { + eb_release_vmas(eb, false); + err = i915_gem_ww_ctx_backoff(&eb->ww); if (!err) - err = flush; + goto repeat_validate; } + if (err == -EAGAIN) + goto repeat; + +out: + if (have_copy) { + const unsigned int count = eb->buffer_count; + unsigned int i; + + for (i = 0; i < count; i++) { + const struct drm_i915_gem_exec_object2 *entry = + &eb->exec[i]; + struct drm_i915_gem_relocation_entry *relocs; + + if (!entry->relocation_count) + continue; + + relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr); + kvfree(relocs); + } + } + + if (rq) + i915_request_put(rq); + return err; } -static int eb_move_to_gpu(struct i915_execbuffer *eb) +static int eb_relocate_parse(struct i915_execbuffer *eb) { - const unsigned int count = eb->buffer_count; - struct ww_acquire_ctx acquire; - unsigned int i; - int err = 0; + int err; + struct i915_request *rq = NULL; + bool throttle = true; - ww_acquire_init(&acquire, &reservation_ww_class); +retry: + rq = eb_pin_engine(eb, throttle); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + rq = NULL; + if (err != -EDEADLK) + return err; - for (i = 0; i < count; i++) { - struct eb_vma *ev = &eb->vma[i]; - struct i915_vma *vma = ev->vma; + goto err; + } + + if (rq) { + bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; + + /* Need to drop all locks now for throttling, take slowpath */ + err = i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, 0); + if (err == -ETIME) { + if (nonblock) { + err = -EWOULDBLOCK; + i915_request_put(rq); + goto err; + } + goto slow; + } + i915_request_put(rq); + rq = NULL; + } - err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire); - if (err == -EDEADLK) { - GEM_BUG_ON(i == 0); - do { - int j = i - 1; + /* only throttle once, even if we didn't need to throttle */ + throttle = false; - ww_mutex_unlock(&eb->vma[j].vma->resv->lock); + err = eb_validate_vmas(eb); + if (err == -EAGAIN) + goto slow; + else if (err) + goto err; - swap(eb->vma[i], eb->vma[j]); - } while (--i); + /* The objects are in their final locations, apply the relocations. */ + if (eb->args->flags & __EXEC_HAS_RELOC) { + struct eb_vma *ev; - err = ww_mutex_lock_slow_interruptible(&vma->resv->lock, - &acquire); + list_for_each_entry(ev, &eb->relocs, reloc_link) { + err = eb_relocate_vma(eb, ev); + if (err) + break; } - if (err) - break; + + if (err == -EDEADLK) + goto err; + else if (err) + goto slow; } - ww_acquire_done(&acquire); + + if (!err) + err = eb_parse(eb); + +err: + if (err == -EDEADLK) { + eb_release_vmas(eb, false); + err = i915_gem_ww_ctx_backoff(&eb->ww); + if (!err) + goto retry; + } + + return err; + +slow: + err = eb_relocate_parse_slow(eb, rq); + if (err) + /* + * If the user expects the execobject.offset and + * reloc.presumed_offset to be an exact match, + * as for using NO_RELOC, then we cannot update + * the execobject.offset until we have completed + * relocation. + */ + eb->args->flags &= ~__EXEC_HAS_RELOC; + + return err; +} + +static int eb_move_to_gpu(struct i915_execbuffer *eb) +{ + const unsigned int count = eb->buffer_count; + unsigned int i = count; + int err = 0; while (i--) { struct eb_vma *ev = &eb->vma[i]; @@ -1596,13 +2177,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) if (err == 0) err = i915_vma_move_to_active(vma, eb->request, flags); - - i915_vma_unlock(vma); - eb_unreserve_vma(ev); } - ww_acquire_fini(&acquire); - - eb_vma_array_put(fetch_and_zero(&eb->array)); if (unlikely(err)) goto err_skip; @@ -1622,7 +2197,8 @@ static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) return -EINVAL; /* Kernel clipping was a DRI1 misfeature */ - if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) { + if (!(exec->flags & (I915_EXEC_FENCE_ARRAY | + I915_EXEC_USE_EXTENSIONS))) { if (exec->num_cliprects || exec->cliprects_ptr) return -EINVAL; } @@ -1666,7 +2242,8 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq) } static struct i915_vma * -shadow_batch_pin(struct drm_i915_gem_object *obj, +shadow_batch_pin(struct i915_execbuffer *eb, + struct drm_i915_gem_object *obj, struct i915_address_space *vm, unsigned int flags) { @@ -1677,7 +2254,7 @@ shadow_batch_pin(struct drm_i915_gem_object *obj, if (IS_ERR(vma)) return vma; - err = i915_vma_pin(vma, 0, 0, flags); + err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); if (err) return ERR_PTR(err); @@ -1729,7 +2306,7 @@ __parser_mark_active(struct i915_vma *vma, { struct intel_gt_buffer_pool_node *node = vma->private; - return i915_active_ref(&node->active, tl, fence); + return i915_active_ref(&node->active, tl->fence_context, fence); } static int @@ -1793,36 +2370,26 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, if (err) goto err_commit; - err = dma_resv_lock_interruptible(pw->batch->resv, NULL); - if (err) - goto err_commit; - err = dma_resv_reserve_shared(pw->batch->resv, 1); if (err) - goto err_commit_unlock; + goto err_commit; /* Wait for all writes (and relocs) into the batch to complete */ err = i915_sw_fence_await_reservation(&pw->base.chain, pw->batch->resv, NULL, false, 0, I915_FENCE_GFP); if (err < 0) - goto err_commit_unlock; + goto err_commit; /* Keep the batch alive and unwritten as we parse */ dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma); - dma_resv_unlock(pw->batch->resv); - /* Force execution to wait for completion of the parser */ - dma_resv_lock(shadow->resv, NULL); dma_resv_add_excl_fence(shadow->resv, &pw->base.dma); - dma_resv_unlock(shadow->resv); dma_fence_work_commit_imm(&pw->base); return 0; -err_commit_unlock: - dma_resv_unlock(pw->batch->resv); err_commit: i915_sw_fence_set_error_once(&pw->base.chain, err); dma_fence_work_commit_imm(&pw->base); @@ -1837,16 +2404,33 @@ err_free: return err; } +static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma) +{ + /* + * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure + * batch" bit. Hence we need to pin secure batches into the global gtt. + * hsw should have this fixed, but bdw mucks it up again. */ + if (eb->batch_flags & I915_DISPATCH_SECURE) + return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); + + return NULL; +} + static int eb_parse(struct i915_execbuffer *eb) { struct drm_i915_private *i915 = eb->i915; - struct intel_gt_buffer_pool_node *pool; - struct i915_vma *shadow, *trampoline; + struct intel_gt_buffer_pool_node *pool = eb->batch_pool; + struct i915_vma *shadow, *trampoline, *batch; unsigned int len; int err; - if (!eb_use_cmdparser(eb)) - return 0; + if (!eb_use_cmdparser(eb)) { + batch = eb_dispatch_secure(eb, eb->batch->vma); + if (IS_ERR(batch)) + return PTR_ERR(batch); + + goto secure_batch; + } len = eb->batch_len; if (!CMDPARSER_USES_GGTT(eb->i915)) { @@ -1863,11 +2447,18 @@ static int eb_parse(struct i915_execbuffer *eb) len += I915_CMD_PARSER_TRAMPOLINE_SIZE; } - pool = intel_gt_get_buffer_pool(eb->engine->gt, len); - if (IS_ERR(pool)) - return PTR_ERR(pool); + if (!pool) { + pool = intel_gt_get_buffer_pool(eb->engine->gt, len); + if (IS_ERR(pool)) + return PTR_ERR(pool); + eb->batch_pool = pool; + } - shadow = shadow_batch_pin(pool->obj, eb->context->vm, PIN_USER); + err = i915_gem_object_lock(pool->obj, &eb->ww); + if (err) + goto err; + + shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); if (IS_ERR(shadow)) { err = PTR_ERR(shadow); goto err; @@ -1879,7 +2470,7 @@ static int eb_parse(struct i915_execbuffer *eb) if (CMDPARSER_USES_GGTT(eb->i915)) { trampoline = shadow; - shadow = shadow_batch_pin(pool->obj, + shadow = shadow_batch_pin(eb, pool->obj, &eb->engine->gt->ggtt->vm, PIN_GLOBAL); if (IS_ERR(shadow)) { @@ -1892,42 +2483,43 @@ static int eb_parse(struct i915_execbuffer *eb) eb->batch_flags |= I915_DISPATCH_SECURE; } + batch = eb_dispatch_secure(eb, shadow); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto err_trampoline; + } + err = eb_parse_pipeline(eb, shadow, trampoline); if (err) - goto err_trampoline; + goto err_unpin_batch; - eb->vma[eb->buffer_count].vma = i915_vma_get(shadow); - eb->vma[eb->buffer_count].flags = __EXEC_OBJECT_HAS_PIN; eb->batch = &eb->vma[eb->buffer_count++]; - eb->vma[eb->buffer_count].vma = NULL; + eb->batch->vma = i915_vma_get(shadow); + eb->batch->flags = __EXEC_OBJECT_HAS_PIN; eb->trampoline = trampoline; eb->batch_start_offset = 0; +secure_batch: + if (batch) { + eb->batch = &eb->vma[eb->buffer_count++]; + eb->batch->flags = __EXEC_OBJECT_HAS_PIN; + eb->batch->vma = i915_vma_get(batch); + } return 0; +err_unpin_batch: + if (batch) + i915_vma_unpin(batch); err_trampoline: if (trampoline) i915_vma_unpin(trampoline); err_shadow: i915_vma_unpin(shadow); err: - intel_gt_buffer_pool_put(pool); return err; } -static void -add_to_client(struct i915_request *rq, struct drm_file *file) -{ - struct drm_i915_file_private *file_priv = file->driver_priv; - - rq->file_priv = file_priv; - - spin_lock(&file_priv->mm.lock); - list_add_tail(&rq->client_link, &file_priv->mm.request_list); - spin_unlock(&file_priv->mm.lock); -} - static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) { int err; @@ -2009,7 +2601,7 @@ static const enum intel_engine_id user_ring_map[] = { [I915_EXEC_VEBOX] = VECS0 }; -static struct i915_request *eb_throttle(struct intel_context *ce) +static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce) { struct intel_ring *ring = ce->ring; struct intel_timeline *tl = ce->timeline; @@ -2043,31 +2635,26 @@ static struct i915_request *eb_throttle(struct intel_context *ce) return i915_request_get(rq); } -static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce) +static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool throttle) { + struct intel_context *ce = eb->context; struct intel_timeline *tl; - struct i915_request *rq; + struct i915_request *rq = NULL; int err; - /* - * ABI: Before userspace accesses the GPU (e.g. execbuffer), report - * EIO if the GPU is already wedged. - */ - err = intel_gt_terminally_wedged(ce->engine->gt); - if (err) - return err; + GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); if (unlikely(intel_context_is_banned(ce))) - return -EIO; + return ERR_PTR(-EIO); /* * Pinning the contexts may generate requests in order to acquire * GGTT space, so do this first before we reserve a seqno for * ourselves. */ - err = intel_context_pin(ce); + err = intel_context_pin_ww(ce, &eb->ww); if (err) - return err; + return ERR_PTR(err); /* * Take a local wakeref for preparing to dispatch the execbuf as @@ -2079,45 +2666,17 @@ static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce) */ tl = intel_context_timeline_lock(ce); if (IS_ERR(tl)) { - err = PTR_ERR(tl); - goto err_unpin; + intel_context_unpin(ce); + return ERR_CAST(tl); } intel_context_enter(ce); - rq = eb_throttle(ce); - + if (throttle) + rq = eb_throttle(eb, ce); intel_context_timeline_unlock(tl); - if (rq) { - bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; - long timeout; - - timeout = MAX_SCHEDULE_TIMEOUT; - if (nonblock) - timeout = 0; - - timeout = i915_request_wait(rq, - I915_WAIT_INTERRUPTIBLE, - timeout); - i915_request_put(rq); - - if (timeout < 0) { - err = nonblock ? -EWOULDBLOCK : timeout; - goto err_exit; - } - } - - eb->engine = ce->engine; - eb->context = ce; - return 0; - -err_exit: - mutex_lock(&tl->mutex); - intel_context_exit(ce); - intel_context_timeline_unlock(tl); -err_unpin: - intel_context_unpin(ce); - return err; + eb->args->flags |= __EXEC_ENGINE_PINNED; + return rq; } static void eb_unpin_engine(struct i915_execbuffer *eb) @@ -2125,6 +2684,11 @@ static void eb_unpin_engine(struct i915_execbuffer *eb) struct intel_context *ce = eb->context; struct intel_timeline *tl = ce->timeline; + if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) + return; + + eb->args->flags &= ~__EXEC_ENGINE_PINNED; + mutex_lock(&tl->mutex); intel_context_exit(ce); mutex_unlock(&tl->mutex); @@ -2133,11 +2697,10 @@ static void eb_unpin_engine(struct i915_execbuffer *eb) } static unsigned int -eb_select_legacy_ring(struct i915_execbuffer *eb, - struct drm_file *file, - struct drm_i915_gem_execbuffer2 *args) +eb_select_legacy_ring(struct i915_execbuffer *eb) { struct drm_i915_private *i915 = eb->i915; + struct drm_i915_gem_execbuffer2 *args = eb->args; unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK; if (user_ring_id != I915_EXEC_BSD && @@ -2152,7 +2715,7 @@ eb_select_legacy_ring(struct i915_execbuffer *eb, unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK; if (bsd_idx == I915_EXEC_BSD_DEFAULT) { - bsd_idx = gen8_dispatch_bsd_engine(i915, file); + bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); } else if (bsd_idx >= I915_EXEC_BSD_RING1 && bsd_idx <= I915_EXEC_BSD_RING2) { bsd_idx >>= I915_EXEC_BSD_SHIFT; @@ -2177,131 +2740,297 @@ eb_select_legacy_ring(struct i915_execbuffer *eb, } static int -eb_pin_engine(struct i915_execbuffer *eb, - struct drm_file *file, - struct drm_i915_gem_execbuffer2 *args) +eb_select_engine(struct i915_execbuffer *eb) { struct intel_context *ce; unsigned int idx; int err; if (i915_gem_context_user_engines(eb->gem_context)) - idx = args->flags & I915_EXEC_RING_MASK; + idx = eb->args->flags & I915_EXEC_RING_MASK; else - idx = eb_select_legacy_ring(eb, file, args); + idx = eb_select_legacy_ring(eb); ce = i915_gem_context_get_engine(eb->gem_context, idx); if (IS_ERR(ce)) return PTR_ERR(ce); - err = __eb_pin_engine(eb, ce); - intel_context_put(ce); + intel_gt_pm_get(ce->engine->gt); + + if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { + err = intel_context_alloc_state(ce); + if (err) + goto err; + } + + /* + * ABI: Before userspace accesses the GPU (e.g. execbuffer), report + * EIO if the GPU is already wedged. + */ + err = intel_gt_terminally_wedged(ce->engine->gt); + if (err) + goto err; + + eb->context = ce; + eb->engine = ce->engine; + + /* + * Make sure engine pool stays alive even if we call intel_context_put + * during ww handling. The pool is destroyed when last pm reference + * is dropped, which breaks our -EDEADLK handling. + */ + return err; +err: + intel_gt_pm_put(ce->engine->gt); + intel_context_put(ce); return err; } static void -__free_fence_array(struct drm_syncobj **fences, unsigned int n) +eb_put_engine(struct i915_execbuffer *eb) { - while (n--) - drm_syncobj_put(ptr_mask_bits(fences[n], 2)); + intel_gt_pm_put(eb->engine->gt); + intel_context_put(eb->context); +} + +static void +__free_fence_array(struct eb_fence *fences, unsigned int n) +{ + while (n--) { + drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2)); + dma_fence_put(fences[n].dma_fence); + kfree(fences[n].chain_fence); + } kvfree(fences); } -static struct drm_syncobj ** -get_fence_array(struct drm_i915_gem_execbuffer2 *args, - struct drm_file *file) +static int +add_timeline_fence_array(struct i915_execbuffer *eb, + const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences) { - const unsigned long nfences = args->num_cliprects; - struct drm_i915_gem_exec_fence __user *user; - struct drm_syncobj **fences; - unsigned long n; - int err; + struct drm_i915_gem_exec_fence __user *user_fences; + u64 __user *user_values; + struct eb_fence *f; + u64 nfences; + int err = 0; - if (!(args->flags & I915_EXEC_FENCE_ARRAY)) - return NULL; + nfences = timeline_fences->fence_count; + if (!nfences) + return 0; /* Check multiplication overflow for access_ok() and kvmalloc_array() */ BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); if (nfences > min_t(unsigned long, - ULONG_MAX / sizeof(*user), - SIZE_MAX / sizeof(*fences))) - return ERR_PTR(-EINVAL); + ULONG_MAX / sizeof(*user_fences), + SIZE_MAX / sizeof(*f)) - eb->num_fences) + return -EINVAL; - user = u64_to_user_ptr(args->cliprects_ptr); - if (!access_ok(user, nfences * sizeof(*user))) - return ERR_PTR(-EFAULT); + user_fences = u64_to_user_ptr(timeline_fences->handles_ptr); + if (!access_ok(user_fences, nfences * sizeof(*user_fences))) + return -EFAULT; + + user_values = u64_to_user_ptr(timeline_fences->values_ptr); + if (!access_ok(user_values, nfences * sizeof(*user_values))) + return -EFAULT; + + f = krealloc(eb->fences, + (eb->num_fences + nfences) * sizeof(*f), + __GFP_NOWARN | GFP_KERNEL); + if (!f) + return -ENOMEM; - fences = kvmalloc_array(nfences, sizeof(*fences), - __GFP_NOWARN | GFP_KERNEL); - if (!fences) - return ERR_PTR(-ENOMEM); + eb->fences = f; + f += eb->num_fences; - for (n = 0; n < nfences; n++) { - struct drm_i915_gem_exec_fence fence; + BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & + ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); + + while (nfences--) { + struct drm_i915_gem_exec_fence user_fence; struct drm_syncobj *syncobj; + struct dma_fence *fence = NULL; + u64 point; - if (__copy_from_user(&fence, user++, sizeof(fence))) { - err = -EFAULT; - goto err; + if (__copy_from_user(&user_fence, + user_fences++, + sizeof(user_fence))) + return -EFAULT; + + if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) + return -EINVAL; + + if (__get_user(point, user_values++)) + return -EFAULT; + + syncobj = drm_syncobj_find(eb->file, user_fence.handle); + if (!syncobj) { + DRM_DEBUG("Invalid syncobj handle provided\n"); + return -ENOENT; } - if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) { - err = -EINVAL; - goto err; + fence = drm_syncobj_fence_get(syncobj); + + if (!fence && user_fence.flags && + !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { + DRM_DEBUG("Syncobj handle has no fence\n"); + drm_syncobj_put(syncobj); + return -EINVAL; + } + + if (fence) + err = dma_fence_chain_find_seqno(&fence, point); + + if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { + DRM_DEBUG("Syncobj handle missing requested point %llu\n", point); + dma_fence_put(fence); + drm_syncobj_put(syncobj); + return err; + } + + /* + * A point might have been signaled already and + * garbage collected from the timeline. In this case + * just ignore the point and carry on. + */ + if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { + drm_syncobj_put(syncobj); + continue; + } + + /* + * For timeline syncobjs we need to preallocate chains for + * later signaling. + */ + if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) { + /* + * Waiting and signaling the same point (when point != + * 0) would break the timeline. + */ + if (user_fence.flags & I915_EXEC_FENCE_WAIT) { + DRM_DEBUG("Trying to wait & signal the same timeline point.\n"); + dma_fence_put(fence); + drm_syncobj_put(syncobj); + return -EINVAL; + } + + f->chain_fence = + kmalloc(sizeof(*f->chain_fence), + GFP_KERNEL); + if (!f->chain_fence) { + drm_syncobj_put(syncobj); + dma_fence_put(fence); + return -ENOMEM; + } + } else { + f->chain_fence = NULL; } - syncobj = drm_syncobj_find(file, fence.handle); + f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); + f->dma_fence = fence; + f->value = point; + f++; + eb->num_fences++; + } + + return 0; +} + +static int add_fence_array(struct i915_execbuffer *eb) +{ + struct drm_i915_gem_execbuffer2 *args = eb->args; + struct drm_i915_gem_exec_fence __user *user; + unsigned long num_fences = args->num_cliprects; + struct eb_fence *f; + + if (!(args->flags & I915_EXEC_FENCE_ARRAY)) + return 0; + + if (!num_fences) + return 0; + + /* Check multiplication overflow for access_ok() and kvmalloc_array() */ + BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); + if (num_fences > min_t(unsigned long, + ULONG_MAX / sizeof(*user), + SIZE_MAX / sizeof(*f) - eb->num_fences)) + return -EINVAL; + + user = u64_to_user_ptr(args->cliprects_ptr); + if (!access_ok(user, num_fences * sizeof(*user))) + return -EFAULT; + + f = krealloc(eb->fences, + (eb->num_fences + num_fences) * sizeof(*f), + __GFP_NOWARN | GFP_KERNEL); + if (!f) + return -ENOMEM; + + eb->fences = f; + f += eb->num_fences; + while (num_fences--) { + struct drm_i915_gem_exec_fence user_fence; + struct drm_syncobj *syncobj; + struct dma_fence *fence = NULL; + + if (__copy_from_user(&user_fence, user++, sizeof(user_fence))) + return -EFAULT; + + if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) + return -EINVAL; + + syncobj = drm_syncobj_find(eb->file, user_fence.handle); if (!syncobj) { DRM_DEBUG("Invalid syncobj handle provided\n"); - err = -ENOENT; - goto err; + return -ENOENT; + } + + if (user_fence.flags & I915_EXEC_FENCE_WAIT) { + fence = drm_syncobj_fence_get(syncobj); + if (!fence) { + DRM_DEBUG("Syncobj handle has no fence\n"); + drm_syncobj_put(syncobj); + return -EINVAL; + } } BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); - fences[n] = ptr_pack_bits(syncobj, fence.flags, 2); + f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); + f->dma_fence = fence; + f->value = 0; + f->chain_fence = NULL; + f++; + eb->num_fences++; } - return fences; - -err: - __free_fence_array(fences, n); - return ERR_PTR(err); + return 0; } -static void -put_fence_array(struct drm_i915_gem_execbuffer2 *args, - struct drm_syncobj **fences) +static void put_fence_array(struct eb_fence *fences, int num_fences) { if (fences) - __free_fence_array(fences, args->num_cliprects); + __free_fence_array(fences, num_fences); } static int -await_fence_array(struct i915_execbuffer *eb, - struct drm_syncobj **fences) +await_fence_array(struct i915_execbuffer *eb) { - const unsigned int nfences = eb->args->num_cliprects; unsigned int n; int err; - for (n = 0; n < nfences; n++) { + for (n = 0; n < eb->num_fences; n++) { struct drm_syncobj *syncobj; - struct dma_fence *fence; unsigned int flags; - syncobj = ptr_unpack_bits(fences[n], &flags, 2); - if (!(flags & I915_EXEC_FENCE_WAIT)) - continue; + syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); - fence = drm_syncobj_fence_get(syncobj); - if (!fence) - return -EINVAL; + if (!eb->fences[n].dma_fence) + continue; - err = i915_request_await_dma_fence(eb->request, fence); - dma_fence_put(fence); + err = i915_request_await_dma_fence(eb->request, + eb->fences[n].dma_fence); if (err < 0) return err; } @@ -2309,26 +3038,47 @@ await_fence_array(struct i915_execbuffer *eb, return 0; } -static void -signal_fence_array(struct i915_execbuffer *eb, - struct drm_syncobj **fences) +static void signal_fence_array(const struct i915_execbuffer *eb) { - const unsigned int nfences = eb->args->num_cliprects; struct dma_fence * const fence = &eb->request->fence; unsigned int n; - for (n = 0; n < nfences; n++) { + for (n = 0; n < eb->num_fences; n++) { struct drm_syncobj *syncobj; unsigned int flags; - syncobj = ptr_unpack_bits(fences[n], &flags, 2); + syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); if (!(flags & I915_EXEC_FENCE_SIGNAL)) continue; - drm_syncobj_replace_fence(syncobj, fence); + if (eb->fences[n].chain_fence) { + drm_syncobj_add_point(syncobj, + eb->fences[n].chain_fence, + fence, + eb->fences[n].value); + /* + * The chain's ownership is transferred to the + * timeline. + */ + eb->fences[n].chain_fence = NULL; + } else { + drm_syncobj_replace_fence(syncobj, fence); + } } } +static int +parse_timeline_fences(struct i915_user_extension __user *ext, void *data) +{ + struct i915_execbuffer *eb = data; + struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences; + + if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences))) + return -EFAULT; + + return add_timeline_fence_array(eb, &timeline_fences); +} + static void retire_requests(struct intel_timeline *tl, struct i915_request *end) { struct i915_request *rq, *rn; @@ -2370,12 +3120,37 @@ static void eb_request_add(struct i915_execbuffer *eb) mutex_unlock(&tl->mutex); } +static const i915_user_extension_fn execbuf_extensions[] = { + [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences, +}; + +static int +parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args, + struct i915_execbuffer *eb) +{ + if (!(args->flags & I915_EXEC_USE_EXTENSIONS)) + return 0; + + /* The execbuf2 extension mechanism reuses cliprects_ptr. So we cannot + * have another flag also using it at the same time. + */ + if (eb->args->flags & I915_EXEC_FENCE_ARRAY) + return -EINVAL; + + if (args->num_cliprects != 0) + return -EINVAL; + + return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr), + execbuf_extensions, + ARRAY_SIZE(execbuf_extensions), + eb); +} + static int i915_gem_do_execbuffer(struct drm_device *dev, struct drm_file *file, struct drm_i915_gem_execbuffer2 *args, - struct drm_i915_gem_exec_object2 *exec, - struct drm_syncobj **fences) + struct drm_i915_gem_exec_object2 *exec) { struct drm_i915_private *i915 = to_i915(dev); struct i915_execbuffer eb; @@ -2392,10 +3167,14 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.i915 = i915; eb.file = file; eb.args = args; - if (!(args->flags & I915_EXEC_NO_RELOC)) + if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) args->flags |= __EXEC_HAS_RELOC; eb.exec = exec; + eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); + eb.vma[0].vma = NULL; + eb.reloc_pool = eb.batch_pool = NULL; + eb.reloc_context = NULL; eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; reloc_cache_init(&eb.reloc_cache, eb.i915); @@ -2405,6 +3184,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.batch_len = args->batch_len; eb.trampoline = NULL; + eb.fences = NULL; + eb.num_fences = 0; + eb.batch_flags = 0; if (args->flags & I915_EXEC_SECURE) { if (INTEL_GEN(i915) >= 11) @@ -2422,14 +3204,24 @@ i915_gem_do_execbuffer(struct drm_device *dev, if (args->flags & I915_EXEC_IS_PINNED) eb.batch_flags |= I915_DISPATCH_PINNED; + err = parse_execbuf2_extensions(args, &eb); + if (err) + goto err_ext; + + err = add_fence_array(&eb); + if (err) + goto err_ext; + #define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT) if (args->flags & IN_FENCES) { if ((args->flags & IN_FENCES) == IN_FENCES) return -EINVAL; in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); - if (!in_fence) - return -EINVAL; + if (!in_fence) { + err = -EINVAL; + goto err_ext; + } } #undef IN_FENCES @@ -2451,11 +3243,19 @@ i915_gem_do_execbuffer(struct drm_device *dev, if (unlikely(err)) goto err_destroy; - err = eb_pin_engine(&eb, file, args); + err = eb_select_engine(&eb); if (unlikely(err)) goto err_context; - err = eb_relocate(&eb); + err = eb_lookup_vmas(&eb); + if (err) { + eb_release_vmas(&eb, true); + goto err_engine; + } + + i915_gem_ww_ctx_init(&eb.ww, true); + + err = eb_relocate_parse(&eb); if (err) { /* * If the user expects the execobject.offset and @@ -2468,54 +3268,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, goto err_vma; } - if (unlikely(eb.batch->flags & EXEC_OBJECT_WRITE)) { - drm_dbg(&i915->drm, - "Attempting to use self-modifying batch buffer\n"); - err = -EINVAL; - goto err_vma; - } - - if (range_overflows_t(u64, - eb.batch_start_offset, eb.batch_len, - eb.batch->vma->size)) { - drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); - err = -EINVAL; - goto err_vma; - } - - if (eb.batch_len == 0) - eb.batch_len = eb.batch->vma->size - eb.batch_start_offset; + ww_acquire_done(&eb.ww.ctx); - err = eb_parse(&eb); - if (err) - goto err_vma; - - /* - * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure - * batch" bit. Hence we need to pin secure batches into the global gtt. - * hsw should have this fixed, but bdw mucks it up again. */ batch = eb.batch->vma; - if (eb.batch_flags & I915_DISPATCH_SECURE) { - struct i915_vma *vma; - - /* - * So on first glance it looks freaky that we pin the batch here - * outside of the reservation loop. But: - * - The batch is already pinned into the relevant ppgtt, so we - * already have the backing storage fully allocated. - * - No other BO uses the global gtt (well contexts, but meh), - * so we don't really have issues with multiple objects not - * fitting due to fragmentation. - * So this is actually safe. - */ - vma = i915_gem_object_ggtt_pin(batch->obj, NULL, 0, 0, 0); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_parse; - } - - batch = vma; - } /* All GPU relocation batches must be submitted prior to the user rq */ GEM_BUG_ON(eb.reloc_cache.rq); @@ -2524,7 +3279,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.request = i915_request_create(eb.context); if (IS_ERR(eb.request)) { err = PTR_ERR(eb.request); - goto err_batch_unpin; + goto err_vma; } if (in_fence) { @@ -2539,8 +3294,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, goto err_request; } - if (fences) { - err = await_fence_array(&eb, fences); + if (eb.fences) { + err = await_fence_array(&eb); if (err) goto err_request; } @@ -2561,18 +3316,17 @@ i915_gem_do_execbuffer(struct drm_device *dev, * to explicitly hold another reference here. */ eb.request->batch = batch; - if (batch->private) - intel_gt_buffer_pool_mark_active(batch->private, eb.request); + if (eb.batch_pool) + intel_gt_buffer_pool_mark_active(eb.batch_pool, eb.request); trace_i915_request_queue(eb.request, eb.batch_flags); err = eb_submit(&eb, batch); err_request: - add_to_client(eb.request, file); i915_request_get(eb.request); eb_request_add(&eb); - if (fences) - signal_fence_array(&eb, fences); + if (eb.fences) + signal_fence_array(&eb); if (out_fence) { if (err == 0) { @@ -2586,16 +3340,21 @@ err_request: } i915_request_put(eb.request); -err_batch_unpin: - if (eb.batch_flags & I915_DISPATCH_SECURE) - i915_vma_unpin(batch); -err_parse: - if (batch->private) - intel_gt_buffer_pool_put(batch->private); err_vma: + eb_release_vmas(&eb, true); if (eb.trampoline) i915_vma_unpin(eb.trampoline); - eb_unpin_engine(&eb); + WARN_ON(err == -EDEADLK); + i915_gem_ww_ctx_fini(&eb.ww); + + if (eb.batch_pool) + intel_gt_buffer_pool_put(eb.batch_pool); + if (eb.reloc_pool) + intel_gt_buffer_pool_put(eb.reloc_pool); + if (eb.reloc_context) + intel_context_put(eb.reloc_context); +err_engine: + eb_put_engine(&eb); err_context: i915_gem_context_put(eb.gem_context); err_destroy: @@ -2605,12 +3364,14 @@ err_out_fence: put_unused_fd(out_fence_fd); err_in_fence: dma_fence_put(in_fence); +err_ext: + put_fence_array(eb.fences, eb.num_fences); return err; } static size_t eb_element_size(void) { - return sizeof(struct drm_i915_gem_exec_object2); + return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma); } static bool check_buffer_count(size_t count) @@ -2666,7 +3427,9 @@ i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, /* Copy in the exec list from userland */ exec_list = kvmalloc_array(count, sizeof(*exec_list), __GFP_NOWARN | GFP_KERNEL); - exec2_list = kvmalloc_array(count, eb_element_size(), + + /* Allocate extra slots for use by the command parser */ + exec2_list = kvmalloc_array(count + 2, eb_element_size(), __GFP_NOWARN | GFP_KERNEL); if (exec_list == NULL || exec2_list == NULL) { drm_dbg(&i915->drm, @@ -2699,7 +3462,7 @@ i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, exec2_list[i].flags = 0; } - err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL); + err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list); if (exec2.flags & __EXEC_HAS_RELOC) { struct drm_i915_gem_exec_object __user *user_exec_list = u64_to_user_ptr(args->buffers_ptr); @@ -2731,7 +3494,6 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_execbuffer2 *args = data; struct drm_i915_gem_exec_object2 *exec2_list; - struct drm_syncobj **fences = NULL; const size_t count = args->buffer_count; int err; @@ -2744,7 +3506,8 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, if (err) return err; - exec2_list = kvmalloc_array(count, eb_element_size(), + /* Allocate extra slots for use by the command parser */ + exec2_list = kvmalloc_array(count + 2, eb_element_size(), __GFP_NOWARN | GFP_KERNEL); if (exec2_list == NULL) { drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n", @@ -2759,15 +3522,7 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, return -EFAULT; } - if (args->flags & I915_EXEC_FENCE_ARRAY) { - fences = get_fence_array(args, file); - if (IS_ERR(fences)) { - kvfree(exec2_list); - return PTR_ERR(fences); - } - } - - err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences); + err = i915_gem_do_execbuffer(dev, file, args, exec2_list); /* * Now that we have begun execution of the batchbuffer, we ignore @@ -2808,7 +3563,6 @@ end:; } args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS; - put_fence_array(args, fences); kvfree(exec2_list); return err; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index b23368529a40..3d69e51f3e4d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -209,7 +209,7 @@ static vm_fault_t i915_error_to_vmf_fault(int err) switch (err) { default: WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); - /* fallthrough */ + fallthrough; case -EIO: /* shmemfs failure from swap device */ case -EFAULT: /* purged object */ case -ENODEV: /* bad object, how did you get here! */ @@ -283,37 +283,46 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) struct intel_runtime_pm *rpm = &i915->runtime_pm; struct i915_ggtt *ggtt = &i915->ggtt; bool write = area->vm_flags & VM_WRITE; + struct i915_gem_ww_ctx ww; intel_wakeref_t wakeref; struct i915_vma *vma; pgoff_t page_offset; int srcu; int ret; - /* Sanity check that we allow writing into this object */ - if (i915_gem_object_is_readonly(obj) && write) - return VM_FAULT_SIGBUS; - /* We don't use vmf->pgoff since that has the fake offset */ page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; trace_i915_gem_object_fault(obj, page_offset, true, write); - ret = i915_gem_object_pin_pages(obj); + wakeref = intel_runtime_pm_get(rpm); + + i915_gem_ww_ctx_init(&ww, true); +retry: + ret = i915_gem_object_lock(obj, &ww); if (ret) - goto err; + goto err_rpm; - wakeref = intel_runtime_pm_get(rpm); + /* Sanity check that we allow writing into this object */ + if (i915_gem_object_is_readonly(obj) && write) { + ret = -EFAULT; + goto err_rpm; + } - ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); + ret = i915_gem_object_pin_pages(obj); if (ret) goto err_rpm; + ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); + if (ret) + goto err_pages; + /* Now pin it into the GTT as needed */ - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, - PIN_MAPPABLE | - PIN_NONBLOCK /* NOWARN */ | - PIN_NOEVICT); - if (IS_ERR(vma)) { + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0, + PIN_MAPPABLE | + PIN_NONBLOCK /* NOWARN */ | + PIN_NOEVICT); + if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { /* Use a partial view if it is bigger than available space */ struct i915_ggtt_view view = compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); @@ -328,11 +337,11 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) * all hope that the hardware is able to track future writes. */ - vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); - if (IS_ERR(vma)) { + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); + if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { flags = PIN_MAPPABLE; view.type = I915_GGTT_VIEW_PARTIAL; - vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); } /* The entire mappable GGTT is pinned? Unexpected! */ @@ -389,10 +398,16 @@ err_unpin: __i915_vma_unpin(vma); err_reset: intel_gt_reset_unlock(ggtt->vm.gt, srcu); +err_pages: + i915_gem_object_unpin_pages(obj); err_rpm: + if (ret == -EDEADLK) { + ret = i915_gem_ww_ctx_backoff(&ww); + if (!ret) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); intel_runtime_pm_put(rpm, wakeref); - i915_gem_object_unpin_pages(obj); -err: return i915_error_to_vmf_fault(ret); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index e5b9276d254c..d46db8d8f38e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -110,20 +110,44 @@ i915_gem_object_put(struct drm_i915_gem_object *obj) #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) -static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj) +static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + bool intr) { - dma_resv_lock(obj->base.resv, NULL); + int ret; + + if (intr) + ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); + else + ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); + + if (!ret && ww) + list_add_tail(&obj->obj_link, &ww->obj_list); + if (ret == -EALREADY) + ret = 0; + + if (ret == -EDEADLK) + ww->contended = obj; + + return ret; } -static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj) +static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww) { - return dma_resv_trylock(obj->base.resv); + return __i915_gem_object_lock(obj, ww, ww && ww->intr); } -static inline int -i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj) +static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww) { - return dma_resv_lock_interruptible(obj->base.resv, NULL); + WARN_ON(ww && !ww->intr); + return __i915_gem_object_lock(obj, ww, true); +} + +static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj) +{ + return dma_resv_trylock(obj->base.resv); } static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) @@ -258,6 +282,10 @@ struct page * i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n); +struct page * +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n); + dma_addr_t i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, unsigned long n, @@ -408,7 +436,6 @@ static inline void i915_gem_object_finish_access(struct drm_i915_gem_object *obj) { i915_gem_object_unpin_pages(obj); - i915_gem_object_unlock(obj); } static inline struct intel_engine_cs * @@ -431,6 +458,7 @@ i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, unsigned int cache_level); void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); +void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); int __must_check i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c index bfdb32d46877..d93eb36160c9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c @@ -14,6 +14,7 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, struct i915_vma *vma, + struct i915_gem_ww_ctx *ww, u32 value) { struct drm_i915_private *i915 = ce->vm->i915; @@ -39,10 +40,24 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, goto out_pm; } + err = i915_gem_object_lock(pool->obj, ww); + if (err) + goto out_put; + + batch = i915_vma_instance(pool->obj, ce->vm, NULL); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto out_put; + } + + err = i915_vma_pin_ww(batch, ww, 0, 0, PIN_USER); + if (unlikely(err)) + goto out_put; + cmd = i915_gem_object_pin_map(pool->obj, I915_MAP_WC); if (IS_ERR(cmd)) { err = PTR_ERR(cmd); - goto out_put; + goto out_unpin; } rem = vma->size; @@ -84,19 +99,11 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, intel_gt_chipset_flush(ce->vm->gt); - batch = i915_vma_instance(pool->obj, ce->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put; - } - - err = i915_vma_pin(batch, 0, 0, PIN_USER); - if (unlikely(err)) - goto out_put; - batch->private = pool; return batch; +out_unpin: + i915_vma_unpin(batch); out_put: intel_gt_buffer_pool_put(pool); out_pm: @@ -108,11 +115,9 @@ int intel_emit_vma_mark_active(struct i915_vma *vma, struct i915_request *rq) { int err; - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, false); if (err == 0) err = i915_vma_move_to_active(vma, rq, 0); - i915_vma_unlock(vma); if (unlikely(err)) return err; @@ -141,6 +146,7 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, struct intel_context *ce, u32 value) { + struct i915_gem_ww_ctx ww; struct i915_request *rq; struct i915_vma *batch; struct i915_vma *vma; @@ -150,17 +156,28 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, if (IS_ERR(vma)) return PTR_ERR(vma); - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (unlikely(err)) - return err; + i915_gem_ww_ctx_init(&ww, true); + intel_engine_pm_get(ce->engine); +retry: + err = i915_gem_object_lock(obj, &ww); + if (err) + goto out; - batch = intel_emit_vma_fill_blt(ce, vma, value); + err = intel_context_pin_ww(ce, &ww); + if (err) + goto out; + + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); + if (err) + goto out_ctx; + + batch = intel_emit_vma_fill_blt(ce, vma, &ww, value); if (IS_ERR(batch)) { err = PTR_ERR(batch); - goto out_unpin; + goto out_vma; } - rq = intel_context_create_request(ce); + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out_batch; @@ -170,11 +187,9 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, if (unlikely(err)) goto out_request; - i915_vma_lock(vma); err = move_obj_to_gpu(vma->obj, rq, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); if (unlikely(err)) goto out_request; @@ -193,8 +208,18 @@ out_request: i915_request_add(rq); out_batch: intel_emit_vma_release(ce, batch); -out_unpin: +out_vma: i915_vma_unpin(vma); +out_ctx: + intel_context_unpin(ce); +out: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + intel_engine_pm_put(ce->engine); return err; } @@ -210,6 +235,7 @@ static bool wa_1209644611_applies(struct drm_i915_private *i915, u32 size) } struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, struct i915_vma *src, struct i915_vma *dst) { @@ -236,10 +262,24 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, goto out_pm; } + err = i915_gem_object_lock(pool->obj, ww); + if (err) + goto out_put; + + batch = i915_vma_instance(pool->obj, ce->vm, NULL); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto out_put; + } + + err = i915_vma_pin_ww(batch, ww, 0, 0, PIN_USER); + if (unlikely(err)) + goto out_put; + cmd = i915_gem_object_pin_map(pool->obj, I915_MAP_WC); if (IS_ERR(cmd)) { err = PTR_ERR(cmd); - goto out_put; + goto out_unpin; } rem = src->size; @@ -296,20 +336,11 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, i915_gem_object_unpin_map(pool->obj); intel_gt_chipset_flush(ce->vm->gt); - - batch = i915_vma_instance(pool->obj, ce->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put; - } - - err = i915_vma_pin(batch, 0, 0, PIN_USER); - if (unlikely(err)) - goto out_put; - batch->private = pool; return batch; +out_unpin: + i915_vma_unpin(batch); out_put: intel_gt_buffer_pool_put(pool); out_pm: @@ -321,10 +352,9 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, struct drm_i915_gem_object *dst, struct intel_context *ce) { - struct drm_gem_object *objs[] = { &src->base, &dst->base }; struct i915_address_space *vm = ce->vm; struct i915_vma *vma[2], *batch; - struct ww_acquire_ctx acquire; + struct i915_gem_ww_ctx ww; struct i915_request *rq; int err, i; @@ -332,25 +362,36 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, if (IS_ERR(vma[0])) return PTR_ERR(vma[0]); - err = i915_vma_pin(vma[0], 0, 0, PIN_USER); - if (unlikely(err)) - return err; - vma[1] = i915_vma_instance(dst, vm, NULL); if (IS_ERR(vma[1])) - goto out_unpin_src; + return PTR_ERR(vma); - err = i915_vma_pin(vma[1], 0, 0, PIN_USER); + i915_gem_ww_ctx_init(&ww, true); + intel_engine_pm_get(ce->engine); +retry: + err = i915_gem_object_lock(src, &ww); + if (!err) + err = i915_gem_object_lock(dst, &ww); + if (!err) + err = intel_context_pin_ww(ce, &ww); + if (err) + goto out; + + err = i915_vma_pin_ww(vma[0], &ww, 0, 0, PIN_USER); + if (err) + goto out_ctx; + + err = i915_vma_pin_ww(vma[1], &ww, 0, 0, PIN_USER); if (unlikely(err)) goto out_unpin_src; - batch = intel_emit_vma_copy_blt(ce, vma[0], vma[1]); + batch = intel_emit_vma_copy_blt(ce, &ww, vma[0], vma[1]); if (IS_ERR(batch)) { err = PTR_ERR(batch); goto out_unpin_dst; } - rq = intel_context_create_request(ce); + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out_batch; @@ -360,14 +401,10 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, if (unlikely(err)) goto out_request; - err = drm_gem_lock_reservations(objs, ARRAY_SIZE(objs), &acquire); - if (unlikely(err)) - goto out_request; - for (i = 0; i < ARRAY_SIZE(vma); i++) { err = move_obj_to_gpu(vma[i]->obj, rq, i); if (unlikely(err)) - goto out_unlock; + goto out_request; } for (i = 0; i < ARRAY_SIZE(vma); i++) { @@ -375,20 +412,19 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, err = i915_vma_move_to_active(vma[i], rq, flags); if (unlikely(err)) - goto out_unlock; + goto out_request; } if (rq->engine->emit_init_breadcrumb) { err = rq->engine->emit_init_breadcrumb(rq); if (unlikely(err)) - goto out_unlock; + goto out_request; } err = rq->engine->emit_bb_start(rq, batch->node.start, batch->node.size, 0); -out_unlock: - drm_gem_unlock_reservations(objs, ARRAY_SIZE(objs), &acquire); + out_request: if (unlikely(err)) i915_request_set_error_once(rq, err); @@ -400,6 +436,16 @@ out_unpin_dst: i915_vma_unpin(vma[1]); out_unpin_src: i915_vma_unpin(vma[0]); +out_ctx: + intel_context_unpin(ce); +out: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + intel_engine_pm_put(ce->engine); return err; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h index 8bcd336a90dc..2409fdcccf0e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h @@ -13,12 +13,15 @@ #include "i915_vma.h" struct drm_i915_gem_object; +struct i915_gem_ww_ctx; struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, struct i915_vma *vma, + struct i915_gem_ww_ctx *ww, u32 value); struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, struct i915_vma *src, struct i915_vma *dst); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 5335f799b548..b5c15557cc87 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -123,6 +123,15 @@ struct drm_i915_gem_object { struct list_head lut_list; spinlock_t lut_lock; /* guards lut_list */ + /** + * @obj_link: Link into @i915_gem_ww_ctx.obj_list + * + * When we lock this object through i915_gem_object_lock() with a + * context, we add it to the list to ensure we can unlock everything + * when i915_gem_ww_ctx_backoff() or i915_gem_ww_ctx_fini() are called. + */ + struct list_head obj_link; + /** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen; union { @@ -282,6 +291,7 @@ struct drm_i915_gem_object { } userptr; unsigned long scratch; + u64 encode; void *gvt_info; }; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 7050519c87a4..e8a083743e09 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -276,7 +276,7 @@ static void *i915_gem_object_map(struct drm_i915_gem_object *obj, switch (type) { default: MISSING_CASE(type); - /* fallthrough - to use PAGE_KERNEL anyway */ + fallthrough; /* to use PAGE_KERNEL anyway */ case I915_MAP_WB: pgprot = PAGE_KERNEL; break; @@ -548,6 +548,20 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) return nth_page(sg_page(sg), offset); } +/* Like i915_gem_object_get_page(), but mark the returned page dirty */ +struct page * +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n) +{ + struct page *page; + + page = i915_gem_object_get_page(obj, n); + if (!obj->mm.dirty) + set_page_dirty(page); + + return page; +} + dma_addr_t i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, unsigned long n, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c index 3d215164dd5a..40d3e40500fa 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c @@ -84,7 +84,7 @@ void i915_gem_suspend_late(struct drm_i915_private *i915) spin_unlock_irqrestore(&i915->mm.obj_lock, flags); - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); drm_WARN_ON(&i915->drm, i915_gem_object_set_to_gtt_domain(obj, false)); i915_gem_object_unlock(obj); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index e0f21f12d3ce..0be5e8683337 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -249,7 +249,7 @@ static void vlv_get_stolen_reserved(struct drm_i915_private *i915, switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) { default: MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK); - /* fall through */ + fallthrough; case GEN7_STOLEN_RESERVED_1M: *size = 1024 * 1024; break; @@ -416,7 +416,7 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) case 4: if (!IS_G4X(i915)) break; - /* fall through */ + fallthrough; case 5: g4x_get_stolen_reserved(i915, uncore, &reserved_base, &reserved_size); @@ -445,7 +445,7 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) break; default: MISSING_CASE(INTEL_GEN(i915)); - /* fall-through */ + fallthrough; case 11: case 12: icl_get_stolen_reserved(i915, uncore, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_throttle.c b/drivers/gpu/drm/i915/gem/i915_gem_throttle.c index 540ef0551789..1929d6cf4150 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_throttle.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_throttle.c @@ -9,6 +9,7 @@ #include <drm/drm_file.h> #include "i915_drv.h" +#include "i915_gem_context.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" @@ -35,9 +36,10 @@ int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { + const unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; struct drm_i915_file_private *file_priv = file->driver_priv; - unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; - struct i915_request *request, *target = NULL; + struct i915_gem_context *ctx; + unsigned long idx; long ret; /* ABI: return -EIO if already wedged */ @@ -45,27 +47,54 @@ i915_gem_throttle_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - spin_lock(&file_priv->mm.lock); - list_for_each_entry(request, &file_priv->mm.request_list, client_link) { - if (time_after_eq(request->emitted_jiffies, recent_enough)) - break; + rcu_read_lock(); + xa_for_each(&file_priv->context_xa, idx, ctx) { + struct i915_gem_engines_iter it; + struct intel_context *ce; - if (target && xchg(&target->file_priv, NULL)) - list_del(&target->client_link); + if (!kref_get_unless_zero(&ctx->ref)) + continue; + rcu_read_unlock(); - target = request; - } - if (target) - i915_request_get(target); - spin_unlock(&file_priv->mm.lock); + for_each_gem_engine(ce, + i915_gem_context_lock_engines(ctx), + it) { + struct i915_request *rq, *target = NULL; + + if (!ce->timeline) + continue; + + mutex_lock(&ce->timeline->mutex); + list_for_each_entry_reverse(rq, + &ce->timeline->requests, + link) { + if (i915_request_completed(rq)) + break; - if (!target) - return 0; + if (time_after(rq->emitted_jiffies, + recent_enough)) + continue; - ret = i915_request_wait(target, - I915_WAIT_INTERRUPTIBLE, - MAX_SCHEDULE_TIMEOUT); - i915_request_put(target); + target = i915_request_get(rq); + break; + } + mutex_unlock(&ce->timeline->mutex); + if (!target) + continue; + + ret = i915_request_wait(target, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); + i915_request_put(target); + if (ret < 0) + break; + } + i915_gem_context_unlock_engines(ctx); + i915_gem_context_put(ctx); + + rcu_read_lock(); + } + rcu_read_unlock(); return ret < 0 ? ret : 0; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index ff72ee2fd9cd..ffcaee74a249 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -249,7 +249,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, * whilst executing a fenced command for an untiled object. */ - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); if (i915_gem_object_is_framebuffer(obj)) { i915_gem_object_unlock(obj); return -EBUSY; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 2c2bf24140c9..12b30075134a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -596,14 +596,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); - /* - * Using __get_user_pages_fast() with a read-only - * access is questionable. A read-only page may be - * COW-broken, and then this might end up giving - * the wrong side of the COW.. - * - * We may or may not care. - */ if (pvec) { /* defer to worker if malloc fails */ if (!i915_gem_object_is_readonly(obj)) diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 8291ede6902c..5daf4a2be422 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -393,7 +393,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) */ for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { - unsigned int combination = 0; + unsigned int combination = SZ_4K; /* Required for ppGTT */ for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { if (i & BIT(j)) @@ -947,7 +947,7 @@ static int gpu_write(struct intel_context *ce, { int err; - i915_gem_object_lock(vma->obj); + i915_gem_object_lock(vma->obj, NULL); err = i915_gem_object_set_to_gtt_domain(vma->obj, true); i915_gem_object_unlock(vma->obj); if (err) @@ -964,9 +964,10 @@ __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val) unsigned long n; int err; + i915_gem_object_lock(obj, NULL); err = i915_gem_object_prepare_read(obj, &needs_flush); if (err) - return err; + goto err_unlock; for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); @@ -986,6 +987,8 @@ __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val) } i915_gem_object_finish_access(obj); +err_unlock: + i915_gem_object_unlock(obj); return err; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 299c29e9ad86..4e36d4897ea6 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -75,7 +75,7 @@ static int __igt_client_fill(struct intel_engine_cs *engine) if (err) goto err_unpin; - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_cpu_domain(obj, false); i915_gem_object_unlock(obj); if (err) diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c index 87d7d8aa080f..7049a6bbc03d 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c @@ -27,9 +27,10 @@ static int cpu_set(struct context *ctx, unsigned long offset, u32 v) u32 *cpu; int err; + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_prepare_write(ctx->obj, &needs_clflush); if (err) - return err; + goto out; page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT); map = kmap_atomic(page); @@ -46,7 +47,9 @@ static int cpu_set(struct context *ctx, unsigned long offset, u32 v) kunmap_atomic(map); i915_gem_object_finish_access(ctx->obj); - return 0; +out: + i915_gem_object_unlock(ctx->obj); + return err; } static int cpu_get(struct context *ctx, unsigned long offset, u32 *v) @@ -57,9 +60,10 @@ static int cpu_get(struct context *ctx, unsigned long offset, u32 *v) u32 *cpu; int err; + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_prepare_read(ctx->obj, &needs_clflush); if (err) - return err; + goto out; page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT); map = kmap_atomic(page); @@ -73,7 +77,9 @@ static int cpu_get(struct context *ctx, unsigned long offset, u32 *v) kunmap_atomic(map); i915_gem_object_finish_access(ctx->obj); - return 0; +out: + i915_gem_object_unlock(ctx->obj); + return err; } static int gtt_set(struct context *ctx, unsigned long offset, u32 v) @@ -82,7 +88,7 @@ static int gtt_set(struct context *ctx, unsigned long offset, u32 v) u32 __iomem *map; int err = 0; - i915_gem_object_lock(ctx->obj); + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_set_to_gtt_domain(ctx->obj, true); i915_gem_object_unlock(ctx->obj); if (err) @@ -115,7 +121,7 @@ static int gtt_get(struct context *ctx, unsigned long offset, u32 *v) u32 __iomem *map; int err = 0; - i915_gem_object_lock(ctx->obj); + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_set_to_gtt_domain(ctx->obj, false); i915_gem_object_unlock(ctx->obj); if (err) @@ -147,7 +153,7 @@ static int wc_set(struct context *ctx, unsigned long offset, u32 v) u32 *map; int err; - i915_gem_object_lock(ctx->obj); + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_set_to_wc_domain(ctx->obj, true); i915_gem_object_unlock(ctx->obj); if (err) @@ -170,7 +176,7 @@ static int wc_get(struct context *ctx, unsigned long offset, u32 *v) u32 *map; int err; - i915_gem_object_lock(ctx->obj); + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_set_to_wc_domain(ctx->obj, false); i915_gem_object_unlock(ctx->obj); if (err) @@ -193,27 +199,27 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v) u32 *cs; int err; - i915_gem_object_lock(ctx->obj); + i915_gem_object_lock(ctx->obj, NULL); err = i915_gem_object_set_to_gtt_domain(ctx->obj, true); - i915_gem_object_unlock(ctx->obj); if (err) - return err; + goto out_unlock; vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0); - if (IS_ERR(vma)) - return PTR_ERR(vma); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto out_unlock; + } rq = intel_engine_create_kernel_request(ctx->engine); if (IS_ERR(rq)) { - i915_vma_unpin(vma); - return PTR_ERR(rq); + err = PTR_ERR(rq); + goto out_unpin; } cs = intel_ring_begin(rq, 4); if (IS_ERR(cs)) { - i915_request_add(rq); - i915_vma_unpin(vma); - return PTR_ERR(cs); + err = PTR_ERR(cs); + goto out_rq; } if (INTEL_GEN(ctx->engine->i915) >= 8) { @@ -234,14 +240,16 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v) } intel_ring_advance(rq, cs); - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); - i915_vma_unpin(vma); +out_rq: i915_request_add(rq); +out_unpin: + i915_vma_unpin(vma); +out_unlock: + i915_gem_object_unlock(ctx->obj); return err; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 7ffc3c751432..99becb86abd3 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -461,9 +461,10 @@ static int cpu_fill(struct drm_i915_gem_object *obj, u32 value) unsigned int n, m, need_flush; int err; + i915_gem_object_lock(obj, NULL); err = i915_gem_object_prepare_write(obj, &need_flush); if (err) - return err; + goto out; for (n = 0; n < real_page_count(obj); n++) { u32 *map; @@ -479,7 +480,9 @@ static int cpu_fill(struct drm_i915_gem_object *obj, u32 value) i915_gem_object_finish_access(obj); obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU; obj->write_domain = 0; - return 0; +out: + i915_gem_object_unlock(obj); + return err; } static noinline int cpu_check(struct drm_i915_gem_object *obj, @@ -488,9 +491,10 @@ static noinline int cpu_check(struct drm_i915_gem_object *obj, unsigned int n, m, needs_flush; int err; + i915_gem_object_lock(obj, NULL); err = i915_gem_object_prepare_read(obj, &needs_flush); if (err) - return err; + goto out_unlock; for (n = 0; n < real_page_count(obj); n++) { u32 *map; @@ -527,6 +531,8 @@ out_unmap: } i915_gem_object_finish_access(obj); +out_unlock: + i915_gem_object_unlock(obj); return err; } @@ -887,24 +893,15 @@ out_file: return err; } -static struct i915_vma *rpcs_query_batch(struct i915_vma *vma) +static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *vma) { - struct drm_i915_gem_object *obj; u32 *cmd; - int err; - if (INTEL_GEN(vma->vm->i915) < 8) - return ERR_PTR(-EINVAL); + GEM_BUG_ON(INTEL_GEN(vma->vm->i915) < 8); - obj = i915_gem_object_create_internal(vma->vm->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } + cmd = i915_gem_object_pin_map(rpcs, I915_MAP_WB); + if (IS_ERR(cmd)) + return PTR_ERR(cmd); *cmd++ = MI_STORE_REGISTER_MEM_GEN8; *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); @@ -912,26 +909,12 @@ static struct i915_vma *rpcs_query_batch(struct i915_vma *vma) *cmd++ = upper_32_bits(vma->node.start); *cmd = MI_BATCH_BUFFER_END; - __i915_gem_object_flush_map(obj, 0, 64); - i915_gem_object_unpin_map(obj); + __i915_gem_object_flush_map(rpcs, 0, 64); + i915_gem_object_unpin_map(rpcs); intel_gt_chipset_flush(vma->vm->gt); - vma = i915_vma_instance(obj, vma->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto err; - - return vma; - -err: - i915_gem_object_put(obj); - return ERR_PTR(err); + return 0; } static int @@ -939,52 +922,68 @@ emit_rpcs_query(struct drm_i915_gem_object *obj, struct intel_context *ce, struct i915_request **rq_out) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_request *rq; + struct i915_gem_ww_ctx ww; struct i915_vma *batch; struct i915_vma *vma; + struct drm_i915_gem_object *rpcs; int err; GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine)); + if (INTEL_GEN(i915) < 8) + return -EINVAL; + vma = i915_vma_instance(obj, ce->vm, NULL); if (IS_ERR(vma)) return PTR_ERR(vma); - i915_gem_object_lock(obj); - err = i915_gem_object_set_to_gtt_domain(obj, false); - i915_gem_object_unlock(obj); - if (err) - return err; - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; + rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(rpcs)) + return PTR_ERR(rpcs); - batch = rpcs_query_batch(vma); + batch = i915_vma_instance(rpcs, ce->vm, NULL); if (IS_ERR(batch)) { err = PTR_ERR(batch); - goto err_vma; + goto err_put; } + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(obj, &ww); + if (!err) + err = i915_gem_object_lock(rpcs, &ww); + if (!err) + err = i915_gem_object_set_to_gtt_domain(obj, false); + if (!err) + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); + if (err) + goto err_put; + + err = i915_vma_pin_ww(batch, &ww, 0, 0, PIN_USER); + if (err) + goto err_vma; + + err = rpcs_query_batch(rpcs, vma); + if (err) + goto err_batch; + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_batch; } - i915_vma_lock(batch); err = i915_request_await_object(rq, batch->obj, false); if (err == 0) err = i915_vma_move_to_active(batch, rq, 0); - i915_vma_unlock(batch); if (err) goto skip_request; - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); if (err) goto skip_request; @@ -1000,23 +999,24 @@ emit_rpcs_query(struct drm_i915_gem_object *obj, if (err) goto skip_request; - i915_vma_unpin_and_release(&batch, 0); - i915_vma_unpin(vma); - *rq_out = i915_request_get(rq); - i915_request_add(rq); - - return 0; - skip_request: - i915_request_set_error_once(rq, err); + if (err) + i915_request_set_error_once(rq, err); i915_request_add(rq); err_batch: - i915_vma_unpin_and_release(&batch, 0); + i915_vma_unpin(batch); err_vma: i915_vma_unpin(vma); - +err_put: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + i915_gem_object_put(rpcs); return err; } @@ -1709,7 +1709,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, i915_request_add(rq); - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_cpu_domain(obj, false); i915_gem_object_unlock(obj); if (err) @@ -1748,7 +1748,7 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) if (!vm) return -ENODEV; - page = vm->scratch[0].base.page; + page = __px_page(vm->scratch[0]); if (!page) { pr_err("No scratch page!\n"); return -EINVAL; @@ -1914,8 +1914,8 @@ static int mock_context_barrier(void *arg) return -ENOMEM; counter = 0; - err = context_barrier_task(ctx, 0, - NULL, NULL, mock_barrier_task, &counter); + err = context_barrier_task(ctx, 0, NULL, NULL, NULL, + mock_barrier_task, &counter); if (err) { pr_err("Failed at line %d, err=%d\n", __LINE__, err); goto out; @@ -1927,11 +1927,8 @@ static int mock_context_barrier(void *arg) } counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, - skip_unused_engines, - NULL, - mock_barrier_task, - &counter); + err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines, + NULL, NULL, mock_barrier_task, &counter); if (err) { pr_err("Failed at line %d, err=%d\n", __LINE__, err); goto out; @@ -1951,8 +1948,8 @@ static int mock_context_barrier(void *arg) counter = 0; context_barrier_inject_fault = BIT(RCS0); - err = context_barrier_task(ctx, ALL_ENGINES, - NULL, NULL, mock_barrier_task, &counter); + err = context_barrier_task(ctx, ALL_ENGINES, NULL, NULL, NULL, + mock_barrier_task, &counter); context_barrier_inject_fault = 0; if (err == -ENXIO) err = 0; @@ -1966,11 +1963,8 @@ static int mock_context_barrier(void *arg) goto out; counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, - skip_unused_engines, - NULL, - mock_barrier_task, - &counter); + err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines, + NULL, NULL, mock_barrier_task, &counter); if (err) { pr_err("Failed at line %d, err=%d\n", __LINE__, err); goto out; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c index 57c14d3340cd..e1d50a5a1477 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c @@ -32,37 +32,39 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb, if (IS_ERR(vma)) return PTR_ERR(vma); - err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH); + err = i915_gem_object_lock(obj, &eb->ww); + if (err) + return err; + + err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, PIN_USER | PIN_HIGH); if (err) return err; /* 8-Byte aligned */ err = __reloc_entry_gpu(eb, vma, offsets[0] * sizeof(u32), 0); - if (err) - goto unpin_vma; + if (err <= 0) + goto reloc_err; /* !8-Byte aligned */ err = __reloc_entry_gpu(eb, vma, offsets[1] * sizeof(u32), 1); - if (err) - goto unpin_vma; + if (err <= 0) + goto reloc_err; /* Skip to the end of the cmd page */ - i = PAGE_SIZE / sizeof(u32) - RELOC_TAIL - 1; + i = PAGE_SIZE / sizeof(u32) - 1; i -= eb->reloc_cache.rq_size; memset32(eb->reloc_cache.rq_cmd + eb->reloc_cache.rq_size, MI_NOOP, i); eb->reloc_cache.rq_size += i; - /* Force batch chaining */ + /* Force next batch */ err = __reloc_entry_gpu(eb, vma, offsets[2] * sizeof(u32), 2); - if (err) - goto unpin_vma; + if (err <= 0) + goto reloc_err; GEM_BUG_ON(!eb->reloc_cache.rq); rq = i915_request_get(eb->reloc_cache.rq); - err = reloc_gpu_flush(&eb->reloc_cache); - if (err) - goto put_rq; + reloc_gpu_flush(eb, &eb->reloc_cache); GEM_BUG_ON(eb->reloc_cache.rq); err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, HZ / 2); @@ -94,6 +96,11 @@ put_rq: unpin_vma: i915_vma_unpin(vma); return err; + +reloc_err: + if (!err) + err = -EIO; + goto unpin_vma; } static int igt_gpu_reloc(void *arg) @@ -115,6 +122,8 @@ static int igt_gpu_reloc(void *arg) goto err_scratch; } + intel_gt_pm_get(&eb.i915->gt); + for_each_uabi_engine(eb.engine, eb.i915) { reloc_cache_init(&eb.reloc_cache, eb.i915); memset(map, POISON_INUSE, 4096); @@ -125,15 +134,29 @@ static int igt_gpu_reloc(void *arg) err = PTR_ERR(eb.context); goto err_pm; } + eb.reloc_pool = NULL; + eb.reloc_context = NULL; - err = intel_context_pin(eb.context); - if (err) - goto err_put; + i915_gem_ww_ctx_init(&eb.ww, false); +retry: + err = intel_context_pin_ww(eb.context, &eb.ww); + if (!err) { + err = __igt_gpu_reloc(&eb, scratch); + + intel_context_unpin(eb.context); + } + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&eb.ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&eb.ww); - err = __igt_gpu_reloc(&eb, scratch); + if (eb.reloc_pool) + intel_gt_buffer_pool_put(eb.reloc_pool); + if (eb.reloc_context) + intel_context_put(eb.reloc_context); - intel_context_unpin(eb.context); -err_put: intel_context_put(eb.context); err_pm: intel_engine_pm_put(eb.engine); @@ -144,6 +167,7 @@ err_pm: if (igt_flush_test(eb.i915)) err = -EIO; + intel_gt_pm_put(&eb.i915->gt); err_scratch: i915_gem_object_put(scratch); return err; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 9c7402ce5bf9..d27d87a678c8 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -103,7 +103,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj, GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_gtt_domain(obj, true); i915_gem_object_unlock(obj); if (err) { @@ -188,7 +188,7 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj, GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_gtt_domain(obj, true); i915_gem_object_unlock(obj); if (err) { @@ -528,31 +528,42 @@ static int make_obj_busy(struct drm_i915_gem_object *obj) for_each_uabi_engine(engine, i915) { struct i915_request *rq; struct i915_vma *vma; + struct i915_gem_ww_ctx ww; int err; vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); if (IS_ERR(vma)) return PTR_ERR(vma); - err = i915_vma_pin(vma, 0, 0, PIN_USER); + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(obj, &ww); + if (!err) + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); if (err) - return err; + goto err; rq = intel_engine_create_kernel_request(engine); if (IS_ERR(rq)) { - i915_vma_unpin(vma); - return PTR_ERR(rq); + err = PTR_ERR(rq); + goto err_unpin; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); i915_request_add(rq); +err_unpin: i915_vma_unpin(vma); +err: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); if (err) return err; } @@ -1123,6 +1134,7 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915, for_each_uabi_engine(engine, i915) { struct i915_request *rq; struct i915_vma *vma; + struct i915_gem_ww_ctx ww; vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL); if (IS_ERR(vma)) { @@ -1130,9 +1142,13 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915, goto out_unmap; } - err = i915_vma_pin(vma, 0, 0, PIN_USER); + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(obj, &ww); + if (!err) + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); if (err) - goto out_unmap; + goto out_ww; rq = i915_request_create(engine->kernel_context); if (IS_ERR(rq)) { @@ -1140,11 +1156,9 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915, goto out_unpin; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, false); if (err == 0) err = i915_vma_move_to_active(vma, rq, 0); - i915_vma_unlock(vma); err = engine->emit_bb_start(rq, vma->node.start, 0, 0); i915_request_get(rq); @@ -1166,6 +1180,13 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915, out_unpin: i915_vma_unpin(vma); +out_ww: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); if (err) goto out_unmap; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c index 34932871b3a5..a94243dc4c5c 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c @@ -44,7 +44,7 @@ static int mock_phys_object(void *arg) } /* Make the object dirty so that put_pages must do copy back the data */ - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_gtt_domain(obj, true); i915_gem_object_unlock(obj); if (err) { diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index cdc0b9c54305..fd0d24d28763 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -16,8 +16,10 @@ static inline void gen6_write_pde(const struct gen6_ppgtt *ppgtt, const unsigned int pde, const struct i915_page_table *pt) { + dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]); + /* Caller needs to make sure the write completes if necessary */ - iowrite32(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID, + iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID, ppgtt->pd_addr + pde); } @@ -79,7 +81,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, { struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); const unsigned int first_entry = start / I915_GTT_PAGE_SIZE; - const gen6_pte_t scratch_pte = vm->scratch[0].encode; + const gen6_pte_t scratch_pte = vm->scratch[0]->encode; unsigned int pde = first_entry / GEN6_PTES; unsigned int pte = first_entry % GEN6_PTES; unsigned int num_entries = length / I915_GTT_PAGE_SIZE; @@ -90,8 +92,6 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, const unsigned int count = min(num_entries, GEN6_PTES - pte); gen6_pte_t *vaddr; - GEM_BUG_ON(px_base(pt) == px_base(&vm->scratch[1])); - num_entries -= count; GEM_BUG_ON(count > atomic_read(&pt->used)); @@ -127,7 +127,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, struct sgt_dma iter = sgt_dma(vma); gen6_pte_t *vaddr; - GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch[1]); + GEM_BUG_ON(!pd->entry[act_pt]); vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); do { @@ -177,39 +177,36 @@ static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end) mutex_unlock(&ppgtt->flush); } -static int gen6_alloc_va_range(struct i915_address_space *vm, - u64 start, u64 length) +static void gen6_alloc_va_range(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + u64 start, u64 length) { struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); struct i915_page_directory * const pd = ppgtt->base.pd; - struct i915_page_table *pt, *alloc = NULL; + struct i915_page_table *pt; bool flush = false; u64 from = start; unsigned int pde; - int ret = 0; spin_lock(&pd->lock); gen6_for_each_pde(pt, pd, start, length, pde) { const unsigned int count = gen6_pte_count(start, length); - if (px_base(pt) == px_base(&vm->scratch[1])) { + if (!pt) { spin_unlock(&pd->lock); - pt = fetch_and_zero(&alloc); - if (!pt) - pt = alloc_pt(vm); - if (IS_ERR(pt)) { - ret = PTR_ERR(pt); - goto unwind_out; - } + pt = stash->pt[0]; + __i915_gem_object_pin_pages(pt->base); + i915_gem_object_make_unshrinkable(pt->base); - fill32_px(pt, vm->scratch[0].encode); + fill32_px(pt, vm->scratch[0]->encode); spin_lock(&pd->lock); - if (pd->entry[pde] == &vm->scratch[1]) { + if (!pd->entry[pde]) { + stash->pt[0] = pt->stash; + atomic_set(&pt->used, 0); pd->entry[pde] = pt; } else { - alloc = pt; pt = pd->entry[pde]; } @@ -226,38 +223,32 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref) gen6_flush_pd(ppgtt, from, start); } - - goto out; - -unwind_out: - gen6_ppgtt_clear_range(vm, from, start - from); -out: - if (alloc) - free_px(vm, alloc); - return ret; } static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) { struct i915_address_space * const vm = &ppgtt->base.vm; - struct i915_page_directory * const pd = ppgtt->base.pd; int ret; - ret = setup_scratch_page(vm, __GFP_HIGHMEM); + ret = setup_scratch_page(vm); if (ret) return ret; - vm->scratch[0].encode = - vm->pte_encode(px_dma(&vm->scratch[0]), + vm->scratch[0]->encode = + vm->pte_encode(px_dma(vm->scratch[0]), I915_CACHE_NONE, PTE_READ_ONLY); - if (unlikely(setup_page_dma(vm, px_base(&vm->scratch[1])))) { - cleanup_scratch_page(vm); - return -ENOMEM; + vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + if (IS_ERR(vm->scratch[1])) + return PTR_ERR(vm->scratch[1]); + + ret = pin_pt_dma(vm, vm->scratch[1]); + if (ret) { + i915_gem_object_put(vm->scratch[1]); + return ret; } - fill32_px(&vm->scratch[1], vm->scratch[0].encode); - memset_p(pd->entry, &vm->scratch[1], I915_PDES); + fill32_px(vm->scratch[1], vm->scratch[0]->encode); return 0; } @@ -265,14 +256,12 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt) { struct i915_page_directory * const pd = ppgtt->base.pd; - struct i915_page_dma * const scratch = - px_base(&ppgtt->base.vm.scratch[1]); struct i915_page_table *pt; u32 pde; gen6_for_all_pdes(pt, pd, pde) - if (px_base(pt) != scratch) - free_px(&ppgtt->base.vm, pt); + if (pt) + free_pt(&ppgtt->base.vm, pt); } static void gen6_ppgtt_cleanup(struct i915_address_space *vm) @@ -286,7 +275,8 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) mutex_destroy(&ppgtt->flush); mutex_destroy(&ppgtt->pin_mutex); - kfree(ppgtt->base.pd); + + free_pd(&ppgtt->base.vm, ppgtt->base.pd); } static int pd_vma_set_pages(struct i915_vma *vma) @@ -302,28 +292,26 @@ static void pd_vma_clear_pages(struct i915_vma *vma) vma->pages = NULL; } -static int pd_vma_bind(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 unused) +static void pd_vma_bind(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 unused) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); struct gen6_ppgtt *ppgtt = vma->private; u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE; - px_base(ppgtt->base.pd)->ggtt_offset = ggtt_offset * sizeof(gen6_pte_t); + ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10; ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total); - return 0; } static void pd_vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) { struct gen6_ppgtt *ppgtt = vma->private; struct i915_page_directory * const pd = ppgtt->base.pd; - struct i915_page_dma * const scratch = - px_base(&ppgtt->base.vm.scratch[1]); struct i915_page_table *pt; unsigned int pde; @@ -332,11 +320,11 @@ static void pd_vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) /* Free all no longer used page tables */ gen6_for_all_pdes(pt, ppgtt->base.pd, pde) { - if (px_base(pt) == scratch || atomic_read(&pt->used)) + if (!pt || atomic_read(&pt->used)) continue; - free_px(&ppgtt->base.vm, pt); - pd->entry[pde] = scratch; + free_pt(&ppgtt->base.vm, pt); + pd->entry[pde] = NULL; } ppgtt->scan_for_unused_pt = false; @@ -380,7 +368,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) return vma; } -int gen6_ppgtt_pin(struct i915_ppgtt *base) +int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww) { struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base); int err; @@ -406,7 +394,7 @@ int gen6_ppgtt_pin(struct i915_ppgtt *base) */ err = 0; if (!atomic_read(&ppgtt->pin_count)) - err = i915_ggtt_pin(ppgtt->vma, GEN6_PD_ALIGN, PIN_HIGH); + err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH); if (!err) atomic_inc(&ppgtt->pin_count); mutex_unlock(&ppgtt->pin_mutex); @@ -448,6 +436,7 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) mutex_init(&ppgtt->pin_mutex); ppgtt_init(&ppgtt->base, gt); + ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t)); ppgtt->base.vm.top = 1; ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND; @@ -456,9 +445,10 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup; + ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma; ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode; - ppgtt->base.pd = __alloc_pd(sizeof(*ppgtt->base.pd)); + ppgtt->base.pd = __alloc_pd(I915_PDES); if (!ppgtt->base.pd) { err = -ENOMEM; goto err_free; @@ -479,7 +469,7 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) err_scratch: free_scratch(&ppgtt->base.vm); err_pd: - kfree(ppgtt->base.pd); + free_pd(&ppgtt->base.vm, ppgtt->base.pd); err_free: mutex_destroy(&ppgtt->pin_mutex); kfree(ppgtt); diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.h b/drivers/gpu/drm/i915/gt/gen6_ppgtt.h index 72e481806c96..3357228f3304 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.h +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.h @@ -8,12 +8,15 @@ #include "intel_gtt.h" +struct i915_gem_ww_ctx; + struct gen6_ppgtt { struct i915_ppgtt base; struct mutex flush; struct i915_vma *vma; gen6_pte_t __iomem *pd_addr; + u32 pp_dir; atomic_t pin_count; struct mutex pin_mutex; @@ -66,7 +69,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base) (pt = i915_pt_entry(pd, iter), true); \ ++iter) -int gen6_ppgtt_pin(struct i915_ppgtt *base); +int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww); void gen6_ppgtt_unpin(struct i915_ppgtt *base); void gen6_ppgtt_unpin_all(struct i915_ppgtt *base); void gen6_ppgtt_enable(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 699125928272..eb64f474a78c 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -181,7 +181,7 @@ static void __gen8_ppgtt_cleanup(struct i915_address_space *vm, } while (pde++, --count); } - free_px(vm, pd); + free_px(vm, &pd->pt, lvl); } static void gen8_ppgtt_cleanup(struct i915_address_space *vm) @@ -199,7 +199,7 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, struct i915_page_directory * const pd, u64 start, const u64 end, int lvl) { - const struct i915_page_scratch * const scratch = &vm->scratch[lvl]; + const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; unsigned int idx, len; GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); @@ -239,7 +239,7 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, vaddr = kmap_atomic_px(pt); memset64(vaddr + gen8_pd_index(start, 0), - vm->scratch[0].encode, + vm->scratch[0]->encode, count); kunmap_atomic(vaddr); @@ -248,7 +248,7 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, } if (release_pd_entry(pd, idx, pt, scratch)) - free_px(vm, pt); + free_px(vm, pt, lvl); } while (idx++, --len); return start; @@ -269,14 +269,12 @@ static void gen8_ppgtt_clear(struct i915_address_space *vm, start, start + length, vm->top); } -static int __gen8_ppgtt_alloc(struct i915_address_space * const vm, - struct i915_page_directory * const pd, - u64 * const start, const u64 end, int lvl) +static void __gen8_ppgtt_alloc(struct i915_address_space * const vm, + struct i915_vm_pt_stash *stash, + struct i915_page_directory * const pd, + u64 * const start, const u64 end, int lvl) { - const struct i915_page_scratch * const scratch = &vm->scratch[lvl]; - struct i915_page_table *alloc = NULL; unsigned int idx, len; - int ret = 0; GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); @@ -297,49 +295,31 @@ static int __gen8_ppgtt_alloc(struct i915_address_space * const vm, DBG("%s(%p):{ lvl:%d, idx:%d } allocating new tree\n", __func__, vm, lvl + 1, idx); - pt = fetch_and_zero(&alloc); - if (lvl) { - if (!pt) { - pt = &alloc_pd(vm)->pt; - if (IS_ERR(pt)) { - ret = PTR_ERR(pt); - goto out; - } - } - - fill_px(pt, vm->scratch[lvl].encode); - } else { - if (!pt) { - pt = alloc_pt(vm); - if (IS_ERR(pt)) { - ret = PTR_ERR(pt); - goto out; - } - } - - if (intel_vgpu_active(vm->i915) || - gen8_pt_count(*start, end) < I915_PDES) - fill_px(pt, vm->scratch[lvl].encode); - } + pt = stash->pt[!!lvl]; + __i915_gem_object_pin_pages(pt->base); + i915_gem_object_make_unshrinkable(pt->base); + + if (lvl || + gen8_pt_count(*start, end) < I915_PDES || + intel_vgpu_active(vm->i915)) + fill_px(pt, vm->scratch[lvl]->encode); spin_lock(&pd->lock); - if (likely(!pd->entry[idx])) + if (likely(!pd->entry[idx])) { + stash->pt[!!lvl] = pt->stash; + atomic_set(&pt->used, 0); set_pd_entry(pd, idx, pt); - else - alloc = pt, pt = pd->entry[idx]; + } else { + pt = pd->entry[idx]; + } } if (lvl) { atomic_inc(&pt->used); spin_unlock(&pd->lock); - ret = __gen8_ppgtt_alloc(vm, as_pd(pt), - start, end, lvl); - if (unlikely(ret)) { - if (release_pd_entry(pd, idx, pt, scratch)) - free_px(vm, pt); - goto out; - } + __gen8_ppgtt_alloc(vm, stash, + as_pd(pt), start, end, lvl); spin_lock(&pd->lock); atomic_dec(&pt->used); @@ -359,18 +339,12 @@ static int __gen8_ppgtt_alloc(struct i915_address_space * const vm, } } while (idx++, --len); spin_unlock(&pd->lock); -out: - if (alloc) - free_px(vm, alloc); - return ret; } -static int gen8_ppgtt_alloc(struct i915_address_space *vm, - u64 start, u64 length) +static void gen8_ppgtt_alloc(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + u64 start, u64 length) { - u64 from; - int err; - GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT))); GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT))); GEM_BUG_ON(range_overflows(start, length, vm->total)); @@ -378,25 +352,9 @@ static int gen8_ppgtt_alloc(struct i915_address_space *vm, start >>= GEN8_PTE_SHIFT; length >>= GEN8_PTE_SHIFT; GEM_BUG_ON(length == 0); - from = start; - err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd, - &start, start + length, vm->top); - if (unlikely(err && from != start)) - __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, - from, start, vm->top); - - return err; -} - -static __always_inline void -write_pte(gen8_pte_t *pte, const gen8_pte_t val) -{ - /* Magic delays? Or can we refine these to flush all in one pass? */ - *pte = val; - wmb(); /* cpu to cache */ - clflush(pte); /* cache to memory */ - wmb(); /* visible to all */ + __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, + &start, start + length, vm->top); } static __always_inline u64 @@ -415,8 +373,7 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); do { GEM_BUG_ON(iter->sg->length < I915_GTT_PAGE_SIZE); - write_pte(&vaddr[gen8_pd_index(idx, 0)], - pte_encode | iter->dma); + vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; iter->dma += I915_GTT_PAGE_SIZE; if (iter->dma >= iter->max) { @@ -439,10 +396,12 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, pd = pdp->entry[gen8_pd_index(idx, 2)]; } + clflush_cache_range(vaddr, PAGE_SIZE); kunmap_atomic(vaddr); vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); } } while (1); + clflush_cache_range(vaddr, PAGE_SIZE); kunmap_atomic(vaddr); return idx; @@ -498,7 +457,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, do { GEM_BUG_ON(iter->sg->length < page_size); - write_pte(&vaddr[index++], encode | iter->dma); + vaddr[index++] = encode | iter->dma; start += page_size; iter->dma += page_size; @@ -523,6 +482,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } } while (rem >= page_size && index < I915_PDES); + clflush_cache_range(vaddr, PAGE_SIZE); kunmap_atomic(vaddr); /* @@ -554,7 +514,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { u16 i; - encode = vma->vm->scratch[0].encode; + encode = vma->vm->scratch[0]->encode; vaddr = kmap_atomic_px(i915_pt_entry(pd, maybe_64K)); for (i = 1; i < index; i += 16) @@ -608,27 +568,37 @@ static int gen8_init_scratch(struct i915_address_space *vm) GEM_BUG_ON(!clone->has_read_only); vm->scratch_order = clone->scratch_order; - memcpy(vm->scratch, clone->scratch, sizeof(vm->scratch)); - px_dma(&vm->scratch[0]) = 0; /* no xfer of ownership */ + for (i = 0; i <= vm->top; i++) + vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); + return 0; } - ret = setup_scratch_page(vm, __GFP_HIGHMEM); + ret = setup_scratch_page(vm); if (ret) return ret; - vm->scratch[0].encode = - gen8_pte_encode(px_dma(&vm->scratch[0]), + vm->scratch[0]->encode = + gen8_pte_encode(px_dma(vm->scratch[0]), I915_CACHE_LLC, vm->has_read_only); for (i = 1; i <= vm->top; i++) { - if (unlikely(setup_page_dma(vm, px_base(&vm->scratch[i])))) + struct drm_i915_gem_object *obj; + + obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + if (IS_ERR(obj)) + goto free_scratch; + + ret = pin_pt_dma(vm, obj); + if (ret) { + i915_gem_object_put(obj); goto free_scratch; + } - fill_px(&vm->scratch[i], vm->scratch[i - 1].encode); - vm->scratch[i].encode = - gen8_pde_encode(px_dma(&vm->scratch[i]), - I915_CACHE_LLC); + fill_px(obj, vm->scratch[i - 1]->encode); + obj->encode = gen8_pde_encode(px_dma(obj), I915_CACHE_LLC); + + vm->scratch[i] = obj; } return 0; @@ -649,12 +619,20 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) { struct i915_page_directory *pde; + int err; pde = alloc_pd(vm); if (IS_ERR(pde)) return PTR_ERR(pde); - fill_px(pde, vm->scratch[1].encode); + err = pin_pt_dma(vm, pde->pt.base); + if (err) { + i915_gem_object_put(pde->pt.base); + free_pd(vm, pde); + return err; + } + + fill_px(pde, vm->scratch[1]->encode); set_pd_entry(pd, idx, pde); atomic_inc(px_used(pde)); /* keep pinned */ } @@ -668,21 +646,32 @@ gen8_alloc_top_pd(struct i915_address_space *vm) { const unsigned int count = gen8_pd_top_count(vm); struct i915_page_directory *pd; + int err; - GEM_BUG_ON(count > ARRAY_SIZE(pd->entry)); + GEM_BUG_ON(count > I915_PDES); - pd = __alloc_pd(offsetof(typeof(*pd), entry[count])); + pd = __alloc_pd(count); if (unlikely(!pd)) return ERR_PTR(-ENOMEM); - if (unlikely(setup_page_dma(vm, px_base(pd)))) { - kfree(pd); - return ERR_PTR(-ENOMEM); + pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + if (IS_ERR(pd->pt.base)) { + err = PTR_ERR(pd->pt.base); + pd->pt.base = NULL; + goto err_pd; } - fill_page_dma(px_base(pd), vm->scratch[vm->top].encode, count); + err = pin_pt_dma(vm, pd->pt.base); + if (err) + goto err_pd; + + fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); atomic_inc(px_used(pd)); /* mark as pinned */ return pd; + +err_pd: + free_pd(vm, pd); + return ERR_PTR(err); } /* @@ -703,6 +692,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) ppgtt_init(ppgtt, gt); ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; + ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); /* * From bdw, there is hw support for read-only pages in the PPGTT. @@ -714,12 +704,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) */ ppgtt->vm.has_read_only = !IS_GEN_RANGE(gt->i915, 11, 12); - /* - * There are only few exceptions for gen >=6. chv and bxt. - * And we are not sure about the latter so play safe for now. - */ - if (IS_CHERRYVIEW(gt->i915) || IS_BROXTON(gt->i915)) - ppgtt->vm.pt_kmap_wc = true; + ppgtt->vm.alloc_pt_dma = alloc_pt_dma; err = gen8_init_scratch(&ppgtt->vm); if (err) diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c index 91786310c114..d8b206e53660 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c @@ -28,6 +28,8 @@ #include "i915_drv.h" #include "i915_trace.h" +#include "intel_breadcrumbs.h" +#include "intel_context.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" @@ -53,33 +55,65 @@ static void irq_disable(struct intel_engine_cs *engine) spin_unlock(&engine->gt->irq_lock); } -static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) +static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) { - struct intel_engine_cs *engine = - container_of(b, struct intel_engine_cs, breadcrumbs); + lockdep_assert_held(&b->irq_lock); + + if (!b->irq_engine || b->irq_armed) + return; + + if (!intel_gt_pm_get_if_awake(b->irq_engine->gt)) + return; + + /* + * The breadcrumb irq will be disarmed on the interrupt after the + * waiters are signaled. This gives us a single interrupt window in + * which we can add a new waiter and avoid the cost of re-enabling + * the irq. + */ + WRITE_ONCE(b->irq_armed, true); + + /* + * Since we are waiting on a request, the GPU should be busy + * and should have its own rpm reference. This is tracked + * by i915->gt.awake, we can forgo holding our own wakref + * for the interrupt as before i915->gt.awake is released (when + * the driver is idle) we disarm the breadcrumbs. + */ + if (!b->irq_enabled++) + irq_enable(b->irq_engine); +} + +static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) +{ lockdep_assert_held(&b->irq_lock); + if (!b->irq_engine || !b->irq_armed) + return; + GEM_BUG_ON(!b->irq_enabled); if (!--b->irq_enabled) - irq_disable(engine); + irq_disable(b->irq_engine); WRITE_ONCE(b->irq_armed, false); - intel_gt_pm_put_async(engine->gt); + intel_gt_pm_put_async(b->irq_engine->gt); } -void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) +static void add_signaling_context(struct intel_breadcrumbs *b, + struct intel_context *ce) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; - unsigned long flags; - - if (!READ_ONCE(b->irq_armed)) - return; + intel_context_get(ce); + list_add_tail(&ce->signal_link, &b->signalers); + if (list_is_first(&ce->signal_link, &b->signalers)) + __intel_breadcrumbs_arm_irq(b); +} - spin_lock_irqsave(&b->irq_lock, flags); - if (b->irq_armed) - __intel_breadcrumbs_disarm_irq(b); - spin_unlock_irqrestore(&b->irq_lock, flags); +static void remove_signaling_context(struct intel_breadcrumbs *b, + struct intel_context *ce) +{ + list_del(&ce->signal_link); + intel_context_put(ce); } static inline bool __request_completed(const struct i915_request *rq) @@ -90,6 +124,9 @@ static inline bool __request_completed(const struct i915_request *rq) __maybe_unused static bool check_signal_order(struct intel_context *ce, struct i915_request *rq) { + if (rq->context != ce) + return false; + if (!list_is_last(&rq->signal_link, &ce->signals) && i915_seqno_passed(rq->fence.seqno, list_next_entry(rq, signal_link)->fence.seqno)) @@ -133,25 +170,21 @@ __dma_fence_signal__notify(struct dma_fence *fence, static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) { - struct intel_engine_cs *engine = - container_of(b, struct intel_engine_cs, breadcrumbs); - - if (unlikely(intel_engine_is_virtual(engine))) - engine = intel_virtual_engine_get_sibling(engine, 0); - - intel_engine_add_retire(engine, tl); + if (b->irq_engine) + intel_engine_add_retire(b->irq_engine, tl); } -static void __signal_request(struct i915_request *rq, struct list_head *signals) +static bool __signal_request(struct i915_request *rq, struct list_head *signals) { - GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); - if (!__dma_fence_signal(&rq->fence)) - return; + if (!__dma_fence_signal(&rq->fence)) { + i915_request_put(rq); + return false; + } - i915_request_get(rq); list_add_tail(&rq->signal_link, signals); + return true; } static void signal_irq_work(struct irq_work *work) @@ -164,7 +197,7 @@ static void signal_irq_work(struct irq_work *work) spin_lock(&b->irq_lock); - if (b->irq_armed && list_empty(&b->signalers)) + if (list_empty(&b->signalers)) __intel_breadcrumbs_disarm_irq(b); list_splice_init(&b->signaled_requests, &signal); @@ -197,8 +230,8 @@ static void signal_irq_work(struct irq_work *work) /* Advance the list to the first incomplete request */ __list_del_many(&ce->signals, pos); if (&ce->signals == pos) { /* now empty */ - list_del_init(&ce->signal_link); add_retire(b, ce->timeline); + remove_signaling_context(b, ce); } } } @@ -220,116 +253,89 @@ static void signal_irq_work(struct irq_work *work) } } -static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) +struct intel_breadcrumbs * +intel_breadcrumbs_create(struct intel_engine_cs *irq_engine) { - struct intel_engine_cs *engine = - container_of(b, struct intel_engine_cs, breadcrumbs); - - lockdep_assert_held(&b->irq_lock); - if (b->irq_armed) - return true; - - if (!intel_gt_pm_get_if_awake(engine->gt)) - return false; - - /* - * The breadcrumb irq will be disarmed on the interrupt after the - * waiters are signaled. This gives us a single interrupt window in - * which we can add a new waiter and avoid the cost of re-enabling - * the irq. - */ - WRITE_ONCE(b->irq_armed, true); - - /* - * Since we are waiting on a request, the GPU should be busy - * and should have its own rpm reference. This is tracked - * by i915->gt.awake, we can forgo holding our own wakref - * for the interrupt as before i915->gt.awake is released (when - * the driver is idle) we disarm the breadcrumbs. - */ - - if (!b->irq_enabled++) - irq_enable(engine); + struct intel_breadcrumbs *b; - return true; -} - -void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; + b = kzalloc(sizeof(*b), GFP_KERNEL); + if (!b) + return NULL; spin_lock_init(&b->irq_lock); INIT_LIST_HEAD(&b->signalers); INIT_LIST_HEAD(&b->signaled_requests); init_irq_work(&b->irq_work, signal_irq_work); + + b->irq_engine = irq_engine; + + return b; } -void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) +void intel_breadcrumbs_reset(struct intel_breadcrumbs *b) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; unsigned long flags; + if (!b->irq_engine) + return; + spin_lock_irqsave(&b->irq_lock, flags); if (b->irq_enabled) - irq_enable(engine); + irq_enable(b->irq_engine); else - irq_disable(engine); + irq_disable(b->irq_engine); spin_unlock_irqrestore(&b->irq_lock, flags); } -void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine, - struct intel_context *ce) +void intel_breadcrumbs_park(struct intel_breadcrumbs *b) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; unsigned long flags; - spin_lock_irqsave(&b->irq_lock, flags); - if (!list_empty(&ce->signals)) { - struct i915_request *rq, *next; - - /* Queue for executing the signal callbacks in the irq_work */ - list_for_each_entry_safe(rq, next, &ce->signals, signal_link) { - GEM_BUG_ON(rq->engine != engine); - GEM_BUG_ON(!__request_completed(rq)); - - __signal_request(rq, &b->signaled_requests); - } + if (!READ_ONCE(b->irq_armed)) + return; - INIT_LIST_HEAD(&ce->signals); - list_del_init(&ce->signal_link); + spin_lock_irqsave(&b->irq_lock, flags); + __intel_breadcrumbs_disarm_irq(b); + spin_unlock_irqrestore(&b->irq_lock, flags); + if (!list_empty(&b->signalers)) irq_work_queue(&b->irq_work); - } - spin_unlock_irqrestore(&b->irq_lock, flags); } -void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) +void intel_breadcrumbs_free(struct intel_breadcrumbs *b) { + kfree(b); } -bool i915_request_enable_breadcrumb(struct i915_request *rq) +static void insert_breadcrumb(struct i915_request *rq, + struct intel_breadcrumbs *b) { - lockdep_assert_held(&rq->lock); - - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) - return true; + struct intel_context *ce = rq->context; + struct list_head *pos; - if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { - struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; - struct intel_context *ce = rq->context; - struct list_head *pos; - - spin_lock(&b->irq_lock); + if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) + return; - if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) - goto unlock; + i915_request_get(rq); - if (!__intel_breadcrumbs_arm_irq(b)) - goto unlock; + /* + * If the request is already completed, we can transfer it + * straight onto a signaled list, and queue the irq worker for + * its signal completion. + */ + if (__request_completed(rq)) { + if (__signal_request(rq, &b->signaled_requests)) + irq_work_queue(&b->irq_work); + return; + } + if (list_empty(&ce->signals)) { + add_signaling_context(b, ce); + pos = &ce->signals; + } else { /* * We keep the seqno in retirement order, so we can break * inside intel_engine_signal_breadcrumbs as soon as we've @@ -351,24 +357,75 @@ bool i915_request_enable_breadcrumb(struct i915_request *rq) if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) break; } - list_add(&rq->signal_link, pos); - if (pos == &ce->signals) /* catch transitions from empty list */ - list_move_tail(&ce->signal_link, &b->signalers); - GEM_BUG_ON(!check_signal_order(ce, rq)); + } + list_add(&rq->signal_link, pos); + GEM_BUG_ON(!check_signal_order(ce, rq)); + set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + + /* Check after attaching to irq, interrupt may have already fired. */ + if (__request_completed(rq)) + irq_work_queue(&b->irq_work); +} - set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); -unlock: +bool i915_request_enable_breadcrumb(struct i915_request *rq) +{ + struct intel_breadcrumbs *b; + + /* Serialises with i915_request_retire() using rq->lock */ + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) + return true; + + /* + * Peek at i915_request_submit()/i915_request_unsubmit() status. + * + * If the request is not yet active (and not signaled), we will + * attach the breadcrumb later. + */ + if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) + return true; + + /* + * rq->engine is locked by rq->engine->active.lock. That however + * is not known until after rq->engine has been dereferenced and + * the lock acquired. Hence we acquire the lock and then validate + * that rq->engine still matches the lock we hold for it. + * + * Here, we are using the breadcrumb lock as a proxy for the + * rq->engine->active.lock, and we know that since the breadcrumb + * will be serialised within i915_request_submit/i915_request_unsubmit, + * the engine cannot change while active as long as we hold the + * breadcrumb lock on that engine. + * + * From the dma_fence_enable_signaling() path, we are outside of the + * request submit/unsubmit path, and so we must be more careful to + * acquire the right lock. + */ + b = READ_ONCE(rq->engine)->breadcrumbs; + spin_lock(&b->irq_lock); + while (unlikely(b != READ_ONCE(rq->engine)->breadcrumbs)) { spin_unlock(&b->irq_lock); + b = READ_ONCE(rq->engine)->breadcrumbs; + spin_lock(&b->irq_lock); } - return !__request_completed(rq); + /* + * Now that we are finally serialised with request submit/unsubmit, + * [with b->irq_lock] and with i915_request_retire() [via checking + * SIGNALED with rq->lock] confirm the request is indeed active. If + * it is no longer active, the breadcrumb will be attached upon + * i915_request_submit(). + */ + if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) + insert_breadcrumb(rq, b); + + spin_unlock(&b->irq_lock); + + return true; } void i915_request_cancel_breadcrumb(struct i915_request *rq) { - struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; - - lockdep_assert_held(&rq->lock); + struct intel_breadcrumbs *b = rq->engine->breadcrumbs; /* * We must wait for b->irq_lock so that we know the interrupt handler @@ -382,23 +439,19 @@ void i915_request_cancel_breadcrumb(struct i915_request *rq) list_del(&rq->signal_link); if (list_empty(&ce->signals)) - list_del_init(&ce->signal_link); + remove_signaling_context(b, ce); clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + i915_request_put(rq); } spin_unlock(&b->irq_lock); } -void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, - struct drm_printer *p) +static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; struct intel_context *ce; struct i915_request *rq; - if (list_empty(&b->signalers)) - return; - drm_printf(p, "Signals:\n"); spin_lock_irq(&b->irq_lock); @@ -414,3 +467,17 @@ void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, } spin_unlock_irq(&b->irq_lock); } + +void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, + struct drm_printer *p) +{ + struct intel_breadcrumbs *b; + + b = engine->breadcrumbs; + if (!b) + return; + + drm_printf(p, "IRQ: %s\n", enableddisabled(b->irq_armed)); + if (!list_empty(&b->signalers)) + print_signals(b, p); +} diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h new file mode 100644 index 000000000000..ed3d1deabfbd --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __INTEL_BREADCRUMBS__ +#define __INTEL_BREADCRUMBS__ + +#include <linux/irq_work.h> + +#include "intel_engine_types.h" + +struct drm_printer; +struct i915_request; +struct intel_breadcrumbs; + +struct intel_breadcrumbs * +intel_breadcrumbs_create(struct intel_engine_cs *irq_engine); +void intel_breadcrumbs_free(struct intel_breadcrumbs *b); + +void intel_breadcrumbs_reset(struct intel_breadcrumbs *b); +void intel_breadcrumbs_park(struct intel_breadcrumbs *b); + +static inline void +intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) +{ + irq_work_queue(&engine->breadcrumbs->irq_work); +} + +void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, + struct drm_printer *p); + +bool i915_request_enable_breadcrumb(struct i915_request *request); +void i915_request_cancel_breadcrumb(struct i915_request *request); + +#endif /* __INTEL_BREADCRUMBS__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h new file mode 100644 index 000000000000..8e53b9942695 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __INTEL_BREADCRUMBS_TYPES__ +#define __INTEL_BREADCRUMBS_TYPES__ + +#include <linux/irq_work.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/types.h> + +/* + * Rather than have every client wait upon all user interrupts, + * with the herd waking after every interrupt and each doing the + * heavyweight seqno dance, we delegate the task (of being the + * bottom-half of the user interrupt) to the first client. After + * every interrupt, we wake up one client, who does the heavyweight + * coherent seqno read and either goes back to sleep (if incomplete), + * or wakes up all the completed clients in parallel, before then + * transferring the bottom-half status to the next client in the queue. + * + * Compared to walking the entire list of waiters in a single dedicated + * bottom-half, we reduce the latency of the first waiter by avoiding + * a context switch, but incur additional coherent seqno reads when + * following the chain of request breadcrumbs. Since it is most likely + * that we have a single client waiting on each seqno, then reducing + * the overhead of waking that client is much preferred. + */ +struct intel_breadcrumbs { + spinlock_t irq_lock; /* protects the lists used in hardirq context */ + + /* Not all breadcrumbs are attached to physical HW */ + struct intel_engine_cs *irq_engine; + + struct list_head signalers; + struct list_head signaled_requests; + + struct irq_work irq_work; /* for use from inside irq_lock */ + + unsigned int irq_enabled; + + bool irq_armed; +}; + +#endif /* __INTEL_BREADCRUMBS_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 52db2bde44a3..d301dda1b261 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -93,57 +93,210 @@ static void intel_context_active_release(struct intel_context *ce) i915_active_release(&ce->active); } -int __intel_context_do_pin(struct intel_context *ce) +static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww) +{ + unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS; + int err; + + err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH); + if (err) + return err; + + err = i915_active_acquire(&vma->active); + if (err) + goto err_unpin; + + /* + * And mark it as a globally pinned object to let the shrinker know + * it cannot reclaim the object until we release it. + */ + i915_vma_make_unshrinkable(vma); + vma->obj->mm.dirty = true; + + return 0; + +err_unpin: + i915_vma_unpin(vma); + return err; +} + +static void __context_unpin_state(struct i915_vma *vma) +{ + i915_vma_make_shrinkable(vma); + i915_active_release(&vma->active); + __i915_vma_unpin(vma); +} + +static int __ring_active(struct intel_ring *ring, + struct i915_gem_ww_ctx *ww) +{ + int err; + + err = intel_ring_pin(ring, ww); + if (err) + return err; + + err = i915_active_acquire(&ring->vma->active); + if (err) + goto err_pin; + + return 0; + +err_pin: + intel_ring_unpin(ring); + return err; +} + +static void __ring_retire(struct intel_ring *ring) +{ + i915_active_release(&ring->vma->active); + intel_ring_unpin(ring); +} + +static int intel_context_pre_pin(struct intel_context *ce, + struct i915_gem_ww_ctx *ww) { int err; + CE_TRACE(ce, "active\n"); + + err = __ring_active(ce->ring, ww); + if (err) + return err; + + err = intel_timeline_pin(ce->timeline, ww); + if (err) + goto err_ring; + + if (!ce->state) + return 0; + + err = __context_pin_state(ce->state, ww); + if (err) + goto err_timeline; + + + return 0; + +err_timeline: + intel_timeline_unpin(ce->timeline); +err_ring: + __ring_retire(ce->ring); + return err; +} + +static void intel_context_post_unpin(struct intel_context *ce) +{ + if (ce->state) + __context_unpin_state(ce->state); + + intel_timeline_unpin(ce->timeline); + __ring_retire(ce->ring); +} + +int __intel_context_do_pin_ww(struct intel_context *ce, + struct i915_gem_ww_ctx *ww) +{ + bool handoff = false; + void *vaddr; + int err = 0; + if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { err = intel_context_alloc_state(ce); if (err) return err; } - err = i915_active_acquire(&ce->active); + /* + * We always pin the context/ring/timeline here, to ensure a pin + * refcount for __intel_context_active(), which prevent a lock + * inversion of ce->pin_mutex vs dma_resv_lock(). + */ + + err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww); + if (!err && ce->ring->vma->obj) + err = i915_gem_object_lock(ce->ring->vma->obj, ww); + if (!err && ce->state) + err = i915_gem_object_lock(ce->state->obj, ww); + if (!err) + err = intel_context_pre_pin(ce, ww); if (err) return err; - if (mutex_lock_interruptible(&ce->pin_mutex)) { - err = -EINTR; - goto out_release; - } + err = i915_active_acquire(&ce->active); + if (err) + goto err_ctx_unpin; + + err = ce->ops->pre_pin(ce, ww, &vaddr); + if (err) + goto err_release; + + err = mutex_lock_interruptible(&ce->pin_mutex); + if (err) + goto err_post_unpin; if (unlikely(intel_context_is_closed(ce))) { err = -ENOENT; - goto out_unlock; + goto err_unlock; } if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) { err = intel_context_active_acquire(ce); if (unlikely(err)) - goto out_unlock; + goto err_unlock; - err = ce->ops->pin(ce); - if (unlikely(err)) - goto err_active; + err = ce->ops->pin(ce, vaddr); + if (err) { + intel_context_active_release(ce); + goto err_unlock; + } CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n", i915_ggtt_offset(ce->ring->vma), ce->ring->head, ce->ring->tail); + handoff = true; smp_mb__before_atomic(); /* flush pin before it is visible */ atomic_inc(&ce->pin_count); } GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ - GEM_BUG_ON(i915_active_is_idle(&ce->active)); - goto out_unlock; -err_active: - intel_context_active_release(ce); -out_unlock: +err_unlock: mutex_unlock(&ce->pin_mutex); -out_release: +err_post_unpin: + if (!handoff) + ce->ops->post_unpin(ce); +err_release: i915_active_release(&ce->active); +err_ctx_unpin: + intel_context_post_unpin(ce); + + /* + * Unlock the hwsp_ggtt object since it's shared. + * In principle we can unlock all the global state locked above + * since it's pinned and doesn't need fencing, and will + * thus remain resident until it is explicitly unpinned. + */ + i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj); + + return err; +} + +int __intel_context_do_pin(struct intel_context *ce) +{ + struct i915_gem_ww_ctx ww; + int err; + + i915_gem_ww_ctx_init(&ww, true); +retry: + err = __intel_context_do_pin_ww(ce, &ww); + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); return err; } @@ -154,6 +307,7 @@ void intel_context_unpin(struct intel_context *ce) CE_TRACE(ce, "unpin\n"); ce->ops->unpin(ce); + ce->ops->post_unpin(ce); /* * Once released, we may asynchronously drop the active reference. @@ -166,65 +320,6 @@ void intel_context_unpin(struct intel_context *ce) intel_context_put(ce); } -static int __context_pin_state(struct i915_vma *vma) -{ - unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS; - int err; - - err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH); - if (err) - return err; - - err = i915_active_acquire(&vma->active); - if (err) - goto err_unpin; - - /* - * And mark it as a globally pinned object to let the shrinker know - * it cannot reclaim the object until we release it. - */ - i915_vma_make_unshrinkable(vma); - vma->obj->mm.dirty = true; - - return 0; - -err_unpin: - i915_vma_unpin(vma); - return err; -} - -static void __context_unpin_state(struct i915_vma *vma) -{ - i915_vma_make_shrinkable(vma); - i915_active_release(&vma->active); - __i915_vma_unpin(vma); -} - -static int __ring_active(struct intel_ring *ring) -{ - int err; - - err = intel_ring_pin(ring); - if (err) - return err; - - err = i915_active_acquire(&ring->vma->active); - if (err) - goto err_pin; - - return 0; - -err_pin: - intel_ring_unpin(ring); - return err; -} - -static void __ring_retire(struct intel_ring *ring) -{ - i915_active_release(&ring->vma->active); - intel_ring_unpin(ring); -} - __i915_active_call static void __intel_context_retire(struct i915_active *active) { @@ -235,48 +330,29 @@ static void __intel_context_retire(struct i915_active *active) intel_context_get_avg_runtime_ns(ce)); set_bit(CONTEXT_VALID_BIT, &ce->flags); - if (ce->state) - __context_unpin_state(ce->state); - - intel_timeline_unpin(ce->timeline); - __ring_retire(ce->ring); - + intel_context_post_unpin(ce); intel_context_put(ce); } static int __intel_context_active(struct i915_active *active) { struct intel_context *ce = container_of(active, typeof(*ce), active); - int err; - - CE_TRACE(ce, "active\n"); intel_context_get(ce); - err = __ring_active(ce->ring); - if (err) - goto err_put; + /* everything should already be activated by intel_context_pre_pin() */ + GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active)); + __intel_ring_pin(ce->ring); - err = intel_timeline_pin(ce->timeline); - if (err) - goto err_ring; + __intel_timeline_pin(ce->timeline); - if (!ce->state) - return 0; - - err = __context_pin_state(ce->state); - if (err) - goto err_timeline; + if (ce->state) { + GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active)); + __i915_vma_pin(ce->state); + i915_vma_make_unshrinkable(ce->state); + } return 0; - -err_timeline: - intel_timeline_unpin(ce->timeline); -err_ring: - __ring_retire(ce->ring); -err_put: - intel_context_put(ce); - return err; } void @@ -382,15 +458,37 @@ int intel_context_prepare_remote_request(struct intel_context *ce, struct i915_request *intel_context_create_request(struct intel_context *ce) { + struct i915_gem_ww_ctx ww; struct i915_request *rq; int err; - err = intel_context_pin(ce); - if (unlikely(err)) - return ERR_PTR(err); + i915_gem_ww_ctx_init(&ww, true); +retry: + err = intel_context_pin_ww(ce, &ww); + if (!err) { + rq = i915_request_create(ce); + intel_context_unpin(ce); + } else if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } else { + rq = ERR_PTR(err); + } + + i915_gem_ww_ctx_fini(&ww); - rq = i915_request_create(ce); - intel_context_unpin(ce); + if (IS_ERR(rq)) + return rq; + + /* + * timeline->mutex should be the inner lock, but is used as outer lock. + * Hack around this to shut up lockdep in selftests.. + */ + lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie); + mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_); + mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_); + rq->cookie = lockdep_pin_lock(&ce->timeline->mutex); return rq; } diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index 07be021882cc..fda2eba81e22 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -25,6 +25,8 @@ ##__VA_ARGS__); \ } while (0) +struct i915_gem_ww_ctx; + void intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine); void intel_context_fini(struct intel_context *ce); @@ -81,6 +83,8 @@ static inline void intel_context_unlock_pinned(struct intel_context *ce) } int __intel_context_do_pin(struct intel_context *ce); +int __intel_context_do_pin_ww(struct intel_context *ce, + struct i915_gem_ww_ctx *ww); static inline bool intel_context_pin_if_active(struct intel_context *ce) { @@ -95,6 +99,15 @@ static inline int intel_context_pin(struct intel_context *ce) return __intel_context_do_pin(ce); } +static inline int intel_context_pin_ww(struct intel_context *ce, + struct i915_gem_ww_ctx *ww) +{ + if (likely(intel_context_pin_if_active(ce))) + return 0; + + return __intel_context_do_pin_ww(ce, ww); +} + static inline void __intel_context_pin(struct intel_context *ce) { GEM_BUG_ON(!intel_context_is_pinned(ce)); diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index 4954b0df4864..552cb57a2e8c 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -23,6 +23,7 @@ DECLARE_EWMA(runtime, 3, 8); struct i915_gem_context; +struct i915_gem_ww_ctx; struct i915_vma; struct intel_context; struct intel_ring; @@ -30,8 +31,10 @@ struct intel_ring; struct intel_context_ops { int (*alloc)(struct intel_context *ce); - int (*pin)(struct intel_context *ce); + int (*pre_pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void **vaddr); + int (*pin)(struct intel_context *ce, void *vaddr); void (*unpin)(struct intel_context *ce); + void (*post_unpin)(struct intel_context *ce); void (*enter)(struct intel_context *ce); void (*exit)(struct intel_context *ce); diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index a9249a23903a..08e2c000dcc3 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -223,26 +223,6 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, void intel_engine_init_execlists(struct intel_engine_cs *engine); -void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); -void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); - -void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine); - -static inline void -intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) -{ - irq_work_queue(&engine->breadcrumbs.irq_work); -} - -void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); -void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); - -void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine, - struct intel_context *ce); - -void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, - struct drm_printer *p); - static inline u32 *__gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) { memset(batch, 0, 6 * sizeof(u32)); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index dd1a42c4d344..5bfb5f7ed02c 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -28,6 +28,7 @@ #include "i915_drv.h" +#include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_engine.h" #include "intel_engine_pm.h" @@ -213,7 +214,7 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) break; default: MISSING_CASE(class); - /* fall through */ + fallthrough; case VIDEO_DECODE_CLASS: case VIDEO_ENHANCEMENT_CLASS: case COPY_ENGINE_CLASS: @@ -634,7 +635,7 @@ static int pin_ggtt_status_page(struct intel_engine_cs *engine, else flags = PIN_HIGH; - return i915_ggtt_pin(vma, 0, flags); + return i915_ggtt_pin(vma, NULL, 0, flags); } static int init_status_page(struct intel_engine_cs *engine) @@ -700,8 +701,13 @@ static int engine_setup_common(struct intel_engine_cs *engine) if (err) return err; + engine->breadcrumbs = intel_breadcrumbs_create(engine); + if (!engine->breadcrumbs) { + err = -ENOMEM; + goto err_status; + } + intel_engine_init_active(engine, ENGINE_PHYSICAL); - intel_engine_init_breadcrumbs(engine); intel_engine_init_execlists(engine); intel_engine_init_cmd_parser(engine); intel_engine_init__pm(engine); @@ -716,6 +722,10 @@ static int engine_setup_common(struct intel_engine_cs *engine) intel_engine_init_ctx_wa(engine); return 0; + +err_status: + cleanup_status_page(engine); + return err; } struct measure_breadcrumb { @@ -785,9 +795,11 @@ intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass) } static struct intel_context * -create_kernel_context(struct intel_engine_cs *engine) +create_pinned_context(struct intel_engine_cs *engine, + unsigned int hwsp, + struct lock_class_key *key, + const char *name) { - static struct lock_class_key kernel; struct intel_context *ce; int err; @@ -796,6 +808,7 @@ create_kernel_context(struct intel_engine_cs *engine) return ce; __set_bit(CONTEXT_BARRIER_BIT, &ce->flags); + ce->timeline = page_pack_bits(NULL, hwsp); err = intel_context_pin(ce); /* perma-pin so it is always available */ if (err) { @@ -809,11 +822,20 @@ create_kernel_context(struct intel_engine_cs *engine) * should we need to inject GPU operations during their request * construction. */ - lockdep_set_class(&ce->timeline->mutex, &kernel); + lockdep_set_class_and_name(&ce->timeline->mutex, key, name); return ce; } +static struct intel_context * +create_kernel_context(struct intel_engine_cs *engine) +{ + static struct lock_class_key kernel; + + return create_pinned_context(engine, I915_GEM_HWS_SEQNO_ADDR, + &kernel, "kernel_context"); +} + /** * intel_engines_init_common - initialize cengine state which might require hw access * @engine: Engine to initialize. @@ -902,9 +924,9 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) tasklet_kill(&engine->execlists.tasklet); /* flush the callback */ cleanup_status_page(engine); + intel_breadcrumbs_free(engine->breadcrumbs); intel_engine_fini_retire(engine); - intel_engine_fini_breadcrumbs(engine); intel_engine_cleanup_cmd_parser(engine); if (engine->default_state) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 8ec3eecf3e39..f7b2e07e2229 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -6,6 +6,7 @@ #include "i915_drv.h" +#include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_engine.h" #include "intel_engine_heartbeat.h" @@ -247,7 +248,7 @@ static int __engine_park(struct intel_wakeref *wf) call_idle_barriers(engine); /* cleanup after wedging */ intel_engine_park_heartbeat(engine); - intel_engine_disarm_breadcrumbs(engine); + intel_breadcrumbs_park(engine->breadcrumbs); /* Must be reset upon idling, or we may miss the busy wakeup. */ GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 8de92fd7d392..c400aaa2287b 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -22,6 +22,7 @@ #include "i915_pmu.h" #include "i915_priolist_types.h" #include "i915_selftest.h" +#include "intel_breadcrumbs_types.h" #include "intel_sseu.h" #include "intel_timeline_types.h" #include "intel_uncore.h" @@ -373,34 +374,8 @@ struct intel_engine_cs { */ struct ewma__engine_latency latency; - /* Rather than have every client wait upon all user interrupts, - * with the herd waking after every interrupt and each doing the - * heavyweight seqno dance, we delegate the task (of being the - * bottom-half of the user interrupt) to the first client. After - * every interrupt, we wake up one client, who does the heavyweight - * coherent seqno read and either goes back to sleep (if incomplete), - * or wakes up all the completed clients in parallel, before then - * transferring the bottom-half status to the next client in the queue. - * - * Compared to walking the entire list of waiters in a single dedicated - * bottom-half, we reduce the latency of the first waiter by avoiding - * a context switch, but incur additional coherent seqno reads when - * following the chain of request breadcrumbs. Since it is most likely - * that we have a single client waiting on each seqno, then reducing - * the overhead of waking that client is much preferred. - */ - struct intel_breadcrumbs { - spinlock_t irq_lock; - struct list_head signalers; - - struct list_head signaled_requests; - - struct irq_work irq_work; /* for use from inside irq_lock */ - - unsigned int irq_enabled; - - bool irq_armed; - } breadcrumbs; + /* Keep track of all the seqno used, a trail of breadcrumbs */ + struct intel_breadcrumbs *breadcrumbs; struct intel_engine_pmu { /** diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 62979ea591f0..81c05f551b9c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -78,8 +78,6 @@ int i915_ggtt_init_hw(struct drm_i915_private *i915) { int ret; - stash_init(&i915->mm.wc_stash); - /* * Note that we use page colouring to enforce a guard page at the * end of the address space. This is required as the CS may prefetch @@ -232,7 +230,7 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, /* Fill the allocated but "unused" space beyond the end of the buffer */ while (gte < end) - gen8_set_pte(gte++, vm->scratch[0].encode); + gen8_set_pte(gte++, vm->scratch[0]->encode); /* * We want to flush the TLBs only after we're certain all the PTE @@ -283,7 +281,7 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, /* Fill the allocated but "unused" space beyond the end of the buffer */ while (gte < end) - iowrite32(vm->scratch[0].encode, gte++); + iowrite32(vm->scratch[0]->encode, gte++); /* * We want to flush the TLBs only after we're certain all the PTE @@ -303,7 +301,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned int first_entry = start / I915_GTT_PAGE_SIZE; unsigned int num_entries = length / I915_GTT_PAGE_SIZE; - const gen8_pte_t scratch_pte = vm->scratch[0].encode; + const gen8_pte_t scratch_pte = vm->scratch[0]->encode; gen8_pte_t __iomem *gtt_base = (gen8_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; @@ -401,7 +399,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, first_entry, num_entries, max_entries)) num_entries = max_entries; - scratch_pte = vm->scratch[0].encode; + scratch_pte = vm->scratch[0]->encode; for (i = 0; i < num_entries; i++) iowrite32(scratch_pte, >t_base[i]); } @@ -436,16 +434,17 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm, intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); } -static int ggtt_bind_vma(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static void ggtt_bind_vma(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags; if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK)) - return 0; + return; /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ pte_flags = 0; @@ -454,8 +453,6 @@ static int ggtt_bind_vma(struct i915_address_space *vm, vm->insert_entries(vm, vma, cache_level, pte_flags); vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; - - return 0; } static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) @@ -568,31 +565,25 @@ err: return ret; } -static int aliasing_gtt_bind_vma(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static void aliasing_gtt_bind_vma(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { u32 pte_flags; - int ret; /* Currently applicable only to VLV */ pte_flags = 0; if (i915_gem_object_is_readonly(vma->obj)) pte_flags |= PTE_READ_ONLY; - if (flags & I915_VMA_LOCAL_BIND) { - struct i915_ppgtt *alias = i915_vm_to_ggtt(vm)->alias; - - ret = ppgtt_bind_vma(&alias->vm, vma, cache_level, flags); - if (ret) - return ret; - } + if (flags & I915_VMA_LOCAL_BIND) + ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, + stash, vma, cache_level, flags); if (flags & I915_VMA_GLOBAL_BIND) vm->insert_entries(vm, vma, cache_level, pte_flags); - - return 0; } static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, @@ -607,6 +598,7 @@ static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) { + struct i915_vm_pt_stash stash = {}; struct i915_ppgtt *ppgtt; int err; @@ -619,15 +611,21 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) goto err_ppgtt; } + err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); + if (err) + goto err_ppgtt; + + err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + if (err) + goto err_stash; + /* * Note we only pre-allocate as far as the end of the global * GTT. On 48b / 4-level page-tables, the difference is very, * very significant! We have to preallocate as GVT/vgpu does * not like the page directory disappearing. */ - err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total); - if (err) - goto err_ppgtt; + ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); ggtt->alias = ppgtt; ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; @@ -638,8 +636,11 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma); ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; + i915_vm_free_pt_stash(&ppgtt->vm, &stash); return 0; +err_stash: + i915_vm_free_pt_stash(&ppgtt->vm, &stash); err_ppgtt: i915_vm_put(&ppgtt->vm); return err; @@ -715,18 +716,11 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) void i915_ggtt_driver_release(struct drm_i915_private *i915) { struct i915_ggtt *ggtt = &i915->ggtt; - struct pagevec *pvec; fini_aliasing_ppgtt(ggtt); intel_ggtt_fini_fences(ggtt); ggtt_cleanup_hw(ggtt); - - pvec = &i915->mm.wc_stash.pvec; - if (pvec->nr) { - set_pages_array_wb(pvec->pages, pvec->nr); - __pagevec_release(pvec); - } } static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) @@ -789,7 +783,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) return -ENOMEM; } - ret = setup_scratch_page(&ggtt->vm, GFP_DMA32); + ret = setup_scratch_page(&ggtt->vm); if (ret) { drm_err(&i915->drm, "Scratch setup failed\n"); /* iounmap will also get called at remove, but meh */ @@ -797,8 +791,8 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) return ret; } - ggtt->vm.scratch[0].encode = - ggtt->vm.pte_encode(px_dma(&ggtt->vm.scratch[0]), + ggtt->vm.scratch[0]->encode = + ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), I915_CACHE_NONE, 0); return 0; @@ -824,7 +818,7 @@ static void gen6_gmch_remove(struct i915_address_space *vm) struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); iounmap(ggtt->gsm); - cleanup_scratch_page(vm); + free_scratch(vm); } static struct resource pci_resource(struct pci_dev *pdev, int bar) @@ -852,6 +846,8 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) else size = gen8_get_total_gtt_size(snb_gmch_ctl); + ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; ggtt->vm.cleanup = gen6_gmch_remove; ggtt->vm.insert_page = gen8_ggtt_insert_page; @@ -1000,6 +996,8 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt) size = gen6_get_total_gtt_size(snb_gmch_ctl); ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; + ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->vm.clear_range = nop_clear_range; if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915)) ggtt->vm.clear_range = gen6_ggtt_clear_range; @@ -1050,6 +1048,8 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt) ggtt->gmadr = (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end); + ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->do_idle_maps = needs_idle_maps(i915); ggtt->vm.insert_page = i915_ggtt_insert_page; ggtt->vm.insert_entries = i915_ggtt_insert_entries; @@ -1165,11 +1165,6 @@ void i915_ggtt_disable_guc(struct i915_ggtt *ggtt) ggtt->invalidate(ggtt); } -static unsigned int clear_bind(struct i915_vma *vma) -{ - return atomic_fetch_and(~I915_VMA_BIND_MASK, &vma->flags); -} - void i915_ggtt_resume(struct i915_ggtt *ggtt) { struct i915_vma *vma; @@ -1187,11 +1182,13 @@ void i915_ggtt_resume(struct i915_ggtt *ggtt) /* clflush objects bound into the GGTT and rebind them. */ list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) { struct drm_i915_gem_object *obj = vma->obj; - unsigned int was_bound = clear_bind(vma); + unsigned int was_bound = + atomic_read(&vma->flags) & I915_VMA_BIND_MASK; - WARN_ON(i915_vma_bind(vma, - obj ? obj->cache_level : 0, - was_bound, NULL)); + GEM_BUG_ON(!was_bound); + vma->ops->bind_vma(&ggtt->vm, NULL, vma, + obj ? obj->cache_level : 0, + was_bound); if (obj) { /* only used during resume => exclusive access */ flush |= fetch_and_zero(&obj->write_domain); obj->read_domains |= I915_GEM_DOMAIN_GTT; @@ -1437,7 +1434,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) switch (vma->ggtt_view.type) { default: GEM_BUG_ON(vma->ggtt_view.type); - /* fall through */ + fallthrough; case I915_GGTT_VIEW_NORMAL: vma->pages = vma->obj->mm.pages; return 0; diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index e0755f1a904b..39b428c5049c 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -356,7 +356,7 @@ static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) goto err_unref; } - ret = i915_ggtt_pin(vma, 0, PIN_HIGH); + ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH); if (ret) goto err_unref; @@ -406,21 +406,20 @@ static int __engines_record_defaults(struct intel_gt *gt) /* We must be able to switch to something! */ GEM_BUG_ON(!engine->kernel_context); - err = intel_renderstate_init(&so, engine); - if (err) - goto out; - ce = intel_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; } - rq = intel_context_create_request(ce); + err = intel_renderstate_init(&so, ce); + if (err) + goto err; + + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); - intel_context_put(ce); - goto out; + goto err_fini; } err = intel_engine_emit_ctx_wa(rq); @@ -434,9 +433,13 @@ static int __engines_record_defaults(struct intel_gt *gt) err_rq: requests[id] = i915_request_get(rq); i915_request_add(rq); - intel_renderstate_fini(&so); - if (err) +err_fini: + intel_renderstate_fini(&so, ce); +err: + if (err) { + intel_context_put(ce); goto out; + } } /* Flush the default context image to memory, and enable powersaving. */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index 418ae184cecf..4b7671ac5dca 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -35,39 +35,65 @@ static void node_free(struct intel_gt_buffer_pool_node *node) { i915_gem_object_put(node->obj); i915_active_fini(&node->active); - kfree(node); + kfree_rcu(node, rcu); } -static void pool_free_work(struct work_struct *wrk) +static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep) { - struct intel_gt_buffer_pool *pool = - container_of(wrk, typeof(*pool), work.work); - struct intel_gt_buffer_pool_node *node, *next; - unsigned long old = jiffies - HZ; + struct intel_gt_buffer_pool_node *node, *stale = NULL; bool active = false; - LIST_HEAD(stale); int n; /* Free buffers that have not been used in the past second */ - spin_lock_irq(&pool->lock); for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { struct list_head *list = &pool->cache_list[n]; - /* Most recent at head; oldest at tail */ - list_for_each_entry_safe_reverse(node, next, list, link) { - if (time_before(node->age, old)) - break; + if (list_empty(list)) + continue; + + if (spin_trylock_irq(&pool->lock)) { + struct list_head *pos; + + /* Most recent at head; oldest at tail */ + list_for_each_prev(pos, list) { + unsigned long age; + + node = list_entry(pos, typeof(*node), link); + + age = READ_ONCE(node->age); + if (!age || jiffies - age < keep) + break; + + /* Check we are the first to claim this node */ + if (!xchg(&node->age, 0)) + break; - list_move(&node->link, &stale); + node->free = stale; + stale = node; + } + if (!list_is_last(pos, list)) + __list_del_many(pos, list); + + spin_unlock_irq(&pool->lock); } + active |= !list_empty(list); } - spin_unlock_irq(&pool->lock); - list_for_each_entry_safe(node, next, &stale, link) + while ((node = stale)) { + stale = stale->free; node_free(node); + } + + return active; +} + +static void pool_free_work(struct work_struct *wrk) +{ + struct intel_gt_buffer_pool *pool = + container_of(wrk, typeof(*pool), work.work); - if (active) + if (pool_free_older_than(pool, HZ)) schedule_delayed_work(&pool->work, round_jiffies_up_relative(HZ)); } @@ -109,8 +135,8 @@ static void pool_retire(struct i915_active *ref) i915_gem_object_make_purgeable(node->obj); spin_lock_irqsave(&pool->lock, flags); - node->age = jiffies; - list_add(&node->link, list); + list_add_rcu(&node->link, list); + WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */ spin_unlock_irqrestore(&pool->lock, flags); schedule_delayed_work(&pool->work, @@ -151,20 +177,30 @@ intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size) struct intel_gt_buffer_pool *pool = >->buffer_pool; struct intel_gt_buffer_pool_node *node; struct list_head *list; - unsigned long flags; int ret; size = PAGE_ALIGN(size); list = bucket_for_size(pool, size); - spin_lock_irqsave(&pool->lock, flags); - list_for_each_entry(node, list, link) { + rcu_read_lock(); + list_for_each_entry_rcu(node, list, link) { + unsigned long age; + if (node->obj->base.size < size) continue; - list_del(&node->link); - break; + + age = READ_ONCE(node->age); + if (!age) + continue; + + if (cmpxchg(&node->age, age, 0) == age) { + spin_lock_irq(&pool->lock); + list_del_rcu(&node->link); + spin_unlock_irq(&pool->lock); + break; + } } - spin_unlock_irqrestore(&pool->lock, flags); + rcu_read_unlock(); if (&node->link == list) { node = node_create(pool, size); @@ -192,28 +228,13 @@ void intel_gt_init_buffer_pool(struct intel_gt *gt) INIT_DELAYED_WORK(&pool->work, pool_free_work); } -static void pool_free_imm(struct intel_gt_buffer_pool *pool) -{ - int n; - - spin_lock_irq(&pool->lock); - for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { - struct intel_gt_buffer_pool_node *node, *next; - struct list_head *list = &pool->cache_list[n]; - - list_for_each_entry_safe(node, next, list, link) - node_free(node); - INIT_LIST_HEAD(list); - } - spin_unlock_irq(&pool->lock); -} - void intel_gt_flush_buffer_pool(struct intel_gt *gt) { struct intel_gt_buffer_pool *pool = >->buffer_pool; do { - pool_free_imm(pool); + while (pool_free_older_than(pool, 0)) + ; } while (cancel_delayed_work_sync(&pool->work)); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h index e28bdda771ed..bcf1658c9633 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h @@ -25,7 +25,11 @@ struct intel_gt_buffer_pool_node { struct i915_active active; struct drm_i915_gem_object *obj; struct list_head link; - struct intel_gt_buffer_pool *pool; + union { + struct intel_gt_buffer_pool *pool; + struct intel_gt_buffer_pool_node *free; + struct rcu_head rcu; + }; unsigned long age; }; diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index b05da68e52f4..257063a57101 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -8,6 +8,7 @@ #include "i915_drv.h" #include "i915_irq.h" +#include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_irq.h" #include "intel_uncore.h" diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 2a72cce63fd9..3f1114b58b01 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -11,160 +11,24 @@ #include "intel_gt.h" #include "intel_gtt.h" -void stash_init(struct pagestash *stash) +struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) { - pagevec_init(&stash->pvec); - spin_lock_init(&stash->lock); -} - -static struct page *stash_pop_page(struct pagestash *stash) -{ - struct page *page = NULL; - - spin_lock(&stash->lock); - if (likely(stash->pvec.nr)) - page = stash->pvec.pages[--stash->pvec.nr]; - spin_unlock(&stash->lock); - - return page; -} - -static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec) -{ - unsigned int nr; - - spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING); - - nr = min_t(typeof(nr), pvec->nr, pagevec_space(&stash->pvec)); - memcpy(stash->pvec.pages + stash->pvec.nr, - pvec->pages + pvec->nr - nr, - sizeof(pvec->pages[0]) * nr); - stash->pvec.nr += nr; - - spin_unlock(&stash->lock); - - pvec->nr -= nr; -} - -static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp) -{ - struct pagevec stack; - struct page *page; - if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1))) i915_gem_shrink_all(vm->i915); - page = stash_pop_page(&vm->free_pages); - if (page) - return page; - - if (!vm->pt_kmap_wc) - return alloc_page(gfp); - - /* Look in our global stash of WC pages... */ - page = stash_pop_page(&vm->i915->mm.wc_stash); - if (page) - return page; - - /* - * Otherwise batch allocate pages to amortize cost of set_pages_wc. - * - * We have to be careful as page allocation may trigger the shrinker - * (via direct reclaim) which will fill up the WC stash underneath us. - * So we add our WB pages into a temporary pvec on the stack and merge - * them into the WC stash after all the allocations are complete. - */ - pagevec_init(&stack); - do { - struct page *page; - - page = alloc_page(gfp); - if (unlikely(!page)) - break; - - stack.pages[stack.nr++] = page; - } while (pagevec_space(&stack)); - - if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) { - page = stack.pages[--stack.nr]; - - /* Merge spare WC pages to the global stash */ - if (stack.nr) - stash_push_pagevec(&vm->i915->mm.wc_stash, &stack); - - /* Push any surplus WC pages onto the local VM stash */ - if (stack.nr) - stash_push_pagevec(&vm->free_pages, &stack); - } - - /* Return unwanted leftovers */ - if (unlikely(stack.nr)) { - WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr)); - __pagevec_release(&stack); - } - - return page; + return i915_gem_object_create_internal(vm->i915, sz); } -static void vm_free_pages_release(struct i915_address_space *vm, - bool immediate) +int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { - struct pagevec *pvec = &vm->free_pages.pvec; - struct pagevec stack; - - lockdep_assert_held(&vm->free_pages.lock); - GEM_BUG_ON(!pagevec_count(pvec)); - - if (vm->pt_kmap_wc) { - /* - * When we use WC, first fill up the global stash and then - * only if full immediately free the overflow. - */ - stash_push_pagevec(&vm->i915->mm.wc_stash, pvec); - - /* - * As we have made some room in the VM's free_pages, - * we can wait for it to fill again. Unless we are - * inside i915_address_space_fini() and must - * immediately release the pages! - */ - if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1)) - return; + int err; - /* - * We have to drop the lock to allow ourselves to sleep, - * so take a copy of the pvec and clear the stash for - * others to use it as we sleep. - */ - stack = *pvec; - pagevec_reinit(pvec); - spin_unlock(&vm->free_pages.lock); - - pvec = &stack; - set_pages_array_wb(pvec->pages, pvec->nr); - - spin_lock(&vm->free_pages.lock); - } + err = i915_gem_object_pin_pages(obj); + if (err) + return err; - __pagevec_release(pvec); -} - -static void vm_free_page(struct i915_address_space *vm, struct page *page) -{ - /* - * On !llc, we need to change the pages back to WB. We only do so - * in bulk, so we rarely need to change the page attributes here, - * but doing so requires a stop_machine() from deep inside arch/x86/mm. - * To make detection of the possible sleep more likely, use an - * unconditional might_sleep() for everybody. - */ - might_sleep(); - spin_lock(&vm->free_pages.lock); - while (!pagevec_space(&vm->free_pages.pvec)) - vm_free_pages_release(vm, false); - GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec) >= PAGEVEC_SIZE); - pagevec_add(&vm->free_pages.pvec, page); - spin_unlock(&vm->free_pages.lock); + i915_gem_object_make_unshrinkable(obj); + return 0; } void __i915_vm_close(struct i915_address_space *vm) @@ -194,14 +58,7 @@ void __i915_vm_close(struct i915_address_space *vm) void i915_address_space_fini(struct i915_address_space *vm) { - spin_lock(&vm->free_pages.lock); - if (pagevec_count(&vm->free_pages.pvec)) - vm_free_pages_release(vm, true); - GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec)); - spin_unlock(&vm->free_pages.lock); - drm_mm_takedown(&vm->mm); - mutex_destroy(&vm->mutex); } @@ -246,8 +103,6 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) drm_mm_init(&vm->mm, 0, vm->total); vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; - stash_init(&vm->free_pages); - INIT_LIST_HEAD(&vm->bound_list); } @@ -264,64 +119,50 @@ void clear_pages(struct i915_vma *vma) memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); } -static int __setup_page_dma(struct i915_address_space *vm, - struct i915_page_dma *p, - gfp_t gfp) -{ - p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL); - if (unlikely(!p->page)) - return -ENOMEM; - - p->daddr = dma_map_page_attrs(vm->dma, - p->page, 0, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL, - DMA_ATTR_SKIP_CPU_SYNC | - DMA_ATTR_NO_WARN); - if (unlikely(dma_mapping_error(vm->dma, p->daddr))) { - vm_free_page(vm, p->page); - return -ENOMEM; - } - - return 0; -} - -int setup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p) +dma_addr_t __px_dma(struct drm_i915_gem_object *p) { - return __setup_page_dma(vm, p, __GFP_HIGHMEM); + GEM_BUG_ON(!i915_gem_object_has_pages(p)); + return sg_dma_address(p->mm.pages->sgl); } -void cleanup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p) +struct page *__px_page(struct drm_i915_gem_object *p) { - dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - vm_free_page(vm, p->page); + GEM_BUG_ON(!i915_gem_object_has_pages(p)); + return sg_page(p->mm.pages->sgl); } void -fill_page_dma(const struct i915_page_dma *p, const u64 val, unsigned int count) +fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count) { - kunmap_atomic(memset64(kmap_atomic(p->page), val, count)); + struct page *page = __px_page(p); + void *vaddr; + + vaddr = kmap(page); + memset64(vaddr, val, count); + clflush_cache_range(vaddr, PAGE_SIZE); + kunmap(page); } -static void poison_scratch_page(struct page *page, unsigned long size) +static void poison_scratch_page(struct drm_i915_gem_object *scratch) { - if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) - return; + struct sgt_iter sgt; + struct page *page; + u8 val; - GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); + val = 0; + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + val = POISON_FREE; - do { + for_each_sgt_page(page, sgt, scratch->mm.pages) { void *vaddr; vaddr = kmap(page); - memset(vaddr, POISON_FREE, PAGE_SIZE); + memset(vaddr, val, PAGE_SIZE); kunmap(page); - - page = pfn_to_page(page_to_pfn(page) + 1); - size -= PAGE_SIZE; - } while (size); + } } -int setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) +int setup_scratch_page(struct i915_address_space *vm) { unsigned long size; @@ -338,21 +179,27 @@ int setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) */ size = I915_GTT_PAGE_SIZE_4K; if (i915_vm_is_4lvl(vm) && - HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) { + HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) size = I915_GTT_PAGE_SIZE_64K; - gfp |= __GFP_NOWARN; - } - gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL; do { - unsigned int order = get_order(size); - struct page *page; - dma_addr_t addr; + struct drm_i915_gem_object *obj; - page = alloc_pages(gfp, order); - if (unlikely(!page)) + obj = vm->alloc_pt_dma(vm, size); + if (IS_ERR(obj)) goto skip; + if (pin_pt_dma(vm, obj)) + goto skip_obj; + + /* We need a single contiguous page for our scratch */ + if (obj->mm.page_sizes.sg < size) + goto skip_obj; + + /* And it needs to be correspondingly aligned */ + if (__px_dma(obj) & (size - 1)) + goto skip_obj; + /* * Use a non-zero scratch page for debugging. * @@ -362,61 +209,28 @@ int setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) * should it ever be accidentally used, the effect should be * fairly benign. */ - poison_scratch_page(page, size); - - addr = dma_map_page_attrs(vm->dma, - page, 0, size, - PCI_DMA_BIDIRECTIONAL, - DMA_ATTR_SKIP_CPU_SYNC | - DMA_ATTR_NO_WARN); - if (unlikely(dma_mapping_error(vm->dma, addr))) - goto free_page; - - if (unlikely(!IS_ALIGNED(addr, size))) - goto unmap_page; - - vm->scratch[0].base.page = page; - vm->scratch[0].base.daddr = addr; - vm->scratch_order = order; + poison_scratch_page(obj); + + vm->scratch[0] = obj; + vm->scratch_order = get_order(size); return 0; -unmap_page: - dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL); -free_page: - __free_pages(page, order); +skip_obj: + i915_gem_object_put(obj); skip: if (size == I915_GTT_PAGE_SIZE_4K) return -ENOMEM; size = I915_GTT_PAGE_SIZE_4K; - gfp &= ~__GFP_NOWARN; } while (1); } -void cleanup_scratch_page(struct i915_address_space *vm) -{ - struct i915_page_dma *p = px_base(&vm->scratch[0]); - unsigned int order = vm->scratch_order; - - dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT, - PCI_DMA_BIDIRECTIONAL); - __free_pages(p->page, order); -} - void free_scratch(struct i915_address_space *vm) { int i; - if (!px_dma(&vm->scratch[0])) /* set to 0 on clones */ - return; - - for (i = 1; i <= vm->top; i++) { - if (!px_dma(&vm->scratch[i])) - break; - cleanup_page_dma(vm, px_base(&vm->scratch[i])); - } - - cleanup_scratch_page(vm); + for (i = 0; i <= vm->top; i++) + i915_gem_object_put(vm->scratch[i]); } void gtt_write_workarounds(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index f2b75078e05f..c13c650ced22 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -134,38 +134,29 @@ typedef u64 gen8_pte_t; #define GEN8_PDE_IPS_64K BIT(11) #define GEN8_PDE_PS_2M BIT(7) +enum i915_cache_level; + +struct drm_i915_file_private; +struct drm_i915_gem_object; struct i915_fence_reg; +struct i915_vma; +struct intel_gt; #define for_each_sgt_daddr(__dp, __iter, __sgt) \ __for_each_sgt_daddr(__dp, __iter, __sgt, I915_GTT_PAGE_SIZE) -struct i915_page_dma { - struct page *page; +struct i915_page_table { + struct drm_i915_gem_object *base; union { - dma_addr_t daddr; - - /* - * For gen6/gen7 only. This is the offset in the GGTT - * where the page directory entries for PPGTT begin - */ - u32 ggtt_offset; + atomic_t used; + struct i915_page_table *stash; }; }; -struct i915_page_scratch { - struct i915_page_dma base; - u64 encode; -}; - -struct i915_page_table { - struct i915_page_dma base; - atomic_t used; -}; - struct i915_page_directory { struct i915_page_table pt; spinlock_t lock; - void *entry[512]; + void **entry; }; #define __px_choose_expr(x, type, expr, other) \ @@ -176,12 +167,14 @@ struct i915_page_directory { other) #define px_base(px) \ - __px_choose_expr(px, struct i915_page_dma *, __x, \ - __px_choose_expr(px, struct i915_page_scratch *, &__x->base, \ - __px_choose_expr(px, struct i915_page_table *, &__x->base, \ - __px_choose_expr(px, struct i915_page_directory *, &__x->pt.base, \ - (void)0)))) -#define px_dma(px) (px_base(px)->daddr) + __px_choose_expr(px, struct drm_i915_gem_object *, __x, \ + __px_choose_expr(px, struct i915_page_table *, __x->base, \ + __px_choose_expr(px, struct i915_page_directory *, __x->pt.base, \ + (void)0))) + +struct page *__px_page(struct drm_i915_gem_object *p); +dma_addr_t __px_dma(struct drm_i915_gem_object *p); +#define px_dma(px) (__px_dma(px_base(px))) #define px_pt(px) \ __px_choose_expr(px, struct i915_page_table *, __x, \ @@ -189,19 +182,18 @@ struct i915_page_directory { (void)0)) #define px_used(px) (&px_pt(px)->used) -enum i915_cache_level; - -struct drm_i915_file_private; -struct drm_i915_gem_object; -struct i915_vma; -struct intel_gt; +struct i915_vm_pt_stash { + /* preallocated chains of page tables/directories */ + struct i915_page_table *pt[2]; +}; struct i915_vma_ops { /* Map an object into an address space with the given cache flags. */ - int (*bind_vma)(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags); + void (*bind_vma)(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags); /* * Unmap an object from an address space. This usually consists of * setting the valid PTE entries to a reserved scratch page. @@ -213,13 +205,6 @@ struct i915_vma_ops { void (*clear_pages)(struct i915_vma *vma); }; -struct pagestash { - spinlock_t lock; - struct pagevec pvec; -}; - -void stash_init(struct pagestash *stash); - struct i915_address_space { struct kref ref; struct rcu_work rcu; @@ -256,33 +241,33 @@ struct i915_address_space { #define VM_CLASS_GGTT 0 #define VM_CLASS_PPGTT 1 - struct i915_page_scratch scratch[4]; - unsigned int scratch_order; - unsigned int top; - + struct drm_i915_gem_object *scratch[4]; /** * List of vma currently bound. */ struct list_head bound_list; - struct pagestash free_pages; - /* Global GTT */ bool is_ggtt:1; - /* Some systems require uncached updates of the page directories */ - bool pt_kmap_wc:1; - /* Some systems support read-only mappings for GGTT and/or PPGTT */ bool has_read_only:1; + u8 top; + u8 pd_shift; + u8 scratch_order; + + struct drm_i915_gem_object * + (*alloc_pt_dma)(struct i915_address_space *vm, int sz); + u64 (*pte_encode)(dma_addr_t addr, enum i915_cache_level level, u32 flags); /* Create a valid PTE */ #define PTE_READ_ONLY BIT(0) - int (*allocate_va_range)(struct i915_address_space *vm, - u64 start, u64 length); + void (*allocate_va_range)(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + u64 start, u64 length); void (*clear_range)(struct i915_address_space *vm, u64 start, u64 length); void (*insert_page)(struct i915_address_space *vm, @@ -490,9 +475,9 @@ i915_pd_entry(const struct i915_page_directory * const pdp, static inline dma_addr_t i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) { - struct i915_page_dma *pt = ppgtt->pd->entry[n]; + struct i915_page_table *pt = ppgtt->pd->entry[n]; - return px_dma(pt ?: px_base(&ppgtt->vm.scratch[ppgtt->vm.top])); + return __px_dma(pt ? px_base(pt) : ppgtt->vm.scratch[ppgtt->vm.top]); } void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt); @@ -517,13 +502,10 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); -int setup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p); -void cleanup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p); - -#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page) +#define kmap_atomic_px(px) kmap_atomic(__px_page(px_base(px))) void -fill_page_dma(const struct i915_page_dma *p, const u64 val, unsigned int count); +fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count); #define fill_px(px, v) fill_page_dma(px_base(px), (v), PAGE_SIZE / sizeof(u64)) #define fill32_px(px, v) do { \ @@ -531,47 +513,51 @@ fill_page_dma(const struct i915_page_dma *p, const u64 val, unsigned int count); fill_px((px), v__ << 32 | v__); \ } while (0) -int setup_scratch_page(struct i915_address_space *vm, gfp_t gfp); -void cleanup_scratch_page(struct i915_address_space *vm); +int setup_scratch_page(struct i915_address_space *vm); void free_scratch(struct i915_address_space *vm); +struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); struct i915_page_table *alloc_pt(struct i915_address_space *vm); struct i915_page_directory *alloc_pd(struct i915_address_space *vm); -struct i915_page_directory *__alloc_pd(size_t sz); +struct i915_page_directory *__alloc_pd(int npde); -void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd); +int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); -#define free_px(vm, px) free_pd(vm, px_base(px)) +void free_px(struct i915_address_space *vm, + struct i915_page_table *pt, int lvl); +#define free_pt(vm, px) free_px(vm, px, 0) +#define free_pd(vm, px) free_px(vm, px_pt(px), 1) void __set_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, - struct i915_page_dma * const to, + struct i915_page_table *pt, u64 (*encode)(const dma_addr_t, const enum i915_cache_level)); #define set_pd_entry(pd, idx, to) \ - __set_pd_entry((pd), (idx), px_base(to), gen8_pde_encode) + __set_pd_entry((pd), (idx), px_pt(to), gen8_pde_encode) void clear_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, - const struct i915_page_scratch * const scratch); + const struct drm_i915_gem_object * const scratch); bool release_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, struct i915_page_table * const pt, - const struct i915_page_scratch * const scratch); + const struct drm_i915_gem_object * const scratch); void gen6_ggtt_invalidate(struct i915_ggtt *ggtt); int ggtt_set_pages(struct i915_vma *vma); int ppgtt_set_pages(struct i915_vma *vma); void clear_pages(struct i915_vma *vma); -int ppgtt_bind_vma(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags); +void ppgtt_bind_vma(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags); void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma); @@ -579,6 +565,14 @@ void gtt_write_workarounds(struct intel_gt *gt); void setup_private_pat(struct intel_uncore *uncore); +int i915_vm_alloc_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + u64 size); +int i915_vm_pin_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash); +void i915_vm_free_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash); + static inline struct sgt_dma { struct scatterlist *sg; dma_addr_t dma, max; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 24322ef08aa4..0412a44f25f2 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -137,6 +137,7 @@ #include "i915_perf.h" #include "i915_trace.h" #include "i915_vgpu.h" +#include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_engine_pm.h" #include "intel_gt.h" @@ -1148,20 +1149,6 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) } else { struct intel_engine_cs *owner = rq->context->engine; - /* - * Decouple the virtual breadcrumb before moving it - * back to the virtual engine -- we don't want the - * request to complete in the background and try - * and cancel the breadcrumb on the virtual engine - * (instead of the old engine where it is linked)! - */ - if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, - &rq->fence.flags)) { - spin_lock_nested(&rq->lock, - SINGLE_DEPTH_NESTING); - i915_request_cancel_breadcrumb(rq); - spin_unlock(&rq->lock); - } WRITE_ONCE(rq->engine, owner); owner->submit_request(rq); active = NULL; @@ -1819,16 +1806,31 @@ static bool virtual_matches(const struct virtual_engine *ve, return true; } -static void virtual_xfer_breadcrumbs(struct virtual_engine *ve) +static void virtual_xfer_context(struct virtual_engine *ve, + struct intel_engine_cs *engine) { + unsigned int n; + + if (likely(engine == ve->siblings[0])) + return; + + GEM_BUG_ON(READ_ONCE(ve->context.inflight)); + if (!intel_engine_has_relative_mmio(engine)) + virtual_update_register_offsets(ve->context.lrc_reg_state, + engine); + /* - * All the outstanding signals on ve->siblings[0] must have - * been completed, just pending the interrupt handler. As those - * signals still refer to the old sibling (via rq->engine), we must - * transfer those to the old irq_worker to keep our locking - * consistent. + * Move the bound engine to the top of the list for + * future execution. We then kick this tasklet first + * before checking others, so that we preferentially + * reuse this set of bound registers. */ - intel_engine_transfer_stale_breadcrumbs(ve->siblings[0], &ve->context); + for (n = 1; n < ve->num_siblings; n++) { + if (ve->siblings[n] == engine) { + swap(ve->siblings[n], ve->siblings[0]); + break; + } + } } #define for_each_waiter(p__, rq__) \ @@ -2060,6 +2062,14 @@ static inline void clear_ports(struct i915_request **ports, int count) memset_p((void **)ports, NULL, count); } +static inline void +copy_ports(struct i915_request **dst, struct i915_request **src, int count) +{ + /* A memcpy_p() would be very useful here! */ + while (count--) + WRITE_ONCE(*dst++, *src++); /* avoid write tearing */ +} + static void execlists_dequeue(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -2271,38 +2281,23 @@ static void execlists_dequeue(struct intel_engine_cs *engine) GEM_BUG_ON(!(rq->execution_mask & engine->mask)); WRITE_ONCE(rq->engine, engine); - if (engine != ve->siblings[0]) { - u32 *regs = ve->context.lrc_reg_state; - unsigned int n; - - GEM_BUG_ON(READ_ONCE(ve->context.inflight)); - - if (!intel_engine_has_relative_mmio(engine)) - virtual_update_register_offsets(regs, - engine); - - if (!list_empty(&ve->context.signals)) - virtual_xfer_breadcrumbs(ve); - + if (__i915_request_submit(rq)) { /* - * Move the bound engine to the top of the list - * for future execution. We then kick this - * tasklet first before checking others, so that - * we preferentially reuse this set of bound - * registers. + * Only after we confirm that we will submit + * this request (i.e. it has not already + * completed), do we want to update the context. + * + * This serves two purposes. It avoids + * unnecessary work if we are resubmitting an + * already completed request after timeslicing. + * But more importantly, it prevents us altering + * ve->siblings[] on an idle context, where + * we may be using ve->siblings[] in + * virtual_context_enter / virtual_context_exit. */ - for (n = 1; n < ve->num_siblings; n++) { - if (ve->siblings[n] == engine) { - swap(ve->siblings[n], - ve->siblings[0]); - break; - } - } - + virtual_xfer_context(ve, engine); GEM_BUG_ON(ve->siblings[0] != engine); - } - if (__i915_request_submit(rq)) { submit = true; last = rq; } @@ -2648,10 +2643,9 @@ static void process_csb(struct intel_engine_cs *engine) /* switch pending to inflight */ GEM_BUG_ON(!assert_pending_valid(execlists, "promote")); - memcpy(execlists->inflight, - execlists->pending, - execlists_num_ports(execlists) * - sizeof(*execlists->pending)); + copy_ports(execlists->inflight, + execlists->pending, + execlists_num_ports(execlists)); smp_wmb(); /* complete the seqlock */ WRITE_ONCE(execlists->active, execlists->inflight); @@ -3309,7 +3303,10 @@ static void execlists_context_unpin(struct intel_context *ce) { check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, ce->engine); +} +static void execlists_context_post_unpin(struct intel_context *ce) +{ i915_gem_object_unpin_map(ce->state->obj); } @@ -3471,20 +3468,24 @@ __execlists_update_reg_state(const struct intel_context *ce, } static int -__execlists_context_pin(struct intel_context *ce, - struct intel_engine_cs *engine) +execlists_context_pre_pin(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, void **vaddr) { - void *vaddr; - GEM_BUG_ON(!ce->state); GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); - vaddr = i915_gem_object_pin_map(ce->state->obj, - i915_coherent_map_type(engine->i915) | + *vaddr = i915_gem_object_pin_map(ce->state->obj, + i915_coherent_map_type(ce->engine->i915) | I915_MAP_OVERRIDE); - if (IS_ERR(vaddr)) - return PTR_ERR(vaddr); + return PTR_ERR_OR_ZERO(*vaddr); +} + +static int +__execlists_context_pin(struct intel_context *ce, + struct intel_engine_cs *engine, + void *vaddr) +{ ce->lrc.lrca = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE; ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; __execlists_update_reg_state(ce, engine, ce->ring->tail); @@ -3492,9 +3493,9 @@ __execlists_context_pin(struct intel_context *ce, return 0; } -static int execlists_context_pin(struct intel_context *ce) +static int execlists_context_pin(struct intel_context *ce, void *vaddr) { - return __execlists_context_pin(ce, ce->engine); + return __execlists_context_pin(ce, ce->engine, vaddr); } static int execlists_context_alloc(struct intel_context *ce) @@ -3520,8 +3521,10 @@ static void execlists_context_reset(struct intel_context *ce) static const struct intel_context_ops execlists_context_ops = { .alloc = execlists_context_alloc, + .pre_pin = execlists_context_pre_pin, .pin = execlists_context_pin, .unpin = execlists_context_unpin, + .post_unpin = execlists_context_post_unpin, .enter = intel_context_enter_engine, .exit = intel_context_exit_engine, @@ -3885,7 +3888,7 @@ static int lrc_setup_wa_ctx(struct intel_engine_cs *engine) goto err; } - err = i915_ggtt_pin(vma, 0, PIN_HIGH); + err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH); if (err) goto err; @@ -4126,7 +4129,7 @@ static int execlists_resume(struct intel_engine_cs *engine) { intel_mocs_init_engine(engine); - intel_engine_reset_breadcrumbs(engine); + intel_breadcrumbs_reset(engine->breadcrumbs); if (GEM_SHOW_DEBUG() && unexpected_starting_state(engine)) { struct drm_printer p = drm_debug_printer(__func__); @@ -4555,7 +4558,7 @@ static int gen8_emit_flush_render(struct i915_request *request, vf_flush_wa = true; /* WaForGAMHang:kbl */ - if (IS_KBL_REVID(request->engine->i915, 0, KBL_REVID_B0)) + if (IS_KBL_GT_REVID(request->engine->i915, 0, KBL_REVID_B0)) dc_flush_wa = true; } @@ -4757,14 +4760,21 @@ static int gen12_emit_flush(struct i915_request *request, u32 mode) intel_engine_mask_t aux_inv = 0; u32 cmd, *cs; + cmd = 4; + if (mode & EMIT_INVALIDATE) + cmd += 2; if (mode & EMIT_INVALIDATE) aux_inv = request->engine->mask & ~BIT(BCS0); + if (aux_inv) + cmd += 2 * hweight8(aux_inv) + 2; - cs = intel_ring_begin(request, - 4 + (aux_inv ? 2 * hweight8(aux_inv) + 2 : 0)); + cs = intel_ring_begin(request, cmd); if (IS_ERR(cs)) return PTR_ERR(cs); + if (mode & EMIT_INVALIDATE) + *cs++ = preparser_disable(true); + cmd = MI_FLUSH_DW + 1; /* We always require a command barrier so that subsequent @@ -4797,6 +4807,10 @@ static int gen12_emit_flush(struct i915_request *request, u32 mode) } *cs++ = MI_NOOP; } + + if (mode & EMIT_INVALIDATE) + *cs++ = preparser_disable(false); + intel_ring_advance(request, cs); return 0; @@ -5295,6 +5309,14 @@ populate_lr_context(struct intel_context *ce, return 0; } +static struct intel_timeline *pinned_timeline(struct intel_context *ce) +{ + struct intel_timeline *tl = fetch_and_zero(&ce->timeline); + + return intel_timeline_create_from_engine(ce->engine, + page_unmask_bits(tl)); +} + static int __execlists_context_alloc(struct intel_context *ce, struct intel_engine_cs *engine) { @@ -5325,19 +5347,17 @@ static int __execlists_context_alloc(struct intel_context *ce, goto error_deref_obj; } - if (!ce->timeline) { + if (!page_mask_bits(ce->timeline)) { struct intel_timeline *tl; - struct i915_vma *hwsp; /* * Use the static global HWSP for the kernel context, and * a dynamically allocated cacheline for everyone else. */ - hwsp = NULL; - if (unlikely(intel_context_is_barrier(ce))) - hwsp = engine->status_page.vma; - - tl = intel_timeline_create(engine->gt, hwsp); + if (unlikely(ce->timeline)) + tl = pinned_timeline(ce); + else + tl = intel_timeline_create(engine->gt); if (IS_ERR(tl)) { ret = PTR_ERR(tl); goto error_deref_obj; @@ -5443,12 +5463,12 @@ static int virtual_context_alloc(struct intel_context *ce) return __execlists_context_alloc(ce, ve->siblings[0]); } -static int virtual_context_pin(struct intel_context *ce) +static int virtual_context_pin(struct intel_context *ce, void *vaddr) { struct virtual_engine *ve = container_of(ce, typeof(*ve), context); /* Note: we must use a real engine class for setting up reg state */ - return __execlists_context_pin(ce, ve->siblings[0]); + return __execlists_context_pin(ce, ve->siblings[0], vaddr); } static void virtual_context_enter(struct intel_context *ce) @@ -5476,8 +5496,10 @@ static void virtual_context_exit(struct intel_context *ce) static const struct intel_context_ops virtual_context_ops = { .alloc = virtual_context_alloc, + .pre_pin = execlists_context_pre_pin, .pin = virtual_context_pin, .unpin = execlists_context_unpin, + .post_unpin = execlists_context_post_unpin, .enter = virtual_context_enter, .exit = virtual_context_exit, @@ -5711,9 +5733,7 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings, snprintf(ve->base.name, sizeof(ve->base.name), "virtual"); intel_engine_init_active(&ve->base, ENGINE_VIRTUAL); - intel_engine_init_breadcrumbs(&ve->base); intel_engine_init_execlists(&ve->base); - ve->base.breadcrumbs.irq_armed = true; /* fake HW, used for irq_work */ ve->base.cops = &virtual_context_ops; ve->base.request_alloc = execlists_request_alloc; @@ -5730,6 +5750,12 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings, intel_context_init(&ve->context, &ve->base); + ve->base.breadcrumbs = intel_breadcrumbs_create(NULL); + if (!ve->base.breadcrumbs) { + err = -ENOMEM; + goto err_put; + } + for (n = 0; n < count; n++) { struct intel_engine_cs *sibling = siblings[n]; diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index f0862e924d11..46d9aceda64c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -18,7 +18,8 @@ struct i915_page_table *alloc_pt(struct i915_address_space *vm) if (unlikely(!pt)) return ERR_PTR(-ENOMEM); - if (unlikely(setup_page_dma(vm, &pt->base))) { + pt->base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + if (IS_ERR(pt->base)) { kfree(pt); return ERR_PTR(-ENOMEM); } @@ -27,14 +28,20 @@ struct i915_page_table *alloc_pt(struct i915_address_space *vm) return pt; } -struct i915_page_directory *__alloc_pd(size_t sz) +struct i915_page_directory *__alloc_pd(int count) { struct i915_page_directory *pd; - pd = kzalloc(sz, I915_GFP_ALLOW_FAIL); + pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); if (unlikely(!pd)) return NULL; + pd->entry = kcalloc(count, sizeof(*pd->entry), I915_GFP_ALLOW_FAIL); + if (unlikely(!pd->entry)) { + kfree(pd); + return NULL; + } + spin_lock_init(&pd->lock); return pd; } @@ -43,11 +50,13 @@ struct i915_page_directory *alloc_pd(struct i915_address_space *vm) { struct i915_page_directory *pd; - pd = __alloc_pd(sizeof(*pd)); + pd = __alloc_pd(I915_PDES); if (unlikely(!pd)) return ERR_PTR(-ENOMEM); - if (unlikely(setup_page_dma(vm, px_base(pd)))) { + pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + if (IS_ERR(pd->pt.base)) { + kfree(pd->entry); kfree(pd); return ERR_PTR(-ENOMEM); } @@ -55,41 +64,52 @@ struct i915_page_directory *alloc_pd(struct i915_address_space *vm) return pd; } -void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd) +void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl) { - cleanup_page_dma(vm, pd); - kfree(pd); + BUILD_BUG_ON(offsetof(struct i915_page_directory, pt)); + + if (lvl) { + struct i915_page_directory *pd = + container_of(pt, typeof(*pd), pt); + kfree(pd->entry); + } + + if (pt->base) + i915_gem_object_put(pt->base); + + kfree(pt); } static inline void -write_dma_entry(struct i915_page_dma * const pdma, +write_dma_entry(struct drm_i915_gem_object * const pdma, const unsigned short idx, const u64 encoded_entry) { - u64 * const vaddr = kmap_atomic(pdma->page); + u64 * const vaddr = kmap_atomic(__px_page(pdma)); vaddr[idx] = encoded_entry; + clflush_cache_range(&vaddr[idx], sizeof(u64)); kunmap_atomic(vaddr); } void __set_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, - struct i915_page_dma * const to, + struct i915_page_table * const to, u64 (*encode)(const dma_addr_t, const enum i915_cache_level)) { /* Each thread pre-pins the pd, and we may have a thread per pde. */ - GEM_BUG_ON(atomic_read(px_used(pd)) > NALLOC * ARRAY_SIZE(pd->entry)); + GEM_BUG_ON(atomic_read(px_used(pd)) > NALLOC * I915_PDES); atomic_inc(px_used(pd)); pd->entry[idx] = to; - write_dma_entry(px_base(pd), idx, encode(to->daddr, I915_CACHE_LLC)); + write_dma_entry(px_base(pd), idx, encode(px_dma(to), I915_CACHE_LLC)); } void clear_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, - const struct i915_page_scratch * const scratch) + const struct drm_i915_gem_object * const scratch) { GEM_BUG_ON(atomic_read(px_used(pd)) == 0); @@ -102,7 +122,7 @@ bool release_pd_entry(struct i915_page_directory * const pd, const unsigned short idx, struct i915_page_table * const pt, - const struct i915_page_scratch * const scratch) + const struct drm_i915_gem_object * const scratch) { bool free = false; @@ -155,19 +175,16 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt) return ppgtt; } -int ppgtt_bind_vma(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +void ppgtt_bind_vma(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { u32 pte_flags; - int err; if (!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) { - err = vm->allocate_va_range(vm, vma->node.start, vma->size); - if (err) - return err; - + vm->allocate_va_range(vm, stash, vma->node.start, vma->size); set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)); } @@ -178,8 +195,6 @@ int ppgtt_bind_vma(struct i915_address_space *vm, vm->insert_entries(vm, vma, cache_level, pte_flags); wmb(); - - return 0; } void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) @@ -188,12 +203,93 @@ void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) vm->clear_range(vm, vma->node.start, vma->size); } +static unsigned long pd_count(u64 size, int shift) +{ + /* Beware later misalignment */ + return (size + 2 * (BIT_ULL(shift) - 1)) >> shift; +} + +int i915_vm_alloc_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + u64 size) +{ + unsigned long count; + int shift, n; + + shift = vm->pd_shift; + if (!shift) + return 0; + + count = pd_count(size, shift); + while (count--) { + struct i915_page_table *pt; + + pt = alloc_pt(vm); + if (IS_ERR(pt)) { + i915_vm_free_pt_stash(vm, stash); + return PTR_ERR(pt); + } + + pt->stash = stash->pt[0]; + stash->pt[0] = pt; + } + + for (n = 1; n < vm->top; n++) { + shift += ilog2(I915_PDES); /* Each PD holds 512 entries */ + count = pd_count(size, shift); + while (count--) { + struct i915_page_directory *pd; + + pd = alloc_pd(vm); + if (IS_ERR(pd)) { + i915_vm_free_pt_stash(vm, stash); + return PTR_ERR(pd); + } + + pd->pt.stash = stash->pt[1]; + stash->pt[1] = &pd->pt; + } + } + + return 0; +} + +int i915_vm_pin_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash) +{ + struct i915_page_table *pt; + int n, err; + + for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { + for (pt = stash->pt[n]; pt; pt = pt->stash) { + err = pin_pt_dma(vm, pt->base); + if (err) + return err; + } + } + + return 0; +} + +void i915_vm_free_pt_stash(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash) +{ + struct i915_page_table *pt; + int n; + + for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { + while ((pt = stash->pt[n])) { + stash->pt[n] = pt->stash; + free_px(vm, pt, n); + } + } +} + int ppgtt_set_pages(struct i915_vma *vma) { GEM_BUG_ON(vma->pages); vma->pages = vma->obj->mm.pages; - vma->page_sizes = vma->obj->mm.page_sizes; return 0; diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.c b/drivers/gpu/drm/i915/gt/intel_renderstate.c index 1bfad589c63b..ea2a77c7b469 100644 --- a/drivers/gpu/drm/i915/gt/intel_renderstate.c +++ b/drivers/gpu/drm/i915/gt/intel_renderstate.c @@ -27,6 +27,7 @@ #include "i915_drv.h" #include "intel_renderstate.h" +#include "gt/intel_context.h" #include "intel_ring.h" static const struct intel_renderstate_rodata * @@ -157,33 +158,47 @@ out: #undef OUT_BATCH int intel_renderstate_init(struct intel_renderstate *so, - struct intel_engine_cs *engine) + struct intel_context *ce) { - struct drm_i915_gem_object *obj; + struct intel_engine_cs *engine = ce->engine; + struct drm_i915_gem_object *obj = NULL; int err; memset(so, 0, sizeof(*so)); so->rodata = render_state_get_rodata(engine); - if (!so->rodata) - return 0; + if (so->rodata) { + if (so->rodata->batch_items * 4 > PAGE_SIZE) + return -EINVAL; + + obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + so->vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); + if (IS_ERR(so->vma)) { + err = PTR_ERR(so->vma); + goto err_obj; + } + } - if (so->rodata->batch_items * 4 > PAGE_SIZE) - return -EINVAL; + i915_gem_ww_ctx_init(&so->ww, true); +retry: + err = intel_context_pin_ww(ce, &so->ww); + if (err) + goto err_fini; - obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); + /* return early if there's nothing to setup */ + if (!err && !so->rodata) + return 0; - so->vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); - if (IS_ERR(so->vma)) { - err = PTR_ERR(so->vma); - goto err_obj; - } + err = i915_gem_object_lock(so->vma->obj, &so->ww); + if (err) + goto err_context; err = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (err) - goto err_obj; + goto err_context; err = render_state_setup(so, engine->i915); if (err) @@ -193,8 +208,18 @@ int intel_renderstate_init(struct intel_renderstate *so, err_unpin: i915_vma_unpin(so->vma); +err_context: + intel_context_unpin(ce); +err_fini: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&so->ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&so->ww); err_obj: - i915_gem_object_put(obj); + if (obj) + i915_gem_object_put(obj); so->vma = NULL; return err; } @@ -208,11 +233,9 @@ int intel_renderstate_emit(struct intel_renderstate *so, if (!so->vma) return 0; - i915_vma_lock(so->vma); err = i915_request_await_object(rq, so->vma->obj, false); if (err == 0) err = i915_vma_move_to_active(so->vma, rq, 0); - i915_vma_unlock(so->vma); if (err) return err; @@ -233,7 +256,17 @@ int intel_renderstate_emit(struct intel_renderstate *so, return 0; } -void intel_renderstate_fini(struct intel_renderstate *so) +void intel_renderstate_fini(struct intel_renderstate *so, + struct intel_context *ce) { - i915_vma_unpin_and_release(&so->vma, 0); + if (so->vma) { + i915_vma_unpin(so->vma); + i915_vma_close(so->vma); + } + + intel_context_unpin(ce); + i915_gem_ww_ctx_fini(&so->ww); + + if (so->vma) + i915_gem_object_put(so->vma->obj); } diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.h b/drivers/gpu/drm/i915/gt/intel_renderstate.h index 5700be69a05a..713aa1e86c80 100644 --- a/drivers/gpu/drm/i915/gt/intel_renderstate.h +++ b/drivers/gpu/drm/i915/gt/intel_renderstate.h @@ -25,9 +25,10 @@ #define _INTEL_RENDERSTATE_H_ #include <linux/types.h> +#include "i915_gem.h" struct i915_request; -struct intel_engine_cs; +struct intel_context; struct i915_vma; struct intel_renderstate_rodata { @@ -49,6 +50,7 @@ extern const struct intel_renderstate_rodata gen8_null_state; extern const struct intel_renderstate_rodata gen9_null_state; struct intel_renderstate { + struct i915_gem_ww_ctx ww; const struct intel_renderstate_rodata *rodata; struct i915_vma *vma; u32 batch_offset; @@ -58,9 +60,10 @@ struct intel_renderstate { }; int intel_renderstate_init(struct intel_renderstate *so, - struct intel_engine_cs *engine); + struct intel_context *ce); int intel_renderstate_emit(struct intel_renderstate *so, struct i915_request *rq); -void intel_renderstate_fini(struct intel_renderstate *so); +void intel_renderstate_fini(struct intel_renderstate *so, + struct intel_context *ce); #endif /* _INTEL_RENDERSTATE_H_ */ diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 46a5ceffc22f..ac36b67fb46b 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -15,6 +15,7 @@ #include "i915_drv.h" #include "i915_gpu_error.h" #include "i915_irq.h" +#include "intel_breadcrumbs.h" #include "intel_engine_pm.h" #include "intel_gt.h" #include "intel_gt_pm.h" diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index bdb324167ef3..4034a4bac7f0 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -21,7 +21,13 @@ unsigned int intel_ring_update_space(struct intel_ring *ring) return space; } -int intel_ring_pin(struct intel_ring *ring) +void __intel_ring_pin(struct intel_ring *ring) +{ + GEM_BUG_ON(!atomic_read(&ring->pin_count)); + atomic_inc(&ring->pin_count); +} + +int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww) { struct i915_vma *vma = ring->vma; unsigned int flags; @@ -39,7 +45,7 @@ int intel_ring_pin(struct intel_ring *ring) else flags |= PIN_HIGH; - ret = i915_ggtt_pin(vma, 0, flags); + ret = i915_ggtt_pin(vma, ww, 0, flags); if (unlikely(ret)) goto err_unpin; diff --git a/drivers/gpu/drm/i915/gt/intel_ring.h b/drivers/gpu/drm/i915/gt/intel_ring.h index cc0ebca65167..1700579bdc93 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.h +++ b/drivers/gpu/drm/i915/gt/intel_ring.h @@ -21,7 +21,8 @@ int intel_ring_cacheline_align(struct i915_request *rq); unsigned int intel_ring_update_space(struct intel_ring *ring); -int intel_ring_pin(struct intel_ring *ring); +void __intel_ring_pin(struct intel_ring *ring); +int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww); void intel_ring_unpin(struct intel_ring *ring); void intel_ring_reset(struct intel_ring *ring, u32 tail); diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 94915f668715..16b48e72c369 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -32,6 +32,7 @@ #include "gen6_ppgtt.h" #include "gen7_renderclear.h" #include "i915_drv.h" +#include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_gt.h" #include "intel_reset.h" @@ -100,7 +101,7 @@ static void set_hwsp(struct intel_engine_cs *engine, u32 offset) */ default: GEM_BUG_ON(engine->id); - /* fallthrough */ + fallthrough; case RCS0: hwsp = RENDER_HWS_PGA_GEN7; break; @@ -201,16 +202,18 @@ static struct i915_address_space *vm_alias(struct i915_address_space *vm) return vm; } +static u32 pp_dir(struct i915_address_space *vm) +{ + return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir; +} + static void set_pp_dir(struct intel_engine_cs *engine) { struct i915_address_space *vm = vm_alias(engine->gt->vm); if (vm) { - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G); - ENGINE_WRITE(engine, RING_PP_DIR_BASE, - px_base(ppgtt->pd)->ggtt_offset << 10); + ENGINE_WRITE(engine, RING_PP_DIR_BASE, pp_dir(vm)); } } @@ -255,7 +258,7 @@ static int xcs_resume(struct intel_engine_cs *engine) else ring_setup_status_page(engine); - intel_engine_reset_breadcrumbs(engine); + intel_breadcrumbs_reset(engine->breadcrumbs); /* Enforce ordering by reading HEAD register back */ ENGINE_POSTING_READ(engine, RING_HEAD); @@ -474,14 +477,16 @@ static void ring_context_destroy(struct kref *ref) intel_context_free(ce); } -static int __context_pin_ppgtt(struct intel_context *ce) +static int ring_context_pre_pin(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, + void **unused) { struct i915_address_space *vm; int err = 0; vm = vm_alias(ce->vm); if (vm) - err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm))); + err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww); return err; } @@ -497,6 +502,10 @@ static void __context_unpin_ppgtt(struct intel_context *ce) static void ring_context_unpin(struct intel_context *ce) { +} + +static void ring_context_post_unpin(struct intel_context *ce) +{ __context_unpin_ppgtt(ce); } @@ -584,9 +593,9 @@ static int ring_context_alloc(struct intel_context *ce) return 0; } -static int ring_context_pin(struct intel_context *ce) +static int ring_context_pin(struct intel_context *ce, void *unused) { - return __context_pin_ppgtt(ce); + return 0; } static void ring_context_reset(struct intel_context *ce) @@ -597,8 +606,10 @@ static void ring_context_reset(struct intel_context *ce) static const struct intel_context_ops ring_context_ops = { .alloc = ring_context_alloc, + .pre_pin = ring_context_pre_pin, .pin = ring_context_pin, .unpin = ring_context_unpin, + .post_unpin = ring_context_post_unpin, .enter = intel_context_enter_engine, .exit = intel_context_exit_engine, @@ -608,7 +619,7 @@ static const struct intel_context_ops ring_context_ops = { }; static int load_pd_dir(struct i915_request *rq, - const struct i915_ppgtt *ppgtt, + struct i915_address_space *vm, u32 valid) { const struct intel_engine_cs * const engine = rq->engine; @@ -624,7 +635,7 @@ static int load_pd_dir(struct i915_request *rq, *cs++ = MI_LOAD_REGISTER_IMM(1); *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); - *cs++ = px_base(ppgtt->pd)->ggtt_offset << 10; + *cs++ = pp_dir(vm); /* Stall until the page table load is complete? */ *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; @@ -826,7 +837,7 @@ static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) * post-sync op, this extra pass appears vital before a * mm switch! */ - ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm), PP_DIR_DCLV_2G); + ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G); if (ret) return ret; @@ -1250,14 +1261,15 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine) return -ENODEV; } - timeline = intel_timeline_create(engine->gt, engine->status_page.vma); + timeline = intel_timeline_create_from_engine(engine, + I915_GEM_HWS_SEQNO_ADDR); if (IS_ERR(timeline)) { err = PTR_ERR(timeline); goto err; } GEM_BUG_ON(timeline->has_initial_breadcrumb); - err = intel_timeline_pin(timeline); + err = intel_timeline_pin(timeline, NULL); if (err) goto err_timeline; @@ -1267,7 +1279,7 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine) goto err_timeline_unpin; } - err = intel_ring_pin(ring); + err = intel_ring_pin(ring, NULL); if (err) goto err_ring; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 97ba14ad52e4..e6a00eea0631 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -7,6 +7,7 @@ #include <drm/i915_drm.h> #include "i915_drv.h" +#include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" #include "intel_gt_irq.h" diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index 46d20f5f3ddc..a2f74cefe4c3 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -215,7 +215,8 @@ static void cacheline_free(struct intel_timeline_cacheline *cl) static int intel_timeline_init(struct intel_timeline *timeline, struct intel_gt *gt, - struct i915_vma *hwsp) + struct i915_vma *hwsp, + unsigned int offset) { void *vaddr; @@ -246,8 +247,7 @@ static int intel_timeline_init(struct intel_timeline *timeline, vaddr = page_mask_bits(cl->vaddr); } else { - timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR; - + timeline->hwsp_offset = offset; vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); @@ -297,7 +297,9 @@ static void intel_timeline_fini(struct intel_timeline *timeline) } struct intel_timeline * -intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp) +__intel_timeline_create(struct intel_gt *gt, + struct i915_vma *global_hwsp, + unsigned int offset) { struct intel_timeline *timeline; int err; @@ -306,7 +308,7 @@ intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp) if (!timeline) return ERR_PTR(-ENOMEM); - err = intel_timeline_init(timeline, gt, global_hwsp); + err = intel_timeline_init(timeline, gt, global_hwsp, offset); if (err) { kfree(timeline); return ERR_PTR(err); @@ -315,14 +317,20 @@ intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp) return timeline; } -int intel_timeline_pin(struct intel_timeline *tl) +void __intel_timeline_pin(struct intel_timeline *tl) +{ + GEM_BUG_ON(!atomic_read(&tl->pin_count)); + atomic_inc(&tl->pin_count); +} + +int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww) { int err; if (atomic_add_unless(&tl->pin_count, 1, 0)) return 0; - err = i915_ggtt_pin(tl->hwsp_ggtt, 0, PIN_HIGH); + err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH); if (err) return err; @@ -465,7 +473,7 @@ __intel_timeline_get_seqno(struct intel_timeline *tl, goto err_rollback; } - err = i915_ggtt_pin(vma, 0, PIN_HIGH); + err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH); if (err) { __idle_hwsp_free(vma->private, cacheline); goto err_rollback; @@ -484,7 +492,9 @@ __intel_timeline_get_seqno(struct intel_timeline *tl, * free it after the current request is retired, which ensures that * all writes into the cacheline from previous requests are complete. */ - err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence); + err = i915_active_ref(&tl->hwsp_cacheline->active, + tl->fence_context, + &rq->fence); if (err) goto err_cacheline; diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.h b/drivers/gpu/drm/i915/gt/intel_timeline.h index 4298b9ac7327..9882cd911d8e 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.h +++ b/drivers/gpu/drm/i915/gt/intel_timeline.h @@ -29,10 +29,27 @@ #include "i915_active.h" #include "i915_syncmap.h" -#include "gt/intel_timeline_types.h" +#include "intel_timeline_types.h" struct intel_timeline * -intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp); +__intel_timeline_create(struct intel_gt *gt, + struct i915_vma *global_hwsp, + unsigned int offset); + +static inline struct intel_timeline * +intel_timeline_create(struct intel_gt *gt) +{ + return __intel_timeline_create(gt, NULL, 0); +} + +static inline struct intel_timeline * +intel_timeline_create_from_engine(struct intel_engine_cs *engine, + unsigned int offset) +{ + return __intel_timeline_create(engine->gt, + engine->status_page.vma, + offset); +} static inline struct intel_timeline * intel_timeline_get(struct intel_timeline *timeline) @@ -71,7 +88,8 @@ static inline bool intel_timeline_sync_is_later(struct intel_timeline *tl, return __intel_timeline_sync_is_later(tl, fence->context, fence->seqno); } -int intel_timeline_pin(struct intel_timeline *tl); +void __intel_timeline_pin(struct intel_timeline *tl); +int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww); void intel_timeline_enter(struct intel_timeline *tl); int intel_timeline_get_seqno(struct intel_timeline *tl, struct i915_request *rq, diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 5726cd0a37e0..a3f72b75c61e 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -52,6 +52,24 @@ * - Public functions to init or apply the given workaround type. */ +/* + * KBL revision ID ordering is bizarre; higher revision ID's map to lower + * steppings in some cases. So rather than test against the revision ID + * directly, let's map that into our own range of increasing ID's that we + * can test against in a regular manner. + */ + +const struct i915_rev_steppings kbl_revids[] = { + [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 }, + [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 }, + [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 }, + [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 }, + [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 }, + [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 }, + [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 }, + [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 }, +}; + static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) { wal->name = name; @@ -470,7 +488,7 @@ static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, gen9_ctx_workarounds_init(engine, wal); /* WaToEnableHwFixForPushConstHWBug:kbl */ - if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER)) + if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); @@ -596,8 +614,8 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); } -static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, - struct i915_wa_list *wal) +static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) { /* * Wa_1409142259:tgl @@ -607,12 +625,28 @@ static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, * Wa_1409207793:tgl * Wa_1409178076:tgl * Wa_1408979724:tgl + * Wa_14010443199:rkl + * Wa_14010698770:rkl */ WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, GEN12_DISABLE_CPS_AWARE_COLOR_PIPE); + /* WaDisableGPGPUMidThreadPreemption:gen12 */ + WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, + GEN9_PREEMPT_GPGPU_LEVEL_MASK, + GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); +} + +static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + gen12_ctx_workarounds_init(engine, wal); + /* - * Wa_1604555607:gen12 and Wa_1608008084:gen12 + * Wa_1604555607:tgl,rkl + * + * Note that the implementation of this workaround is further modified + * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. * FF_MODE2 register will return the wrong value when read. The default * value for this register is zero for all fields and there are no bit * masks. So instead of doing a RMW we should just write the GS Timer @@ -623,11 +657,6 @@ static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK, FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128, 0); - - /* WaDisableGPGPUMidThreadPreemption:tgl */ - WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, - GEN9_PREEMPT_GPGPU_LEVEL_MASK, - GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); } static void @@ -642,8 +671,10 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, wa_init_start(wal, name, engine->name); - if (IS_GEN(i915, 12)) + if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) tgl_ctx_workarounds_init(engine, wal); + else if (IS_GEN(i915, 12)) + gen12_ctx_workarounds_init(engine, wal); else if (IS_GEN(i915, 11)) icl_ctx_workarounds_init(engine, wal); else if (IS_CANNONLAKE(i915)) @@ -995,7 +1026,7 @@ kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) gen9_gt_workarounds_init(i915, wal); /* WaDisableDynamicCreditSharing:kbl */ - if (IS_KBL_REVID(i915, 0, KBL_REVID_B0)) + if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0)) wa_write_or(wal, GAMT_CHKN_BIT_REG, GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); @@ -1176,9 +1207,16 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +gen12_gt_workarounds_init(struct drm_i915_private *i915, + struct i915_wa_list *wal) { wa_init_mcr(i915, wal); +} + +static void +tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + gen12_gt_workarounds_init(i915, wal); /* Wa_1409420604:tgl */ if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) @@ -1196,8 +1234,10 @@ tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) static void gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) { - if (IS_GEN(i915, 12)) + if (IS_TIGERLAKE(i915)) tgl_gt_workarounds_init(i915, wal); + else if (IS_GEN(i915, 12)) + gen12_gt_workarounds_init(i915, wal); else if (IS_GEN(i915, 11)) icl_gt_workarounds_init(i915, wal); else if (IS_CANNONLAKE(i915)) @@ -1630,18 +1670,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN12_DISABLE_POSH_BUSY_FF_DOP_CG); /* - * Wa_1607030317:tgl - * Wa_1607186500:tgl - * Wa_1607297627:tgl there is 3 entries for this WA on BSpec, 2 - * of then says it is fixed on B0 the other one says it is - * permanent - */ - wa_masked_en(wal, - GEN6_RC_SLEEP_PSMI_CONTROL, - GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | - GEN8_RC_SEMA_IDLE_MSG_DISABLE); - - /* * Wa_1606679103:tgl * (see also Wa_1606682166:icl) */ @@ -1654,22 +1682,17 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) VSUNIT_CLKGATE_DIS_TGL); } - if (IS_TIGERLAKE(i915)) { - /* Wa_1606931601:tgl */ + if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { + /* Wa_1606931601:tgl,rkl */ wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); - /* Wa_1409804808:tgl */ + /* Wa_1409804808:tgl,rkl */ wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_PUSH_CONST_DEREF_HOLD_DIS); - /* Wa_1606700617:tgl */ - wa_masked_en(wal, - GEN9_CS_DEBUG_MODE1, - FF_DOP_CLOCK_GATE_DISABLE); - /* * Wa_1409085225:tgl - * Wa_14010229206:tgl + * Wa_14010229206:tgl,rkl */ wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); @@ -1677,9 +1700,29 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * Wa_1407928979:tgl A* * Wa_18011464164:tgl B0+ * Wa_22010931296:tgl B0+ + * Wa_14010919138:rkl */ wa_write_or(wal, GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE); + + /* + * Wa_1607030317:tgl + * Wa_1607186500:tgl + * Wa_1607297627:tgl,rkl there are multiple entries for this + * WA in the BSpec; some indicate this is an A0-only WA, + * others indicate it applies to all steppings. + */ + wa_masked_en(wal, + GEN6_RC_SLEEP_PSMI_CONTROL, + GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + GEN8_RC_SEMA_IDLE_MSG_DISABLE); + } + + if (IS_TIGERLAKE(i915)) { + /* Wa_1606700617:tgl */ + wa_masked_en(wal, + GEN9_CS_DEBUG_MODE1, + FF_DOP_CLOCK_GATE_DISABLE); } if (IS_GEN(i915, 11)) { @@ -1898,7 +1941,7 @@ xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) struct drm_i915_private *i915 = engine->i915; /* WaKBLVECSSemaphoreWaitPoll:kbl */ - if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) { + if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) { wa_write(wal, RING_SEMA_WAIT_POLL(engine->mmio_base), 1); @@ -2045,6 +2088,7 @@ static int engine_wa_list_verify(struct intel_context *ce, const struct i915_wa *wa; struct i915_request *rq; struct i915_vma *vma; + struct i915_gem_ww_ctx ww; unsigned int i; u32 *results; int err; @@ -2057,29 +2101,34 @@ static int engine_wa_list_verify(struct intel_context *ce, return PTR_ERR(vma); intel_engine_pm_get(ce->engine); - rq = intel_context_create_request(ce); - intel_engine_pm_put(ce->engine); + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(vma->obj, &ww); + if (err == 0) + err = intel_context_pin_ww(ce, &ww); + if (err) + goto err_pm; + + rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); - goto err_vma; + goto err_unpin; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(vma); - if (err) { - i915_request_add(rq); - goto err_vma; - } - - err = wa_list_srm(rq, wal, vma); - if (err) - goto err_vma; + if (err == 0) + err = wa_list_srm(rq, wal, vma); i915_request_get(rq); + if (err) + i915_request_set_error_once(rq, err); i915_request_add(rq); + + if (err) + goto err_rq; + if (i915_request_wait(rq, 0, HZ / 5) < 0) { err = -ETIME; goto err_rq; @@ -2104,7 +2153,16 @@ static int engine_wa_list_verify(struct intel_context *ce, err_rq: i915_request_put(rq); -err_vma: +err_unpin: + intel_context_unpin(ce); +err_pm: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + intel_engine_pm_put(ce->engine); i915_vma_unpin(vma); i915_vma_put(vma); return err; diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index b8dd3cbc8696..dfd1cfb8a7ec 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -131,6 +131,10 @@ static void mock_context_unpin(struct intel_context *ce) { } +static void mock_context_post_unpin(struct intel_context *ce) +{ +} + static void mock_context_destroy(struct kref *ref) { struct intel_context *ce = container_of(ref, typeof(*ce), ref); @@ -152,8 +156,7 @@ static int mock_context_alloc(struct intel_context *ce) if (!ce->ring) return -ENOMEM; - GEM_BUG_ON(ce->timeline); - ce->timeline = intel_timeline_create(ce->engine->gt, NULL); + ce->timeline = intel_timeline_create(ce->engine->gt); if (IS_ERR(ce->timeline)) { kfree(ce->engine); return PTR_ERR(ce->timeline); @@ -164,7 +167,13 @@ static int mock_context_alloc(struct intel_context *ce) return 0; } -static int mock_context_pin(struct intel_context *ce) +static int mock_context_pre_pin(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, void **unused) +{ + return 0; +} + +static int mock_context_pin(struct intel_context *ce, void *unused) { return 0; } @@ -176,8 +185,10 @@ static void mock_context_reset(struct intel_context *ce) static const struct intel_context_ops mock_context_ops = { .alloc = mock_context_alloc, + .pre_pin = mock_context_pre_pin, .pin = mock_context_pin, .unpin = mock_context_unpin, + .post_unpin = mock_context_post_unpin, .enter = intel_context_enter_engine, .exit = intel_context_exit_engine, @@ -261,11 +272,12 @@ static void mock_engine_release(struct intel_engine_cs *engine) GEM_BUG_ON(timer_pending(&mock->hw_delay)); + intel_breadcrumbs_free(engine->breadcrumbs); + intel_context_unpin(engine->kernel_context); intel_context_put(engine->kernel_context); intel_engine_fini_retire(engine); - intel_engine_fini_breadcrumbs(engine); } struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, @@ -323,20 +335,26 @@ int mock_engine_init(struct intel_engine_cs *engine) struct intel_context *ce; intel_engine_init_active(engine, ENGINE_MOCK); - intel_engine_init_breadcrumbs(engine); intel_engine_init_execlists(engine); intel_engine_init__pm(engine); intel_engine_init_retire(engine); + engine->breadcrumbs = intel_breadcrumbs_create(NULL); + if (!engine->breadcrumbs) + return -ENOMEM; + ce = create_kernel_context(engine); if (IS_ERR(ce)) goto err_breadcrumbs; + /* We insist the kernel context is using the status_page */ + engine->status_page.vma = ce->timeline->hwsp_ggtt; + engine->kernel_context = ce; return 0; err_breadcrumbs: - intel_engine_fini_breadcrumbs(engine); + intel_breadcrumbs_free(engine->breadcrumbs); return -ENOMEM; } diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c index 52af1cee9a94..1f4020e906a8 100644 --- a/drivers/gpu/drm/i915/gt/selftest_context.c +++ b/drivers/gpu/drm/i915/gt/selftest_context.c @@ -68,6 +68,8 @@ static int context_sync(struct intel_context *ce) } while (!err); mutex_unlock(&tl->mutex); + /* Wait for all barriers to complete (remote CPU) before we check */ + i915_active_unlock_wait(&ce->active); return err; } diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index 73243ba59c7d..e73854dd2fe0 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -47,7 +47,10 @@ static int pulse_active(struct i915_active *active) static void pulse_free(struct kref *kref) { - kfree(container_of(kref, struct pulse, kref)); + struct pulse *p = container_of(kref, typeof(*p), kref); + + i915_active_fini(&p->active); + kfree(p); } static void pulse_put(struct pulse *p) diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 3fc5de961280..95d41c01d0e0 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -2729,7 +2729,7 @@ static int create_gang(struct intel_engine_cs *engine, i915_gem_object_put(obj); intel_context_put(ce); - rq->client_link.next = &(*prev)->client_link; + rq->mock.link.next = &(*prev)->mock.link; *prev = rq; return 0; @@ -2970,8 +2970,7 @@ static int live_preempt_gang(void *arg) } while (rq) { /* wait for each rq from highest to lowest prio */ - struct i915_request *n = - list_next_entry(rq, client_link); + struct i915_request *n = list_next_entry(rq, mock.link); if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) { struct drm_printer p = @@ -3090,7 +3089,7 @@ static struct i915_vma *create_global(struct intel_gt *gt, size_t sz) return vma; } - err = i915_ggtt_pin(vma, 0, 0); + err = i915_ggtt_pin(vma, NULL, 0, 0); if (err) { i915_vma_put(vma); return ERR_PTR(err); @@ -4997,6 +4996,7 @@ static int __live_lrc_state(struct intel_engine_cs *engine, { struct intel_context *ce; struct i915_request *rq; + struct i915_gem_ww_ctx ww; enum { RING_START_IDX = 0, RING_TAIL_IDX, @@ -5011,7 +5011,11 @@ static int __live_lrc_state(struct intel_engine_cs *engine, if (IS_ERR(ce)) return PTR_ERR(ce); - err = intel_context_pin(ce); + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(scratch->obj, &ww); + if (!err) + err = intel_context_pin_ww(ce, &ww); if (err) goto err_put; @@ -5040,11 +5044,9 @@ static int __live_lrc_state(struct intel_engine_cs *engine, *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); *cs++ = 0; - i915_vma_lock(scratch); err = i915_request_await_object(rq, scratch->obj, true); if (!err) err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); - i915_vma_unlock(scratch); i915_request_get(rq); i915_request_add(rq); @@ -5081,6 +5083,12 @@ err_rq: err_unpin: intel_context_unpin(ce); err_put: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); intel_context_put(ce); return err; } diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 8624f5d2a1f3..3540ba9bd459 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -77,20 +77,20 @@ create_spin_counter(struct intel_engine_cs *engine, vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { - i915_gem_object_put(obj); - return vma; + err = PTR_ERR(vma); + goto err_put; } err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) { - i915_vma_put(vma); - return ERR_PTR(err); - } + if (err) + goto err_unlock; + + i915_vma_lock(vma); base = i915_gem_object_pin_map(obj, I915_MAP_WC); if (IS_ERR(base)) { - i915_gem_object_put(obj); - return ERR_CAST(base); + err = PTR_ERR(base); + goto err_unpin; } cs = base; @@ -134,6 +134,14 @@ create_spin_counter(struct intel_engine_cs *engine, *cancel = base + loop; *counter = srm ? memset32(base + end, 0, 1) : NULL; return vma; + +err_unpin: + i915_vma_unpin(vma); +err_unlock: + i915_vma_unlock(vma); +err_put: + i915_gem_object_put(obj); + return ERR_PTR(err); } static u8 wait_for_freq(struct intel_rps *rps, u8 freq, int timeout_ms) @@ -639,7 +647,6 @@ int live_rps_frequency_cs(void *arg) goto err_vma; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, false); if (!err) err = i915_vma_move_to_active(vma, rq, 0); @@ -647,7 +654,6 @@ int live_rps_frequency_cs(void *arg) err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0); - i915_vma_unlock(vma); i915_request_add(rq); if (err) goto err_vma; @@ -700,7 +706,7 @@ int live_rps_frequency_cs(void *arg) f = act; /* may skip ahead [pcu granularity] */ } - err = -EINVAL; + err = -EINTR; /* ignore error, continue on with test */ } err_vma: @@ -708,6 +714,7 @@ err_vma: i915_gem_object_flush_map(vma->obj); i915_gem_object_unpin_map(vma->obj); i915_vma_unpin(vma); + i915_vma_unlock(vma); i915_vma_put(vma); st_engine_heartbeat_enable(engine); @@ -781,7 +788,6 @@ int live_rps_frequency_srm(void *arg) goto err_vma; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, false); if (!err) err = i915_vma_move_to_active(vma, rq, 0); @@ -789,7 +795,6 @@ int live_rps_frequency_srm(void *arg) err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0); - i915_vma_unlock(vma); i915_request_add(rq); if (err) goto err_vma; @@ -841,7 +846,7 @@ int live_rps_frequency_srm(void *arg) f = act; /* may skip ahead [pcu granularity] */ } - err = -EINVAL; + err = -EINTR; /* ignore error, continue on with test */ } err_vma: @@ -849,6 +854,7 @@ err_vma: i915_gem_object_flush_map(vma->obj); i915_gem_object_unpin_map(vma->obj); i915_vma_unpin(vma); + i915_vma_unlock(vma); i915_vma_put(vma); st_engine_heartbeat_enable(engine); diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index fb5b7d3498a6..96d164a3841d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -72,7 +72,7 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, unsigned long cacheline; int err; - tl = intel_timeline_create(state->gt, NULL); + tl = intel_timeline_create(state->gt); if (IS_ERR(tl)) return PTR_ERR(tl); @@ -455,7 +455,7 @@ tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value) struct i915_request *rq; int err; - err = intel_timeline_pin(tl); + err = intel_timeline_pin(tl, NULL); if (err) { rq = ERR_PTR(err); goto out; @@ -487,11 +487,11 @@ checked_intel_timeline_create(struct intel_gt *gt) { struct intel_timeline *tl; - tl = intel_timeline_create(gt, NULL); + tl = intel_timeline_create(gt); if (IS_ERR(tl)) return tl; - if (*tl->hwsp_seqno != tl->seqno) { + if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) { pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", *tl->hwsp_seqno, tl->seqno); intel_timeline_put(tl); @@ -561,9 +561,9 @@ static int live_hwsp_engine(void *arg) for (n = 0; n < count; n++) { struct intel_timeline *tl = timelines[n]; - if (!err && *tl->hwsp_seqno != n) { - pr_err("Invalid seqno stored in timeline %lu @ %x, found 0x%x\n", - n, tl->hwsp_offset, *tl->hwsp_seqno); + if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { + GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", + n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } @@ -633,9 +633,9 @@ out: for (n = 0; n < count; n++) { struct intel_timeline *tl = timelines[n]; - if (!err && *tl->hwsp_seqno != n) { - pr_err("Invalid seqno stored in timeline %lu @ %x, found 0x%x\n", - n, tl->hwsp_offset, *tl->hwsp_seqno); + if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { + GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", + n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } @@ -660,14 +660,14 @@ static int live_hwsp_wrap(void *arg) * foreign GPU references. */ - tl = intel_timeline_create(gt, NULL); + tl = intel_timeline_create(gt); if (IS_ERR(tl)) return PTR_ERR(tl); if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline) goto out_free; - err = intel_timeline_pin(tl); + err = intel_timeline_pin(tl, NULL); if (err) goto out_free; @@ -733,7 +733,8 @@ static int live_hwsp_wrap(void *arg) goto out; } - if (*hwsp_seqno[0] != seqno[0] || *hwsp_seqno[1] != seqno[1]) { + if (READ_ONCE(*hwsp_seqno[0]) != seqno[0] || + READ_ONCE(*hwsp_seqno[1]) != seqno[1]) { pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n", *hwsp_seqno[0], *hwsp_seqno[1], seqno[0], seqno[1]); @@ -966,9 +967,10 @@ static int live_hwsp_recycle(void *arg) break; } - if (*tl->hwsp_seqno != count) { - pr_err("Invalid seqno stored in timeline %lu @ tl->hwsp_offset, found 0x%x\n", - count, *tl->hwsp_seqno); + if (READ_ONCE(*tl->hwsp_seqno) != count) { + GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n", + count, tl->fence_context, + tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index febc9e6692ba..61a0532d0f3d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -214,7 +214,7 @@ static int check_whitelist(struct i915_gem_context *ctx, return PTR_ERR(results); err = 0; - i915_gem_object_lock(results); + i915_gem_object_lock(results, NULL); intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ err = i915_gem_object_set_to_cpu_domain(results, false); i915_gem_object_unlock(results); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 861657897c0f..942c7c187adb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -677,7 +677,7 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size) goto err; flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma); - ret = i915_ggtt_pin(vma, 0, flags); + ret = i915_ggtt_pin(vma, NULL, 0, flags); if (ret) { vma = ERR_PTR(ret); goto err; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index 59b27aba15c6..80e8b6c3bc8c 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -51,8 +51,8 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * Note that RKL uses the same firmware as TGL. */ #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ - fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 0, 12)) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 0, 12)) \ + fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ fw_def(COMETLAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index 072725a448db..ad86c5eb5bba 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -70,6 +70,7 @@ static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off, { u8 *cfg_base = vgpu_cfg_space(vgpu); u8 mask, new, old; + pci_power_t pwr; int i = 0; for (; i < bytes && (off + i < sizeof(pci_cfg_space_rw_bmp)); i++) { @@ -91,6 +92,15 @@ static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off, /* For other configuration space directly copy as it is. */ if (i < bytes) memcpy(cfg_base + off + i, src + i, bytes - i); + + if (off == vgpu->cfg_space.pmcsr_off && vgpu->cfg_space.pmcsr_off) { + pwr = (pci_power_t __force)(*(u16*)(&vgpu_cfg_space(vgpu)[off]) + & PCI_PM_CTRL_STATE_MASK); + if (pwr == PCI_D3hot) + vgpu->d3_entered = true; + gvt_dbg_core("vgpu-%d power status changed to %d\n", + vgpu->id, pwr); + } } /** @@ -366,6 +376,7 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; u16 *gmch_ctl; + u8 next; memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space, info->cfg_space_size); @@ -401,6 +412,19 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, pci_resource_len(gvt->gt->i915->drm.pdev, 2); memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4); + + /* PM Support */ + vgpu->cfg_space.pmcsr_off = 0; + if (vgpu_cfg_space(vgpu)[PCI_STATUS] & PCI_STATUS_CAP_LIST) { + next = vgpu_cfg_space(vgpu)[PCI_CAPABILITY_LIST]; + do { + if (vgpu_cfg_space(vgpu)[next + PCI_CAP_LIST_ID] == PCI_CAP_ID_PM) { + vgpu->cfg_space.pmcsr_off = next + PCI_PM_CTRL; + break; + } + next = vgpu_cfg_space(vgpu)[next + PCI_CAP_LIST_NEXT]; + } while (next); + } } /** diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index f1940939260a..d0a599b51bfe 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -1923,6 +1923,7 @@ static int perform_bb_shadow(struct parser_exec_state *s) if (ret) goto err_unmap; + i915_gem_object_unlock(bb->obj); INIT_LIST_HEAD(&bb->list); list_add(&bb->list, &s->workload->shadow_bb); @@ -2982,7 +2983,7 @@ static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx) goto put_obj; } - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); ret = i915_gem_object_set_to_cpu_domain(obj, false); i915_gem_object_unlock(obj); if (ret) { diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 210016192ce7..a3a4305eda01 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -2501,7 +2501,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) return create_scratch_page_tree(vgpu); } -static void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu) +void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu) { struct list_head *pos, *n; struct intel_vgpu_mm *mm; diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index 320b8d6ad92f..52d0d88abd86 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -279,4 +279,6 @@ int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, void *p_data, unsigned int bytes); +void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu); + #endif /* _GVT_GTT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index a4a6db6b7f90..ff7f2515a6fe 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -106,6 +106,7 @@ struct intel_vgpu_pci_bar { struct intel_vgpu_cfg_space { unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE]; struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM]; + u32 pmcsr_off; }; #define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space) @@ -198,6 +199,8 @@ struct intel_vgpu { struct intel_vgpu_submission submission; struct radix_tree_root page_track_tree; u32 hws_pga[I915_NUM_ENGINES]; + /* Set on PCI_D3, reset on DMLR, not reflecting the actual PM state */ + bool d3_entered; struct dentry *debugfs; diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 63bba7b4bb2f..05f3bc98d242 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -1226,7 +1226,7 @@ static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification) switch (notification) { case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE: root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; - /* fall through */ + fallthrough; case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE: mm = intel_vgpu_get_ppgtt_mm(vgpu, root_entry_type, pdps); return PTR_ERR_OR_ZERO(mm); diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 3c3b9842bbbd..1570eb8aa978 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -403,6 +403,14 @@ static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) wa_ctx->indirect_ctx.shadow_va = NULL; } +static void set_dma_address(struct i915_page_directory *pd, dma_addr_t addr) +{ + struct scatterlist *sg = pd->pt.base->mm.pages->sgl; + + /* This is not a good idea */ + sg->dma_address = addr; +} + static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, struct intel_context *ce) { @@ -411,7 +419,7 @@ static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, int i = 0; if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { - px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0]; + set_dma_address(ppgtt->pd, mm->ppgtt_mm.shadow_pdps[0]); } else { for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { struct i915_page_directory * const pd = @@ -421,7 +429,8 @@ static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, shadow ppgtt. */ if (!pd) break; - px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i]; + + set_dma_address(pd, mm->ppgtt_mm.shadow_pdps[i]); } } } @@ -1240,13 +1249,13 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s, int i; if (i915_vm_is_4lvl(&ppgtt->vm)) { - px_dma(ppgtt->pd) = s->i915_context_pml4; + set_dma_address(ppgtt->pd, s->i915_context_pml4); } else { for (i = 0; i < GEN8_3LVL_PDPES; i++) { struct i915_page_directory * const pd = i915_pd_entry(ppgtt->pd, i); - px_dma(pd) = s->i915_context_pdps[i]; + set_dma_address(pd, s->i915_context_pdps[i]); } } } diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 7d361623ff67..8fa9b31a2484 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -257,6 +257,7 @@ void intel_gvt_release_vgpu(struct intel_vgpu *vgpu) intel_gvt_deactivate_vgpu(vgpu); mutex_lock(&vgpu->vgpu_lock); + vgpu->d3_entered = false; intel_vgpu_clean_workloads(vgpu, ALL_ENGINES); intel_vgpu_dmabuf_cleanup(vgpu); mutex_unlock(&vgpu->vgpu_lock); @@ -393,6 +394,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); idr_init(&vgpu->object_idr); intel_vgpu_init_cfg_space(vgpu, param->primary); + vgpu->d3_entered = false; ret = intel_vgpu_init_mmio(vgpu); if (ret) @@ -557,10 +559,15 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, /* full GPU reset or device model level reset */ if (engine_mask == ALL_ENGINES || dmlr) { intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); - intel_vgpu_invalidate_ppgtt(vgpu); + if (engine_mask == ALL_ENGINES) + intel_vgpu_invalidate_ppgtt(vgpu); /*fence will not be reset during virtual reset */ if (dmlr) { - intel_vgpu_reset_gtt(vgpu); + if(!vgpu->d3_entered) { + intel_vgpu_invalidate_ppgtt(vgpu); + intel_vgpu_destroy_all_ppgtt_mm(vgpu); + } + intel_vgpu_reset_ggtt(vgpu, true); intel_vgpu_reset_resource(vgpu); } @@ -572,7 +579,14 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, intel_vgpu_reset_cfg_space(vgpu); /* only reset the failsafe mode when dmlr reset */ vgpu->failsafe = false; - vgpu->pv_notified = false; + /* + * PCI_D0 is set before dmlr, so reset d3_entered here + * after done using. + */ + if(vgpu->d3_entered) + vgpu->d3_entered = false; + else + vgpu->pv_notified = false; } } diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index d960d0be5bd2..b0a6522be3d1 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -28,12 +28,14 @@ static struct i915_global_active { } global; struct active_node { + struct rb_node node; struct i915_active_fence base; struct i915_active *ref; - struct rb_node node; u64 timeline; }; +#define fetch_node(x) rb_entry(READ_ONCE(x), typeof(struct active_node), node) + static inline struct active_node * node_from_active(struct i915_active_fence *active) { @@ -128,8 +130,8 @@ static inline void debug_active_assert(struct i915_active *ref) { } static void __active_retire(struct i915_active *ref) { + struct rb_root root = RB_ROOT; struct active_node *it, *n; - struct rb_root root; unsigned long flags; GEM_BUG_ON(i915_active_is_idle(ref)); @@ -141,9 +143,25 @@ __active_retire(struct i915_active *ref) GEM_BUG_ON(rcu_access_pointer(ref->excl.fence)); debug_active_deactivate(ref); - root = ref->tree; - ref->tree = RB_ROOT; - ref->cache = NULL; + /* Even if we have not used the cache, we may still have a barrier */ + if (!ref->cache) + ref->cache = fetch_node(ref->tree.rb_node); + + /* Keep the MRU cached node for reuse */ + if (ref->cache) { + /* Discard all other nodes in the tree */ + rb_erase(&ref->cache->node, &ref->tree); + root = ref->tree; + + /* Rebuild the tree with only the cached node */ + rb_link_node(&ref->cache->node, NULL, &ref->tree.rb_node); + rb_insert_color(&ref->cache->node, &ref->tree); + GEM_BUG_ON(ref->tree.rb_node != &ref->cache->node); + + /* Make the cached node available for reuse with any timeline */ + if (IS_ENABLED(CONFIG_64BIT)) + ref->cache->timeline = 0; /* needs cmpxchg(u64) */ + } spin_unlock_irqrestore(&ref->tree_lock, flags); @@ -154,6 +172,7 @@ __active_retire(struct i915_active *ref) /* ... except if you wait on it, you must manage your own references! */ wake_up_var(ref); + /* Finally free the discarded timeline tree */ rbtree_postorder_for_each_entry_safe(it, n, &root, node) { GEM_BUG_ON(i915_active_fence_isset(&it->base)); kmem_cache_free(global.slab_cache, it); @@ -216,12 +235,11 @@ excl_retire(struct dma_fence *fence, struct dma_fence_cb *cb) active_retire(container_of(cb, struct i915_active, excl.cb)); } -static struct i915_active_fence * -active_instance(struct i915_active *ref, struct intel_timeline *tl) +static struct active_node *__active_lookup(struct i915_active *ref, u64 idx) { - struct active_node *node, *prealloc; - struct rb_node **p, *parent; - u64 idx = tl->fence_context; + struct active_node *it; + + GEM_BUG_ON(idx == 0); /* 0 is the unordered timeline, rsvd for cache */ /* * We track the most recently used timeline to skip a rbtree search @@ -230,8 +248,59 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl) * after the previous activity has been retired, or if it matches the * current timeline. */ - node = READ_ONCE(ref->cache); - if (node && node->timeline == idx) + it = READ_ONCE(ref->cache); + if (it) { + u64 cached = READ_ONCE(it->timeline); + + /* Once claimed, this slot will only belong to this idx */ + if (cached == idx) + return it; + +#ifdef CONFIG_64BIT /* for cmpxchg(u64) */ + /* + * An unclaimed cache [.timeline=0] can only be claimed once. + * + * If the value is already non-zero, some other thread has + * claimed the cache and we know that is does not match our + * idx. If, and only if, the timeline is currently zero is it + * worth competing to claim it atomically for ourselves (for + * only the winner of that race will cmpxchg return the old + * value of 0). + */ + if (!cached && !cmpxchg(&it->timeline, 0, idx)) + return it; +#endif + } + + BUILD_BUG_ON(offsetof(typeof(*it), node)); + + /* While active, the tree can only be built; not destroyed */ + GEM_BUG_ON(i915_active_is_idle(ref)); + + it = fetch_node(ref->tree.rb_node); + while (it) { + if (it->timeline < idx) { + it = fetch_node(it->node.rb_right); + } else if (it->timeline > idx) { + it = fetch_node(it->node.rb_left); + } else { + WRITE_ONCE(ref->cache, it); + break; + } + } + + /* NB: If the tree rotated beneath us, we may miss our target. */ + return it; +} + +static struct i915_active_fence * +active_instance(struct i915_active *ref, u64 idx) +{ + struct active_node *node, *prealloc; + struct rb_node **p, *parent; + + node = __active_lookup(ref, idx); + if (likely(node)) return &node->base; /* Preallocate a replacement, just in case */ @@ -268,10 +337,9 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl) rb_insert_color(&node->node, &ref->tree); out: - ref->cache = node; + WRITE_ONCE(ref->cache, node); spin_unlock_irq(&ref->tree_lock); - BUILD_BUG_ON(offsetof(typeof(*node), base)); return &node->base; } @@ -353,69 +421,116 @@ __active_del_barrier(struct i915_active *ref, struct active_node *node) return ____active_del_barrier(ref, node, barrier_to_engine(node)); } -int i915_active_ref(struct i915_active *ref, - struct intel_timeline *tl, - struct dma_fence *fence) +static bool +replace_barrier(struct i915_active *ref, struct i915_active_fence *active) +{ + if (!is_barrier(active)) /* proto-node used by our idle barrier? */ + return false; + + /* + * This request is on the kernel_context timeline, and so + * we can use it to substitute for the pending idle-barrer + * request that we want to emit on the kernel_context. + */ + __active_del_barrier(ref, node_from_active(active)); + return true; +} + +int i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence) { struct i915_active_fence *active; int err; - lockdep_assert_held(&tl->mutex); - /* Prevent reaping in case we malloc/wait while building the tree */ err = i915_active_acquire(ref); if (err) return err; - active = active_instance(ref, tl); + active = active_instance(ref, idx); if (!active) { err = -ENOMEM; goto out; } - if (is_barrier(active)) { /* proto-node used by our idle barrier */ - /* - * This request is on the kernel_context timeline, and so - * we can use it to substitute for the pending idle-barrer - * request that we want to emit on the kernel_context. - */ - __active_del_barrier(ref, node_from_active(active)); + if (replace_barrier(ref, active)) { RCU_INIT_POINTER(active->fence, NULL); atomic_dec(&ref->count); } if (!__i915_active_fence_set(active, fence)) - atomic_inc(&ref->count); + __i915_active_acquire(ref); out: i915_active_release(ref); return err; } -struct dma_fence * -i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f) +static struct dma_fence * +__i915_active_set_fence(struct i915_active *ref, + struct i915_active_fence *active, + struct dma_fence *fence) { struct dma_fence *prev; - /* We expect the caller to manage the exclusive timeline ordering */ - GEM_BUG_ON(i915_active_is_idle(ref)); + if (replace_barrier(ref, active)) { + RCU_INIT_POINTER(active->fence, fence); + return NULL; + } rcu_read_lock(); - prev = __i915_active_fence_set(&ref->excl, f); + prev = __i915_active_fence_set(active, fence); if (prev) prev = dma_fence_get_rcu(prev); else - atomic_inc(&ref->count); + __i915_active_acquire(ref); rcu_read_unlock(); return prev; } +static struct i915_active_fence * +__active_fence(struct i915_active *ref, u64 idx) +{ + struct active_node *it; + + it = __active_lookup(ref, idx); + if (unlikely(!it)) { /* Contention with parallel tree builders! */ + spin_lock_irq(&ref->tree_lock); + it = __active_lookup(ref, idx); + spin_unlock_irq(&ref->tree_lock); + } + GEM_BUG_ON(!it); /* slot must be preallocated */ + + return &it->base; +} + +struct dma_fence * +__i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence) +{ + /* Only valid while active, see i915_active_acquire_for_context() */ + return __i915_active_set_fence(ref, __active_fence(ref, idx), fence); +} + +struct dma_fence * +i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f) +{ + /* We expect the caller to manage the exclusive timeline ordering */ + return __i915_active_set_fence(ref, &ref->excl, f); +} + bool i915_active_acquire_if_busy(struct i915_active *ref) { debug_active_assert(ref); return atomic_add_unless(&ref->count, 1, 0); } +static void __i915_active_activate(struct i915_active *ref) +{ + spin_lock_irq(&ref->tree_lock); /* __active_retire() */ + if (!atomic_fetch_inc(&ref->count)) + debug_active_activate(ref); + spin_unlock_irq(&ref->tree_lock); +} + int i915_active_acquire(struct i915_active *ref) { int err; @@ -423,19 +538,19 @@ int i915_active_acquire(struct i915_active *ref) if (i915_active_acquire_if_busy(ref)) return 0; + if (!ref->active) { + __i915_active_activate(ref); + return 0; + } + err = mutex_lock_interruptible(&ref->mutex); if (err) return err; if (likely(!i915_active_acquire_if_busy(ref))) { - if (ref->active) - err = ref->active(ref); - if (!err) { - spin_lock_irq(&ref->tree_lock); /* __active_retire() */ - debug_active_activate(ref); - atomic_inc(&ref->count); - spin_unlock_irq(&ref->tree_lock); - } + err = ref->active(ref); + if (!err) + __i915_active_activate(ref); } mutex_unlock(&ref->mutex); @@ -443,6 +558,24 @@ int i915_active_acquire(struct i915_active *ref) return err; } +int i915_active_acquire_for_context(struct i915_active *ref, u64 idx) +{ + struct i915_active_fence *active; + int err; + + err = i915_active_acquire(ref); + if (err) + return err; + + active = active_instance(ref, idx); + if (!active) { + i915_active_release(ref); + return -ENOMEM; + } + + return 0; /* return with active ref */ +} + void i915_active_release(struct i915_active *ref) { debug_active_assert(ref); @@ -651,16 +784,16 @@ int i915_sw_fence_await_active(struct i915_sw_fence *fence, return await_active(ref, flags, sw_await_fence, fence, fence); } -#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) void i915_active_fini(struct i915_active *ref) { debug_active_fini(ref); GEM_BUG_ON(atomic_read(&ref->count)); GEM_BUG_ON(work_pending(&ref->work)); - GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree)); mutex_destroy(&ref->mutex); + + if (ref->cache) + kmem_cache_free(global.slab_cache, ref->cache); } -#endif static inline bool is_idle_barrier(struct active_node *node, u64 idx) { @@ -674,7 +807,6 @@ static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx) if (RB_EMPTY_ROOT(&ref->tree)) return NULL; - spin_lock_irq(&ref->tree_lock); GEM_BUG_ON(i915_active_is_idle(ref)); /* @@ -700,9 +832,9 @@ static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx) prev = p; if (node->timeline < idx) - p = p->rb_right; + p = READ_ONCE(p->rb_right); else - p = p->rb_left; + p = READ_ONCE(p->rb_left); } /* @@ -739,14 +871,13 @@ static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx) goto match; } - spin_unlock_irq(&ref->tree_lock); - return NULL; match: + spin_lock_irq(&ref->tree_lock); rb_erase(p, &ref->tree); /* Hide from waits and sibling allocations */ if (p == &ref->cache->node) - ref->cache = NULL; + WRITE_ONCE(ref->cache, NULL); spin_unlock_irq(&ref->tree_lock); return rb_entry(p, struct active_node, node); @@ -758,7 +889,6 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, intel_engine_mask_t tmp, mask = engine->mask; struct llist_node *first = NULL, *last = NULL; struct intel_gt *gt = engine->gt; - int err; GEM_BUG_ON(i915_active_is_idle(ref)); @@ -778,13 +908,13 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, struct llist_node *prev = first; struct active_node *node; + rcu_read_lock(); node = reuse_idle_barrier(ref, idx); + rcu_read_unlock(); if (!node) { node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL); - if (!node) { - err = ENOMEM; + if (!node) goto unwind; - } RCU_INIT_POINTER(node->base.fence, NULL); node->base.cb.func = node_retire; @@ -804,7 +934,7 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, */ RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN)); node->base.cb.node.prev = (void *)engine; - atomic_inc(&ref->count); + __i915_active_acquire(ref); } GEM_BUG_ON(rcu_access_pointer(node->base.fence) != ERR_PTR(-EAGAIN)); @@ -832,7 +962,7 @@ unwind: kmem_cache_free(global.slab_cache, node); } - return err; + return -ENOMEM; } void i915_active_acquire_barrier(struct i915_active *ref) diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h index cf4058150966..fb165d3f01cf 100644 --- a/drivers/gpu/drm/i915/i915_active.h +++ b/drivers/gpu/drm/i915/i915_active.h @@ -163,14 +163,16 @@ void __i915_active_init(struct i915_active *ref, __i915_active_init(ref, active, retire, &__mkey, &__wkey); \ } while (0) -int i915_active_ref(struct i915_active *ref, - struct intel_timeline *tl, - struct dma_fence *fence); +struct dma_fence * +__i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence); +int i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence); static inline int i915_active_add_request(struct i915_active *ref, struct i915_request *rq) { - return i915_active_ref(ref, i915_request_timeline(rq), &rq->fence); + return i915_active_ref(ref, + i915_request_timeline(rq)->fence_context, + &rq->fence); } struct dma_fence * @@ -198,7 +200,9 @@ int i915_request_await_active(struct i915_request *rq, #define I915_ACTIVE_AWAIT_BARRIER BIT(2) int i915_active_acquire(struct i915_active *ref); +int i915_active_acquire_for_context(struct i915_active *ref, u64 idx); bool i915_active_acquire_if_busy(struct i915_active *ref); + void i915_active_release(struct i915_active *ref); static inline void __i915_active_acquire(struct i915_active *ref) @@ -213,11 +217,7 @@ i915_active_is_idle(const struct i915_active *ref) return !atomic_read(&ref->count); } -#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) void i915_active_fini(struct i915_active *ref); -#else -static inline void i915_active_fini(struct i915_active *ref) { } -#endif int i915_active_acquire_preallocate_barrier(struct i915_active *ref, struct intel_engine_cs *engine); @@ -231,4 +231,19 @@ struct i915_active *i915_active_create(void); struct i915_active *i915_active_get(struct i915_active *ref); void i915_active_put(struct i915_active *ref); +static inline int __i915_request_await_exclusive(struct i915_request *rq, + struct i915_active *active) +{ + struct dma_fence *fence; + int err = 0; + + fence = i915_active_fence_get(&active->excl); + if (fence) { + err = i915_request_await_dma_fence(rq, fence); + dma_fence_put(fence); + } + + return err; +} + #endif /* _I915_ACTIVE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index 372354d33f55..5ac4a999f05a 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1204,6 +1204,12 @@ static u32 *copy_batch(struct drm_i915_gem_object *dst_obj, return dst; } +static inline bool cmd_desc_is(const struct drm_i915_cmd_descriptor * const desc, + const u32 cmd) +{ + return desc->cmd.value == (cmd & desc->cmd.mask); +} + static bool check_cmd(const struct intel_engine_cs *engine, const struct drm_i915_cmd_descriptor *desc, const u32 *cmd, u32 length) @@ -1242,19 +1248,19 @@ static bool check_cmd(const struct intel_engine_cs *engine, * allowed mask/value pair given in the whitelist entry. */ if (reg->mask) { - if (desc->cmd.value == MI_LOAD_REGISTER_MEM) { + if (cmd_desc_is(desc, MI_LOAD_REGISTER_MEM)) { DRM_DEBUG("CMD: Rejected LRM to masked register 0x%08X\n", reg_addr); return false; } - if (desc->cmd.value == MI_LOAD_REGISTER_REG) { + if (cmd_desc_is(desc, MI_LOAD_REGISTER_REG)) { DRM_DEBUG("CMD: Rejected LRR to masked register 0x%08X\n", reg_addr); return false; } - if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) && + if (cmd_desc_is(desc, MI_LOAD_REGISTER_IMM(1)) && (offset + 2 > length || (cmd[offset + 1] & reg->mask) != reg->value)) { DRM_DEBUG("CMD: Rejected LRI to masked register 0x%08X\n", @@ -1478,7 +1484,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, break; } - if (desc->cmd.value == MI_BATCH_BUFFER_START) { + if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { ret = check_bbstart(cmd, offset, length, batch_length, batch_addr, shadow_addr, jump_whitelist); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 5fd5af4bc855..00292a849c34 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -392,7 +392,7 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) pre |= IS_HSW_EARLY_SDV(dev_priv); pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); - pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0); + pre |= IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_A0); pre |= IS_GLK_REVID(dev_priv, 0, GLK_REVID_A2); if (pre) { @@ -1075,6 +1075,7 @@ static void i915_driver_release(struct drm_device *dev) intel_memory_regions_driver_release(dev_priv); i915_ggtt_driver_release(dev_priv); + i915_gem_drain_freed_objects(dev_priv); i915_driver_mmio_release(dev_priv); @@ -1119,7 +1120,6 @@ static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) struct drm_i915_file_private *file_priv = file->driver_priv; i915_gem_context_close(file); - i915_gem_release(dev, file); kfree_rcu(file_priv, rcu); @@ -1846,7 +1846,8 @@ static struct drm_driver driver = { */ .driver_features = DRIVER_GEM | - DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ, + DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ | + DRIVER_SYNCOBJ_TIMELINE, .release = i915_driver_release, .open = i915_driver_open, .lastclose = i915_driver_lastclose, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index e4f7f6518945..ab17084af0ff 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -108,8 +108,8 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20200715" -#define DRIVER_TIMESTAMP 1594811881 +#define DRIVER_DATE "20200824" +#define DRIVER_TIMESTAMP 1598293597 struct drm_i915_gem_object; @@ -203,11 +203,6 @@ struct drm_i915_file_private { struct rcu_head rcu; }; - struct { - spinlock_t lock; - struct list_head request_list; - } mm; - struct xarray context_xa; struct xarray vm_xa; @@ -506,6 +501,7 @@ struct i915_psr { bool link_standby; bool colorimetry_support; bool psr2_enabled; + bool psr2_sel_fetch_enabled; u8 sink_sync_latency; ktime_t last_entry_attempt; ktime_t last_exit; @@ -592,11 +588,6 @@ struct i915_gem_mm { atomic_t free_count; /** - * Small stash of WC pages - */ - struct pagestash wc_stash; - - /** * tmpfs instance used for shmem backed objects */ struct vfsmount *gemfs; @@ -1045,6 +1036,14 @@ struct drm_i915_private { struct intel_l3_parity l3_parity; /* + * HTI (aka HDPORT) state read during initial hw readout. Most + * platforms don't have HTI, so this will just stay 0. Those that do + * will use this later to figure out which PLLs and PHYs are unavailable + * for driver usage. + */ + u32 hti_state; + + /* * edram size in MB. * Cannot be determined by PCIID. You must always read a register. */ @@ -1489,6 +1488,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_ICL_WITH_PORT_F(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF) +#define IS_TGL_U(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULT) + +#define IS_TGL_Y(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULX) + #define SKL_REVID_A0 0x0 #define SKL_REVID_B0 0x1 #define SKL_REVID_C0 0x2 @@ -1509,14 +1514,34 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_BXT_REVID(dev_priv, since, until) \ (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) -#define KBL_REVID_A0 0x0 -#define KBL_REVID_B0 0x1 -#define KBL_REVID_C0 0x2 -#define KBL_REVID_D0 0x3 -#define KBL_REVID_E0 0x4 +enum { + KBL_REVID_A0, + KBL_REVID_B0, + KBL_REVID_B1, + KBL_REVID_C0, + KBL_REVID_D0, + KBL_REVID_D1, + KBL_REVID_E0, + KBL_REVID_F0, + KBL_REVID_G0, +}; + +struct i915_rev_steppings { + u8 gt_stepping; + u8 disp_stepping; +}; + +/* Defined in intel_workarounds.c */ +extern const struct i915_rev_steppings kbl_revids[]; -#define IS_KBL_REVID(dev_priv, since, until) \ - (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until)) +#define IS_KBL_GT_REVID(dev_priv, since, until) \ + (IS_KABYLAKE(dev_priv) && \ + kbl_revids[INTEL_REVID(dev_priv)].gt_stepping >= since && \ + kbl_revids[INTEL_REVID(dev_priv)].gt_stepping <= until) +#define IS_KBL_DISP_REVID(dev_priv, since, until) \ + (IS_KABYLAKE(dev_priv) && \ + kbl_revids[INTEL_REVID(dev_priv)].disp_stepping >= since && \ + kbl_revids[INTEL_REVID(dev_priv)].disp_stepping <= until) #define GLK_REVID_A0 0x0 #define GLK_REVID_A1 0x1 @@ -1665,6 +1690,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) #define HAS_PSR_HW_TRACKING(dev_priv) \ (INTEL_INFO(dev_priv)->display.has_psr_hw_tracking) +#define HAS_PSR2_SEL_FETCH(dev_priv) (INTEL_GEN(dev_priv) >= 12) #define HAS_TRANSCODER(dev_priv, trans) ((INTEL_INFO(dev_priv)->cpu_transcoder_mask & BIT(trans)) != 0) #define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) @@ -1790,11 +1816,18 @@ static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915) } struct i915_vma * __must_check +i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + const struct i915_ggtt_view *view, + u64 size, u64 alignment, u64 flags); + +static inline struct i915_vma * __must_check i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view, - u64 size, - u64 alignment, - u64 flags); + u64 size, u64 alignment, u64 flags) +{ + return i915_gem_object_ggtt_pin_ww(obj, NULL, view, size, alignment, flags); +} int i915_gem_object_unbind(struct drm_i915_gem_object *obj, unsigned long flags); @@ -1831,7 +1864,6 @@ void i915_gem_suspend_late(struct drm_i915_private *dev_priv); void i915_gem_resume(struct drm_i915_private *dev_priv); int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); -void i915_gem_release(struct drm_device *dev, struct drm_file *file); int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 9aa3066cb75d..bb0c12975f38 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -335,12 +335,20 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, u64 remain; int ret; - ret = i915_gem_object_prepare_read(obj, &needs_clflush); + ret = i915_gem_object_lock_interruptible(obj, NULL); if (ret) return ret; + ret = i915_gem_object_prepare_read(obj, &needs_clflush); + if (ret) { + i915_gem_object_unlock(obj); + return ret; + } + fence = i915_gem_object_lock_fence(obj); i915_gem_object_finish_access(obj); + i915_gem_object_unlock(obj); + if (!fence) return -ENOMEM; @@ -420,7 +428,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, GEM_BUG_ON(!drm_mm_node_allocated(&node)); } - ret = i915_gem_object_lock_interruptible(obj); + ret = i915_gem_object_lock_interruptible(obj, NULL); if (ret) goto out_unpin; @@ -619,7 +627,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, GEM_BUG_ON(!drm_mm_node_allocated(&node)); } - ret = i915_gem_object_lock_interruptible(obj); + ret = i915_gem_object_lock_interruptible(obj, NULL); if (ret) goto out_unpin; @@ -734,12 +742,20 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, u64 remain; int ret; - ret = i915_gem_object_prepare_write(obj, &needs_clflush); + ret = i915_gem_object_lock_interruptible(obj, NULL); if (ret) return ret; + ret = i915_gem_object_prepare_write(obj, &needs_clflush); + if (ret) { + i915_gem_object_unlock(obj); + return ret; + } + fence = i915_gem_object_lock_fence(obj); i915_gem_object_finish_access(obj); + i915_gem_object_unlock(obj); + if (!fence) return -ENOMEM; @@ -946,11 +962,10 @@ static void discard_ggtt_vma(struct i915_vma *vma) } struct i915_vma * -i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, - const struct i915_ggtt_view *view, - u64 size, - u64 alignment, - u64 flags) +i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + const struct i915_ggtt_view *view, + u64 size, u64 alignment, u64 flags) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_ggtt *ggtt = &i915->ggtt; @@ -1016,7 +1031,7 @@ new_vma: return ERR_PTR(ret); } - ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); + ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL); if (ret) return ERR_PTR(ret); @@ -1290,7 +1305,7 @@ int i915_gem_freeze_late(struct drm_i915_private *i915) i915_gem_drain_freed_objects(i915); list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) { - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); drm_WARN_ON(&i915->drm, i915_gem_object_set_to_cpu_domain(obj, true)); i915_gem_object_unlock(obj); @@ -1301,21 +1316,6 @@ int i915_gem_freeze_late(struct drm_i915_private *i915) return 0; } -void i915_gem_release(struct drm_device *dev, struct drm_file *file) -{ - struct drm_i915_file_private *file_priv = file->driver_priv; - struct i915_request *request; - - /* Clean up our request list when the client is going away, so that - * later retire_requests won't dereference our soon-to-be-gone - * file_priv. - */ - spin_lock(&file_priv->mm.lock); - list_for_each_entry(request, &file_priv->mm.request_list, client_link) - request->file_priv = NULL; - spin_unlock(&file_priv->mm.lock); -} - int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) { struct drm_i915_file_private *file_priv; @@ -1331,9 +1331,6 @@ int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) file_priv->dev_priv = i915; file_priv->file = file; - spin_lock_init(&file_priv->mm.lock); - INIT_LIST_HEAD(&file_priv->mm.request_list); - file_priv->bsd_engine = -1; file_priv->hang_timestamp = jiffies; @@ -1344,6 +1341,58 @@ int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) return ret; } +void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr) +{ + ww_acquire_init(&ww->ctx, &reservation_ww_class); + INIT_LIST_HEAD(&ww->obj_list); + ww->intr = intr; + ww->contended = NULL; +} + +static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww) +{ + struct drm_i915_gem_object *obj; + + while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) { + list_del(&obj->obj_link); + i915_gem_object_unlock(obj); + } +} + +void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj) +{ + list_del(&obj->obj_link); + i915_gem_object_unlock(obj); +} + +void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww) +{ + i915_gem_ww_ctx_unlock_all(ww); + WARN_ON(ww->contended); + ww_acquire_fini(&ww->ctx); +} + +int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww) +{ + int ret = 0; + + if (WARN_ON(!ww->contended)) + return -EINVAL; + + i915_gem_ww_ctx_unlock_all(ww); + if (ww->intr) + ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx); + else + dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx); + + if (!ret) + list_add_tail(&ww->contended->obj_link, &ww->obj_list); + + ww->contended = NULL; + + return ret; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/mock_gem_device.c" #include "selftests/i915_gem.c" diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index f333e88a2b6e..a4cad3f154ca 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -116,4 +116,16 @@ static inline bool __tasklet_is_scheduled(struct tasklet_struct *t) return test_bit(TASKLET_STATE_SCHED, &t->state); } +struct i915_gem_ww_ctx { + struct ww_acquire_ctx ctx; + struct list_head obj_list; + bool intr; + struct drm_i915_gem_object *contended; +}; + +void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ctx, bool intr); +void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ctx); +int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ctx); +void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj); + #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 421613219ae9..f96032c60a12 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -132,6 +132,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, case I915_PARAM_HAS_EXEC_BATCH_FIRST: case I915_PARAM_HAS_EXEC_FENCE_ARRAY: case I915_PARAM_HAS_EXEC_SUBMIT_FENCE: + case I915_PARAM_HAS_EXEC_TIMELINE_FENCES: /* For the time being all of these are always true; * if some supported hardware does not have one of these * features this value needs to be provided from diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 6a3a2ce0b394..3e6cbb0d1150 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1159,7 +1159,7 @@ static void engine_record_registers(struct intel_engine_coredump *ee) switch (engine->id) { default: MISSING_CASE(engine->id); - /* fall through */ + fallthrough; case RCS0: mmio = RENDER_HWS_PGA_GEN7; break; diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 1fa67700d8f4..f113fe44572b 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -41,6 +41,7 @@ #include "display/intel_lpe_audio.h" #include "display/intel_psr.h" +#include "gt/intel_breadcrumbs.h" #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 8d8db9ff0a48..7f139ea4a90b 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -102,6 +102,11 @@ i915_param_named(psr_safest_params, bool, 0400, "is helpful to detect if PSR issues are related to bad values set in " " VBT. (0=use VBT parameters, 1=use safest parameters)"); +i915_param_named_unsafe(enable_psr2_sel_fetch, bool, 0400, + "Enable PSR2 selective fetch " + "(0=disabled, 1=enabled) " + "Default: 0"); + i915_param_named_unsafe(force_probe, charp, 0400, "Force probe the driver for specified devices. " "See CONFIG_DRM_I915_FORCE_PROBE for details."); diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 53fb5ba8fbed..330c03e2b4f7 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -54,6 +54,7 @@ struct drm_printer; param(int, enable_fbc, -1, 0600) \ param(int, enable_psr, -1, 0600) \ param(bool, psr_safest_params, false, 0600) \ + param(bool, enable_psr2_sel_fetch, false, 0600) \ param(int, disable_power_well, -1, 0400) \ param(int, enable_ips, 1, 0600) \ param(int, invert_brightness, 0, 0600) \ diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 2338f92ce490..366ddfc8df6b 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -890,6 +890,7 @@ static const struct intel_device_info rkl_info = { .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), .require_force_probe = 1, + .display.has_hti = 1, .display.has_psr_hw_tracking = 0, .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0), diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index c6f6370283cf..e94976976571 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -1195,24 +1195,39 @@ static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) struct i915_gem_engines_iter it; struct i915_gem_context *ctx = stream->ctx; struct intel_context *ce; - int err; + struct i915_gem_ww_ctx ww; + int err = -ENODEV; for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { if (ce->engine != stream->engine) /* first match! */ continue; - /* - * As the ID is the gtt offset of the context's vma we - * pin the vma to ensure the ID remains fixed. - */ - err = intel_context_pin(ce); - if (err == 0) { - stream->pinned_ctx = ce; - break; - } + err = 0; + break; } i915_gem_context_unlock_engines(ctx); + if (err) + return ERR_PTR(err); + + i915_gem_ww_ctx_init(&ww, true); +retry: + /* + * As the ID is the gtt offset of the context's vma we + * pin the vma to ensure the ID remains fixed. + */ + err = intel_context_pin_ww(ce, &ww); + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + + if (err) + return ERR_PTR(err); + + stream->pinned_ctx = ce; return stream->pinned_ctx; } @@ -1923,15 +1938,22 @@ emit_oa_config(struct i915_perf_stream *stream, { struct i915_request *rq; struct i915_vma *vma; + struct i915_gem_ww_ctx ww; int err; vma = get_oa_vma(stream, oa_config); if (IS_ERR(vma)) return PTR_ERR(vma); - err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH); + i915_gem_ww_ctx_init(&ww, true); +retry: + err = i915_gem_object_lock(vma->obj, &ww); + if (err) + goto err; + + err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_GLOBAL | PIN_HIGH); if (err) - goto err_vma_put; + goto err; intel_engine_pm_get(ce->engine); rq = i915_request_create(ce); @@ -1953,11 +1975,9 @@ emit_oa_config(struct i915_perf_stream *stream, goto err_add_request; } - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, 0); if (!err) err = i915_vma_move_to_active(vma, rq, 0); - i915_vma_unlock(vma); if (err) goto err_add_request; @@ -1971,7 +1991,14 @@ err_add_request: i915_request_add(rq); err_vma_unpin: i915_vma_unpin(vma); -err_vma_put: +err: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + + i915_gem_ww_ctx_fini(&ww); i915_vma_put(vma); return err; } diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 28bc5f13ae52..69c0fa20eba1 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -445,8 +445,6 @@ static void i915_pmu_event_destroy(struct perf_event *event) container_of(event->pmu, typeof(*i915), pmu.base); drm_WARN_ON(&i915->drm, event->parent); - - module_put(THIS_MODULE); } static int @@ -476,7 +474,7 @@ config_status(struct drm_i915_private *i915, u64 config) if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) /* Requires a mutex for sampling! */ return -ENODEV; - /* Fall-through. */ + fallthrough; case I915_PMU_REQUESTED_FREQUENCY: if (INTEL_GEN(i915) < 6) return -ENODEV; @@ -538,10 +536,8 @@ static int i915_pmu_event_init(struct perf_event *event) if (ret) return ret; - if (!event->parent) { - __module_get(THIS_MODULE); + if (!event->parent) event->destroy = i915_pmu_event_destroy; - } return 0; } @@ -1130,6 +1126,7 @@ void i915_pmu_register(struct drm_i915_private *i915) if (!pmu->base.attr_groups) goto err_attr; + pmu->base.module = THIS_MODULE; pmu->base.task_ctx_nr = perf_invalid_context; pmu->base.event_init = i915_pmu_event_init; pmu->base.add = i915_pmu_event_add; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 4e796ff4d7d0..ac691927e29d 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -1898,6 +1898,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define PWR_DOWN_LN_3_1_0 (0xb << 4) #define PWR_DOWN_LN_MASK (0xf << 4) #define PWR_DOWN_LN_SHIFT 4 +#define EDP4K2K_MODE_OVRD_EN (1 << 3) +#define EDP4K2K_MODE_OVRD_OPTIMIZED (1 << 2) #define ICL_PORT_CL_DW12(phy) _MMIO(_ICL_PORT_CL_DW(12, phy)) #define ICL_LANE_ENABLE_AUX (1 << 0) @@ -2919,6 +2921,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MBUS_BBOX_CTL_S1 _MMIO(0x45040) #define MBUS_BBOX_CTL_S2 _MMIO(0x45044) +#define HDPORT_STATE _MMIO(0x45050) +#define HDPORT_DPLL_USED_MASK REG_GENMASK(14, 12) +#define HDPORT_PHY_USED_DP(phy) REG_BIT(2 * (phy) + 2) +#define HDPORT_PHY_USED_HDMI(phy) REG_BIT(2 * (phy) + 1) +#define HDPORT_ENABLED REG_BIT(0) + /* Make render/texture TLB fetches lower priorty than associated data * fetches. This is not turned on by default */ @@ -7870,6 +7878,7 @@ enum { # define CHICKEN3_DGMG_DONE_FIX_DISABLE (1 << 2) #define CHICKEN_PAR1_1 _MMIO(0x42080) +#define DIS_RAM_BYPASS_PSR2_MAN_TRACK (1 << 16) #define SKL_DE_COMPRESSED_HASH_MODE (1 << 15) #define DPA_MASK_VBLANK_SRD (1 << 15) #define FORCE_ARB_IDLE_PLANES (1 << 14) @@ -8711,6 +8720,7 @@ enum { #define PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 31) #define PCH_DPLUNIT_CLOCK_GATE_DISABLE (1 << 30) #define PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1 << 29) +#define PCH_DPMGUNIT_CLOCK_GATE_DISABLE (1 << 15) #define PCH_CPUNIT_CLOCK_GATE_DISABLE (1 << 14) #define CNP_PWM_CGE_GATING_DISABLE (1 << 13) #define PCH_LP_PARTITION_LEVEL_DISABLE (1 << 12) @@ -9217,8 +9227,8 @@ enum { #define DISPLAY_IPS_CONTROL 0x19 #define TGL_PCODE_TCCOLD 0x26 #define TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED REG_BIT(0) -#define TGL_PCODE_EXIT_TCCOLD_DATA_H_BLOCK_REQ 0 -#define TGL_PCODE_EXIT_TCCOLD_DATA_H_UNBLOCK_REQ REG_BIT(0) +#define TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ 0 +#define TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ REG_BIT(0) /* See also IPS_CTL */ #define IPS_PCODE_CONTROL (1 << 30) #define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A @@ -10277,12 +10287,18 @@ enum skl_power_gate { #define ICL_DPCLKA_CFGCR0 _MMIO(0x164280) #define ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) (1 << _PICK(phy, 10, 11, 24)) +#define RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) REG_BIT((phy) + 10) #define ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port) (1 << ((tc_port) < PORT_TC4 ? \ (tc_port) + 12 : \ (tc_port) - PORT_TC4 + 21)) #define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy) ((phy) * 2) #define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy) (3 << ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) #define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy) ((pll) << ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) +#define RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy) _PICK(phy, 0, 2, 4, 27) +#define RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy) \ + (3 << RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) +#define RKL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy) \ + ((pll) << RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) /* CNL PLL */ #define DPLL0_ENABLE 0x46010 @@ -10503,19 +10519,21 @@ enum skl_power_gate { #define _TGL_DPLL0_CFGCR0 0x164284 #define _TGL_DPLL1_CFGCR0 0x16428C -/* TODO: add DPLL4 */ #define _TGL_TBTPLL_CFGCR0 0x16429C #define TGL_DPLL_CFGCR0(pll) _MMIO_PLL3(pll, _TGL_DPLL0_CFGCR0, \ _TGL_DPLL1_CFGCR0, \ _TGL_TBTPLL_CFGCR0) +#define RKL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _TGL_DPLL0_CFGCR0, \ + _TGL_DPLL1_CFGCR0) #define _TGL_DPLL0_CFGCR1 0x164288 #define _TGL_DPLL1_CFGCR1 0x164290 -/* TODO: add DPLL4 */ #define _TGL_TBTPLL_CFGCR1 0x1642A0 #define TGL_DPLL_CFGCR1(pll) _MMIO_PLL3(pll, _TGL_DPLL0_CFGCR1, \ _TGL_DPLL1_CFGCR1, \ _TGL_TBTPLL_CFGCR1) +#define RKL_DPLL_CFGCR1(pll) _MMIO_PLL(pll, _TGL_DPLL0_CFGCR1, \ + _TGL_DPLL1_CFGCR1) #define _DKL_PHY1_BASE 0x168000 #define _DKL_PHY2_BASE 0x169000 @@ -12336,4 +12354,10 @@ enum skl_power_gate { #define DSB_ENABLE (1 << 31) #define DSB_STATUS (1 << 0) +#define TGL_ROOT_DEVICE_ID 0x9A00 +#define TGL_ROOT_DEVICE_MASK 0xFF00 +#define TGL_ROOT_DEVICE_SKU_MASK 0xF +#define TGL_ROOT_DEVICE_SKU_ULX 0x2 +#define TGL_ROOT_DEVICE_SKU_ULT 0x4 + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 0b2fe55e6194..11e272422fb7 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -31,6 +31,7 @@ #include <linux/sched/signal.h> #include "gem/i915_gem_context.h" +#include "gt/intel_breadcrumbs.h" #include "gt/intel_context.h" #include "gt/intel_ring.h" #include "gt/intel_rps.h" @@ -186,48 +187,34 @@ static void irq_execute_cb_hook(struct irq_work *wrk) irq_execute_cb(wrk); } -static void __notify_execute_cb(struct i915_request *rq) +static __always_inline void +__notify_execute_cb(struct i915_request *rq, bool (*fn)(struct irq_work *wrk)) { struct execute_cb *cb, *cn; - lockdep_assert_held(&rq->lock); - - GEM_BUG_ON(!i915_request_is_active(rq)); if (llist_empty(&rq->execute_cb)) return; - llist_for_each_entry_safe(cb, cn, rq->execute_cb.first, work.llnode) - irq_work_queue(&cb->work); - - /* - * XXX Rollback on __i915_request_unsubmit() - * - * In the future, perhaps when we have an active time-slicing scheduler, - * it will be interesting to unsubmit parallel execution and remove - * busywaits from the GPU until their master is restarted. This is - * quite hairy, we have to carefully rollback the fence and do a - * preempt-to-idle cycle on the target engine, all the while the - * master execute_cb may refire. - */ - init_llist_head(&rq->execute_cb); + llist_for_each_entry_safe(cb, cn, + llist_del_all(&rq->execute_cb), + work.llnode) + fn(&cb->work); } -static inline void -remove_from_client(struct i915_request *request) +static void __notify_execute_cb_irq(struct i915_request *rq) { - struct drm_i915_file_private *file_priv; + __notify_execute_cb(rq, irq_work_queue); +} - if (!READ_ONCE(request->file_priv)) - return; +static bool irq_work_imm(struct irq_work *wrk) +{ + wrk->func(wrk); + return false; +} - rcu_read_lock(); - file_priv = xchg(&request->file_priv, NULL); - if (file_priv) { - spin_lock(&file_priv->mm.lock); - list_del(&request->client_link); - spin_unlock(&file_priv->mm.lock); - } - rcu_read_unlock(); +static void __notify_execute_cb_imm(struct i915_request *rq) +{ + __notify_execute_cb(rq, irq_work_imm); } static void free_capture_list(struct i915_request *request) @@ -274,9 +261,16 @@ static void remove_from_engine(struct i915_request *rq) locked = engine; } list_del_init(&rq->sched.link); + clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags); + + /* Prevent further __await_execution() registering a cb, then flush */ + set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); + spin_unlock_irq(&locked->active.lock); + + __notify_execute_cb_imm(rq); } bool i915_request_retire(struct i915_request *rq) @@ -288,6 +282,7 @@ bool i915_request_retire(struct i915_request *rq) GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit)); trace_i915_request_retire(rq); + i915_request_mark_complete(rq); /* * We know the GPU must have read the request to have @@ -305,32 +300,30 @@ bool i915_request_retire(struct i915_request *rq) __i915_request_fill(rq, POISON_FREE); rq->ring->head = rq->postfix; + if (!i915_request_signaled(rq)) { + spin_lock_irq(&rq->lock); + dma_fence_signal_locked(&rq->fence); + spin_unlock_irq(&rq->lock); + } + + if (i915_request_has_waitboost(rq)) { + GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters)); + atomic_dec(&rq->engine->gt->rps.num_waiters); + } + /* * We only loosely track inflight requests across preemption, * and so we may find ourselves attempting to retire a _completed_ * request that we have removed from the HW and put back on a run * queue. + * + * As we set I915_FENCE_FLAG_ACTIVE on the request, this should be + * after removing the breadcrumb and signaling it, so that we do not + * inadvertently attach the breadcrumb to a completed request. */ remove_from_engine(rq); - - spin_lock_irq(&rq->lock); - i915_request_mark_complete(rq); - if (!i915_request_signaled(rq)) - dma_fence_signal_locked(&rq->fence); - if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) - i915_request_cancel_breadcrumb(rq); - if (i915_request_has_waitboost(rq)) { - GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters)); - atomic_dec(&rq->engine->gt->rps.num_waiters); - } - if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { - set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); - __notify_execute_cb(rq); - } GEM_BUG_ON(!llist_empty(&rq->execute_cb)); - spin_unlock_irq(&rq->lock); - remove_from_client(rq); __list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */ intel_context_exit(rq->context); @@ -357,12 +350,6 @@ void i915_request_retire_upto(struct i915_request *rq) } while (i915_request_retire(tmp) && tmp != rq); } -static void __llist_add(struct llist_node *node, struct llist_head *head) -{ - node->next = head->first; - head->first = node; -} - static struct i915_request * const * __engine_active(struct intel_engine_cs *engine) { @@ -388,17 +375,38 @@ static bool __request_in_flight(const struct i915_request *signal) * As we know that there are always preemption points between * requests, we know that only the currently executing request * may be still active even though we have cleared the flag. - * However, we can't rely on our tracking of ELSP[0] to known + * However, we can't rely on our tracking of ELSP[0] to know * which request is currently active and so maybe stuck, as * the tracking maybe an event behind. Instead assume that * if the context is still inflight, then it is still active * even if the active flag has been cleared. + * + * To further complicate matters, if there a pending promotion, the HW + * may either perform a context switch to the second inflight execlists, + * or it may switch to the pending set of execlists. In the case of the + * latter, it may send the ACK and we process the event copying the + * pending[] over top of inflight[], _overwriting_ our *active. Since + * this implies the HW is arbitrating and not struck in *active, we do + * not worry about complete accuracy, but we do require no read/write + * tearing of the pointer [the read of the pointer must be valid, even + * as the array is being overwritten, for which we require the writes + * to avoid tearing.] + * + * Note that the read of *execlists->active may race with the promotion + * of execlists->pending[] to execlists->inflight[], overwritting + * the value at *execlists->active. This is fine. The promotion implies + * that we received an ACK from the HW, and so the context is not + * stuck -- if we do not see ourselves in *active, the inflight status + * is valid. If instead we see ourselves being copied into *active, + * we are inflight and may signal the callback. */ if (!intel_context_inflight(signal->context)) return false; rcu_read_lock(); - for (port = __engine_active(signal->engine); (rq = *port); port++) { + for (port = __engine_active(signal->engine); + (rq = READ_ONCE(*port)); /* may race with promotion of pending[] */ + port++) { if (rq->context == signal->context) { inflight = i915_seqno_passed(rq->fence.seqno, signal->fence.seqno); @@ -439,18 +447,24 @@ __await_execution(struct i915_request *rq, cb->work.func = irq_execute_cb_hook; } - spin_lock_irq(&signal->lock); - if (i915_request_is_active(signal) || __request_in_flight(signal)) { - if (hook) { - hook(rq, &signal->fence); - i915_request_put(signal); - } - i915_sw_fence_complete(cb->fence); - kmem_cache_free(global.slab_execute_cbs, cb); - } else { - __llist_add(&cb->work.llnode, &signal->execute_cb); + /* + * Register the callback first, then see if the signaler is already + * active. This ensures that if we race with the + * __notify_execute_cb from i915_request_submit() and we are not + * included in that list, we get a second bite of the cherry and + * execute it ourselves. After this point, a future + * i915_request_submit() will notify us. + * + * In i915_request_retire() we set the ACTIVE bit on a completed + * request (then flush the execute_cb). So by registering the + * callback first, then checking the ACTIVE bit, we serialise with + * the completed/retired request. + */ + if (llist_add(&cb->work.llnode, &signal->execute_cb)) { + if (i915_request_is_active(signal) || + __request_in_flight(signal)) + __notify_execute_cb_imm(signal); } - spin_unlock_irq(&signal->lock); return 0; } @@ -566,18 +580,28 @@ xfer: clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags); } + /* + * XXX Rollback bonded-execution on __i915_request_unsubmit()? + * + * In the future, perhaps when we have an active time-slicing scheduler, + * it will be interesting to unsubmit parallel execution and remove + * busywaits from the GPU until their master is restarted. This is + * quite hairy, we have to carefully rollback the fence and do a + * preempt-to-idle cycle on the target engine, all the while the + * master execute_cb may refire. + */ + __notify_execute_cb_irq(request); + /* We may be recursing from the signal callback of another i915 fence */ if (!i915_request_signaled(request)) { spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); - __notify_execute_cb(request); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags) && !i915_request_enable_breadcrumb(request)) intel_engine_signal_breadcrumbs(engine); spin_unlock(&request->lock); - GEM_BUG_ON(!llist_empty(&request->execute_cb)); } return result; @@ -600,27 +624,27 @@ void __i915_request_unsubmit(struct i915_request *request) { struct intel_engine_cs *engine = request->engine; + /* + * Only unwind in reverse order, required so that the per-context list + * is kept in seqno/ring order. + */ RQ_TRACE(request, "\n"); GEM_BUG_ON(!irqs_disabled()); lockdep_assert_held(&engine->active.lock); /* - * Only unwind in reverse order, required so that the per-context list - * is kept in seqno/ring order. + * Before we remove this breadcrumb from the signal list, we have + * to ensure that a concurrent dma_fence_enable_signaling() does not + * attach itself. We first mark the request as no longer active and + * make sure that is visible to other cores, and then remove the + * breadcrumb if attached. */ - - /* We may be recursing from the signal callback of another i915 fence */ - spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); - + GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); + clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) i915_request_cancel_breadcrumb(request); - GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); - clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); - - spin_unlock(&request->lock); - /* We've already spun, don't charge on resubmitting. */ if (request->sched.semaphores && i915_request_started(request)) request->sched.semaphores = 0; @@ -757,7 +781,6 @@ static void __i915_request_ctor(void *arg) dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock, 0, 0); - rq->file_priv = NULL; rq->capture_list = NULL; init_llist_head(&rq->execute_cb); @@ -847,7 +870,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) /* No zalloc, everything must be cleared after use */ rq->batch = NULL; - GEM_BUG_ON(rq->file_priv); GEM_BUG_ON(rq->capture_list); GEM_BUG_ON(!llist_empty(&rq->execute_cb)); @@ -1640,7 +1662,7 @@ static bool busywait_stop(unsigned long timeout, unsigned int cpu) return this_cpu != cpu; } -static bool __i915_spin_request(const struct i915_request * const rq, int state) +static bool __i915_spin_request(struct i915_request * const rq, int state) { unsigned long timeout_ns; unsigned int cpu; @@ -1673,7 +1695,7 @@ static bool __i915_spin_request(const struct i915_request * const rq, int state) timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns); timeout_ns += local_clock_ns(&cpu); do { - if (i915_request_completed(rq)) + if (dma_fence_is_signaled(&rq->fence)) return true; if (signal_pending_state(state, current)) @@ -1697,7 +1719,7 @@ static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb) { struct request_wait *wait = container_of(cb, typeof(*wait), cb); - wake_up_process(wait->tsk); + wake_up_process(fetch_and_zero(&wait->tsk)); } /** @@ -1766,10 +1788,8 @@ long i915_request_wait(struct i915_request *rq, * duration, which we currently lack. */ if (IS_ACTIVE(CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT) && - __i915_spin_request(rq, state)) { - dma_fence_signal(&rq->fence); + __i915_spin_request(rq, state)) goto out; - } /* * This client is about to stall waiting for the GPU. In many cases @@ -1783,25 +1803,36 @@ long i915_request_wait(struct i915_request *rq, * but at a cost of spending more power processing the workload * (bad for battery). */ - if (flags & I915_WAIT_PRIORITY) { - if (!i915_request_started(rq) && - INTEL_GEN(rq->engine->i915) >= 6) - intel_rps_boost(rq); - } + if (flags & I915_WAIT_PRIORITY && !i915_request_started(rq)) + intel_rps_boost(rq); wait.tsk = current; if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake)) goto out; + /* + * Flush the submission tasklet, but only if it may help this request. + * + * We sometimes experience some latency between the HW interrupts and + * tasklet execution (mostly due to ksoftirqd latency, but it can also + * be due to lazy CS events), so lets run the tasklet manually if there + * is a chance it may submit this request. If the request is not ready + * to run, as it is waiting for other fences to be signaled, flushing + * the tasklet is busy work without any advantage for this client. + * + * If the HW is being lazy, this is the last chance before we go to + * sleep to catch any pending events. We will check periodically in + * the heartbeat to flush the submission tasklets as a last resort + * for unhappy HW. + */ + if (i915_request_is_ready(rq)) + intel_engine_flush_submission(rq->engine); + for (;;) { set_current_state(state); - if (i915_request_completed(rq)) { - dma_fence_signal(&rq->fence); + if (dma_fence_is_signaled(&rq->fence)) break; - } - - intel_engine_flush_submission(rq->engine); if (signal_pending_state(state, current)) { timeout = -ERESTARTSYS; @@ -1817,7 +1848,9 @@ long i915_request_wait(struct i915_request *rq, } __set_current_state(TASK_RUNNING); - dma_fence_remove_callback(&rq->fence, &wait.cb); + if (READ_ONCE(wait.tsk)) + dma_fence_remove_callback(&rq->fence, &wait.cb); + GEM_BUG_ON(!list_empty(&wait.cb.node)); out: mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_); diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 590762820761..16b721080195 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -284,10 +284,6 @@ struct i915_request { /** timeline->request entry for this request */ struct list_head link; - struct drm_i915_file_private *file_priv; - /** file_priv list entry for this request */ - struct list_head client_link; - I915_SELFTEST_DECLARE(struct { struct list_head link; unsigned long delay; @@ -365,10 +361,6 @@ void i915_request_submit(struct i915_request *request); void __i915_request_unsubmit(struct i915_request *request); void i915_request_unsubmit(struct i915_request *request); -/* Note: part of the intel_breadcrumbs family */ -bool i915_request_enable_breadcrumb(struct i915_request *request); -void i915_request_cancel_breadcrumb(struct i915_request *request); - long i915_request_wait(struct i915_request *rq, unsigned int flags, long timeout) diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index 295b9829e2da..4cd2038cbe35 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -164,9 +164,13 @@ static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, do { list_for_each_entry_safe(pos, next, &x->head, entry) { - pos->func(pos, - TASK_NORMAL, fence->error, - &extra); + int wake_flags; + + wake_flags = fence->error; + if (pos->func == autoremove_wake_function) + wake_flags = 0; + + pos->func(pos, TASK_NORMAL, wake_flags, &extra); } if (list_empty(&extra)) diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index bc64f773dcdb..495d28f6d160 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -291,6 +291,8 @@ i915_vma_instance(struct drm_i915_gem_object *obj, struct i915_vma_work { struct dma_fence_work base; + struct i915_address_space *vm; + struct i915_vm_pt_stash stash; struct i915_vma *vma; struct drm_i915_gem_object *pinned; struct i915_sw_dma_fence_cb cb; @@ -302,13 +304,10 @@ static int __vma_bind(struct dma_fence_work *work) { struct i915_vma_work *vw = container_of(work, typeof(*vw), base); struct i915_vma *vma = vw->vma; - int err; - - err = vma->ops->bind_vma(vma->vm, vma, vw->cache_level, vw->flags); - if (err) - atomic_or(I915_VMA_ERROR, &vma->flags); - return err; + vma->ops->bind_vma(vw->vm, &vw->stash, + vma, vw->cache_level, vw->flags); + return 0; } static void __vma_release(struct dma_fence_work *work) @@ -317,6 +316,9 @@ static void __vma_release(struct dma_fence_work *work) if (vw->pinned) __i915_gem_object_unpin_pages(vw->pinned); + + i915_vm_free_pt_stash(vw->vm, &vw->stash); + i915_vm_put(vw->vm); } static const struct dma_fence_work_ops bind_ops = { @@ -376,7 +378,6 @@ int i915_vma_bind(struct i915_vma *vma, { u32 bind_flags; u32 vma_flags; - int ret; GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); GEM_BUG_ON(vma->size > vma->node.size); @@ -433,9 +434,7 @@ int i915_vma_bind(struct i915_vma *vma, work->pinned = vma->obj; } } else { - ret = vma->ops->bind_vma(vma->vm, vma, cache_level, bind_flags); - if (ret) - return ret; + vma->ops->bind_vma(vma->vm, NULL, vma, cache_level, bind_flags); } atomic_or(bind_flags, &vma->flags); @@ -853,13 +852,19 @@ static void vma_unbind_pages(struct i915_vma *vma) __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS); } -int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, + u64 size, u64 alignment, u64 flags) { struct i915_vma_work *work = NULL; intel_wakeref_t wakeref = 0; unsigned int bound; int err; +#ifdef CONFIG_PROVE_LOCKING + if (debug_locks && lockdep_is_held(&vma->vm->i915->drm.struct_mutex)) + WARN_ON(!ww); +#endif + BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); @@ -873,16 +878,30 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) if (err) return err; + if (flags & PIN_GLOBAL) + wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); + if (flags & vma->vm->bind_async_flags) { work = i915_vma_work(); if (!work) { err = -ENOMEM; - goto err_pages; + goto err_rpm; } - } - if (flags & PIN_GLOBAL) - wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); + work->vm = i915_vm_get(vma->vm); + + /* Allocate enough page directories to used PTE */ + if (vma->vm->allocate_va_range) { + i915_vm_alloc_pt_stash(vma->vm, + &work->stash, + vma->size); + + err = i915_vm_pin_pt_stash(vma->vm, + &work->stash); + if (err) + goto err_fence; + } + } /* * Differentiate between user/kernel vma inside the aliasing-ppgtt. @@ -971,9 +990,9 @@ err_unlock: err_fence: if (work) dma_fence_work_commit_imm(&work->base); +err_rpm: if (wakeref) intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); -err_pages: vma_put_pages(vma); return err; } @@ -989,7 +1008,8 @@ static void flush_idle_contexts(struct intel_gt *gt) intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); } -int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags) +int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, + u32 align, unsigned int flags) { struct i915_address_space *vm = vma->vm; int err; @@ -997,7 +1017,7 @@ int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags) GEM_BUG_ON(!i915_vma_is_ggtt(vma)); do { - err = i915_vma_pin(vma, 0, align, flags | PIN_GLOBAL); + err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL); if (err != -ENOSPC) { if (!err) { err = i915_vma_wait_for_bind(vma); @@ -1167,6 +1187,12 @@ void i915_vma_revoke_mmap(struct i915_vma *vma) list_del(&vma->obj->userfault_link); } +static int +__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma) +{ + return __i915_request_await_exclusive(rq, &vma->active); +} + int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) { int err; @@ -1174,8 +1200,7 @@ int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) GEM_BUG_ON(!i915_vma_is_pinned(vma)); /* Wait for the vma to be bound before we start! */ - err = i915_request_await_active(rq, &vma->active, - I915_ACTIVE_AWAIT_EXCL); + err = __i915_request_await_bind(rq, vma); if (err) return err; diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index d0d01f909548..5b3a3c653454 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -237,8 +237,17 @@ static inline void i915_vma_unlock(struct i915_vma *vma) } int __must_check -i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags); -int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags); +i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, + u64 size, u64 alignment, u64 flags); + +static inline int __must_check +i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +{ + return i915_vma_pin_ww(vma, NULL, size, alignment, flags); +} + +int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, + u32 align, unsigned int flags); static inline int i915_vma_pin_count(const struct i915_vma *vma) { diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 40c590db3c76..e2aa5bc3a6e0 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -346,6 +346,25 @@ void intel_device_info_subplatform_init(struct drm_i915_private *i915) mask = BIT(INTEL_SUBPLATFORM_PORTF); } + if (IS_TIGERLAKE(i915)) { + struct pci_dev *root, *pdev = i915->drm.pdev; + + root = list_first_entry(&pdev->bus->devices, typeof(*root), bus_list); + + drm_WARN_ON(&i915->drm, mask); + drm_WARN_ON(&i915->drm, (root->device & TGL_ROOT_DEVICE_MASK) != + TGL_ROOT_DEVICE_ID); + + switch (root->device & TGL_ROOT_DEVICE_SKU_MASK) { + case TGL_ROOT_DEVICE_SKU_ULX: + mask = BIT(INTEL_SUBPLATFORM_ULX); + break; + case TGL_ROOT_DEVICE_SKU_ULT: + mask = BIT(INTEL_SUBPLATFORM_ULT); + break; + } + } + GEM_BUG_ON(mask & ~INTEL_SUBPLATFORM_BITS); RUNTIME_INFO(i915)->platform_mask[pi] |= mask; diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index fd2385457ab6..6a3d607218aa 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -146,6 +146,7 @@ enum intel_ppgtt_type { func(has_gmch); \ func(has_hdcp); \ func(has_hotplug); \ + func(has_hti); \ func(has_ipc); \ func(has_modular_fia); \ func(has_overlay); \ diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index cfabbe0481ab..b4bd19266b8c 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -100,12 +100,6 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) */ I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | DISP_FBC_MEMORY_WAKE); - - if (IS_SKYLAKE(dev_priv)) { - /* WaDisableDopClockGating */ - I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL) - & ~GEN7_DOP_CLOCK_GATE_ENABLE); - } } static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7223,12 +7217,12 @@ static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:kbl */ - if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0)) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* WaDisableGamClockGating:kbl */ - if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0)) I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | GEN6_GAMUNIT_CLOCK_GATE_DISABLE); @@ -7251,6 +7245,10 @@ static void skl_init_clock_gating(struct drm_i915_private *dev_priv) { gen9_init_clock_gating(dev_priv); + /* WaDisableDopClockGating:skl */ + I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL) & + ~GEN7_DOP_CLOCK_GATE_ENABLE); + /* WAC6entrylatency:skl */ I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | FBC_LLC_FULLY_OPEN); diff --git a/drivers/gpu/drm/i915/selftests/i915_buddy.c b/drivers/gpu/drm/i915/selftests/i915_buddy.c index 939a6caebb03..632b912b0bc9 100644 --- a/drivers/gpu/drm/i915/selftests/i915_buddy.c +++ b/drivers/gpu/drm/i915/selftests/i915_buddy.c @@ -8,8 +8,6 @@ #include "../i915_selftest.h" #include "i915_random.h" -#define SZ_8G (1ULL << 33) - static void __igt_dump_block(struct i915_buddy_mm *mm, struct i915_buddy_block *block, bool buddy) @@ -281,18 +279,22 @@ static int igt_check_mm(struct i915_buddy_mm *mm) static void igt_mm_config(u64 *size, u64 *chunk_size) { I915_RND_STATE(prng); - u64 s, ms; + u32 s, ms; /* Nothing fancy, just try to get an interesting bit pattern */ prandom_seed_state(&prng, i915_selftest.random_seed); - s = i915_prandom_u64_state(&prng) & (SZ_8G - 1); - ms = BIT_ULL(12 + (prandom_u32_state(&prng) % ilog2(s >> 12))); - s = max(s & -ms, ms); + /* Let size be a random number of pages up to 8 GB (2M pages) */ + s = 1 + i915_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng); + /* Let the chunk size be a random power of 2 less than size */ + ms = BIT(i915_prandom_u32_max_state(ilog2(s), &prng)); + /* Round size down to the chunk size */ + s &= -ms; - *chunk_size = ms; - *size = s; + /* Convert from pages to bytes */ + *chunk_size = (u64)ms << 12; + *size = (u64)s << 12; } static int igt_buddy_alloc_smoke(void *arg) diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index 88d400b9df88..23a6132c5f4e 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -199,11 +199,52 @@ out: return err; } +static int igt_gem_ww_ctx(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct drm_i915_gem_object *obj, *obj2; + struct i915_gem_ww_ctx ww; + int err = 0; + + obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + obj2 = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto put1; + } + + i915_gem_ww_ctx_init(&ww, true); +retry: + /* Lock the objects, twice for good measure (-EALREADY handling) */ + err = i915_gem_object_lock(obj, &ww); + if (!err) + err = i915_gem_object_lock_interruptible(obj, &ww); + if (!err) + err = i915_gem_object_lock_interruptible(obj2, &ww); + if (!err) + err = i915_gem_object_lock(obj2, &ww); + + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + i915_gem_object_put(obj2); +put1: + i915_gem_object_put(obj); + return err; +} + int i915_gem_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_gem_suspend), SUBTEST(igt_gem_hibernate), + SUBTEST(igt_gem_ww_ctx), }; if (intel_gt_is_wedged(&i915->gt)) diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 0016ffc7d914..af8205a2bd8f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -172,35 +172,45 @@ static int igt_ppgtt_alloc(void *arg) /* Check we can allocate the entire range */ for (size = 4096; size <= limit; size <<= 2) { - err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size); + struct i915_vm_pt_stash stash = {}; + + err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size); + if (err) + goto err_ppgtt_cleanup; + + err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); if (err) { - if (err == -ENOMEM) { - pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n", - size, ilog2(size)); - err = 0; /* virtual space too large! */ - } + i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; } + ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size); cond_resched(); ppgtt->vm.clear_range(&ppgtt->vm, 0, size); + + i915_vm_free_pt_stash(&ppgtt->vm, &stash); } /* Check we can incrementally allocate the entire range */ for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { - err = ppgtt->vm.allocate_va_range(&ppgtt->vm, - last, size - last); + struct i915_vm_pt_stash stash = {}; + + err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last); + if (err) + goto err_ppgtt_cleanup; + + err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); if (err) { - if (err == -ENOMEM) { - pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n", - last, size - last, ilog2(size)); - err = 0; /* virtual space too large! */ - } + i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; } + ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, + last, size - last); cond_resched(); + + i915_vm_free_pt_stash(&ppgtt->vm, &stash); } err_ppgtt_cleanup: @@ -284,9 +294,23 @@ static int lowlevel_hole(struct i915_address_space *vm, break; } - if (vm->allocate_va_range && - vm->allocate_va_range(vm, addr, BIT_ULL(size))) - break; + if (vm->allocate_va_range) { + struct i915_vm_pt_stash stash = {}; + + if (i915_vm_alloc_pt_stash(vm, &stash, + BIT_ULL(size))) + break; + + if (i915_vm_pin_pt_stash(vm, &stash)) { + i915_vm_free_pt_stash(vm, &stash); + break; + } + + vm->allocate_va_range(vm, &stash, + addr, BIT_ULL(size)); + + i915_vm_free_pt_stash(vm, &stash); + } mock_vma->pages = obj->mm.pages; mock_vma->node.size = BIT_ULL(size); @@ -1881,6 +1905,7 @@ static int igt_cs_tlb(void *arg) continue; while (!__igt_timeout(end_time, NULL)) { + struct i915_vm_pt_stash stash = {}; struct i915_request *rq; u64 offset; @@ -1888,10 +1913,6 @@ static int igt_cs_tlb(void *arg) 0, vm->total - PAGE_SIZE, chunk_size, PAGE_SIZE); - err = vm->allocate_va_range(vm, offset, chunk_size); - if (err) - goto end; - memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); vma = i915_vma_instance(bbe, vm, NULL); @@ -1904,6 +1925,20 @@ static int igt_cs_tlb(void *arg) if (err) goto end; + err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size); + if (err) + goto end; + + err = i915_vm_pin_pt_stash(vm, &stash); + if (err) { + i915_vm_free_pt_stash(vm, &stash); + goto end; + } + + vm->allocate_va_range(vm, &stash, offset, chunk_size); + + i915_vm_free_pt_stash(vm, &stash); + /* Prime the TLB with the dummy pages */ for (i = 0; i < count; i++) { vma->node.start = offset + i * PAGE_SIZE; diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index c2d001d9c0ec..debbac660519 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -307,7 +307,7 @@ static int live_noa_gpr(void *arg) } /* Poison the ce->vm so we detect writes not to the GGTT gt->scratch */ - scratch = kmap(ce->vm->scratch[0].base.page); + scratch = kmap(__px_page(ce->vm->scratch[0])); memset(scratch, POISON_FREE, PAGE_SIZE); rq = intel_context_create_request(ce); @@ -405,7 +405,7 @@ static int live_noa_gpr(void *arg) out_rq: i915_request_put(rq); out_ce: - kunmap(ce->vm->scratch[0].base.page); + kunmap(__px_page(ce->vm->scratch[0])); intel_context_put(ce); out: stream_destroy(stream); diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index 57dd6f5122ee..3092ca763789 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -331,7 +331,7 @@ static int __igt_breadcrumbs_smoketest(void *arg) if (!wait) { i915_sw_fence_commit(submit); heap_fence_put(submit); - err = ENOMEM; + err = -ENOMEM; break; } @@ -862,6 +862,8 @@ static int live_all_engines(void *arg) goto out_free; } + i915_vma_lock(batch); + idx = 0; for_each_uabi_engine(engine, i915) { request[idx] = intel_engine_create_kernel_request(engine); @@ -872,11 +874,9 @@ static int live_all_engines(void *arg) goto out_request; } - i915_vma_lock(batch); err = i915_request_await_object(request[idx], batch->obj, 0); if (err == 0) err = i915_vma_move_to_active(batch, request[idx], 0); - i915_vma_unlock(batch); GEM_BUG_ON(err); err = engine->emit_bb_start(request[idx], @@ -891,6 +891,8 @@ static int live_all_engines(void *arg) idx++; } + i915_vma_unlock(batch); + idx = 0; for_each_uabi_engine(engine, i915) { if (i915_request_completed(request[idx])) { @@ -981,12 +983,13 @@ static int live_sequential_engines(void *arg) goto out_free; } + i915_vma_lock(batch); request[idx] = intel_engine_create_kernel_request(engine); if (IS_ERR(request[idx])) { err = PTR_ERR(request[idx]); pr_err("%s: Request allocation failed for %s with err=%d\n", __func__, engine->name, err); - goto out_request; + goto out_unlock; } if (prev) { @@ -996,16 +999,14 @@ static int live_sequential_engines(void *arg) i915_request_add(request[idx]); pr_err("%s: Request await failed for %s with err=%d\n", __func__, engine->name, err); - goto out_request; + goto out_unlock; } } - i915_vma_lock(batch); err = i915_request_await_object(request[idx], batch->obj, false); if (err == 0) err = i915_vma_move_to_active(batch, request[idx], 0); - i915_vma_unlock(batch); GEM_BUG_ON(err); err = engine->emit_bb_start(request[idx], @@ -1020,6 +1021,11 @@ static int live_sequential_engines(void *arg) prev = request[idx]; idx++; + +out_unlock: + i915_vma_unlock(batch); + if (err) + goto out_request; } idx = 0; diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index af89c7fc8f59..88c5e9acb84c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -892,7 +892,7 @@ static int igt_vma_remapped_gtt(void *arg) unsigned int x, y; int err; - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_gtt_domain(obj, true); i915_gem_object_unlock(obj); if (err) diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 6e80d99048e4..93a38a323584 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -509,7 +509,7 @@ static int igt_lmem_write_cpu(void *arg) if (err) goto out_unpin; - i915_gem_object_lock(obj); + i915_gem_object_lock(obj, NULL); err = i915_gem_object_set_to_wc_domain(obj, true); i915_gem_object_unlock(obj); if (err) @@ -522,9 +522,9 @@ static int igt_lmem_write_cpu(void *arg) goto out_unpin; } - /* We want to throw in a random width/align */ - bytes[0] = igt_random_offset(&prng, 0, PAGE_SIZE, sizeof(u32), - sizeof(u32)); + /* A random multiple of u32, picked between [64, PAGE_SIZE - 64] */ + bytes[0] = igt_random_offset(&prng, 64, PAGE_SIZE - 64, 0, sizeof(u32)); + GEM_BUG_ON(!IS_ALIGNED(bytes[0], sizeof(u32))); i = 0; do { diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index b9810bf156c3..f127e633f7ca 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -78,6 +78,7 @@ static void mock_device_release(struct drm_device *dev) drm_mode_config_cleanup(&i915->drm); out: + i915_params_free(&i915->params); put_device(&i915->drm.pdev->dev); i915->drm.pdev = NULL; } @@ -165,6 +166,8 @@ struct drm_i915_private *mock_gem_device(void) i915->drm.pdev = pdev; drmm_add_final_kfree(&i915->drm, i915); + i915_params_copy(&i915->params, &i915_modparams); + intel_runtime_pm_init_early(&i915->runtime_pm); /* Using the global GTT may ask questions about KMS users, so prepare */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index b173086411ef..7270fc8ca801 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -38,14 +38,14 @@ static void mock_insert_entries(struct i915_address_space *vm, { } -static int mock_bind_ppgtt(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static void mock_bind_ppgtt(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { GEM_BUG_ON(flags & I915_VMA_GLOBAL_BIND); set_bit(I915_VMA_LOCAL_BIND_BIT, __i915_vma_flags(vma)); - return 0; } static void mock_unbind_ppgtt(struct i915_address_space *vm, @@ -74,9 +74,12 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) ppgtt->vm.i915 = i915; ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE); ppgtt->vm.file = ERR_PTR(-ENODEV); + ppgtt->vm.dma = &i915->drm.pdev->dev; i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); + ppgtt->vm.alloc_pt_dma = alloc_pt_dma; + ppgtt->vm.clear_range = mock_clear_range; ppgtt->vm.insert_page = mock_insert_page; ppgtt->vm.insert_entries = mock_insert_entries; @@ -90,13 +93,12 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) return ppgtt; } -static int mock_bind_ggtt(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static void mock_bind_ggtt(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { - atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags); - return 0; } static void mock_unbind_ggtt(struct i915_address_space *vm, @@ -116,6 +118,8 @@ void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) ggtt->mappable_end = resource_size(&ggtt->gmadr); ggtt->vm.total = 4096 * PAGE_SIZE; + ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->vm.clear_range = mock_clear_range; ggtt->vm.insert_page = mock_insert_page; ggtt->vm.insert_entries = mock_insert_entries; diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c index 6776ebb3246d..8a4235d9d9f1 100644 --- a/drivers/gpu/drm/imx/ipuv3-plane.c +++ b/drivers/gpu/drm/imx/ipuv3-plane.c @@ -447,7 +447,7 @@ static int ipu_plane_atomic_check(struct drm_plane *plane, if (fb->pitches[1] != fb->pitches[2]) return -EINVAL; - /* fall-through */ + fallthrough; case DRM_FORMAT_NV12: case DRM_FORMAT_NV16: ubo = drm_plane_state_to_ubo(state); diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c index 5dab9c3d0a52..a3d1617d7c67 100644 --- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c +++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c @@ -686,7 +686,7 @@ static void ingenic_drm_unbind_all(void *d) component_unbind_all(priv->dev, &priv->drm); } -static int ingenic_drm_bind(struct device *dev) +static int ingenic_drm_bind(struct device *dev, bool has_components) { struct platform_device *pdev = to_platform_device(dev); const struct jz_soc_info *soc_info; @@ -821,7 +821,7 @@ static int ingenic_drm_bind(struct device *dev) return ret; } - if (IS_ENABLED(CONFIG_DRM_INGENIC_IPU)) { + if (IS_ENABLED(CONFIG_DRM_INGENIC_IPU) && has_components) { ret = component_bind_all(dev, drm); if (ret) { if (ret != -EPROBE_DEFER) @@ -952,6 +952,11 @@ err_pixclk_disable: return ret; } +static int ingenic_drm_bind_with_components(struct device *dev) +{ + return ingenic_drm_bind(dev, true); +} + static int compare_of(struct device *dev, void *data) { return dev->of_node == data; @@ -970,7 +975,7 @@ static void ingenic_drm_unbind(struct device *dev) } static const struct component_master_ops ingenic_master_ops = { - .bind = ingenic_drm_bind, + .bind = ingenic_drm_bind_with_components, .unbind = ingenic_drm_unbind, }; @@ -981,16 +986,15 @@ static int ingenic_drm_probe(struct platform_device *pdev) struct device_node *np; if (!IS_ENABLED(CONFIG_DRM_INGENIC_IPU)) - return ingenic_drm_bind(dev); + return ingenic_drm_bind(dev, false); /* IPU is at port address 8 */ np = of_graph_get_remote_node(dev->of_node, 8, 0); - if (!np) { - dev_err(dev, "Unable to get IPU node\n"); - return -EINVAL; - } + if (!np) + return ingenic_drm_bind(dev, false); drm_of_component_match_add(dev, &match, compare_of, np); + of_node_put(np); return component_master_add_with_match(dev, &ingenic_master_ops, match); } diff --git a/drivers/gpu/drm/meson/meson_osd_afbcd.c b/drivers/gpu/drm/meson/meson_osd_afbcd.c index f12e0271f166..ffc6b584dbf8 100644 --- a/drivers/gpu/drm/meson/meson_osd_afbcd.c +++ b/drivers/gpu/drm/meson/meson_osd_afbcd.c @@ -205,7 +205,7 @@ static int meson_g12a_afbcd_pixel_fmt(u64 modifier, uint32_t format) /* YTR is forbidden for non XBGR formats */ if (modifier & AFBC_FORMAT_MOD_YTR) return -EINVAL; - /* fall through */ + fallthrough; case DRM_FORMAT_XBGR8888: case DRM_FORMAT_ABGR8888: return MAFBC_FMT_RGBA8888; diff --git a/drivers/gpu/drm/meson/meson_overlay.c b/drivers/gpu/drm/meson/meson_overlay.c index a8bcc70644df..1ffbbecafa22 100644 --- a/drivers/gpu/drm/meson/meson_overlay.c +++ b/drivers/gpu/drm/meson/meson_overlay.c @@ -654,7 +654,7 @@ static void meson_overlay_atomic_update(struct drm_plane *plane, priv->viu.vd1_addr2, priv->viu.vd1_stride2, priv->viu.vd1_height2); - /* fallthrough */ + fallthrough; case 2: gem = drm_fb_cma_get_gem_obj(fb, 1); priv->viu.vd1_addr1 = gem->paddr + fb->offsets[1]; @@ -666,7 +666,7 @@ static void meson_overlay_atomic_update(struct drm_plane *plane, priv->viu.vd1_addr1, priv->viu.vd1_stride1, priv->viu.vd1_height1); - /* fallthrough */ + fallthrough; case 1: gem = drm_fb_cma_get_gem_obj(fb, 0); priv->viu.vd1_addr0 = gem->paddr + fb->offsets[0]; diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index 6021f8d9efd1..48fa49f69d6d 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -164,6 +164,11 @@ static int a2xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; + gpu_write(gpu, REG_AXXX_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); + + gpu_write(gpu, REG_AXXX_CP_RB_BASE, lower_32_bits(gpu->rb[0]->iova)); + /* NOTE: PM4/micro-engine firmware registers look to be the same * for a2xx and a3xx.. we could possibly push that part down to * adreno_gpu base class. Or push both PM4 and PFP but diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c index 0a5ea9f56cb8..f6471145a7a6 100644 --- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c @@ -211,6 +211,16 @@ static int a3xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; + /* + * Use the default ringbuffer size and block size but disable the RPTR + * shadow + */ + gpu_write(gpu, REG_AXXX_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); + + /* Set the ringbuffer address */ + gpu_write(gpu, REG_AXXX_CP_RB_BASE, lower_32_bits(gpu->rb[0]->iova)); + /* setup access protection: */ gpu_write(gpu, REG_A3XX_CP_PROTECT_CTRL, 0x00000007); diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c index b9b26b2bf9c5..954753600625 100644 --- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c @@ -267,6 +267,16 @@ static int a4xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; + /* + * Use the default ringbuffer size and block size but disable the RPTR + * shadow + */ + gpu_write(gpu, REG_A4XX_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); + + /* Set the ringbuffer address */ + gpu_write(gpu, REG_A4XX_CP_RB_BASE, lower_32_bits(gpu->rb[0]->iova)); + /* Load PM4: */ ptr = (uint32_t *)(adreno_gpu->fw[ADRENO_FW_PM4]->data); len = adreno_gpu->fw[ADRENO_FW_PM4]->size / 4; diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index 9e63a190642c..91726da82ed6 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -59,7 +59,7 @@ static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: if (priv->lastctx == ctx) break; - /* fall-thru */ + fallthrough; case MSM_SUBMIT_CMD_BUF: /* copy commands into RB: */ obj = submit->bos[submit->cmd[i].idx].obj; @@ -150,7 +150,7 @@ static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: if (priv->lastctx == ctx) break; - /* fall-thru */ + fallthrough; case MSM_SUBMIT_CMD_BUF: OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); @@ -703,8 +703,6 @@ static int a5xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; - a5xx_preempt_hw_init(gpu); - if (!adreno_is_a510(adreno_gpu)) a5xx_gpmu_ucode_init(gpu); @@ -712,6 +710,15 @@ static int a5xx_hw_init(struct msm_gpu *gpu) if (ret) return ret; + /* Set the ringbuffer address */ + gpu_write64(gpu, REG_A5XX_CP_RB_BASE, REG_A5XX_CP_RB_BASE_HI, + gpu->rb[0]->iova); + + gpu_write(gpu, REG_A5XX_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); + + a5xx_preempt_hw_init(gpu); + /* Disable the interrupts through the initial bringup stage */ gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK); @@ -1511,7 +1518,8 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev) check_speed_bin(&pdev->dev); - ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4); + /* Restricting nr_rings to 1 to temporarily disable preemption */ + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); if (ret) { a5xx_destroy(&(a5xx_gpu->base.base)); return ERR_PTR(ret); diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h index 54868d4e3958..1e5b1a15a70f 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h @@ -31,6 +31,7 @@ struct a5xx_gpu { struct msm_ringbuffer *next_ring; struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS]; + struct drm_gem_object *preempt_counters_bo[MSM_GPU_MAX_RINGS]; struct a5xx_preempt_record *preempt[MSM_GPU_MAX_RINGS]; uint64_t preempt_iova[MSM_GPU_MAX_RINGS]; diff --git a/drivers/gpu/drm/msm/adreno/a5xx_preempt.c b/drivers/gpu/drm/msm/adreno/a5xx_preempt.c index 9cf9353a7ff1..9f3fe177b00e 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_preempt.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_preempt.c @@ -226,19 +226,31 @@ static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, struct adreno_gpu *adreno_gpu = &a5xx_gpu->base; struct msm_gpu *gpu = &adreno_gpu->base; struct a5xx_preempt_record *ptr; - struct drm_gem_object *bo = NULL; - u64 iova = 0; + void *counters; + struct drm_gem_object *bo = NULL, *counters_bo = NULL; + u64 iova = 0, counters_iova = 0; ptr = msm_gem_kernel_new(gpu->dev, A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE, - MSM_BO_UNCACHED, gpu->aspace, &bo, &iova); + MSM_BO_UNCACHED | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova); if (IS_ERR(ptr)) return PTR_ERR(ptr); + /* The buffer to store counters needs to be unprivileged */ + counters = msm_gem_kernel_new(gpu->dev, + A5XX_PREEMPT_COUNTER_SIZE, + MSM_BO_UNCACHED, gpu->aspace, &counters_bo, &counters_iova); + if (IS_ERR(counters)) { + msm_gem_kernel_put(bo, gpu->aspace, true); + return PTR_ERR(counters); + } + msm_gem_object_set_name(bo, "preempt"); + msm_gem_object_set_name(counters_bo, "preempt_counters"); a5xx_gpu->preempt_bo[ring->id] = bo; + a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo; a5xx_gpu->preempt_iova[ring->id] = iova; a5xx_gpu->preempt[ring->id] = ptr; @@ -249,7 +261,7 @@ static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, ptr->data = 0; ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; ptr->rptr_addr = rbmemptr(ring, rptr); - ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE; + ptr->counter = counters_iova; return 0; } @@ -260,8 +272,11 @@ void a5xx_preempt_fini(struct msm_gpu *gpu) struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); int i; - for (i = 0; i < gpu->nr_rings; i++) + for (i = 0; i < gpu->nr_rings; i++) { msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true); + msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], + gpu->aspace, true); + } } void a5xx_preempt_init(struct msm_gpu *gpu) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index b67b38c8fadf..e1c7bcd1b1eb 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -133,7 +133,7 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp) if (!gmu->legacy) { a6xx_hfi_set_freq(gmu, perf_index); - icc_set_bw(gpu->icc_path, 0, MBps_to_icc(7216)); + dev_pm_opp_set_bw(&gpu->pdev->dev, opp); pm_runtime_put(gmu->dev); return; } @@ -157,11 +157,7 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp) if (ret) dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret); - /* - * Eventually we will want to scale the path vote with the frequency but - * for now leave it at max so that the performance is nominal. - */ - icc_set_bw(gpu->icc_path, 0, MBps_to_icc(7216)); + dev_pm_opp_set_bw(&gpu->pdev->dev, opp); pm_runtime_put(gmu->dev); } @@ -204,6 +200,16 @@ static int a6xx_gmu_start(struct a6xx_gmu *gmu) { int ret; u32 val; + u32 mask, reset_val; + + val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8); + if (val <= 0x20010004) { + mask = 0xffffffff; + reset_val = 0xbabeface; + } else { + mask = 0x1ff; + reset_val = 0x100; + } gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); @@ -215,7 +221,7 @@ static int a6xx_gmu_start(struct a6xx_gmu *gmu) gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0); ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val, - val == 0xbabeface, 100, 10000); + (val & mask) == reset_val, 100, 10000); if (ret) DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n"); @@ -602,7 +608,7 @@ static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE | A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE); - /* Fall through */ + fallthrough; case GMU_IDLE_STATE_SPTP: gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST, GMU_PWR_COL_HYST); @@ -845,10 +851,24 @@ static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) if (IS_ERR_OR_NULL(gpu_opp)) return; + gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */ a6xx_gmu_set_freq(gpu, gpu_opp); dev_pm_opp_put(gpu_opp); } +static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu) +{ + struct dev_pm_opp *gpu_opp; + unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index]; + + gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true); + if (IS_ERR_OR_NULL(gpu_opp)) + return; + + dev_pm_opp_set_bw(&gpu->pdev->dev, gpu_opp); + dev_pm_opp_put(gpu_opp); +} + int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; @@ -882,7 +902,7 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) } /* Set the bus quota to a reasonable value for boot */ - icc_set_bw(gpu->icc_path, 0, MBps_to_icc(3072)); + a6xx_gmu_set_initial_bw(gpu, gmu); /* Enable the GMU interrupt */ gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0); @@ -1051,7 +1071,7 @@ int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu) a6xx_gmu_shutdown(gmu); /* Remove the bus vote */ - icc_set_bw(gpu->icc_path, 0, 0); + dev_pm_opp_set_bw(&gpu->pdev->dev, NULL); /* * Make sure the GX domain is off before turning off the GMU (CX) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index c5a3e4d4c007..66a95e22b7b3 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -117,7 +117,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: if (priv->lastctx == ctx) break; - /* fall-thru */ + fallthrough; case MSM_SUBMIT_CMD_BUF: OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); @@ -678,7 +678,8 @@ static int a6xx_hw_init(struct msm_gpu *gpu) A6XX_PROTECT_RDONLY(0x980, 0x4)); gpu_write(gpu, REG_A6XX_CP_PROTECT(25), A6XX_PROTECT_RW(0xa630, 0x0)); - if (adreno_is_a650(adreno_gpu)) { + /* Enable expanded apriv for targets that support it */ + if (gpu->hw_apriv) { gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL, (1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1)); } @@ -694,6 +695,13 @@ static int a6xx_hw_init(struct msm_gpu *gpu) if (ret) goto out; + /* Set the ringbuffer address */ + gpu_write64(gpu, REG_A6XX_CP_RB_BASE, REG_A6XX_CP_RB_BASE_HI, + gpu->rb[0]->iova); + + gpu_write(gpu, REG_A6XX_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); + /* Always come up on rb 0 */ a6xx_gpu->cur_ring = gpu->rb[0]; @@ -1056,6 +1064,9 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) adreno_gpu->registers = NULL; adreno_gpu->reg_offsets = a6xx_register_offsets; + if (adreno_is_a650(adreno_gpu)) + adreno_gpu->base.hw_apriv = true; + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); if (ret) { a6xx_destroy(&(a6xx_gpu->base.base)); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 959656ad6987..b12f5b4a1bea 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -938,7 +938,8 @@ struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu) msm_gem_kernel_put(dumper.bo, gpu->aspace, true); } - a6xx_get_debugbus(gpu, a6xx_state); + if (snapshot_debugbus) + a6xx_get_debugbus(gpu, a6xx_state); return &a6xx_state->base; } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h index 846fd5b54c23..2fb58b7098e4 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h @@ -372,7 +372,7 @@ static const struct a6xx_indexed_registers { u32 data; u32 count; } a6xx_indexed_reglist[] = { - { "CP_SEQ_STAT", REG_A6XX_CP_SQE_STAT_ADDR, + { "CP_SQE_STAT", REG_A6XX_CP_SQE_STAT_ADDR, REG_A6XX_CP_SQE_STAT_DATA, 0x33 }, { "CP_DRAW_STATE", REG_A6XX_CP_DRAW_STATE_ADDR, REG_A6XX_CP_DRAW_STATE_DATA, 0x100 }, diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 4e84f3c76f4f..9eeb46bf2a5d 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -14,6 +14,10 @@ bool hang_debug = false; MODULE_PARM_DESC(hang_debug, "Dump registers when hang is detected (can be slow!)"); module_param_named(hang_debug, hang_debug, bool, 0600); +bool snapshot_debugbus = false; +MODULE_PARM_DESC(snapshot_debugbus, "Include debugbus sections in GPU devcoredump (if not fused off)"); +module_param_named(snapshot_debugbus, snapshot_debugbus, bool, 0600); + static const struct adreno_info gpulist[] = { { .rev = ADRENO_REV(2, 0, 0, 0), diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index e23641a5ec84..862dd35b27d3 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -396,30 +396,10 @@ int adreno_hw_init(struct msm_gpu *gpu) ring->next = ring->start; /* reset completed fence seqno: */ - ring->memptrs->fence = ring->seqno; + ring->memptrs->fence = ring->fctx->completed_fence; ring->memptrs->rptr = 0; } - /* - * Setup REG_CP_RB_CNTL. The same value is used across targets (with - * the excpetion of A430 that disables the RPTR shadow) - the cacluation - * for the ringbuffer size and block size is moved to msm_gpu.h for the - * pre-processor to deal with and the A430 variant is ORed in here - */ - adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_CNTL, - MSM_GPU_RB_CNTL_DEFAULT | - (adreno_is_a430(adreno_gpu) ? AXXX_CP_RB_CNTL_NO_UPDATE : 0)); - - /* Setup ringbuffer address - use ringbuffer[0] for GPU init */ - adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_BASE, - REG_ADRENO_CP_RB_BASE_HI, gpu->rb[0]->iova); - - if (!adreno_is_a430(adreno_gpu)) { - adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_RPTR_ADDR, - REG_ADRENO_CP_RB_RPTR_ADDR_HI, - rbmemptr(gpu->rb[0], rptr)); - } - return 0; } @@ -427,11 +407,8 @@ int adreno_hw_init(struct msm_gpu *gpu) static uint32_t get_rptr(struct adreno_gpu *adreno_gpu, struct msm_ringbuffer *ring) { - if (adreno_is_a430(adreno_gpu)) - return ring->memptrs->rptr = adreno_gpu_read( - adreno_gpu, REG_ADRENO_CP_RB_RPTR); - else - return ring->memptrs->rptr; + return ring->memptrs->rptr = adreno_gpu_read( + adreno_gpu, REG_ADRENO_CP_RB_RPTR); } struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu) @@ -474,7 +451,7 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, /* ignore if there has not been a ctx switch: */ if (priv->lastctx == ctx) break; - /* fall-thru */ + fallthrough; case MSM_SUBMIT_CMD_BUF: OUT_PKT3(ring, adreno_is_a4xx(adreno_gpu) ? CP_INDIRECT_BUFFER_PFE : CP_INDIRECT_BUFFER_PFD, 2); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 99bb468f5f24..e55abae365b5 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -21,6 +21,8 @@ #define REG_SKIP ~0 #define REG_ADRENO_SKIP(_offset) [_offset] = REG_SKIP +extern bool snapshot_debugbus; + /** * adreno_regs: List of registers that are used in across all * 3D devices. Each device type has different offset value for the same diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index f272a8d0f95b..c2729f71e2fa 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -827,7 +827,7 @@ static void dpu_crtc_enable(struct drm_crtc *crtc, { struct dpu_crtc *dpu_crtc; struct drm_encoder *encoder; - bool request_bandwidth; + bool request_bandwidth = false; if (!crtc) { DPU_ERROR("invalid crtc\n"); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index a97f6d2e5a08..bd6def436c65 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -599,7 +599,10 @@ static int dpu_encoder_virt_atomic_check( dpu_kms = to_dpu_kms(priv->kms); mode = &crtc_state->mode; adj_mode = &crtc_state->adjusted_mode; - global_state = dpu_kms_get_existing_global_state(dpu_kms); + global_state = dpu_kms_get_global_state(crtc_state->state); + if (IS_ERR(global_state)) + return PTR_ERR(global_state); + trace_dpu_enc_atomic_check(DRMID(drm_enc)); /* perform atomic check on the first physical encoder (master) */ @@ -625,12 +628,15 @@ static int dpu_encoder_virt_atomic_check( /* Reserve dynamic resources now. */ if (!ret) { /* - * Avoid reserving resources when mode set is pending. Topology - * info may not be available to complete reservation. + * Release and Allocate resources on every modeset + * Dont allocate when active is false. */ if (drm_atomic_crtc_needs_modeset(crtc_state)) { - ret = dpu_rm_reserve(&dpu_kms->rm, global_state, - drm_enc, crtc_state, topology); + dpu_rm_release(global_state, drm_enc); + + if (!crtc_state->active_changed || crtc_state->active) + ret = dpu_rm_reserve(&dpu_kms->rm, global_state, + drm_enc, crtc_state, topology); } } @@ -1181,7 +1187,6 @@ static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc) struct dpu_encoder_virt *dpu_enc = NULL; struct msm_drm_private *priv; struct dpu_kms *dpu_kms; - struct dpu_global_state *global_state; int i = 0; if (!drm_enc) { @@ -1200,7 +1205,6 @@ static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc) priv = drm_enc->dev->dev_private; dpu_kms = to_dpu_kms(priv->kms); - global_state = dpu_kms_get_existing_global_state(dpu_kms); trace_dpu_enc_disable(DRMID(drm_enc)); @@ -1230,8 +1234,6 @@ static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc) DPU_DEBUG_ENC(dpu_enc, "encoder disabled\n"); - dpu_rm_release(global_state, drm_enc); - mutex_unlock(&dpu_enc->enc_lock); } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index 33f6c56f01ed..29e373d2e7b5 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -866,9 +866,9 @@ static int dpu_plane_atomic_check(struct drm_plane *plane, crtc_state = drm_atomic_get_new_crtc_state(state->state, state->crtc); - min_scale = FRAC_16_16(1, pdpu->pipe_sblk->maxdwnscale); + min_scale = FRAC_16_16(1, pdpu->pipe_sblk->maxupscale); ret = drm_atomic_helper_check_plane_state(state, crtc_state, min_scale, - pdpu->pipe_sblk->maxupscale << 16, + pdpu->pipe_sblk->maxdwnscale << 16, true, true); if (ret) { DPU_DEBUG_PLANE(pdpu, "Check plane state failed (%d)\n", ret); diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c index 5ccfad794c6a..561bfa48841c 100644 --- a/drivers/gpu/drm/msm/msm_atomic.c +++ b/drivers/gpu/drm/msm/msm_atomic.c @@ -27,6 +27,34 @@ int msm_atomic_prepare_fb(struct drm_plane *plane, return msm_framebuffer_prepare(new_state->fb, kms->aspace); } +/* + * Helpers to control vblanks while we flush.. basically just to ensure + * that vblank accounting is switched on, so we get valid seqn/timestamp + * on pageflip events (if requested) + */ + +static void vblank_get(struct msm_kms *kms, unsigned crtc_mask) +{ + struct drm_crtc *crtc; + + for_each_crtc_mask(kms->dev, crtc, crtc_mask) { + if (!crtc->state->active) + continue; + drm_crtc_vblank_get(crtc); + } +} + +static void vblank_put(struct msm_kms *kms, unsigned crtc_mask) +{ + struct drm_crtc *crtc; + + for_each_crtc_mask(kms->dev, crtc, crtc_mask) { + if (!crtc->state->active) + continue; + drm_crtc_vblank_put(crtc); + } +} + static void msm_atomic_async_commit(struct msm_kms *kms, int crtc_idx) { unsigned crtc_mask = BIT(crtc_idx); @@ -44,6 +72,8 @@ static void msm_atomic_async_commit(struct msm_kms *kms, int crtc_idx) kms->funcs->enable_commit(kms); + vblank_get(kms, crtc_mask); + /* * Flush hardware updates: */ @@ -58,6 +88,8 @@ static void msm_atomic_async_commit(struct msm_kms *kms, int crtc_idx) kms->funcs->wait_flush(kms, crtc_mask); trace_msm_atomic_wait_flush_finish(crtc_mask); + vblank_put(kms, crtc_mask); + mutex_lock(&kms->commit_lock); kms->funcs->complete_commit(kms, crtc_mask); mutex_unlock(&kms->commit_lock); @@ -221,6 +253,8 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state) */ kms->pending_crtc_mask &= ~crtc_mask; + vblank_get(kms, crtc_mask); + /* * Flush hardware updates: */ @@ -235,6 +269,8 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state) kms->funcs->wait_flush(kms, crtc_mask); trace_msm_atomic_wait_flush_finish(crtc_mask); + vblank_put(kms, crtc_mask); + mutex_lock(&kms->commit_lock); kms->funcs->complete_commit(kms, crtc_mask); mutex_unlock(&kms->commit_lock); diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 7d641c7e3514..79333842f70a 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -1320,6 +1320,13 @@ static int msm_pdev_remove(struct platform_device *pdev) return 0; } +static void msm_pdev_shutdown(struct platform_device *pdev) +{ + struct drm_device *drm = platform_get_drvdata(pdev); + + drm_atomic_helper_shutdown(drm); +} + static const struct of_device_id dt_match[] = { { .compatible = "qcom,mdp4", .data = (void *)KMS_MDP4 }, { .compatible = "qcom,mdss", .data = (void *)KMS_MDP5 }, @@ -1332,6 +1339,7 @@ MODULE_DEVICE_TABLE(of, dt_match); static struct platform_driver msm_platform_driver = { .probe = msm_pdev_probe, .remove = msm_pdev_remove, + .shutdown = msm_pdev_shutdown, .driver = { .name = "msm", .of_match_table = dt_match, diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index d5645472b25d..57ddc9438351 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -908,7 +908,7 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, memptrs = msm_gem_kernel_new(drm, sizeof(struct msm_rbmemptrs) * nr_rings, - MSM_BO_UNCACHED, gpu->aspace, &gpu->memptrs_bo, + check_apriv(gpu, MSM_BO_UNCACHED), gpu->aspace, &gpu->memptrs_bo, &memptrs_iova); if (IS_ERR(memptrs)) { diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index 0db117a7339b..37cffac4cbe3 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -15,6 +15,7 @@ #include "msm_drv.h" #include "msm_fence.h" #include "msm_ringbuffer.h" +#include "msm_gem.h" struct msm_gem_submit; struct msm_gpu_perfcntr; @@ -139,6 +140,8 @@ struct msm_gpu { } devfreq; struct msm_gpu_state *crashstate; + /* True if the hardware supports expanded apriv (a650 and newer) */ + bool hw_apriv; }; /* It turns out that all targets use the same ringbuffer size */ @@ -327,4 +330,12 @@ static inline void msm_gpu_crashstate_put(struct msm_gpu *gpu) mutex_unlock(&gpu->dev->struct_mutex); } +/* + * Simple macro to semi-cleanly add the MAP_PRIV flag for targets that can + * support expanded privileges + */ +#define check_apriv(gpu, flags) \ + (((gpu)->hw_apriv ? MSM_BO_MAP_PRIV : 0) | (flags)) + + #endif /* __MSM_GPU_H__ */ diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index e397c44cc011..935bf9b1d941 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -27,7 +27,8 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, ring->id = id; ring->start = msm_gem_kernel_new(gpu->dev, MSM_GPU_RINGBUFFER_SZ, - MSM_BO_WC, gpu->aspace, &ring->bo, &ring->iova); + check_apriv(gpu, MSM_BO_WC | MSM_BO_GPU_READONLY), + gpu->aspace, &ring->bo, &ring->iova); if (IS_ERR(ring->start)) { ret = PTR_ERR(ring->start); diff --git a/drivers/gpu/drm/nouveau/dispnv04/dac.c b/drivers/gpu/drm/nouveau/dispnv04/dac.c index ffdd447d8706..22d10f328559 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/dac.c +++ b/drivers/gpu/drm/nouveau/dispnv04/dac.c @@ -419,7 +419,7 @@ static void nv04_dac_commit(struct drm_encoder *encoder) helper->dpms(encoder, DRM_MODE_DPMS_ON); NV_DEBUG(drm, "Output %s is running on CRTC %d using output %c\n", - nouveau_encoder_connector_get(nv_encoder)->base.name, + nv04_encoder_get_connector(nv_encoder)->base.name, nv_crtc->index, '@' + ffs(nv_encoder->dcb->or)); } diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c index f9f4482c79b5..42687ea2a4ca 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c +++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c @@ -184,7 +184,8 @@ static bool nv04_dfp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *adjusted_mode) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nouveau_connector *nv_connector = nouveau_encoder_connector_get(nv_encoder); + struct nouveau_connector *nv_connector = + nv04_encoder_get_connector(nv_encoder); if (!nv_connector->native_mode || nv_connector->scaling_mode == DRM_MODE_SCALE_NONE || @@ -478,7 +479,7 @@ static void nv04_dfp_commit(struct drm_encoder *encoder) helper->dpms(encoder, DRM_MODE_DPMS_ON); NV_DEBUG(drm, "Output %s is running on CRTC %d using output %c\n", - nouveau_encoder_connector_get(nv_encoder)->base.name, + nv04_encoder_get_connector(nv_encoder)->base.name, nv_crtc->index, '@' + ffs(nv_encoder->dcb->or)); } @@ -591,7 +592,7 @@ static void nv04_dfp_restore(struct drm_encoder *encoder) if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS) { struct nouveau_connector *connector = - nouveau_encoder_connector_get(nv_encoder); + nv04_encoder_get_connector(nv_encoder); if (connector && connector->native_mode) call_lvds_script(dev, nv_encoder->dcb, head, diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c index 41d990cca685..7739f46470d3 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c @@ -35,9 +35,28 @@ #include <nvif/if0004.h> +struct nouveau_connector * +nv04_encoder_get_connector(struct nouveau_encoder *encoder) +{ + struct drm_device *dev = to_drm_encoder(encoder)->dev; + struct drm_connector *connector; + struct drm_connector_list_iter conn_iter; + struct nouveau_connector *nv_connector = NULL; + + drm_connector_list_iter_begin(dev, &conn_iter); + drm_for_each_connector_iter(connector, &conn_iter) { + if (connector->encoder == to_drm_encoder(encoder)) + nv_connector = nouveau_connector(connector); + } + drm_connector_list_iter_end(&conn_iter); + + return nv_connector; +} + static void -nv04_display_fini(struct drm_device *dev, bool suspend) +nv04_display_fini(struct drm_device *dev, bool runtime, bool suspend) { + struct nouveau_drm *drm = nouveau_drm(dev); struct nv04_display *disp = nv04_display(dev); struct drm_crtc *crtc; @@ -49,6 +68,9 @@ nv04_display_fini(struct drm_device *dev, bool suspend) if (nv_two_heads(dev)) NVWriteCRTC(dev, 1, NV_PCRTC_INTR_EN_0, 0); + if (!runtime) + cancel_work_sync(&drm->hpd_work); + if (!suspend) return; diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.h b/drivers/gpu/drm/nouveau/dispnv04/disp.h index 495d3284e876..5ace5e906949 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.h +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.h @@ -6,6 +6,8 @@ #include "nouveau_display.h" +struct nouveau_encoder; + enum nv04_fp_display_regs { FP_DISPLAY_END, FP_TOTAL, @@ -93,6 +95,8 @@ nv04_display(struct drm_device *dev) /* nv04_display.c */ int nv04_display_create(struct drm_device *); +struct nouveau_connector * +nv04_encoder_get_connector(struct nouveau_encoder *nv_encoder); /* nv04_crtc.c */ int nv04_crtc_create(struct drm_device *, int index); diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c index b701a4d8fe76..3ba7b59580d5 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c +++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c @@ -172,7 +172,7 @@ static void nv04_tv_commit(struct drm_encoder *encoder) helper->dpms(encoder, DRM_MODE_DPMS_ON); NV_DEBUG(drm, "Output %s is running on CRTC %d using output %c\n", - nouveau_encoder_connector_get(nv_encoder)->base.name, + nv04_encoder_get_connector(nv_encoder)->base.name, nv_crtc->index, '@' + ffs(nv_encoder->dcb->or)); } diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c index 3a9489ed6544..be28e7bd7490 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c +++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c @@ -599,7 +599,7 @@ static void nv17_tv_commit(struct drm_encoder *encoder) helper->dpms(encoder, DRM_MODE_DPMS_ON); NV_INFO(drm, "Output %s is running on CRTC %d using output %c\n", - nouveau_encoder_connector_get(nv_encoder)->base.name, + nv04_encoder_get_connector(nv_encoder)->base.name, nv_crtc->index, '@' + ffs(nv_encoder->dcb->or)); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/core507d.c b/drivers/gpu/drm/nouveau/dispnv50/core507d.c index ad1f09a143aa..248edf69e168 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/core507d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/core507d.c @@ -50,7 +50,10 @@ core507d_update(struct nv50_core *core, u32 *interlock, bool ntfy) interlock[NV50_DISP_INTERLOCK_OVLY] | NVDEF(NV507D, UPDATE, NOT_DRIVER_FRIENDLY, FALSE) | NVDEF(NV507D, UPDATE, NOT_DRIVER_UNFRIENDLY, FALSE) | - NVDEF(NV507D, UPDATE, INHIBIT_INTERRUPTS, FALSE)); + NVDEF(NV507D, UPDATE, INHIBIT_INTERRUPTS, FALSE), + + SET_NOTIFIER_CONTROL, + NVDEF(NV507D, SET_NOTIFIER_CONTROL, NOTIFY, DISABLE)); return PUSH_KICK(push); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/crcc37d.c b/drivers/gpu/drm/nouveau/dispnv50/crcc37d.c index 9afe9a87bde0..814e5bd97446 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/crcc37d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/crcc37d.c @@ -6,7 +6,7 @@ #include "disp.h" #include "head.h" -#include <nvif/push507c.h> +#include <nvif/pushc37b.h> #include <nvhw/class/clc37d.h> diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 9dfb577f148a..852e1b56f3a7 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -257,6 +257,12 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, dmac->push->end = dmac->push->bgn; dmac->max = 0x1000/4 - 1; + /* EVO channels are affected by a HW bug where the last 12 DWORDs + * of the push buffer aren't able to be used safely. + */ + if (disp->oclass < GV100_DISP) + dmac->max -= 12; + args->pushbuf = nvif_handle(&dmac->_push.mem.object); ret = nv50_chan_create(device, disp, oclass, head, data, size, @@ -411,6 +417,40 @@ nv50_outp_atomic_check(struct drm_encoder *encoder, return 0; } +struct nouveau_connector * +nv50_outp_get_new_connector(struct nouveau_encoder *outp, + struct drm_atomic_state *state) +{ + struct drm_connector *connector; + struct drm_connector_state *connector_state; + struct drm_encoder *encoder = to_drm_encoder(outp); + int i; + + for_each_new_connector_in_state(state, connector, connector_state, i) { + if (connector_state->best_encoder == encoder) + return nouveau_connector(connector); + } + + return NULL; +} + +struct nouveau_connector * +nv50_outp_get_old_connector(struct nouveau_encoder *outp, + struct drm_atomic_state *state) +{ + struct drm_connector *connector; + struct drm_connector_state *connector_state; + struct drm_encoder *encoder = to_drm_encoder(outp); + int i; + + for_each_old_connector_in_state(state, connector, connector_state, i) { + if (connector_state->best_encoder == encoder) + return nouveau_connector(connector); + } + + return NULL; +} + /****************************************************************************** * DAC *****************************************************************************/ @@ -552,16 +592,31 @@ nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id, struct nouveau_drm *drm = nouveau_drm(drm_dev); struct drm_encoder *encoder; struct nouveau_encoder *nv_encoder; - struct nouveau_connector *nv_connector; + struct drm_connector *connector; struct nouveau_crtc *nv_crtc; + struct drm_connector_list_iter conn_iter; int ret = 0; *enabled = false; + drm_for_each_encoder(encoder, drm->dev) { + struct nouveau_connector *nv_connector = NULL; + nv_encoder = nouveau_encoder(encoder); - nv_connector = nouveau_encoder_connector_get(nv_encoder); + + drm_connector_list_iter_begin(drm_dev, &conn_iter); + drm_for_each_connector_iter(connector, &conn_iter) { + if (connector->state->best_encoder == encoder) { + nv_connector = nouveau_connector(connector); + break; + } + } + drm_connector_list_iter_end(&conn_iter); + if (!nv_connector) + continue; + nv_crtc = nouveau_crtc(encoder->crtc); - if (!nv_connector || !nv_crtc || nv_encoder->or != port || + if (!nv_crtc || nv_encoder->or != port || nv_crtc->index != dev_id) continue; *enabled = nv_encoder->audio; @@ -572,6 +627,7 @@ nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id, } break; } + return ret; } @@ -665,7 +721,8 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) } static void -nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) +nv50_audio_enable(struct drm_encoder *encoder, struct drm_atomic_state *state, + struct drm_display_mode *mode) { struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); @@ -686,7 +743,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) (0x0100 << nv_crtc->index), }; - nv_connector = nouveau_encoder_connector_get(nv_encoder); + nv_connector = nv50_outp_get_new_connector(nv_encoder, state); if (!drm_detect_monitor_audio(nv_connector->edid)) return; @@ -723,7 +780,8 @@ nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) } static void -nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) +nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_atomic_state *state, + struct drm_display_mode *mode) { struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); @@ -752,7 +810,7 @@ nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) int ret; int size; - nv_connector = nouveau_encoder_connector_get(nv_encoder); + nv_connector = nv50_outp_get_new_connector(nv_encoder, state); if (!drm_detect_hdmi_monitor(nv_connector->edid)) return; @@ -798,7 +856,7 @@ nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) + args.pwr.vendor_infoframe_length; nvif_mthd(&disp->disp->object, 0, &args, size); - nv50_audio_enable(encoder, mode); + nv50_audio_enable(encoder, state, mode); /* If SCDC is supported by the downstream monitor, update * divider / scrambling settings to what we programmed above. @@ -827,16 +885,6 @@ nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) #define nv50_mstc(p) container_of((p), struct nv50_mstc, connector) #define nv50_msto(p) container_of((p), struct nv50_msto, encoder) -struct nv50_mstm { - struct nouveau_encoder *outp; - - struct drm_dp_mst_topology_mgr mgr; - - bool modified; - bool disabled; - int links; -}; - struct nv50_mstc { struct nv50_mstm *mstm; struct drm_dp_mst_port *port; @@ -1216,7 +1264,10 @@ nv50_mstc_detect(struct drm_connector *connector, ret = drm_dp_mst_detect_port(connector, ctx, mstc->port->mgr, mstc->port); + if (ret != connector_status_connected) + goto out; +out: pm_runtime_mark_last_busy(connector->dev->dev); pm_runtime_put_autosuspend(connector->dev->dev); return ret; @@ -1365,41 +1416,51 @@ nv50_mstm = { .add_connector = nv50_mstm_add_connector, }; -void -nv50_mstm_service(struct nv50_mstm *mstm) +bool +nv50_mstm_service(struct nouveau_drm *drm, + struct nouveau_connector *nv_connector, + struct nv50_mstm *mstm) { - struct drm_dp_aux *aux = mstm ? mstm->mgr.aux : NULL; - bool handled = true; - int ret; + struct drm_dp_aux *aux = &nv_connector->aux; + bool handled = true, ret = true; + int rc; u8 esi[8] = {}; - if (!aux) - return; - while (handled) { - ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8); - if (ret != 8) { - drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); - return; + rc = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8); + if (rc != 8) { + ret = false; + break; } drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled); if (!handled) break; - drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], 3); + rc = drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], + 3); + if (rc != 3) { + ret = false; + break; + } } + + if (!ret) + NV_DEBUG(drm, "Failed to handle ESI on %s: %d\n", + nv_connector->base.name, rc); + + return ret; } void nv50_mstm_remove(struct nv50_mstm *mstm) { - if (mstm) - drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); + mstm->is_mst = false; + drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); } static int -nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state) +nv50_mstm_enable(struct nv50_mstm *mstm, int state) { struct nouveau_encoder *outp = mstm->outp; struct { @@ -1414,106 +1475,85 @@ nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state) }; struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev); struct nvif_object *disp = &drm->display->disp.object; - int ret; - - if (dpcd >= 0x12) { - /* Even if we're enabling MST, start with disabling the - * branching unit to clear any sink-side MST topology state - * that wasn't set by us - */ - ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, 0); - if (ret < 0) - return ret; - - if (state) { - /* Now, start initializing */ - ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, - DP_MST_EN); - if (ret < 0) - return ret; - } - } return nvif_mthd(disp, 0, &args, sizeof(args)); } int -nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow) +nv50_mstm_detect(struct nouveau_encoder *outp) { + struct nv50_mstm *mstm = outp->dp.mstm; struct drm_dp_aux *aux; int ret; - bool old_state, new_state; - u8 mstm_ctrl; - if (!mstm) + if (!mstm || !mstm->can_mst) return 0; - mutex_lock(&mstm->mgr.lock); - - old_state = mstm->mgr.mst_state; - new_state = old_state; aux = mstm->mgr.aux; - if (old_state) { - /* Just check that the MST hub is still as we expect it */ - ret = drm_dp_dpcd_readb(aux, DP_MSTM_CTRL, &mstm_ctrl); - if (ret < 0 || !(mstm_ctrl & DP_MST_EN)) { - DRM_DEBUG_KMS("Hub gone, disabling MST topology\n"); - new_state = false; - } - } else if (dpcd[0] >= 0x12) { - ret = drm_dp_dpcd_readb(aux, DP_MSTM_CAP, &dpcd[1]); - if (ret < 0) - goto probe_error; - - if (!(dpcd[1] & DP_MST_CAP)) - dpcd[0] = 0x11; - else - new_state = allow; - } - - if (new_state == old_state) { - mutex_unlock(&mstm->mgr.lock); - return new_state; - } - - ret = nv50_mstm_enable(mstm, dpcd[0], new_state); - if (ret) - goto probe_error; - - mutex_unlock(&mstm->mgr.lock); + /* Clear any leftover MST state we didn't set ourselves by first + * disabling MST if it was already enabled + */ + ret = drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0); + if (ret < 0) + return ret; - ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, new_state); + /* And start enabling */ + ret = nv50_mstm_enable(mstm, true); if (ret) - return nv50_mstm_enable(mstm, dpcd[0], 0); + return ret; - return new_state; + ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, true); + if (ret) { + nv50_mstm_enable(mstm, false); + return ret; + } -probe_error: - mutex_unlock(&mstm->mgr.lock); - return ret; + mstm->is_mst = true; + return 1; } static void -nv50_mstm_fini(struct nv50_mstm *mstm) +nv50_mstm_fini(struct nouveau_encoder *outp) { - if (mstm && mstm->mgr.mst_state) + struct nv50_mstm *mstm = outp->dp.mstm; + + if (!mstm) + return; + + /* Don't change the MST state of this connector until we've finished + * resuming, since we can't safely grab hpd_irq_lock in our resume + * path to protect mstm->is_mst without potentially deadlocking + */ + mutex_lock(&outp->dp.hpd_irq_lock); + mstm->suspended = true; + mutex_unlock(&outp->dp.hpd_irq_lock); + + if (mstm->is_mst) drm_dp_mst_topology_mgr_suspend(&mstm->mgr); } static void -nv50_mstm_init(struct nv50_mstm *mstm, bool runtime) +nv50_mstm_init(struct nouveau_encoder *outp, bool runtime) { - int ret; + struct nv50_mstm *mstm = outp->dp.mstm; + int ret = 0; - if (!mstm || !mstm->mgr.mst_state) + if (!mstm) return; - ret = drm_dp_mst_topology_mgr_resume(&mstm->mgr, !runtime); - if (ret == -1) { - drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); - drm_kms_helper_hotplug_event(mstm->mgr.dev); + if (mstm->is_mst) { + ret = drm_dp_mst_topology_mgr_resume(&mstm->mgr, !runtime); + if (ret == -1) + nv50_mstm_remove(mstm); } + + mutex_lock(&outp->dp.hpd_irq_lock); + mstm->suspended = false; + mutex_unlock(&outp->dp.hpd_irq_lock); + + if (ret == -1) + drm_kms_helper_hotplug_event(mstm->mgr.dev); } static void @@ -1535,17 +1575,6 @@ nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max, struct drm_device *dev = outp->base.base.dev; struct nv50_mstm *mstm; int ret; - u8 dpcd; - - /* This is a workaround for some monitors not functioning - * correctly in MST mode on initial module load. I think - * some bad interaction with the VBIOS may be responsible. - * - * A good ol' off and on again seems to work here ;) - */ - ret = drm_dp_dpcd_readb(aux, DP_DPCD_REV, &dpcd); - if (ret >= 0 && dpcd >= 0x12) - drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0); if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL))) return -ENOMEM; @@ -1584,23 +1613,27 @@ nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head, } static void -nv50_sor_disable(struct drm_encoder *encoder) +nv50_sor_disable(struct drm_encoder *encoder, + struct drm_atomic_state *state) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc); + struct nouveau_connector *nv_connector = + nv50_outp_get_old_connector(nv_encoder, state); nv_encoder->crtc = NULL; if (nv_crtc) { - struct nvkm_i2c_aux *aux = nv_encoder->aux; + struct drm_dp_aux *aux = &nv_connector->aux; u8 pwr; - if (aux) { - int ret = nvkm_rdaux(aux, DP_SET_POWER, &pwr, 1); + if (nv_encoder->dcb->type == DCB_OUTPUT_DP) { + int ret = drm_dp_dpcd_readb(aux, DP_SET_POWER, &pwr); + if (ret == 0) { pwr &= ~DP_SET_POWER_MASK; pwr |= DP_SET_POWER_D3; - nvkm_wraux(aux, DP_SET_POWER, &pwr, 1); + drm_dp_dpcd_writeb(aux, DP_SET_POWER, pwr); } } @@ -1612,7 +1645,8 @@ nv50_sor_disable(struct drm_encoder *encoder) } static void -nv50_sor_enable(struct drm_encoder *encoder) +nv50_sor_enable(struct drm_encoder *encoder, + struct drm_atomic_state *state) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); @@ -1636,7 +1670,7 @@ nv50_sor_enable(struct drm_encoder *encoder) u8 proto = NV507D_SOR_SET_CONTROL_PROTOCOL_CUSTOM; u8 depth = NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT; - nv_connector = nouveau_encoder_connector_get(nv_encoder); + nv_connector = nv50_outp_get_new_connector(nv_encoder, state); nv_encoder->crtc = encoder->crtc; if ((disp->disp->object.oclass == GT214_DISP || @@ -1663,7 +1697,7 @@ nv50_sor_enable(struct drm_encoder *encoder) proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_B; } - nv50_hdmi_enable(&nv_encoder->base.base, mode); + nv50_hdmi_enable(&nv_encoder->base.base, state, mode); break; case DCB_OUTPUT_LVDS: proto = NV507D_SOR_SET_CONTROL_PROTOCOL_LVDS_CUSTOM; @@ -1704,7 +1738,7 @@ nv50_sor_enable(struct drm_encoder *encoder) else proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_B; - nv50_audio_enable(encoder, mode); + nv50_audio_enable(encoder, state, mode); break; default: BUG(); @@ -1717,8 +1751,8 @@ nv50_sor_enable(struct drm_encoder *encoder) static const struct drm_encoder_helper_funcs nv50_sor_help = { .atomic_check = nv50_outp_atomic_check, - .enable = nv50_sor_enable, - .disable = nv50_sor_disable, + .atomic_enable = nv50_sor_enable, + .atomic_disable = nv50_sor_disable, }; static void @@ -1727,6 +1761,10 @@ nv50_sor_destroy(struct drm_encoder *encoder) struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); nv50_mstm_del(&nv_encoder->dp.mstm); drm_encoder_cleanup(encoder); + + if (nv_encoder->dcb->type == DCB_OUTPUT_DP) + mutex_destroy(&nv_encoder->dp.hpd_irq_lock); + kfree(encoder); } @@ -1786,6 +1824,8 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe) struct nvkm_i2c_aux *aux = nvkm_i2c_aux_find(i2c, dcbe->i2c_index); + mutex_init(&nv_encoder->dp.hpd_irq_lock); + if (aux) { if (disp->disp->object.oclass < GF110_DISP) { /* HW has no support for address-only @@ -2077,7 +2117,7 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) outp->clr.mask, outp->set.mask); if (outp->clr.mask) { - help->disable(encoder); + help->atomic_disable(encoder, state); interlock[NV50_DISP_INTERLOCK_CORE] |= 1; if (outp->flush_disable) { nv50_disp_atomic_commit_wndw(state, interlock); @@ -2116,7 +2156,7 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) outp->set.mask, outp->clr.mask); if (outp->set.mask) { - help->enable(encoder); + help->atomic_enable(encoder, state); interlock[NV50_DISP_INTERLOCK_CORE] = 1; } @@ -2484,9 +2524,9 @@ nv50_disp_func = { *****************************************************************************/ static void -nv50_display_fini(struct drm_device *dev, bool suspend) +nv50_display_fini(struct drm_device *dev, bool runtime, bool suspend) { - struct nouveau_encoder *nv_encoder; + struct nouveau_drm *drm = nouveau_drm(dev); struct drm_encoder *encoder; struct drm_plane *plane; @@ -2498,11 +2538,12 @@ nv50_display_fini(struct drm_device *dev, bool suspend) } list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { - if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { - nv_encoder = nouveau_encoder(encoder); - nv50_mstm_fini(nv_encoder->dp.mstm); - } + if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) + nv50_mstm_fini(nouveau_encoder(encoder)); } + + if (!runtime) + cancel_work_sync(&drm->hpd_work); } static int @@ -2519,7 +2560,7 @@ nv50_display_init(struct drm_device *dev, bool resume, bool runtime) if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - nv50_mstm_init(nv_encoder->dp.mstm, runtime); + nv50_mstm_init(nv_encoder, runtime); } } diff --git a/drivers/gpu/drm/nouveau/include/nvif/push507c.h b/drivers/gpu/drm/nouveau/include/nvif/push507c.h index 889467f13fd9..7917bead4845 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/push507c.h +++ b/drivers/gpu/drm/nouveau/include/nvif/push507c.h @@ -20,6 +20,6 @@ PUSH_ASSERT(!((o) & ~DRF_SMASK(NV507C_DMA_JUMP_OFFSET)), "offset"); \ PUSH_DATA__((p), NVDEF(NV507C, DMA, OPCODE, JUMP) | \ NVVAL(NV507C, DMA, JUMP_OFFSET, (o) >> 2), \ - "jump 0x%08x - %s", (u32)(o), __func__); \ + " jump 0x%08x - %s", (u32)(o), __func__); \ } while(0) #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index 7674025a4bfe..49dd0cbc332f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -391,20 +391,6 @@ find_encoder(struct drm_connector *connector, int type) return NULL; } -struct nouveau_connector * -nouveau_encoder_connector_get(struct nouveau_encoder *encoder) -{ - struct drm_device *dev = to_drm_encoder(encoder)->dev; - struct drm_connector *drm_connector; - - list_for_each_entry(drm_connector, &dev->mode_config.connector_list, head) { - if (drm_connector->encoder == to_drm_encoder(encoder)) - return nouveau_connector(drm_connector); - } - - return NULL; -} - static void nouveau_connector_destroy(struct drm_connector *connector) { @@ -435,7 +421,8 @@ nouveau_connector_ddc_detect(struct drm_connector *connector) switch (nv_encoder->dcb->type) { case DCB_OUTPUT_DP: - ret = nouveau_dp_detect(nv_encoder); + ret = nouveau_dp_detect(nouveau_connector(connector), + nv_encoder); if (ret == NOUVEAU_DP_MST) return NULL; else if (ret == NOUVEAU_DP_SST) @@ -541,6 +528,17 @@ nouveau_connector_set_encoder(struct drm_connector *connector, } } +static void +nouveau_connector_set_edid(struct nouveau_connector *nv_connector, + struct edid *edid) +{ + struct edid *old_edid = nv_connector->edid; + + drm_connector_update_edid_property(&nv_connector->base, edid); + kfree(old_edid); + nv_connector->edid = edid; +} + static enum drm_connector_status nouveau_connector_detect(struct drm_connector *connector, bool force) { @@ -554,13 +552,6 @@ nouveau_connector_detect(struct drm_connector *connector, bool force) int ret; enum drm_connector_status conn_status = connector_status_disconnected; - /* Cleanup the previous EDID block. */ - if (nv_connector->edid) { - drm_connector_update_edid_property(connector, NULL); - kfree(nv_connector->edid); - nv_connector->edid = NULL; - } - /* Outputs are only polled while runtime active, so resuming the * device here is unnecessary (and would deadlock upon runtime suspend * because it waits for polling to finish). We do however, want to @@ -573,22 +564,23 @@ nouveau_connector_detect(struct drm_connector *connector, bool force) ret = pm_runtime_get_sync(dev->dev); if (ret < 0 && ret != -EACCES) { pm_runtime_put_autosuspend(dev->dev); + nouveau_connector_set_edid(nv_connector, NULL); return conn_status; } } nv_encoder = nouveau_connector_ddc_detect(connector); if (nv_encoder && (i2c = nv_encoder->i2c) != NULL) { + struct edid *new_edid; + if ((vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) && nv_connector->type == DCB_CONNECTOR_LVDS) - nv_connector->edid = drm_get_edid_switcheroo(connector, - i2c); + new_edid = drm_get_edid_switcheroo(connector, i2c); else - nv_connector->edid = drm_get_edid(connector, i2c); + new_edid = drm_get_edid(connector, i2c); - drm_connector_update_edid_property(connector, - nv_connector->edid); + nouveau_connector_set_edid(nv_connector, new_edid); if (!nv_connector->edid) { NV_ERROR(drm, "DDC responded, but no EDID for %s\n", connector->name); @@ -622,6 +614,8 @@ nouveau_connector_detect(struct drm_connector *connector, bool force) conn_status = connector_status_connected; drm_dp_cec_set_edid(&nv_connector->aux, nv_connector->edid); goto out; + } else { + nouveau_connector_set_edid(nv_connector, NULL); } nv_encoder = nouveau_connector_of_detect(connector); @@ -646,10 +640,11 @@ detect_analog: conn_status = connector_status_connected; goto out; } - } out: + if (!nv_connector->edid) + drm_dp_cec_unset_edid(&nv_connector->aux); pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); @@ -664,18 +659,12 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force) struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_connector *nv_connector = nouveau_connector(connector); struct nouveau_encoder *nv_encoder = NULL; + struct edid *edid = NULL; enum drm_connector_status status = connector_status_disconnected; - /* Cleanup the previous EDID block. */ - if (nv_connector->edid) { - drm_connector_update_edid_property(connector, NULL); - kfree(nv_connector->edid); - nv_connector->edid = NULL; - } - nv_encoder = find_encoder(connector, DCB_OUTPUT_LVDS); if (!nv_encoder) - return connector_status_disconnected; + goto out; /* Try retrieving EDID via DDC */ if (!drm->vbios.fp_no_ddc) { @@ -694,7 +683,8 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force) * valid - it's not (rh#613284) */ if (nv_encoder->dcb->lvdsconf.use_acpi_for_edid) { - if ((nv_connector->edid = nouveau_acpi_edid(dev, connector))) { + edid = nouveau_acpi_edid(dev, connector); + if (edid) { status = connector_status_connected; goto out; } @@ -714,12 +704,10 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force) * stored for the panel stored in them. */ if (!drm->vbios.fp_no_ddc) { - struct edid *edid = - (struct edid *)nouveau_bios_embedded_edid(dev); + edid = (struct edid *)nouveau_bios_embedded_edid(dev); if (edid) { - nv_connector->edid = - kmemdup(edid, EDID_LENGTH, GFP_KERNEL); - if (nv_connector->edid) + edid = kmemdup(edid, EDID_LENGTH, GFP_KERNEL); + if (edid) status = connector_status_connected; } } @@ -732,7 +720,7 @@ out: status = connector_status_unknown; #endif - drm_connector_update_edid_property(connector, nv_connector->edid); + nouveau_connector_set_edid(nv_connector, edid); nouveau_connector_set_encoder(connector, nv_encoder); return status; } @@ -1150,59 +1138,39 @@ nouveau_connector_funcs_lvds = { .early_unregister = nouveau_connector_early_unregister, }; +void +nouveau_connector_hpd(struct drm_connector *connector) +{ + struct nouveau_drm *drm = nouveau_drm(connector->dev); + u32 mask = drm_connector_mask(connector); + + mutex_lock(&drm->hpd_lock); + if (!(drm->hpd_pending & mask)) { + drm->hpd_pending |= mask; + schedule_work(&drm->hpd_work); + } + mutex_unlock(&drm->hpd_lock); +} + static int nouveau_connector_hotplug(struct nvif_notify *notify) { struct nouveau_connector *nv_connector = container_of(notify, typeof(*nv_connector), hpd); struct drm_connector *connector = &nv_connector->base; - struct nouveau_drm *drm = nouveau_drm(connector->dev); + struct drm_device *dev = connector->dev; + struct nouveau_drm *drm = nouveau_drm(dev); const struct nvif_notify_conn_rep_v0 *rep = notify->data; - const char *name = connector->name; - struct nouveau_encoder *nv_encoder; - int ret; bool plugged = (rep->mask != NVIF_NOTIFY_CONN_V0_UNPLUG); if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) { - NV_DEBUG(drm, "service %s\n", name); - drm_dp_cec_irq(&nv_connector->aux); - if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP))) - nv50_mstm_service(nv_encoder->dp.mstm); - + nouveau_dp_irq(drm, nv_connector); return NVIF_NOTIFY_KEEP; } - ret = pm_runtime_get(drm->dev->dev); - if (ret == 0) { - /* We can't block here if there's a pending PM request - * running, as we'll deadlock nouveau_display_fini() when it - * calls nvif_put() on our nvif_notify struct. So, simply - * defer the hotplug event until the device finishes resuming - */ - NV_DEBUG(drm, "Deferring HPD on %s until runtime resume\n", - name); - schedule_work(&drm->hpd_work); - - pm_runtime_put_noidle(drm->dev->dev); - return NVIF_NOTIFY_KEEP; - } else if (ret != 1 && ret != -EACCES) { - NV_WARN(drm, "HPD on %s dropped due to RPM failure: %d\n", - name, ret); - return NVIF_NOTIFY_DROP; - } - - if (!plugged) - drm_dp_cec_unset_edid(&nv_connector->aux); - NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", name); - if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP))) { - if (!plugged) - nv50_mstm_remove(nv_encoder->dp.mstm); - } - - drm_helper_hpd_irq_event(connector->dev); + NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", connector->name); + nouveau_connector_hpd(connector); - pm_runtime_mark_last_busy(drm->dev->dev); - pm_runtime_put_autosuspend(drm->dev->dev); return NVIF_NOTIFY_KEEP; } diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h index d6de5cb8e223..d0b859c4a80e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.h +++ b/drivers/gpu/drm/nouveau/nouveau_connector.h @@ -187,6 +187,7 @@ nouveau_crtc_connector_get(struct nouveau_crtc *nv_crtc) struct drm_connector * nouveau_connector_create(struct drm_device *, const struct dcb_output *); +void nouveau_connector_hpd(struct drm_connector *connector); extern int nouveau_tv_disable; extern int nouveau_ignorelid; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 5f31b11ac2e7..bceb48a2dfca 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -457,16 +457,70 @@ static struct nouveau_drm_prop_enum_list dither_depth[] = { } \ } while(0) +void +nouveau_display_hpd_resume(struct drm_device *dev) +{ + struct nouveau_drm *drm = nouveau_drm(dev); + + mutex_lock(&drm->hpd_lock); + drm->hpd_pending = ~0; + mutex_unlock(&drm->hpd_lock); + + schedule_work(&drm->hpd_work); +} + static void nouveau_display_hpd_work(struct work_struct *work) { struct nouveau_drm *drm = container_of(work, typeof(*drm), hpd_work); + struct drm_device *dev = drm->dev; + struct drm_connector *connector; + struct drm_connector_list_iter conn_iter; + u32 pending; + bool changed = false; + + pm_runtime_get_sync(dev->dev); - pm_runtime_get_sync(drm->dev->dev); + mutex_lock(&drm->hpd_lock); + pending = drm->hpd_pending; + drm->hpd_pending = 0; + mutex_unlock(&drm->hpd_lock); - drm_helper_hpd_irq_event(drm->dev); + /* Nothing to do, exit early without updating the last busy counter */ + if (!pending) + goto noop; + + mutex_lock(&dev->mode_config.mutex); + drm_connector_list_iter_begin(dev, &conn_iter); + + nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) { + enum drm_connector_status old_status = connector->status; + u64 old_epoch_counter = connector->epoch_counter; + + if (!(pending & drm_connector_mask(connector))) + continue; + + connector->status = drm_helper_probe_detect(connector, NULL, + false); + if (old_epoch_counter == connector->epoch_counter) + continue; + + changed = true; + drm_dbg_kms(dev, "[CONNECTOR:%d:%s] status updated from %s to %s (epoch counter %llu->%llu)\n", + connector->base.id, connector->name, + drm_get_connector_status_name(old_status), + drm_get_connector_status_name(connector->status), + old_epoch_counter, connector->epoch_counter); + } + + drm_connector_list_iter_end(&conn_iter); + mutex_unlock(&dev->mode_config.mutex); + + if (changed) + drm_kms_helper_hotplug_event(dev); pm_runtime_mark_last_busy(drm->dev->dev); +noop: pm_runtime_put_sync(drm->dev->dev); } @@ -490,12 +544,11 @@ nouveau_display_acpi_ntfy(struct notifier_block *nb, unsigned long val, */ pm_runtime_put_autosuspend(drm->dev->dev); } else if (ret == 0) { - /* This may be the only indication we receive - * of a connector hotplug on a runtime - * suspended GPU, schedule hpd_work to check. + /* We've started resuming the GPU already, so + * it will handle scheduling a full reprobe + * itself */ NV_DEBUG(drm, "ACPI requested connector reprobe\n"); - schedule_work(&drm->hpd_work); pm_runtime_put_noidle(drm->dev->dev); } else { NV_WARN(drm, "Dropped ACPI reprobe event due to RPM error: %d\n", @@ -569,7 +622,7 @@ nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime) cancel_work_sync(&drm->hpd_work); drm_kms_helper_poll_disable(dev); - disp->fini(dev, suspend); + disp->fini(dev, runtime, suspend); } static void @@ -686,6 +739,7 @@ nouveau_display_create(struct drm_device *dev) } INIT_WORK(&drm->hpd_work, nouveau_display_hpd_work); + mutex_init(&drm->hpd_lock); #ifdef CONFIG_ACPI drm->acpi_nb.notifier_call = nouveau_display_acpi_ntfy; register_acpi_notifier(&drm->acpi_nb); @@ -705,9 +759,10 @@ void nouveau_display_destroy(struct drm_device *dev) { struct nouveau_display *disp = nouveau_display(dev); + struct nouveau_drm *drm = nouveau_drm(dev); #ifdef CONFIG_ACPI - unregister_acpi_notifier(&nouveau_drm(dev)->acpi_nb); + unregister_acpi_notifier(&drm->acpi_nb); #endif drm_kms_helper_poll_fini(dev); @@ -719,6 +774,7 @@ nouveau_display_destroy(struct drm_device *dev) nvif_disp_dtor(&disp->disp); nouveau_drm(dev)->display = NULL; + mutex_destroy(&drm->hpd_lock); kfree(disp); } diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h index 6e0d900441d6..616c43427059 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.h +++ b/drivers/gpu/drm/nouveau/nouveau_display.h @@ -18,7 +18,7 @@ struct nouveau_display { void *priv; void (*dtor)(struct drm_device *); int (*init)(struct drm_device *, bool resume, bool runtime); - void (*fini)(struct drm_device *, bool suspend); + void (*fini)(struct drm_device *, bool suspend, bool runtime); struct nvif_disp disp; @@ -45,6 +45,7 @@ nouveau_display(struct drm_device *dev) int nouveau_display_create(struct drm_device *dev); void nouveau_display_destroy(struct drm_device *dev); int nouveau_display_init(struct drm_device *dev, bool resume, bool runtime); +void nouveau_display_hpd_resume(struct drm_device *dev); void nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime); int nouveau_display_suspend(struct drm_device *dev, bool runtime); void nouveau_display_resume(struct drm_device *dev, bool runtime); diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index 8a0f7994e1ae..810bf6956568 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -36,50 +36,123 @@ MODULE_PARM_DESC(mst, "Enable DisplayPort multi-stream (default: enabled)"); static int nouveau_mst = 1; module_param_named(mst, nouveau_mst, int, 0400); -static void -nouveau_dp_probe_oui(struct drm_device *dev, struct nvkm_i2c_aux *aux, u8 *dpcd) +static bool +nouveau_dp_has_sink_count(struct drm_connector *connector, + struct nouveau_encoder *outp) { - struct nouveau_drm *drm = nouveau_drm(dev); - u8 buf[3]; - - if (!(dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) - return; - - if (!nvkm_rdaux(aux, DP_SINK_OUI, buf, 3)) - NV_DEBUG(drm, "Sink OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); - - if (!nvkm_rdaux(aux, DP_BRANCH_OUI, buf, 3)) - NV_DEBUG(drm, "Branch OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); + return drm_dp_read_sink_count_cap(connector, outp->dp.dpcd, &outp->dp.desc); +} +static enum drm_connector_status +nouveau_dp_probe_dpcd(struct nouveau_connector *nv_connector, + struct nouveau_encoder *outp) +{ + struct drm_connector *connector = &nv_connector->base; + struct drm_dp_aux *aux = &nv_connector->aux; + struct nv50_mstm *mstm = NULL; + enum drm_connector_status status = connector_status_disconnected; + int ret; + u8 *dpcd = outp->dp.dpcd; + + ret = drm_dp_read_dpcd_caps(aux, dpcd); + if (ret < 0) + goto out; + + ret = drm_dp_read_desc(aux, &outp->dp.desc, drm_dp_is_branch(dpcd)); + if (ret < 0) + goto out; + + if (nouveau_mst) { + mstm = outp->dp.mstm; + if (mstm) + mstm->can_mst = drm_dp_read_mst_cap(aux, dpcd); + } + + if (nouveau_dp_has_sink_count(connector, outp)) { + ret = drm_dp_read_sink_count(aux); + if (ret < 0) + goto out; + + outp->dp.sink_count = ret; + + /* + * Dongle connected, but no display. Don't bother reading + * downstream port info + */ + if (!outp->dp.sink_count) + return connector_status_disconnected; + } + + ret = drm_dp_read_downstream_info(aux, dpcd, + outp->dp.downstream_ports); + if (ret < 0) + goto out; + + status = connector_status_connected; +out: + if (status != connector_status_connected) { + /* Clear any cached info */ + outp->dp.sink_count = 0; + } + return status; } int -nouveau_dp_detect(struct nouveau_encoder *nv_encoder) +nouveau_dp_detect(struct nouveau_connector *nv_connector, + struct nouveau_encoder *nv_encoder) { struct drm_device *dev = nv_encoder->base.base.dev; struct nouveau_drm *drm = nouveau_drm(dev); - struct nvkm_i2c_aux *aux; - u8 dpcd[8]; - int ret; - - aux = nv_encoder->aux; - if (!aux) - return -ENODEV; - - ret = nvkm_rdaux(aux, DP_DPCD_REV, dpcd, sizeof(dpcd)); - if (ret) - return ret; + struct drm_connector *connector = &nv_connector->base; + struct nv50_mstm *mstm = nv_encoder->dp.mstm; + enum drm_connector_status status; + u8 *dpcd = nv_encoder->dp.dpcd; + int ret = NOUVEAU_DP_NONE; + + /* If we've already read the DPCD on an eDP device, we don't need to + * reread it as it won't change + */ + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP && + dpcd[DP_DPCD_REV] != 0) + return NOUVEAU_DP_SST; - nv_encoder->dp.link_bw = 27000 * dpcd[1]; - nv_encoder->dp.link_nr = dpcd[2] & DP_MAX_LANE_COUNT_MASK; + mutex_lock(&nv_encoder->dp.hpd_irq_lock); + if (mstm) { + /* If we're not ready to handle MST state changes yet, just + * report the last status of the connector. We'll reprobe it + * once we've resumed. + */ + if (mstm->suspended) { + if (mstm->is_mst) + ret = NOUVEAU_DP_MST; + else if (connector->status == + connector_status_connected) + ret = NOUVEAU_DP_SST; + + goto out; + } + } + + status = nouveau_dp_probe_dpcd(nv_connector, nv_encoder); + if (status == connector_status_disconnected) + goto out; + + /* If we're in MST mode, we're done here */ + if (mstm && mstm->can_mst && mstm->is_mst) { + ret = NOUVEAU_DP_MST; + goto out; + } + + nv_encoder->dp.link_bw = 27000 * dpcd[DP_MAX_LINK_RATE]; + nv_encoder->dp.link_nr = + dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK; NV_DEBUG(drm, "display: %dx%d dpcd 0x%02x\n", - nv_encoder->dp.link_nr, nv_encoder->dp.link_bw, dpcd[0]); + nv_encoder->dp.link_nr, nv_encoder->dp.link_bw, + dpcd[DP_DPCD_REV]); NV_DEBUG(drm, "encoder: %dx%d\n", - nv_encoder->dcb->dpconf.link_nr, - nv_encoder->dcb->dpconf.link_bw); + nv_encoder->dcb->dpconf.link_nr, + nv_encoder->dcb->dpconf.link_bw); if (nv_encoder->dcb->dpconf.link_nr < nv_encoder->dp.link_nr) nv_encoder->dp.link_nr = nv_encoder->dcb->dpconf.link_nr; @@ -87,23 +160,68 @@ nouveau_dp_detect(struct nouveau_encoder *nv_encoder) nv_encoder->dp.link_bw = nv_encoder->dcb->dpconf.link_bw; NV_DEBUG(drm, "maximum: %dx%d\n", - nv_encoder->dp.link_nr, nv_encoder->dp.link_bw); + nv_encoder->dp.link_nr, nv_encoder->dp.link_bw); + + if (mstm && mstm->can_mst) { + ret = nv50_mstm_detect(nv_encoder); + if (ret == 1) { + ret = NOUVEAU_DP_MST; + goto out; + } else if (ret != 0) { + goto out; + } + } + ret = NOUVEAU_DP_SST; + +out: + if (mstm && !mstm->suspended && ret != NOUVEAU_DP_MST) + nv50_mstm_remove(mstm); + + mutex_unlock(&nv_encoder->dp.hpd_irq_lock); + return ret; +} - nouveau_dp_probe_oui(dev, aux, dpcd); +void nouveau_dp_irq(struct nouveau_drm *drm, + struct nouveau_connector *nv_connector) +{ + struct drm_connector *connector = &nv_connector->base; + struct nouveau_encoder *outp = find_encoder(connector, DCB_OUTPUT_DP); + struct nv50_mstm *mstm; + int ret; + bool send_hpd = false; - ret = nv50_mstm_detect(nv_encoder->dp.mstm, dpcd, nouveau_mst); - if (ret == 1) - return NOUVEAU_DP_MST; - if (ret == 0) - return NOUVEAU_DP_SST; - return ret; + if (!outp) + return; + + mstm = outp->dp.mstm; + NV_DEBUG(drm, "service %s\n", connector->name); + + mutex_lock(&outp->dp.hpd_irq_lock); + + if (mstm && mstm->is_mst) { + if (!nv50_mstm_service(drm, nv_connector, mstm)) + send_hpd = true; + } else { + drm_dp_cec_irq(&nv_connector->aux); + + if (nouveau_dp_has_sink_count(connector, outp)) { + ret = drm_dp_read_sink_count(&nv_connector->aux); + if (ret != outp->dp.sink_count) + send_hpd = true; + if (ret >= 0) + outp->dp.sink_count = ret; + } + } + + mutex_unlock(&outp->dp.hpd_irq_lock); + + if (send_hpd) + nouveau_connector_hpd(connector); } /* TODO: * - Use the minimum possible BPC here, once we add support for the max bpc * property. - * - Validate the mode against downstream port caps (see - * drm_dp_downstream_max_clock()) * - Validate against the DP caps advertised by the GPU (we don't check these * yet) */ @@ -114,15 +232,19 @@ nv50_dp_mode_valid(struct drm_connector *connector, unsigned *out_clock) { const unsigned min_clock = 25000; - unsigned max_clock, clock; + unsigned max_clock, ds_clock, clock; enum drm_mode_status ret; if (mode->flags & DRM_MODE_FLAG_INTERLACE && !outp->caps.dp_interlace) return MODE_NO_INTERLACE; max_clock = outp->dp.link_nr * outp->dp.link_bw; - clock = mode->clock * (connector->display_info.bpc * 3) / 10; + ds_clock = drm_dp_downstream_max_clock(outp->dp.dpcd, + outp->dp.downstream_ports); + if (ds_clock) + max_clock = min(max_clock, ds_clock); + clock = mode->clock * (connector->display_info.bpc * 3) / 10; ret = nouveau_conn_mode_clock_valid(mode, min_clock, max_clock, &clock); if (out_clock) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 22d246acc5e5..42fc5c813a9b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -953,7 +953,7 @@ nouveau_pmops_resume(struct device *dev) ret = nouveau_do_resume(drm_dev, false); /* Monitors may have been connected / disconnected during suspend */ - schedule_work(&nouveau_drm(drm_dev)->hpd_work); + nouveau_display_hpd_resume(drm_dev); return ret; } @@ -1036,7 +1036,7 @@ nouveau_pmops_runtime_resume(struct device *dev) drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; /* Monitors may have been connected / disconnected during suspend */ - schedule_work(&nouveau_drm(drm_dev)->hpd_work); + nouveau_display_hpd_resume(drm_dev); return ret; } diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 26a2c1090045..b8025507a9e4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -200,6 +200,8 @@ struct nouveau_drm { struct nvbios vbios; struct nouveau_display *display; struct work_struct hpd_work; + struct mutex hpd_lock; + u32 hpd_pending; struct work_struct fbcon_work; int fbcon_new_state; #ifdef CONFIG_ACPI diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index a72c412ac8b1..21937f1c7dd9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -33,6 +33,7 @@ #include <drm/drm_dp_mst_helper.h> #include "dispnv04/disp.h" struct nv50_head_atom; +struct nouveau_connector; #define NV_DPMS_CLEARED 0x80 @@ -64,6 +65,17 @@ struct nouveau_encoder { struct nv50_mstm *mstm; int link_nr; int link_bw; + + /* Protects DP state that needs to be accessed outside + * connector reprobing contexts + */ + struct mutex hpd_irq_lock; + + u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + struct drm_dp_desc desc; + + u8 sink_count; } dp; }; @@ -77,6 +89,21 @@ struct nouveau_encoder { struct nv50_head_atom *, u8 proto, u8 depth); }; +struct nv50_mstm { + struct nouveau_encoder *outp; + + struct drm_dp_mst_topology_mgr mgr; + + /* Protected under nouveau_encoder->dp.hpd_irq_lock */ + bool can_mst; + bool is_mst; + bool suspended; + + bool modified; + bool disabled; + int links; +}; + struct nouveau_encoder * find_encoder(struct drm_connector *connector, int type); @@ -100,20 +127,29 @@ get_slave_funcs(struct drm_encoder *enc) /* nouveau_dp.c */ enum nouveau_dp_status { + NOUVEAU_DP_NONE, NOUVEAU_DP_SST, NOUVEAU_DP_MST, }; -int nouveau_dp_detect(struct nouveau_encoder *); +int nouveau_dp_detect(struct nouveau_connector *, struct nouveau_encoder *); +void nouveau_dp_irq(struct nouveau_drm *drm, + struct nouveau_connector *nv_connector); enum drm_mode_status nv50_dp_mode_valid(struct drm_connector *, struct nouveau_encoder *, const struct drm_display_mode *, unsigned *clock); struct nouveau_connector * -nouveau_encoder_connector_get(struct nouveau_encoder *encoder); - -int nv50_mstm_detect(struct nv50_mstm *, u8 dpcd[8], int allow); -void nv50_mstm_remove(struct nv50_mstm *); -void nv50_mstm_service(struct nv50_mstm *); +nv50_outp_get_new_connector(struct nouveau_encoder *outp, + struct drm_atomic_state *state); +struct nouveau_connector * +nv50_outp_get_old_connector(struct nouveau_encoder *outp, + struct drm_atomic_state *state); + +int nv50_mstm_detect(struct nouveau_encoder *encoder); +void nv50_mstm_remove(struct nv50_mstm *mstm); +bool nv50_mstm_service(struct nouveau_drm *drm, + struct nouveau_connector *nv_connector, + struct nv50_mstm *mstm); #endif /* __NOUVEAU_ENCODER_H__ */ diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c index 0fa8f677b647..5c027c81760f 100644 --- a/drivers/gpu/drm/omapdrm/dss/venc.c +++ b/drivers/gpu/drm/omapdrm/dss/venc.c @@ -597,7 +597,7 @@ static void venc_bridge_mode_set(struct drm_bridge *bridge, switch (venc_mode) { default: WARN_ON_ONCE(1); - /* Fall-through */ + fallthrough; case VENC_MODE_PAL: venc->config = &venc_config_pal_trm; break; diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c index 6d40914675da..328a4a74f534 100644 --- a/drivers/gpu/drm/omapdrm/omap_crtc.c +++ b/drivers/gpu/drm/omapdrm/omap_crtc.c @@ -451,11 +451,12 @@ static void omap_crtc_atomic_enable(struct drm_crtc *crtc, if (omap_state->manually_updated) return; - spin_lock_irq(&crtc->dev->event_lock); drm_crtc_vblank_on(crtc); + ret = drm_crtc_vblank_get(crtc); WARN_ON(ret != 0); + spin_lock_irq(&crtc->dev->event_lock); omap_crtc_arm_event(crtc); spin_unlock_irq(&crtc->dev->event_lock); } diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c index fa79688013b7..6063f3a15329 100644 --- a/drivers/gpu/drm/qxl/qxl_display.c +++ b/drivers/gpu/drm/qxl/qxl_display.c @@ -26,6 +26,7 @@ #include <linux/crc32.h> #include <linux/delay.h> +#include <drm/drm_drv.h> #include <drm/drm_atomic.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_gem_framebuffer_helper.h> @@ -186,7 +187,7 @@ void qxl_display_read_client_monitors_config(struct qxl_device *qdev) DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE, ret); qxl_update_offset_props(qdev); - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); if (!drm_helper_hpd_irq_event(dev)) { /* notify that the monitor configuration changed, to adjust at the arbitrary resolution */ @@ -431,7 +432,7 @@ static int qxl_framebuffer_surface_dirty(struct drm_framebuffer *fb, clips, num_clips, inc, 0); out_lock_end: - DRM_MODESET_LOCK_ALL_END(ctx, ret); + DRM_MODESET_LOCK_ALL_END(fb->dev, ctx, ret); return 0; } diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index ba20c6f03719..886e9959496f 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -4856,7 +4856,7 @@ static void ci_request_link_speed_change_before_state_change(struct radeon_devic pi->force_pcie_gen = RADEON_PCIE_GEN2; if (current_link_speed == RADEON_PCIE_GEN2) break; - /* fall through */ + fallthrough; case RADEON_PCIE_GEN2: if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c index 3b7ead5be5bf..73f67bf222e1 100644 --- a/drivers/gpu/drm/radeon/r300.c +++ b/drivers/gpu/drm/radeon/r300.c @@ -820,7 +820,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p, ((idx_value >> 21) & 0xF)); return -EINVAL; } - /* Fall through. */ + fallthrough; case 6: track->cb[i].cpp = 4; break; @@ -971,7 +971,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p, return -EINVAL; } /* The same rules apply as for DXT3/5. */ - /* Fall through. */ + fallthrough; case R300_TX_FORMAT_DXT3: case R300_TX_FORMAT_DXT5: track->textures[i].cpp = 1; diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c index 1d4c04e0a449..50b89b6d9a6c 100644 --- a/drivers/gpu/drm/radeon/r420.c +++ b/drivers/gpu/drm/radeon/r420.c @@ -115,7 +115,7 @@ void r420_pipes_init(struct radeon_device *rdev) default: /* force to 1 pipe */ num_pipes = 1; - /* fall through */ + fallthrough; case 1: tmp = (0 << 1); break; diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c index 49e8266461f8..390a9621604a 100644 --- a/drivers/gpu/drm/radeon/r600_cs.c +++ b/drivers/gpu/drm/radeon/r600_cs.c @@ -487,7 +487,7 @@ static int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i) return -EINVAL; } } - /* fall through */ + fallthrough; case V_0280A0_CLEAR_ENABLE: { uint32_t block_max = G_028100_CMASK_BLOCK_MAX(track->cb_color_mask[i]); @@ -1535,7 +1535,7 @@ static int r600_check_texture_resource(struct radeon_cs_parser *p, u32 idx, break; case V_038000_SQ_TEX_DIM_2D_ARRAY_MSAA: is_array = true; - /* fall through */ + fallthrough; case V_038000_SQ_TEX_DIM_2D_MSAA: array_check.nsamples = 1 << llevel; llevel = 0; diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c index e0ae911ef427..7b69d6dfe44a 100644 --- a/drivers/gpu/drm/radeon/radeon_display.c +++ b/drivers/gpu/drm/radeon/radeon_display.c @@ -933,7 +933,7 @@ static void avivo_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_div, /* get matching reference and feedback divider */ *ref_div = min(max(den/post_div, 1u), ref_div_max); - *fb_div = DIV_ROUND_CLOSEST(nom * *ref_div * post_div, den); + *fb_div = max(nom * *ref_div * post_div / den, 1u); /* limit fb divider to its maximum */ if (*fb_div > fb_div_max) { diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c index f178ba321715..3808a753127b 100644 --- a/drivers/gpu/drm/radeon/radeon_gart.c +++ b/drivers/gpu/drm/radeon/radeon_gart.c @@ -72,8 +72,8 @@ int radeon_gart_table_ram_alloc(struct radeon_device *rdev) { void *ptr; - ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size, - &rdev->gart.table_addr); + ptr = dma_alloc_coherent(&rdev->pdev->dev, rdev->gart.table_size, + &rdev->gart.table_addr, GFP_KERNEL); if (ptr == NULL) { return -ENOMEM; } @@ -85,7 +85,6 @@ int radeon_gart_table_ram_alloc(struct radeon_device *rdev) } #endif rdev->gart.ptr = ptr; - memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); return 0; } @@ -110,9 +109,8 @@ void radeon_gart_table_ram_free(struct radeon_device *rdev) rdev->gart.table_size >> PAGE_SHIFT); } #endif - pci_free_consistent(rdev->pdev, rdev->gart.table_size, - (void *)rdev->gart.ptr, - rdev->gart.table_addr); + dma_free_coherent(&rdev->pdev->dev, rdev->gart.table_size, + (void *)rdev->gart.ptr, rdev->gart.table_addr); rdev->gart.ptr = NULL; rdev->gart.table_addr = 0; } diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c index 1ad5c3b86b64..57fb3eb3a4b4 100644 --- a/drivers/gpu/drm/radeon/radeon_uvd.c +++ b/drivers/gpu/drm/radeon/radeon_uvd.c @@ -454,7 +454,7 @@ static int radeon_uvd_validate_codec(struct radeon_cs_parser *p, if (p->rdev->family >= CHIP_PALM) return 0; - /* fall through */ + fallthrough; default: DRM_ERROR("UVD codec not supported by hardware %d!\n", stream_type); diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index a167e1c36d24..d1c73e9db889 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -5744,7 +5744,7 @@ static void si_request_link_speed_change_before_state_change(struct radeon_devic si_pi->force_pcie_gen = RADEON_PCIE_GEN2; if (current_link_speed == RADEON_PCIE_GEN2) break; - /* fall through */ + fallthrough; case RADEON_PCIE_GEN2: if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) break; diff --git a/drivers/gpu/drm/radeon/uvd_v1_0.c b/drivers/gpu/drm/radeon/uvd_v1_0.c index f858d8d06347..800721153d51 100644 --- a/drivers/gpu/drm/radeon/uvd_v1_0.c +++ b/drivers/gpu/drm/radeon/uvd_v1_0.c @@ -219,7 +219,7 @@ done: WREG32(RS_DQ_RD_RET_CONF, 0x3f); WREG32(MC_CONFIG, 0x1f); - /* fall through */ + fallthrough; case CHIP_RV670: case CHIP_RV635: diff --git a/drivers/gpu/drm/savage/savage_state.c b/drivers/gpu/drm/savage/savage_state.c index a2ac25c11c90..e0d40ae67d54 100644 --- a/drivers/gpu/drm/savage/savage_state.c +++ b/drivers/gpu/drm/savage/savage_state.c @@ -306,7 +306,7 @@ static int savage_dispatch_dma_prim(drm_savage_private_t * dev_priv, case SAVAGE_PRIM_TRILIST_201: reorder = 1; prim = SAVAGE_PRIM_TRILIST; - /* fall through */ + fallthrough; case SAVAGE_PRIM_TRILIST: if (n % 3 != 0) { DRM_ERROR("wrong number of vertices %u in TRILIST\n", @@ -444,7 +444,7 @@ static int savage_dispatch_vb_prim(drm_savage_private_t * dev_priv, case SAVAGE_PRIM_TRILIST_201: reorder = 1; prim = SAVAGE_PRIM_TRILIST; - /* fall through */ + fallthrough; case SAVAGE_PRIM_TRILIST: if (n % 3 != 0) { DRM_ERROR("wrong number of vertices %u in TRILIST\n", @@ -566,7 +566,7 @@ static int savage_dispatch_dma_idx(drm_savage_private_t * dev_priv, case SAVAGE_PRIM_TRILIST_201: reorder = 1; prim = SAVAGE_PRIM_TRILIST; - /* fall through */ + fallthrough; case SAVAGE_PRIM_TRILIST: if (n % 3 != 0) { DRM_ERROR("wrong number of indices %u in TRILIST\n", n); @@ -705,7 +705,7 @@ static int savage_dispatch_vb_idx(drm_savage_private_t * dev_priv, case SAVAGE_PRIM_TRILIST_201: reorder = 1; prim = SAVAGE_PRIM_TRILIST; - /* fall through */ + fallthrough; case SAVAGE_PRIM_TRILIST: if (n % 3 != 0) { DRM_ERROR("wrong number of indices %u in TRILIST\n", n); @@ -1066,7 +1066,7 @@ int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_ ret = -EINVAL; goto done; } - /* fall through */ + fallthrough; case SAVAGE_CMD_DMA_PRIM: case SAVAGE_CMD_VB_PRIM: if (!first_draw_cmd) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 96f763d888af..9a0d77a68018 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -625,7 +625,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) return NULL; /* Kernel run queue has higher priority than normal run queue*/ - for (i = DRM_SCHED_PRIORITY_MAX - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { entity = drm_sched_rq_select_entity(&sched->sched_rq[i]); if (entity) break; @@ -852,7 +852,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, sched->name = name; sched->timeout = timeout; sched->hang_limit = hang_limit; - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++) + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; i++) drm_sched_rq_init(sched, &sched->sched_rq[i]); init_waitqueue_head(&sched->wake_up_worker); diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c index 008f07923bbc..38a558768e53 100644 --- a/drivers/gpu/drm/sti/sti_hdmi.c +++ b/drivers/gpu/drm/sti/sti_hdmi.c @@ -850,13 +850,13 @@ static int hdmi_audio_configure(struct sti_hdmi *hdmi) switch (info->channels) { case 8: audio_cfg |= HDMI_AUD_CFG_CH78_VALID; - /* fall through */ + fallthrough; case 6: audio_cfg |= HDMI_AUD_CFG_CH56_VALID; - /* fall through */ + fallthrough; case 4: audio_cfg |= HDMI_AUD_CFG_CH34_VALID | HDMI_AUD_CFG_8CH; - /* fall through */ + fallthrough; case 2: audio_cfg |= HDMI_AUD_CFG_CH12_VALID; break; diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index f025534eb30c..2f26f85ef538 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -589,8 +589,7 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine, /* We can't have an alpha plane at the lowest position */ if (!backend->quirks->supports_lowest_plane_alpha && - (plane_states[0]->fb->format->has_alpha || - (plane_states[0]->alpha != DRM_BLEND_ALPHA_OPAQUE))) + (plane_states[0]->alpha != DRM_BLEND_ALPHA_OPAQUE)) return -EINVAL; for (i = 1; i < num_planes; i++) { @@ -995,7 +994,6 @@ static const struct sun4i_backend_quirks sun6i_backend_quirks = { static const struct sun4i_backend_quirks sun7i_backend_quirks = { .needs_output_muxing = true, - .supports_lowest_plane_alpha = true, }; static const struct sun4i_backend_quirks sun8i_a33_backend_quirks = { diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 09ad8506a81b..eaaf5d70e352 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -195,7 +195,7 @@ void sun4i_tcon_set_status(struct sun4i_tcon *tcon, switch (encoder->encoder_type) { case DRM_MODE_ENCODER_LVDS: is_lvds = true; - /* Fallthrough */ + fallthrough; case DRM_MODE_ENCODER_DSI: case DRM_MODE_ENCODER_NONE: channel = 0; @@ -342,7 +342,7 @@ static void sun4i_tcon0_mode_set_dithering(struct sun4i_tcon *tcon, /* R and B components are only 5 bits deep */ val |= SUN4I_TCON0_FRM_CTL_MODE_R; val |= SUN4I_TCON0_FRM_CTL_MODE_B; - /* Fall through */ + fallthrough; case MEDIA_BUS_FMT_RGB666_1X18: case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG: /* Fall through: enable dithering */ @@ -1431,14 +1431,18 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP) && encoder->encoder_type == DRM_MODE_ENCODER_TMDS) { ret = sun8i_tcon_top_set_hdmi_src(&pdev->dev, id); - if (ret) + if (ret) { + put_device(&pdev->dev); return ret; + } } if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP)) { ret = sun8i_tcon_top_de_config(&pdev->dev, tcon->id, id); - if (ret) + if (ret) { + put_device(&pdev->dev); return ret; + } } return 0; diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c index 9abde2817e1b..4f5efcace68e 100644 --- a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c +++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c @@ -889,7 +889,7 @@ static int sun6i_dsi_dcs_write_long(struct sun6i_dsi *dsi, regmap_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(0), sun6i_dsi_dcs_build_pkt_hdr(dsi, msg)); - bounce = kzalloc(msg->tx_len + sizeof(crc), GFP_KERNEL); + bounce = kzalloc(ALIGN(msg->tx_len + sizeof(crc), 4), GFP_KERNEL); if (!bounce) return -ENOMEM; @@ -900,7 +900,7 @@ static int sun6i_dsi_dcs_write_long(struct sun6i_dsi *dsi, memcpy((u8 *)bounce + msg->tx_len, &crc, sizeof(crc)); len += sizeof(crc); - regmap_bulk_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(1), bounce, len); + regmap_bulk_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(1), bounce, DIV_ROUND_UP(len, 4)); regmap_write(dsi->regs, SUN6I_DSI_CMD_CTL_REG, len + 4 - 1); kfree(bounce); @@ -1024,7 +1024,7 @@ static ssize_t sun6i_dsi_transfer(struct mipi_dsi_host *host, ret = sun6i_dsi_dcs_read(dsi, msg); break; } - /* Else, fall through */ + fallthrough; default: ret = -EINVAL; diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c index a360697a4a4a..76393fc976fe 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c +++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c @@ -211,7 +211,7 @@ static int sun8i_vi_layer_update_coord(struct sun8i_mixer *mixer, int channel, return 0; } -static bool sun8i_vi_layer_get_csc_mode(const struct drm_format_info *format) +static u32 sun8i_vi_layer_get_csc_mode(const struct drm_format_info *format) { if (!format->is_yuv) return SUN8I_CSC_MODE_OFF; diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c index 9a0b3240bc58..424ad60b4f38 100644 --- a/drivers/gpu/drm/tegra/dc.c +++ b/drivers/gpu/drm/tegra/dc.c @@ -135,7 +135,7 @@ static inline u32 compute_dda_inc(unsigned int in, unsigned int out, bool v, default: WARN_ON_ONCE(1); - /* fallthrough */ + fallthrough; case 4: max = 4; break; diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c index 1856962411c7..518220bd092a 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c @@ -386,7 +386,7 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc) case DRM_FORMAT_XBGR8888: case DRM_FORMAT_XRGB8888: reg |= LCDC_V2_TFT_24BPP_UNPACK; - /* fallthrough */ + fallthrough; case DRM_FORMAT_BGR888: case DRM_FORMAT_RGB888: reg |= LCDC_V2_TFT_24BPP_MODE; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 0e5497d80686..98a006fc30a5 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -505,7 +505,7 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, if (unlikely(ret != 0)) return ret; } - /* fall through */ + fallthrough; case TTM_PL_TT: ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); break; diff --git a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c index d733bbc4ac0e..17ff24d999d1 100644 --- a/drivers/gpu/drm/tve200/tve200_display.c +++ b/drivers/gpu/drm/tve200/tve200_display.c @@ -14,6 +14,7 @@ #include <linux/version.h> #include <linux/dma-buf.h> #include <linux/of_graph.h> +#include <linux/delay.h> #include <drm/drm_fb_cma_helper.h> #include <drm/drm_fourcc.h> @@ -130,9 +131,25 @@ static void tve200_display_enable(struct drm_simple_display_pipe *pipe, struct drm_connector *connector = priv->connector; u32 format = fb->format->format; u32 ctrl1 = 0; + int retries; clk_prepare_enable(priv->clk); + /* Reset the TVE200 and wait for it to come back online */ + writel(TVE200_CTRL_4_RESET, priv->regs + TVE200_CTRL_4); + for (retries = 0; retries < 5; retries++) { + usleep_range(30000, 50000); + if (readl(priv->regs + TVE200_CTRL_4) & TVE200_CTRL_4_RESET) + continue; + else + break; + } + if (retries == 5 && + readl(priv->regs + TVE200_CTRL_4) & TVE200_CTRL_4_RESET) { + dev_err(drm->dev, "can't get hardware out of reset\n"); + return; + } + /* Function 1 */ ctrl1 |= TVE200_CTRL_CSMODE; /* Interlace mode for CCIR656: parameterize? */ @@ -230,8 +247,9 @@ static void tve200_display_disable(struct drm_simple_display_pipe *pipe) drm_crtc_vblank_off(crtc); - /* Disable and Power Down */ + /* Disable put into reset and Power Down */ writel(0, priv->regs + TVE200_CTRL); + writel(TVE200_CTRL_4_RESET, priv->regs + TVE200_CTRL_4); clk_disable_unprepare(priv->clk); } @@ -279,6 +297,8 @@ static int tve200_display_enable_vblank(struct drm_simple_display_pipe *pipe) struct drm_device *drm = crtc->dev; struct tve200_drm_dev_private *priv = drm->dev_private; + /* Clear any IRQs and enable */ + writel(0xFF, priv->regs + TVE200_INT_CLR); writel(TVE200_INT_V_STATUS, priv->regs + TVE200_INT_EN); return 0; } diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c index 551fa31629af..5771bb53ce6a 100644 --- a/drivers/gpu/drm/via/via_dmablit.c +++ b/drivers/gpu/drm/via/via_dmablit.c @@ -179,21 +179,21 @@ via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg) switch (vsg->state) { case dr_via_device_mapped: via_unmap_blit_from_device(pdev, vsg); - /* fall through */ + fallthrough; case dr_via_desc_pages_alloc: for (i = 0; i < vsg->num_desc_pages; ++i) { if (vsg->desc_pages[i] != NULL) free_page((unsigned long)vsg->desc_pages[i]); } kfree(vsg->desc_pages); - /* fall through */ + fallthrough; case dr_via_pages_locked: unpin_user_pages_dirty_lock(vsg->pages, vsg->num_pages, (vsg->direction == DMA_FROM_DEVICE)); - /* fall through */ + fallthrough; case dr_via_pages_alloc: vfree(vsg->pages); - /* fall through */ + fallthrough; default: vsg->state = dr_via_sg_init; } diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c index 7a2430e34e00..c8da7adc6b30 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c @@ -179,6 +179,7 @@ static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, virtio_gpu_cmd_submit(vgdev, buf, exbuf->size, vfpriv->ctx_id, buflist, out_fence); + dma_fence_put(&out_fence->f); virtio_gpu_notify(vgdev); return 0; diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c index 729f98ad7c02..842f8b61aa89 100644 --- a/drivers/gpu/drm/virtio/virtgpu_object.c +++ b/drivers/gpu/drm/virtio/virtgpu_object.c @@ -79,6 +79,7 @@ void virtio_gpu_cleanup_object(struct virtio_gpu_object *bo) } sg_free_table(shmem->pages); + kfree(shmem->pages); shmem->pages = NULL; drm_gem_shmem_unpin(&bo->base.base); } diff --git a/drivers/gpu/drm/xen/xen_drm_front.c b/drivers/gpu/drm/xen/xen_drm_front.c index 013c9e0e412c..cc93a8c9547b 100644 --- a/drivers/gpu/drm/xen/xen_drm_front.c +++ b/drivers/gpu/drm/xen/xen_drm_front.c @@ -649,9 +649,7 @@ static void displback_changed(struct xenbus_device *xb_dev, switch (backend_state) { case XenbusStateReconfiguring: - /* fall through */ case XenbusStateReconfigured: - /* fall through */ case XenbusStateInitialised: break; @@ -701,7 +699,6 @@ static void displback_changed(struct xenbus_device *xb_dev, break; case XenbusStateUnknown: - /* fall through */ case XenbusStateClosed: if (xb_dev->state == XenbusStateClosed) break; diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c index aed7510e2710..a8aefaa38bd3 100644 --- a/drivers/gpu/drm/xen/xen_drm_front_gem.c +++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c @@ -18,6 +18,7 @@ #include <drm/drm_probe_helper.h> #include <xen/balloon.h> +#include <xen/xen.h> #include "xen_drm_front.h" #include "xen_drm_front_gem.h" @@ -99,8 +100,8 @@ static struct xen_gem_object *gem_create(struct drm_device *dev, size_t size) * allocate ballooned pages which will be used to map * grant references provided by the backend */ - ret = alloc_xenballooned_pages(xen_obj->num_pages, - xen_obj->pages); + ret = xen_alloc_unpopulated_pages(xen_obj->num_pages, + xen_obj->pages); if (ret < 0) { DRM_ERROR("Cannot allocate %zu ballooned pages: %d\n", xen_obj->num_pages, ret); @@ -152,8 +153,8 @@ void xen_drm_front_gem_free_object_unlocked(struct drm_gem_object *gem_obj) } else { if (xen_obj->pages) { if (xen_obj->be_alloc) { - free_xenballooned_pages(xen_obj->num_pages, - xen_obj->pages); + xen_free_unpopulated_pages(xen_obj->num_pages, + xen_obj->pages); gem_free_pages_array(xen_obj); } else { drm_gem_put_pages(&xen_obj->base, diff --git a/drivers/gpu/drm/xlnx/Kconfig b/drivers/gpu/drm/xlnx/Kconfig index aa6cd889bd11..b52c6cdfc0b8 100644 --- a/drivers/gpu/drm/xlnx/Kconfig +++ b/drivers/gpu/drm/xlnx/Kconfig @@ -2,6 +2,7 @@ config DRM_ZYNQMP_DPSUB tristate "ZynqMP DisplayPort Controller Driver" depends on ARCH_ZYNQMP || COMPILE_TEST depends on COMMON_CLK && DRM && OF + depends on DMADEVICES select DMA_ENGINE select DRM_GEM_CMA_HELPER select DRM_KMS_CMA_HELPER diff --git a/drivers/gpu/ipu-v3/ipu-dc.c b/drivers/gpu/ipu-v3/ipu-dc.c index dbcc16721931..34b4075a6a8e 100644 --- a/drivers/gpu/ipu-v3/ipu-dc.c +++ b/drivers/gpu/ipu-v3/ipu-dc.c @@ -141,7 +141,7 @@ static int ipu_bus_format_to_map(u32 fmt) switch (fmt) { default: WARN_ON(1); - /* fall-through */ + fallthrough; case MEDIA_BUS_FMT_RGB888_1X24: return IPU_DC_MAP_RGB24; case MEDIA_BUS_FMT_RGB565_1X16: |