diff options
| author |   Maxime Ripard <maxime@cerno.tech> | 2021-09-14 09:25:30 +0200 |
|---|---|---|
| committer | Maxime Ripard <maxime@cerno.tech> | 2021-09-14 09:25:30 +0200 |
| commit | 2f76520561d01a5f37e6d6ed2c2e441b6a355a96 (patch) | |
| tree | 692f210145f35ec2621e2015d788267f25e673cb /drivers/gpu/drm | |
| parent | drm/ttm: Try to check if new ttm man out of bounds during compile (diff) | |
| parent | Linux 5.15-rc1 (diff) | |
| download | linux-dev-2f76520561d01a5f37e6d6ed2c2e441b6a355a96.tar.xz linux-dev-2f76520561d01a5f37e6d6ed2c2e441b6a355a96.zip | |
Merge drm/drm-next into drm-misc-next
Kickstart new drm-misc-next cycle.
Signed-off-by: Maxime Ripard <maxime@cerno.tech>
Diffstat (limited to 'drivers/gpu/drm')
638 files changed, 28833 insertions, 14121 deletions
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 9dd02aad59d5..b17e231ca6f7 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -256,7 +256,6 @@ config DRM_AMDGPU select HWMON select BACKLIGHT_CLASS_DEVICE select INTERVAL_TREE - select CHASH help Choose this option if you have a recent AMD Radeon graphics card. diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index f089794bbdd5..8d0748184a14 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -58,7 +58,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \ amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \ amdgpu_fw_attestation.o amdgpu_securedisplay.o amdgpu_hdp.o \ - amdgpu_eeprom.o + amdgpu_eeprom.o amdgpu_mca.o amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o @@ -120,7 +120,7 @@ amdgpu-y += \ amdgpu-y += \ dce_v10_0.o \ dce_v11_0.o \ - dce_virtual.o + amdgpu_vkms.o # add GFX block amdgpu-y += \ @@ -189,6 +189,10 @@ amdgpu-y += \ amdgpu-y += \ amdgpu_reset.o +# add MCA block +amdgpu-y += \ + mca_v3_0.o + # add amdkfd interfaces amdgpu-y += amdgpu_amdkfd.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index d10baa3338bc..dc3c6b3a00e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -108,6 +108,7 @@ #include "amdgpu_df.h" #include "amdgpu_smuio.h" #include "amdgpu_fdinfo.h" +#include "amdgpu_mca.h" #define MAX_GPU_INSTANCE 16 @@ -916,6 +917,7 @@ struct amdgpu_device { /* display */ bool enable_virtual_display; + struct amdgpu_vkms_output *amdgpu_vkms_output; struct amdgpu_mode_info mode_info; /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */ struct work_struct hotplug_work; @@ -1008,6 +1010,9 @@ struct amdgpu_device { /* df */ struct amdgpu_df df; + /* MCA */ + struct amdgpu_mca mca; + struct amdgpu_ip_block ip_blocks[AMDGPU_MAX_IP_NUM]; uint32_t harvest_ip_mask; int num_ip_blocks; @@ -1270,6 +1275,8 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define amdgpu_inc_vram_lost(adev) atomic_inc(&((adev)->vram_lost_counter)); +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) + /* Common functions */ bool amdgpu_device_has_job_running(struct amdgpu_device *adev); bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 58520b0ad0bf..4811b0faafd9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1040,7 +1040,7 @@ void amdgpu_acpi_detect(void) */ bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { -#if defined(CONFIG_AMD_PMC) || defined(CONFIG_AMD_PMC_MODULE) +#if IS_ENABLED(CONFIG_AMD_PMC) && IS_ENABLED(CONFIG_SUSPEND) if (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0) { if (adev->flags & AMD_IS_APU) return pm_suspend_target_state == PM_SUSPEND_TO_IDLE; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 7b46ba551cb2..3003ee1c9487 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -714,7 +714,6 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, ret = dma_fence_wait(f, false); err_ib_sched: - dma_fence_put(f); amdgpu_job_free(job); err: return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 0462d4aceab7..ec028cf963f5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -271,7 +271,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv, uint64_t *size); int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( - struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv); + struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv, bool *table_freed); int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv); int amdgpu_amdkfd_gpuvm_sync_memory( diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c index a5434b713856..46cd4ee6bafb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c @@ -44,4 +44,5 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = kgd_gfx_v9_set_vm_context_page_table_base, + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c index 6409d6b1b2df..5a7f680bcb3f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c @@ -305,5 +305,6 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = kgd_gfx_v9_set_vm_context_page_table_base, - .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy + .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 491acdf92f73..960acf68150a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -560,6 +560,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: type = RESET_WAVES; break; + case KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: + type = SAVE_WAVES; + break; default: type = DRAIN_PIPE; break; @@ -754,6 +757,33 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); } +static void program_trap_handler_settings(struct kgd_dev *kgd, + uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + /* + * Program TBA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_LO), + lower_32_bits(tba_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_HI), + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + /* + * Program TMA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_LO), + lower_32_bits(tma_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI), + upper_32_bits(tma_addr >> 8)); + + unlock_srbm(kgd); +} + const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .program_sh_mem_settings = kgd_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, @@ -774,4 +804,5 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = set_vm_context_page_table_base, + .program_trap_handler_settings = program_trap_handler_settings, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c index 1f5620cc3570..dac0d751d5af 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c @@ -537,6 +537,9 @@ static int hqd_destroy_v10_3(struct kgd_dev *kgd, void *mqd, case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: type = RESET_WAVES; break; + case KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: + type = SAVE_WAVES; + break; default: type = DRAIN_PIPE; break; @@ -658,6 +661,33 @@ static void set_vm_context_page_table_base_v10_3(struct kgd_dev *kgd, uint32_t v adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); } +static void program_trap_handler_settings_v10_3(struct kgd_dev *kgd, + uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + /* + * Program TBA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_LO), + lower_32_bits(tba_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_HI), + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + /* + * Program TMA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_LO), + lower_32_bits(tma_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI), + upper_32_bits(tma_addr >> 8)); + + unlock_srbm(kgd); +} + #if 0 uint32_t enable_debug_trap_v10_3(struct kgd_dev *kgd, uint32_t trap_debug_wave_launch_mode, @@ -820,6 +850,7 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .address_watch_get_offset = address_watch_get_offset_v10_3, .get_atc_vmid_pasid_mapping_info = NULL, .set_vm_context_page_table_base = set_vm_context_page_table_base_v10_3, + .program_trap_handler_settings = program_trap_handler_settings_v10_3, #if 0 .enable_debug_trap = enable_debug_trap_v10_3, .disable_debug_trap = disable_debug_trap_v10_3, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index ed3014fbb563..bcc1cbeb8799 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -42,7 +42,8 @@ enum hqd_dequeue_request_type { NO_ACTION = 0, DRAIN_PIPE, - RESET_WAVES + RESET_WAVES, + SAVE_WAVES }; static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) @@ -566,6 +567,9 @@ int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd, case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: type = RESET_WAVES; break; + case KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: + type = SAVE_WAVES; + break; default: type = DRAIN_PIPE; break; @@ -878,6 +882,32 @@ void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid, adev->gfx.cu_info.max_waves_per_simd; } +void kgd_gfx_v9_program_trap_handler_settings(struct kgd_dev *kgd, + uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + /* + * Program TBA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_LO), + lower_32_bits(tba_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_HI), + upper_32_bits(tba_addr >> 8)); + + /* + * Program TMA registers + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_LO), + lower_32_bits(tma_addr >> 8)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI), + upper_32_bits(tma_addr >> 8)); + + unlock_srbm(kgd); +} + const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping, @@ -899,4 +929,5 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = kgd_gfx_v9_set_vm_context_page_table_base, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h index e64deba8900f..c63591106879 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h @@ -65,3 +65,5 @@ void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, uint64_t page_table_base); void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid, int *pasid_wave_cnt, int *max_waves_per_cu); +void kgd_gfx_v9_program_trap_handler_settings(struct kgd_dev *kgd, + uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index c4b3eeeab0a1..2d6b2d77b738 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1057,7 +1057,8 @@ static void unmap_bo_from_gpuvm(struct kgd_mem *mem, static int update_gpuvm_pte(struct kgd_mem *mem, struct kfd_mem_attachment *entry, - struct amdgpu_sync *sync) + struct amdgpu_sync *sync, + bool *table_freed) { struct amdgpu_bo_va *bo_va = entry->bo_va; struct amdgpu_device *adev = entry->adev; @@ -1068,7 +1069,7 @@ static int update_gpuvm_pte(struct kgd_mem *mem, return ret; /* Update the page tables */ - ret = amdgpu_vm_bo_update(adev, bo_va, false); + ret = amdgpu_vm_bo_update(adev, bo_va, false, table_freed); if (ret) { pr_err("amdgpu_vm_bo_update failed\n"); return ret; @@ -1080,7 +1081,8 @@ static int update_gpuvm_pte(struct kgd_mem *mem, static int map_bo_to_gpuvm(struct kgd_mem *mem, struct kfd_mem_attachment *entry, struct amdgpu_sync *sync, - bool no_update_pte) + bool no_update_pte, + bool *table_freed) { int ret; @@ -1097,7 +1099,7 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem, if (no_update_pte) return 0; - ret = update_gpuvm_pte(mem, entry, sync); + ret = update_gpuvm_pte(mem, entry, sync, table_freed); if (ret) { pr_err("update_gpuvm_pte() failed\n"); goto update_gpuvm_pte_failed; @@ -1605,7 +1607,8 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( } int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( - struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv) + struct kgd_dev *kgd, struct kgd_mem *mem, + void *drm_priv, bool *table_freed) { struct amdgpu_device *adev = get_amdgpu_device(kgd); struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); @@ -1693,7 +1696,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( entry->va, entry->va + bo_size, entry); ret = map_bo_to_gpuvm(mem, entry, ctx.sync, - is_invalid_userptr); + is_invalid_userptr, table_freed); if (ret) { pr_err("Failed to map bo to gpuvm\n"); goto out_unreserve; @@ -1717,6 +1720,12 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( true); ret = unreserve_bo_and_vms(&ctx, false, false); + /* Only apply no TLB flush on Aldebaran to + * workaround regressions on other Asics. + */ + if (table_freed && (adev->asic_type != CHIP_ALDEBARAN)) + *table_freed = true; + goto out; out_unreserve: @@ -2143,7 +2152,7 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) continue; kfd_mem_dmaunmap_attachment(mem, attachment); - ret = update_gpuvm_pte(mem, attachment, &sync); + ret = update_gpuvm_pte(mem, attachment, &sync, NULL); if (ret) { pr_err("%s: update PTE failed\n", __func__); /* make sure this gets validated again */ @@ -2349,7 +2358,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) continue; kfd_mem_dmaunmap_attachment(mem, attachment); - ret = update_gpuvm_pte(mem, attachment, &sync_obj); + ret = update_gpuvm_pte(mem, attachment, &sync_obj, NULL); if (ret) { pr_debug("Memory eviction: update PTE failed. Try again\n"); goto validate_map_fail; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c index 3b5d13189073..97178b307ed6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c @@ -468,6 +468,62 @@ bool amdgpu_atomfirmware_dynamic_boot_config_supported(struct amdgpu_device *ade return (fw_cap & ATOM_FIRMWARE_CAP_DYNAMIC_BOOT_CFG_ENABLE) ? true : false; } +/** + * amdgpu_atomfirmware_ras_rom_addr -- Get the RAS EEPROM addr from VBIOS + * adev: amdgpu_device pointer + * i2c_address: pointer to u8; if not NULL, will contain + * the RAS EEPROM address if the function returns true + * + * Return true if VBIOS supports RAS EEPROM address reporting, + * else return false. If true and @i2c_address is not NULL, + * will contain the RAS ROM address. + */ +bool amdgpu_atomfirmware_ras_rom_addr(struct amdgpu_device *adev, + u8 *i2c_address) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + u16 data_offset, size; + union firmware_info *firmware_info; + u8 frev, crev; + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + firmwareinfo); + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, + index, &size, &frev, &crev, + &data_offset)) { + /* support firmware_info 3.4 + */ + if ((frev == 3 && crev >=4) || (frev > 3)) { + firmware_info = (union firmware_info *) + (mode_info->atom_context->bios + data_offset); + /* The ras_rom_i2c_slave_addr should ideally + * be a 19-bit EEPROM address, which would be + * used as is by the driver; see top of + * amdgpu_eeprom.c. + * + * When this is the case, 0 is of course a + * valid RAS EEPROM address, in which case, + * we'll drop the first "if (firm...)" and only + * leave the check for the pointer. + * + * The reason this works right now is because + * ras_rom_i2c_slave_addr contains the EEPROM + * device type qualifier 1010b in the top 4 + * bits. + */ + if (firmware_info->v34.ras_rom_i2c_slave_addr) { + if (i2c_address) + *i2c_address = firmware_info->v34.ras_rom_i2c_slave_addr; + return true; + } + } + } + + return false; +} + + union smu_info { struct atom_smu_info_v3_1 v31; }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h index 1bbbb195015d..751248b253de 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h @@ -36,6 +36,7 @@ int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev); int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev); bool amdgpu_atomfirmware_mem_ecc_supported(struct amdgpu_device *adev); bool amdgpu_atomfirmware_sram_ecc_supported(struct amdgpu_device *adev); +bool amdgpu_atomfirmware_ras_rom_addr(struct amdgpu_device *adev, uint8_t* i2c_address); bool amdgpu_atomfirmware_mem_training_supported(struct amdgpu_device *adev); bool amdgpu_atomfirmware_dynamic_boot_config_supported(struct amdgpu_device *adev); int amdgpu_atomfirmware_get_fw_reserved_fb_size(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 617e39122ea3..0311d799a010 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -799,7 +799,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) if (r) return r; - r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); + r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false, NULL); if (r) return r; @@ -810,7 +810,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { bo_va = fpriv->csa_va; BUG_ON(!bo_va); - r = amdgpu_vm_bo_update(adev, bo_va, false); + r = amdgpu_vm_bo_update(adev, bo_va, false, NULL); if (r) return r; @@ -829,7 +829,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) if (bo_va == NULL) continue; - r = amdgpu_vm_bo_update(adev, bo_va, false); + r = amdgpu_vm_bo_update(adev, bo_va, false, NULL); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 536005bff24a..277128846dd1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -1414,7 +1414,7 @@ no_preempt: continue; } job = to_amdgpu_job(s_job); - if (preempted && job->fence == fence) + if (preempted && (&job->hw_fence) == fence) /* mark the job as preempted */ job->preemption_status |= AMDGPU_IB_PREEMPTED; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 6ca9fe54ee0a..41c6b3aacd37 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2829,12 +2829,11 @@ static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); - mutex_lock(&adev->gfx.gfx_off_mutex); - if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { - if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) - adev->gfx.gfx_off_state = true; - } - mutex_unlock(&adev->gfx.gfx_off_mutex); + WARN_ON_ONCE(adev->gfx.gfx_off_state); + WARN_ON_ONCE(adev->gfx.gfx_off_req_count); + + if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) + adev->gfx.gfx_off_state = true; } /** @@ -3647,9 +3646,9 @@ int amdgpu_device_init(struct amdgpu_device *adev, fence_driver_init: /* Fence driver */ - r = amdgpu_fence_driver_init(adev); + r = amdgpu_fence_driver_sw_init(adev); if (r) { - dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); + dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); goto failed; } @@ -3826,7 +3825,10 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) { dev_info(adev->dev, "amdgpu: finishing device.\n"); flush_delayed_work(&adev->delayed_init_work); - ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); + if (adev->mman.initialized) { + flush_delayed_work(&adev->mman.bdev.wq); + ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); + } adev->shutdown = true; /* make sure IB test finished before entering exclusive mode @@ -3989,7 +3991,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) } amdgpu_fence_driver_hw_init(adev); - r = amdgpu_device_ip_late_init(adev); if (r) return r; @@ -4449,7 +4450,7 @@ int amdgpu_device_mode1_reset(struct amdgpu_device *adev) int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context) { - int i, r = 0; + int i, j, r = 0; struct amdgpu_job *job = NULL; bool need_full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); @@ -4473,6 +4474,17 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; + /*clear job fence from fence drv to avoid force_completion + *leave NULL and vm flush fence in fence drv */ + for (j = 0; j <= ring->fence_drv.num_fences_mask; j++) { + struct dma_fence *old, **ptr; + + ptr = &ring->fence_drv.fences[j]; + old = rcu_dereference_protected(*ptr, 1); + if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) { + RCU_INIT_POINTER(*ptr, NULL); + } + } /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 43e7b61d1c5c..ada7bc19118a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -299,6 +299,9 @@ int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) ip->major, ip->minor, ip->revision); + if (le16_to_cpu(ip->hw_id) == VCN_HWID) + adev->vcn.num_vcn_inst++; + for (k = 0; k < num_base_address; k++) { /* * convert the endianness of base addresses in place, @@ -385,7 +388,7 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev) { struct binary_header *bhdr; struct harvest_table *harvest_info; - int i; + int i, vcn_harvest_count = 0; bhdr = (struct binary_header *)adev->mman.discovery_bin; harvest_info = (struct harvest_table *)(adev->mman.discovery_bin + @@ -397,8 +400,7 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev) switch (le32_to_cpu(harvest_info->list[i].hw_id)) { case VCN_HWID: - adev->harvest_ip_mask |= AMD_HARVEST_IP_VCN_MASK; - adev->harvest_ip_mask |= AMD_HARVEST_IP_JPEG_MASK; + vcn_harvest_count++; break; case DMU_HWID: adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK; @@ -407,6 +409,10 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev) break; } } + if (vcn_harvest_count == adev->vcn.num_vcn_inst) { + adev->harvest_ip_mask |= AMD_HARVEST_IP_VCN_MASK; + adev->harvest_ip_mask |= AMD_HARVEST_IP_JPEG_MASK; + } } int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index 8e5a7ac8c36f..7a7316731911 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -522,6 +522,7 @@ uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev, break; case CHIP_RENOIR: case CHIP_VANGOGH: + case CHIP_YELLOW_CARP: domain |= AMDGPU_GEM_DOMAIN_GTT; break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 2b0b0e8a6acb..f18240f87387 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -1181,7 +1181,12 @@ static const struct pci_device_id pciidlist[] = { {0x1002, 0x73A1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73A5, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73A8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73A9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73AB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73AC, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73AD, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73AE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73AF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73BF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, @@ -1197,6 +1202,11 @@ static const struct pci_device_id pciidlist[] = { {0x1002, 0x73C0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, {0x1002, 0x73C1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, {0x1002, 0x73C3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, + {0x1002, 0x73DA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, + {0x1002, 0x73DB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, + {0x1002, 0x73DC, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, + {0x1002, 0x73DD, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, + {0x1002, 0x73DE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, {0x1002, 0x73DF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVY_FLOUNDER}, /* DIMGREY_CAVEFISH */ @@ -1204,6 +1214,13 @@ static const struct pci_device_id pciidlist[] = { {0x1002, 0x73E1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, {0x1002, 0x73E2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, {0x1002, 0x73E3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73E8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73E9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73EA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73EB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73EC, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73ED, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, + {0x1002, 0x73EF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, {0x1002, 0x73FF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_DIMGREY_CAVEFISH}, /* Aldebaran */ @@ -1215,6 +1232,13 @@ static const struct pci_device_id pciidlist[] = { /* CYAN_SKILLFISH */ {0x1002, 0x13FE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYAN_SKILLFISH|AMD_IS_APU}, + /* BEIGE_GOBY */ + {0x1002, 0x7420, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BEIGE_GOBY}, + {0x1002, 0x7421, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BEIGE_GOBY}, + {0x1002, 0x7422, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BEIGE_GOBY}, + {0x1002, 0x7423, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BEIGE_GOBY}, + {0x1002, 0x743F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BEIGE_GOBY}, + {0, 0, 0} }; @@ -1231,7 +1255,7 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, int ret, retry = 0; bool supports_atomic = false; - if (!amdgpu_virtual_display && + if (amdgpu_virtual_display || amdgpu_device_asic_has_dc_support(flags & AMD_ASIC_MASK)) supports_atomic = true; @@ -1566,6 +1590,8 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) pci_ignore_hotplug(pdev); pci_set_power_state(pdev, PCI_D3cold); drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF; + } else if (amdgpu_device_supports_boco(drm_dev)) { + /* nothing to do */ } else if (amdgpu_device_supports_baco(drm_dev)) { amdgpu_device_baco_enter(drm_dev); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index 09b048647523..cd0acbea75da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -273,9 +273,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper, return 0; out: - if (abo) { - - } if (fb && ret) { drm_gem_object_put(gobj); drm_framebuffer_unregister_private(fb); @@ -344,7 +341,7 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev) } /* disable all the possible outputs/crtcs before entering KMS mode */ - if (!amdgpu_device_has_dc_support(adev)) + if (!amdgpu_device_has_dc_support(adev) && !amdgpu_virtual_display) drm_helper_disable_unused_functions(adev_to_drm(adev)); drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c index d94c5419ec25..5a6857c44bb6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c @@ -59,6 +59,7 @@ void amdgpu_show_fdinfo(struct seq_file *m, struct file *f) uint64_t vram_mem = 0, gtt_mem = 0, cpu_mem = 0; struct drm_file *file = f->private_data; struct amdgpu_device *adev = drm_to_adev(file->minor->dev); + struct amdgpu_bo *root; int ret; ret = amdgpu_file_to_fpriv(f, &fpriv); @@ -69,13 +70,19 @@ void amdgpu_show_fdinfo(struct seq_file *m, struct file *f) dev = PCI_SLOT(adev->pdev->devfn); fn = PCI_FUNC(adev->pdev->devfn); - ret = amdgpu_bo_reserve(fpriv->vm.root.bo, false); + root = amdgpu_bo_ref(fpriv->vm.root.bo); + if (!root) + return; + + ret = amdgpu_bo_reserve(root, false); if (ret) { DRM_ERROR("Fail to reserve bo\n"); return; } amdgpu_vm_get_memory(&fpriv->vm, &vram_mem, >t_mem, &cpu_mem); - amdgpu_bo_unreserve(fpriv->vm.root.bo); + amdgpu_bo_unreserve(root); + amdgpu_bo_unref(&root); + seq_printf(m, "pdev:\t%04x:%02x:%02x.%d\npasid:\t%u\n", domain, bus, dev, fn, fpriv->vm.pasid); seq_printf(m, "vram mem:\t%llu kB\n", vram_mem/1024UL); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index c65994e382bd..a25e192e8a3f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -129,30 +129,50 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring) * * @ring: ring the fence is associated with * @f: resulting fence object + * @job: job the fence is embedded in * @flags: flags to pass into the subordinate .emit_fence() call * * Emits a fence command on the requested ring (all asics). * Returns 0 on success, -ENOMEM on failure. */ -int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job, unsigned flags) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_fence *fence; + struct dma_fence *fence; + struct amdgpu_fence *am_fence; struct dma_fence __rcu **ptr; uint32_t seq; int r; - fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); - if (fence == NULL) - return -ENOMEM; + if (job == NULL) { + /* create a sperate hw fence */ + am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC); + if (am_fence == NULL) + return -ENOMEM; + fence = &am_fence->base; + am_fence->ring = ring; + } else { + /* take use of job-embedded fence */ + fence = &job->hw_fence; + } seq = ++ring->fence_drv.sync_seq; - fence->ring = ring; - dma_fence_init(&fence->base, &amdgpu_fence_ops, - &ring->fence_drv.lock, - adev->fence_context + ring->idx, - seq); + if (job != NULL && job->job_run_counter) { + /* reinit seq for resubmitted jobs */ + fence->seqno = seq; + } else { + dma_fence_init(fence, &amdgpu_fence_ops, + &ring->fence_drv.lock, + adev->fence_context + ring->idx, + seq); + } + + if (job != NULL) { + /* mark this fence has a parent job */ + set_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &fence->flags); + } + amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, seq, flags | AMDGPU_FENCE_FLAG_INT); pm_runtime_get_noresume(adev_to_drm(adev)->dev); @@ -175,9 +195,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, /* This function can't be called concurrently anyway, otherwise * emitting the fence would mess up the hardware ring buffer. */ - rcu_assign_pointer(*ptr, dma_fence_get(&fence->base)); + rcu_assign_pointer(*ptr, dma_fence_get(fence)); - *f = &fence->base; + *f = fence; return 0; } @@ -492,7 +512,7 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, } /** - * amdgpu_fence_driver_init - init the fence driver + * amdgpu_fence_driver_sw_init - init the fence driver * for all possible rings. * * @adev: amdgpu device pointer @@ -503,13 +523,13 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, * amdgpu_fence_driver_start_ring(). * Returns 0 for success. */ -int amdgpu_fence_driver_init(struct amdgpu_device *adev) +int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev) { return 0; } /** - * amdgpu_fence_driver_fini - tear down the fence driver + * amdgpu_fence_driver_hw_fini - tear down the fence driver * for all possible rings. * * @adev: amdgpu device pointer @@ -525,8 +545,10 @@ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev) if (!ring || !ring->fence_drv.initialized) continue; + if (!ring->no_scheduler) - drm_sched_fini(&ring->sched); + drm_sched_stop(&ring->sched, NULL); + /* You can't wait for HW to signal if it's gone */ if (!drm_dev_is_unplugged(&adev->ddev)) r = amdgpu_fence_wait_empty(ring); @@ -554,6 +576,9 @@ void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev) if (!ring || !ring->fence_drv.initialized) continue; + if (!ring->no_scheduler) + drm_sched_fini(&ring->sched); + for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) dma_fence_put(ring->fence_drv.fences[j]); kfree(ring->fence_drv.fences); @@ -583,6 +608,11 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev) if (!ring || !ring->fence_drv.initialized) continue; + if (!ring->no_scheduler) { + drm_sched_resubmit_jobs(&ring->sched); + drm_sched_start(&ring->sched, true); + } + /* enable the interrupt */ if (ring->fence_drv.irq_src) amdgpu_irq_get(adev, ring->fence_drv.irq_src, @@ -613,8 +643,16 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) { - struct amdgpu_fence *fence = to_amdgpu_fence(f); - return (const char *)fence->ring->name; + struct amdgpu_ring *ring; + + if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) { + struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence); + + ring = to_amdgpu_ring(job->base.sched); + } else { + ring = to_amdgpu_fence(f)->ring; + } + return (const char *)ring->name; } /** @@ -627,8 +665,15 @@ static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) */ static bool amdgpu_fence_enable_signaling(struct dma_fence *f) { - struct amdgpu_fence *fence = to_amdgpu_fence(f); - struct amdgpu_ring *ring = fence->ring; + struct amdgpu_ring *ring; + + if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) { + struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence); + + ring = to_amdgpu_ring(job->base.sched); + } else { + ring = to_amdgpu_fence(f)->ring; + } if (!timer_pending(&ring->fence_drv.fallback_timer)) amdgpu_fence_schedule_fallback(ring); @@ -646,8 +691,20 @@ static bool amdgpu_fence_enable_signaling(struct dma_fence *f) static void amdgpu_fence_free(struct rcu_head *rcu) { struct dma_fence *f = container_of(rcu, struct dma_fence, rcu); - struct amdgpu_fence *fence = to_amdgpu_fence(f); - kmem_cache_free(amdgpu_fence_slab, fence); + + if (test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &f->flags)) { + /* free job if fence has a parent job */ + struct amdgpu_job *job; + + job = container_of(f, struct amdgpu_job, hw_fence); + kfree(job); + } else { + /* free fence_slab if it's separated fence*/ + struct amdgpu_fence *fence; + + fence = to_amdgpu_fence(f); + kmem_cache_free(amdgpu_fence_slab, fence); + } } /** @@ -670,6 +727,7 @@ static const struct dma_fence_ops amdgpu_fence_ops = { .release = amdgpu_fence_release, }; + /* * Fence debugfs */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index b36405170ff3..76efd5f8950f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -76,7 +76,7 @@ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) if (adev->dummy_page_addr) return 0; adev->dummy_page_addr = dma_map_page(&adev->pdev->dev, dummy_page, 0, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + PAGE_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(&adev->pdev->dev, adev->dummy_page_addr)) { dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n"); adev->dummy_page_addr = 0; @@ -96,8 +96,8 @@ void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) { if (!adev->dummy_page_addr) return; - pci_unmap_page(adev->pdev, adev->dummy_page_addr, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(&adev->pdev->dev, adev->dummy_page_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); adev->dummy_page_addr = 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 611fd10c3410..d6aa032890ee 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -341,21 +341,18 @@ retry: r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain, flags, ttm_bo_type_device, resv, &gobj); - if (r) { - if (r != -ERESTARTSYS) { - if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { - flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; - goto retry; - } + if (r && r != -ERESTARTSYS) { + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + goto retry; + } - if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { - initial_domain |= AMDGPU_GEM_DOMAIN_GTT; - goto retry; - } - DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", - size, initial_domain, args->in.alignment, r); + if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { + initial_domain |= AMDGPU_GEM_DOMAIN_GTT; + goto retry; } - return r; + DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", + size, initial_domain, args->in.alignment, r); } if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { @@ -621,7 +618,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, if (operation == AMDGPU_VA_OP_MAP || operation == AMDGPU_VA_OP_REPLACE) { - r = amdgpu_vm_bo_update(adev, bo_va, false); + r = amdgpu_vm_bo_update(adev, bo_va, false, NULL); if (r) goto error; } @@ -904,7 +901,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, DIV_ROUND_UP(args->bpp, 8), 0); args->size = (u64)args->pitch * args->height; args->size = ALIGN(args->size, PAGE_SIZE); - domain = amdgpu_bo_get_preferred_pin_domain(adev, + domain = amdgpu_bo_get_preferred_domain(adev, amdgpu_display_supported_domains(adev, flags)); r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, ttm_bo_type_device, NULL, &gobj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index a0be0772c8b3..e7f06bd0f0cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -563,24 +563,38 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) mutex_lock(&adev->gfx.gfx_off_mutex); - if (!enable) - adev->gfx.gfx_off_req_count++; - else if (adev->gfx.gfx_off_req_count > 0) + if (enable) { + /* If the count is already 0, it means there's an imbalance bug somewhere. + * Note that the bug may be in a different caller than the one which triggers the + * WARN_ON_ONCE. + */ + if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0)) + goto unlock; + adev->gfx.gfx_off_req_count--; - if (enable && !adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { - schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); - } else if (!enable && adev->gfx.gfx_off_state) { - if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) { - adev->gfx.gfx_off_state = false; + if (adev->gfx.gfx_off_req_count == 0 && !adev->gfx.gfx_off_state) + schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); + } else { + if (adev->gfx.gfx_off_req_count == 0) { + cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); + + if (adev->gfx.gfx_off_state && + !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) { + adev->gfx.gfx_off_state = false; - if (adev->gfx.funcs->init_spm_golden) { - dev_dbg(adev->dev, "GFXOFF is disabled, re-init SPM golden settings\n"); - amdgpu_gfx_init_spm_golden(adev); + if (adev->gfx.funcs->init_spm_golden) { + dev_dbg(adev->dev, + "GFXOFF is disabled, re-init SPM golden settings\n"); + amdgpu_gfx_init_spm_golden(adev); + } } } + + adev->gfx.gfx_off_req_count++; } +unlock: mutex_unlock(&adev->gfx.gfx_off_mutex); } @@ -615,7 +629,6 @@ int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev) adev->gfx.ras_if->block = AMDGPU_RAS_BLOCK__GFX; adev->gfx.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->gfx.ras_if->sub_block_index = 0; - strcpy(adev->gfx.ras_if->name, "gfx"); } fs_info.head = ih_info.head = *adev->gfx.ras_if; r = amdgpu_ras_late_init(adev, adev->gfx.ras_if, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index d0b8d415b63b..c7797eac83c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -471,6 +471,27 @@ int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) return r; } + if (adev->mca.mp0.ras_funcs && + adev->mca.mp0.ras_funcs->ras_late_init) { + r = adev->mca.mp0.ras_funcs->ras_late_init(adev); + if (r) + return r; + } + + if (adev->mca.mp1.ras_funcs && + adev->mca.mp1.ras_funcs->ras_late_init) { + r = adev->mca.mp1.ras_funcs->ras_late_init(adev); + if (r) + return r; + } + + if (adev->mca.mpio.ras_funcs && + adev->mca.mpio.ras_funcs->ras_late_init) { + r = adev->mca.mpio.ras_funcs->ras_late_init(adev); + if (r) + return r; + } + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index 543000304a1c..675a72ef305d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -118,7 +118,7 @@ bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_resource *res) * @man: TTM memory type manager * @tbo: TTM BO we need this range for * @place: placement flags and restrictions - * @mem: the resulting mem object + * @res: the resulting mem object * * Dummy, allocate the node but no space for it yet. */ @@ -182,7 +182,7 @@ err_out: * amdgpu_gtt_mgr_del - free ranges * * @man: TTM memory type manager - * @mem: TTM memory object + * @res: TTM memory object * * Free the allocated GTT again. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c index 1d50d534d77c..a766e1aad2b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c @@ -41,7 +41,6 @@ int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev) adev->hdp.ras_if->block = AMDGPU_RAS_BLOCK__HDP; adev->hdp.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->hdp.ras_if->sub_block_index = 0; - strcpy(adev->hdp.ras_if->name, "hdp"); } ih_info.head = fs_info.head = *adev->hdp.ras_if; r = amdgpu_ras_late_init(adev, adev->hdp.ras_if, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c index bca4dddd5a15..82608df43396 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c @@ -339,7 +339,7 @@ static void amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus, void amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *amdgpu_connector) { - u8 val; + u8 val = 0; if (!amdgpu_connector->router.ddc_valid) return; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index ec65ab0ddf89..c076a6b9a5a2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -262,7 +262,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, fence_flags | AMDGPU_FENCE_FLAG_64BIT); } - r = amdgpu_fence_emit(ring, f, fence_flags); + r = amdgpu_fence_emit(ring, f, job, fence_flags); if (r) { dev_err(adev->dev, "failed to emit fence (%d)\n", r); if (job && job->vmid) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 3ac39b44a211..cc2e0c9cfe0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -509,7 +509,7 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, } else if ((client_id == AMDGPU_IRQ_CLIENTID_LEGACY) && adev->irq.virq[src_id]) { - generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id)); + generic_handle_domain_irq(adev->irq.domain, src_id); } else if (!adev->irq.client[client_id].sources) { DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index b8609cccc9c1..4e30a09bc887 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -127,11 +127,16 @@ void amdgpu_job_free_resources(struct amdgpu_job *job) { struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched); struct dma_fence *f; + struct dma_fence *hw_fence; unsigned i; - /* use sched fence if available */ - f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; + if (job->hw_fence.ops == NULL) + hw_fence = job->external_hw_fence; + else + hw_fence = &job->hw_fence; + /* use sched fence if available */ + f = job->base.s_fence ? &job->base.s_fence->finished : hw_fence; for (i = 0; i < job->num_ibs; ++i) amdgpu_ib_free(ring->adev, &job->ibs[i], f); } @@ -142,20 +147,27 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job) drm_sched_job_cleanup(s_job); - dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); - kfree(job); + + /* only put the hw fence if has embedded fence */ + if (job->hw_fence.ops != NULL) + dma_fence_put(&job->hw_fence); + else + kfree(job); } void amdgpu_job_free(struct amdgpu_job *job) { amdgpu_job_free_resources(job); - - dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); - kfree(job); + + /* only put the hw fence if has embedded fence */ + if (job->hw_fence.ops != NULL) + dma_fence_put(&job->hw_fence); + else + kfree(job); } int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, @@ -186,11 +198,14 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, job->base.sched = &ring->sched; r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence); - job->fence = dma_fence_get(*fence); + /* record external_hw_fence for direct submit */ + job->external_hw_fence = dma_fence_get(*fence); if (r) return r; amdgpu_job_free(job); + dma_fence_put(*fence); + return 0; } @@ -248,10 +263,12 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) if (r) DRM_ERROR("Error scheduling IBs (%d)\n", r); } - /* if gpu reset, hw fence will be replaced here */ - dma_fence_put(job->fence); - job->fence = dma_fence_get(fence); + if (!job->job_run_counter) + dma_fence_get(fence); + else if (finished->error < 0) + dma_fence_put(&job->hw_fence); + job->job_run_counter++; amdgpu_job_free_resources(job); fence = r ? ERR_PTR(r) : fence; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index 81caac9b958a..9e65730193b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -46,7 +46,8 @@ struct amdgpu_job { struct amdgpu_sync sync; struct amdgpu_sync sched_sync; struct amdgpu_ib *ibs; - struct dma_fence *fence; /* the hw fence */ + struct dma_fence hw_fence; + struct dma_fence *external_hw_fence; uint32_t preamble_status; uint32_t preemption_status; uint32_t num_ibs; @@ -62,6 +63,9 @@ struct amdgpu_job { /* user fence handling */ uint64_t uf_addr; uint64_t uf_sequence; + + /* job_run_counter >= 1 means a resubmit job */ + uint32_t job_run_counter; }; int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c index 8996cb4ed57a..9342aa23ebd2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c @@ -47,8 +47,6 @@ int amdgpu_jpeg_sw_fini(struct amdgpu_device *adev) { int i; - cancel_delayed_work_sync(&adev->jpeg.idle_work); - for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { if (adev->jpeg.harvest_config & (1 << i)) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 20b049ad61c1..7e45640fbee0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -341,27 +341,27 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, switch (query_fw->index) { case TA_FW_TYPE_PSP_XGMI: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_xgmi_ucode_version; + fw_info->feature = adev->psp.xgmi.feature_version; break; case TA_FW_TYPE_PSP_RAS: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_ras_ucode_version; + fw_info->feature = adev->psp.ras.feature_version; break; case TA_FW_TYPE_PSP_HDCP: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_hdcp_ucode_version; + fw_info->feature = adev->psp.hdcp.feature_version; break; case TA_FW_TYPE_PSP_DTM: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_dtm_ucode_version; + fw_info->feature = adev->psp.dtm.feature_version; break; case TA_FW_TYPE_PSP_RAP: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_rap_ucode_version; + fw_info->feature = adev->psp.rap.feature_version; break; case TA_FW_TYPE_PSP_SECUREDISPLAY: fw_info->ver = adev->psp.ta_fw_version; - fw_info->feature = adev->psp.ta_securedisplay_ucode_version; + fw_info->feature = adev->psp.securedisplay.feature_version; break; default: return -EINVAL; @@ -378,8 +378,8 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->feature = adev->psp.sos.feature_version; break; case AMDGPU_INFO_FW_ASD: - fw_info->ver = adev->psp.asd_fw_version; - fw_info->feature = adev->psp.asd_feature_version; + fw_info->ver = adev->psp.asd.fw_version; + fw_info->feature = adev->psp.asd.feature_version; break; case AMDGPU_INFO_FW_DMCU: fw_info->ver = adev->dm.dmcu_fw_version; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c new file mode 100644 index 000000000000..a2d3dbbf7d25 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c @@ -0,0 +1,117 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu_ras.h" +#include "amdgpu.h" +#include "amdgpu_mca.h" + +#include "umc/umc_6_7_0_offset.h" +#include "umc/umc_6_7_0_sh_mask.h" + +void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + unsigned long *error_count) +{ + uint64_t mc_status = RREG64_PCIE(mc_status_addr * 4); + + if (REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && + REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1) + *error_count += 1; +} + +void amdgpu_mca_query_uncorrectable_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + unsigned long *error_count) +{ + uint64_t mc_status = RREG64_PCIE(mc_status_addr * 4); + + if ((REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) && + (REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 || + REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || + REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 || + REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 || + REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1)) + *error_count += 1; +} + +void amdgpu_mca_reset_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr) +{ + WREG64_PCIE(mc_status_addr * 4, 0x0ULL); +} + +void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + void *ras_error_status) +{ + struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; + + amdgpu_mca_query_correctable_error_count(adev, mc_status_addr, &(err_data->ce_count)); + amdgpu_mca_query_uncorrectable_error_count(adev, mc_status_addr, &(err_data->ue_count)); + + amdgpu_mca_reset_error_count(adev, mc_status_addr); +} + +int amdgpu_mca_ras_late_init(struct amdgpu_device *adev, + struct amdgpu_mca_ras *mca_dev) +{ + int r; + struct ras_ih_if ih_info = { + .cb = NULL, + }; + struct ras_fs_if fs_info = { + .sysfs_name = mca_dev->ras_funcs->sysfs_name, + }; + + if (!mca_dev->ras_if) { + mca_dev->ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL); + if (!mca_dev->ras_if) + return -ENOMEM; + mca_dev->ras_if->block = mca_dev->ras_funcs->ras_block; + mca_dev->ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; + mca_dev->ras_if->sub_block_index = 0; + } + ih_info.head = fs_info.head = *mca_dev->ras_if; + r = amdgpu_ras_late_init(adev, mca_dev->ras_if, + &fs_info, &ih_info); + if (r || !amdgpu_ras_is_supported(adev, mca_dev->ras_if->block)) { + kfree(mca_dev->ras_if); + mca_dev->ras_if = NULL; + } + + return r; +} + +void amdgpu_mca_ras_fini(struct amdgpu_device *adev, + struct amdgpu_mca_ras *mca_dev) +{ + struct ras_ih_if ih_info = { + .cb = NULL, + }; + + if (!mca_dev->ras_if) + return; + + amdgpu_ras_late_fini(adev, mca_dev->ras_if, &ih_info); + kfree(mca_dev->ras_if); + mca_dev->ras_if = NULL; +}
\ No newline at end of file diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h new file mode 100644 index 000000000000..f860f2f0e296 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __AMDGPU_MCA_H__ +#define __AMDGPU_MCA_H__ + +struct amdgpu_mca_ras_funcs { + int (*ras_late_init)(struct amdgpu_device *adev); + void (*ras_fini)(struct amdgpu_device *adev); + void (*query_ras_error_count)(struct amdgpu_device *adev, + void *ras_error_status); + void (*query_ras_error_address)(struct amdgpu_device *adev, + void *ras_error_status); + uint32_t ras_block; + const char* sysfs_name; +}; + +struct amdgpu_mca_ras { + struct ras_common_if *ras_if; + const struct amdgpu_mca_ras_funcs *ras_funcs; +}; + +struct amdgpu_mca_funcs { + void (*init)(struct amdgpu_device *adev); +}; + +struct amdgpu_mca { + const struct amdgpu_mca_funcs *funcs; + struct amdgpu_mca_ras mp0; + struct amdgpu_mca_ras mp1; + struct amdgpu_mca_ras mpio; +}; + +void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + unsigned long *error_count); + +void amdgpu_mca_query_uncorrectable_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + unsigned long *error_count); + +void amdgpu_mca_reset_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr); + +void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev, + uint64_t mc_status_addr, + void *ras_error_status); + +int amdgpu_mca_ras_late_init(struct amdgpu_device *adev, + struct amdgpu_mca_ras *mca_dev); + +void amdgpu_mca_ras_fini(struct amdgpu_device *adev, + struct amdgpu_mca_ras *mca_dev); + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c index ead3dc572ec5..24297dc51434 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c @@ -41,7 +41,6 @@ int amdgpu_mmhub_ras_late_init(struct amdgpu_device *adev) adev->mmhub.ras_if->block = AMDGPU_RAS_BLOCK__MMHUB; adev->mmhub.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mmhub.ras_if->sub_block_index = 0; - strcpy(adev->mmhub.ras_if->name, "mmhub"); } ih_info.head = fs_info.head = *adev->mmhub.ras_if; r = amdgpu_ras_late_init(adev, adev->mmhub.ras_if, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c index 6201a5f4b4fa..6afb02fef8cf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c @@ -39,7 +39,6 @@ int amdgpu_nbio_ras_late_init(struct amdgpu_device *adev) adev->nbio.ras_if->block = AMDGPU_RAS_BLOCK__PCIE_BIF; adev->nbio.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->nbio.ras_if->sub_block_index = 0; - strcpy(adev->nbio.ras_if->name, "pcie_bif"); } ih_info.head = fs_info.head = *adev->nbio.ras_if; r = amdgpu_ras_late_init(adev, adev->nbio.ras_if, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index d15eee98204d..01a78c786536 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -920,11 +920,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, return -EINVAL; } - /* This assumes only APU display buffers are pinned with (VRAM|GTT). - * See function amdgpu_display_supported_domains() - */ - domain = amdgpu_bo_get_preferred_pin_domain(adev, domain); - if (bo->tbo.pin_count) { uint32_t mem_type = bo->tbo.resource->mem_type; uint32_t mem_flags = bo->tbo.resource->placement; @@ -949,6 +944,11 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, return 0; } + /* This assumes only APU display buffers are pinned with (VRAM|GTT). + * See function amdgpu_display_supported_domains() + */ + domain = amdgpu_bo_get_preferred_domain(adev, domain); + if (bo->tbo.base.import_attach) dma_buf_pin(bo->tbo.base.import_attach); @@ -1518,14 +1518,14 @@ u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) } /** - * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout + * amdgpu_bo_get_preferred_domain - get preferred domain * @adev: amdgpu device object * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` * * Returns: - * Which of the allowed domains is preferred for pinning the BO for scanout. + * Which of the allowed domains is preferred for allocating the BO. */ -uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, +uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, uint32_t domain) { if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index e72f329e7f18..9d6c001c15f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -333,7 +333,7 @@ void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem, void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo); int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence); -uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, +uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, uint32_t domain); /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c index f2e20666c9c1..4eaec446b49d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c @@ -80,12 +80,17 @@ static void amdgpu_pll_reduce_ratio(unsigned *nom, unsigned *den, * Calculate feedback and reference divider for a given post divider. Makes * sure we stay within the limits. */ -static void amdgpu_pll_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_div, - unsigned fb_div_max, unsigned ref_div_max, - unsigned *fb_div, unsigned *ref_div) +static void amdgpu_pll_get_fb_ref_div(struct amdgpu_device *adev, unsigned int nom, + unsigned int den, unsigned int post_div, + unsigned int fb_div_max, unsigned int ref_div_max, + unsigned int *fb_div, unsigned int *ref_div) { + /* limit reference * post divider to a maximum */ - ref_div_max = min(128 / post_div, ref_div_max); + if (adev->family == AMDGPU_FAMILY_SI) + ref_div_max = min(100 / post_div, ref_div_max); + else + ref_div_max = min(128 / post_div, ref_div_max); /* get matching reference and feedback divider */ *ref_div = min(max(DIV_ROUND_CLOSEST(den, post_div), 1u), ref_div_max); @@ -112,7 +117,8 @@ static void amdgpu_pll_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_ * Try to calculate the PLL parameters to generate the given frequency: * dot_clock = (ref_freq * feedback_div) / (ref_div * post_div) */ -void amdgpu_pll_compute(struct amdgpu_pll *pll, +void amdgpu_pll_compute(struct amdgpu_device *adev, + struct amdgpu_pll *pll, u32 freq, u32 *dot_clock_p, u32 *fb_div_p, @@ -199,7 +205,7 @@ void amdgpu_pll_compute(struct amdgpu_pll *pll, for (post_div = post_div_min; post_div <= post_div_max; ++post_div) { unsigned diff; - amdgpu_pll_get_fb_ref_div(nom, den, post_div, fb_div_max, + amdgpu_pll_get_fb_ref_div(adev, nom, den, post_div, fb_div_max, ref_div_max, &fb_div, &ref_div); diff = abs(target_clock - (pll->reference_freq * fb_div) / (ref_div * post_div)); @@ -214,7 +220,7 @@ void amdgpu_pll_compute(struct amdgpu_pll *pll, post_div = post_div_best; /* get the feedback and reference divider for the optimal value */ - amdgpu_pll_get_fb_ref_div(nom, den, post_div, fb_div_max, ref_div_max, + amdgpu_pll_get_fb_ref_div(adev, nom, den, post_div, fb_div_max, ref_div_max, &fb_div, &ref_div); /* reduce the numbers to a simpler ratio once more */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.h index db6136f68b82..44a583d6c9b4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.h @@ -24,7 +24,8 @@ #ifndef __AMDGPU_PLL_H__ #define __AMDGPU_PLL_H__ -void amdgpu_pll_compute(struct amdgpu_pll *pll, +void amdgpu_pll_compute(struct amdgpu_device *adev, + struct amdgpu_pll *pll, u32 freq, u32 *dot_clock_p, u32 *fb_div_p, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 10b046147190..9b41cb8c3de5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -29,6 +29,7 @@ #include "amdgpu.h" #include "amdgpu_psp.h" #include "amdgpu_ucode.h" +#include "amdgpu_xgmi.h" #include "soc15_common.h" #include "psp_v3_1.h" #include "psp_v10_0.h" @@ -41,8 +42,6 @@ #include "amdgpu_securedisplay.h" #include "amdgpu_atomfirmware.h" -#include <drm/drm_drv.h> - static int psp_sysfs_init(struct amdgpu_device *adev); static void psp_sysfs_fini(struct amdgpu_device *adev); @@ -253,6 +252,12 @@ static int psp_sw_init(void *handle) struct psp_runtime_boot_cfg_entry boot_cfg_entry; struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx; + psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!psp->cmd) { + DRM_ERROR("Failed to allocate memory to command buffer!\n"); + ret = -ENOMEM; + } + if (!amdgpu_sriov_vf(adev)) { ret = psp_init_microcode(psp); if (ret) { @@ -315,25 +320,30 @@ static int psp_sw_init(void *handle) static int psp_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + struct psp_gfx_cmd_resp *cmd = psp->cmd; - psp_memory_training_fini(&adev->psp); - if (adev->psp.sos_fw) { - release_firmware(adev->psp.sos_fw); - adev->psp.sos_fw = NULL; + psp_memory_training_fini(psp); + if (psp->sos_fw) { + release_firmware(psp->sos_fw); + psp->sos_fw = NULL; } - if (adev->psp.asd_fw) { - release_firmware(adev->psp.asd_fw); - adev->psp.asd_fw = NULL; + if (psp->asd_fw) { + release_firmware(psp->asd_fw); + psp->asd_fw = NULL; } - if (adev->psp.ta_fw) { - release_firmware(adev->psp.ta_fw); - adev->psp.ta_fw = NULL; + if (psp->ta_fw) { + release_firmware(psp->ta_fw); + psp->ta_fw = NULL; } if (adev->asic_type == CHIP_NAVI10 || adev->asic_type == CHIP_SIENNA_CICHLID) psp_sysfs_fini(adev); + kfree(cmd); + cmd = NULL; + return 0; } @@ -417,8 +427,6 @@ psp_cmd_submit_buf(struct psp_context *psp, if (!drm_dev_enter(&psp->adev->ddev, &idx)) return 0; - mutex_lock(&psp->mutex); - memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE); memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp)); @@ -461,10 +469,10 @@ psp_cmd_submit_buf(struct psp_context *psp, */ if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) { if (ucode) - DRM_WARN("failed to load ucode (%s) ", - amdgpu_ucode_name(ucode->ucode_id)); - DRM_WARN("psp gfx command (%s) failed and response status is (0x%X)\n", - psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), + DRM_WARN("failed to load ucode %s(0x%X) ", + amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id); + DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n", + psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id, psp->cmd_buf_mem->resp.status); if (!timeout) { ret = -EINVAL; @@ -478,11 +486,26 @@ psp_cmd_submit_buf(struct psp_context *psp, } exit: - mutex_unlock(&psp->mutex); drm_dev_exit(idx); return ret; } +static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp) +{ + struct psp_gfx_cmd_resp *cmd = psp->cmd; + + mutex_lock(&psp->mutex); + + memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); + + return cmd; +} + +void release_psp_cmd_buf(struct psp_context *psp) +{ + mutex_unlock(&psp->mutex); +} + static void psp_prep_tmr_cmd_buf(struct psp_context *psp, struct psp_gfx_cmd_resp *cmd, uint64_t tmr_mc, struct amdgpu_bo *tmr_bo) @@ -517,11 +540,8 @@ static int psp_load_toc(struct psp_context *psp, uint32_t *tmr_size) { int ret; - struct psp_gfx_cmd_resp *cmd; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; /* Copy toc to psp firmware private buffer */ psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); @@ -531,7 +551,9 @@ static int psp_load_toc(struct psp_context *psp, psp->fence_buf_mc_addr); if (!ret) *tmr_size = psp->cmd_buf_mem->resp.tmr_size; - kfree(cmd); + + release_psp_cmd_buf(psp); + return ret; } @@ -596,9 +618,7 @@ static int psp_tmr_load(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo); DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n", @@ -607,13 +627,13 @@ static int psp_tmr_load(struct psp_context *psp) ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp, - struct psp_gfx_cmd_resp *cmd) + struct psp_gfx_cmd_resp *cmd) { if (amdgpu_sriov_vf(psp->adev)) cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR; @@ -624,11 +644,7 @@ static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp, static int psp_tmr_unload(struct psp_context *psp) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); psp_prep_tmr_unload_cmd_buf(psp, cmd); DRM_INFO("free PSP TMR buffer\n"); @@ -636,7 +652,7 @@ static int psp_tmr_unload(struct psp_context *psp) ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -670,9 +686,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION; @@ -684,7 +698,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32); } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -692,13 +706,13 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg) { struct psp_context *psp = &adev->psp; - struct psp_gfx_cmd_resp *cmd = psp->cmd; + struct psp_gfx_cmd_resp *cmd; int ret; if (amdgpu_sriov_vf(adev)) return 0; - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); + cmd = acquire_psp_cmd_buf(psp); cmd->cmd_id = GFX_CMD_ID_BOOT_CFG; cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET; @@ -709,47 +723,59 @@ static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg) (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0; } + release_psp_cmd_buf(psp); + return ret; } static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg) { + int ret; struct psp_context *psp = &adev->psp; - struct psp_gfx_cmd_resp *cmd = psp->cmd; + struct psp_gfx_cmd_resp *cmd; if (amdgpu_sriov_vf(adev)) return 0; - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); + cmd = acquire_psp_cmd_buf(psp); cmd->cmd_id = GFX_CMD_ID_BOOT_CFG; cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET; cmd->cmd.boot_cfg.boot_config = boot_cfg; cmd->cmd.boot_cfg.boot_config_valid = boot_cfg; - return psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + release_psp_cmd_buf(psp); + + return ret; } static int psp_rl_load(struct amdgpu_device *adev) { + int ret; struct psp_context *psp = &adev->psp; - struct psp_gfx_cmd_resp *cmd = psp->cmd; + struct psp_gfx_cmd_resp *cmd; if (!is_psp_fw_valid(psp->rl)) return 0; + cmd = acquire_psp_cmd_buf(psp); + memset(psp->fw_pri_buf, 0, PSP_1_MEG); memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes); - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); - cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr); cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr); cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes; cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST; - return psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + release_psp_cmd_buf(psp); + + return ret; } static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, @@ -774,17 +800,15 @@ static int psp_asd_load(struct psp_context *psp) * add workaround to bypass it for sriov now. * TODO: add version check to make it common */ - if (amdgpu_sriov_vf(psp->adev) || !psp->asd_ucode_size) + if (amdgpu_sriov_vf(psp->adev) || !psp->asd.size_bytes) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_copy_fw(psp, psp->asd_start_addr, psp->asd_ucode_size); + psp_copy_fw(psp, psp->asd.start_addr, psp->asd.size_bytes); psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->asd_ucode_size); + psp->asd.size_bytes); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); @@ -793,7 +817,7 @@ static int psp_asd_load(struct psp_context *psp) psp->asd_context.session_id = cmd->resp.session_id; } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -816,9 +840,7 @@ static int psp_asd_unload(struct psp_context *psp) if (!psp->asd_context.asd_initialized) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id); @@ -827,7 +849,7 @@ static int psp_asd_unload(struct psp_context *psp) if (!ret) psp->asd_context.asd_initialized = false; - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -843,22 +865,21 @@ static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd, int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg, uint32_t value) { - struct psp_gfx_cmd_resp *cmd = NULL; + struct psp_gfx_cmd_resp *cmd; int ret = 0; if (reg >= PSP_REG_LAST) return -EINVAL; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); psp_prep_reg_prog_cmd_buf(cmd, reg, value); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (ret) DRM_ERROR("PSP failed to program reg id %d", reg); - kfree(cmd); + release_psp_cmd_buf(psp); + return ret; } @@ -878,23 +899,37 @@ static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, cmd->cmd.cmd_load_ta.cmd_buf_len = ta_shared_size; } -static int psp_xgmi_init_shared_buf(struct psp_context *psp) +static int psp_ta_init_shared_buf(struct psp_context *psp, + struct ta_mem_context *mem_ctx, + uint32_t shared_mem_size) { int ret; /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for xgmi ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->xgmi_context.xgmi_shared_bo, - &psp->xgmi_context.xgmi_shared_mc_addr, - &psp->xgmi_context.xgmi_shared_buf); + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for ta to host memory + */ + ret = amdgpu_bo_create_kernel(psp->adev, shared_mem_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &mem_ctx->shared_bo, + &mem_ctx->shared_mc_addr, + &mem_ctx->shared_buf); return ret; } +static void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx) +{ + amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr, + &mem_ctx->shared_buf); +} + +static int psp_xgmi_init_shared_buf(struct psp_context *psp) +{ + return psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context, + PSP_XGMI_SHARED_MEM_SIZE); +} + static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd, uint32_t ta_cmd_id, uint32_t session_id) @@ -909,18 +944,14 @@ static int psp_ta_invoke(struct psp_context *psp, uint32_t session_id) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -934,27 +965,25 @@ static int psp_xgmi_load(struct psp_context *psp) * TODO: bypass the loading in sriov for now */ - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_copy_fw(psp, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size); + psp_copy_fw(psp, psp->xgmi.start_addr, psp->xgmi.size_bytes); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_xgmi_ucode_size, - psp->xgmi_context.xgmi_shared_mc_addr, + psp->xgmi.size_bytes, + psp->xgmi_context.context.mem_context.shared_mc_addr, PSP_XGMI_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (!ret) { - psp->xgmi_context.initialized = 1; - psp->xgmi_context.session_id = cmd->resp.session_id; + psp->xgmi_context.context.initialized = true; + psp->xgmi_context.context.session_id = cmd->resp.session_id; } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -974,57 +1003,56 @@ static int psp_xgmi_unload(struct psp_context *psp) * TODO: bypass the unloading in sriov for now */ - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id) { - return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id); + return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.context.session_id); } int psp_xgmi_terminate(struct psp_context *psp) { int ret; - if (!psp->xgmi_context.initialized) + if (!psp->xgmi_context.context.initialized) return 0; ret = psp_xgmi_unload(psp); if (ret) return ret; - psp->xgmi_context.initialized = 0; + psp->xgmi_context.context.initialized = false; /* free xgmi shared memory */ - amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo, - &psp->xgmi_context.xgmi_shared_mc_addr, - &psp->xgmi_context.xgmi_shared_buf); + psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context); return 0; } -int psp_xgmi_initialize(struct psp_context *psp) +int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta) { struct ta_xgmi_shared_memory *xgmi_cmd; int ret; - if (!psp->adev->psp.ta_fw || - !psp->adev->psp.ta_xgmi_ucode_size || - !psp->adev->psp.ta_xgmi_start_addr) + if (!psp->ta_fw || + !psp->xgmi.size_bytes || + !psp->xgmi.start_addr) return -ENOENT; - if (!psp->xgmi_context.initialized) { + if (!load_ta) + goto invoke; + + if (!psp->xgmi_context.context.initialized) { ret = psp_xgmi_init_shared_buf(psp); if (ret) return ret; @@ -1035,9 +1063,11 @@ int psp_xgmi_initialize(struct psp_context *psp) if (ret) return ret; +invoke: /* Initialize XGMI session */ - xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf); + xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf); memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); + xgmi_cmd->flag_extend_link_record = set_extended_data; xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE; ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id); @@ -1050,7 +1080,7 @@ int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id) struct ta_xgmi_shared_memory *xgmi_cmd; int ret; - xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.xgmi_shared_buf; + xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf; memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID; @@ -1070,7 +1100,7 @@ int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id) struct ta_xgmi_shared_memory *xgmi_cmd; int ret; - xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.xgmi_shared_buf; + xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf; memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID; @@ -1088,12 +1118,59 @@ int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id) static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp) { return psp->adev->asic_type == CHIP_ALDEBARAN && - psp->ta_xgmi_ucode_version >= 0x2000000b; + psp->xgmi.feature_version >= 0x2000000b; +} + +/* + * Chips that support extended topology information require the driver to + * reflect topology information in the opposite direction. This is + * because the TA has already exceeded its link record limit and if the + * TA holds bi-directional information, the driver would have to do + * multiple fetches instead of just two. + */ +static void psp_xgmi_reflect_topology_info(struct psp_context *psp, + struct psp_xgmi_node_info node_info) +{ + struct amdgpu_device *mirror_adev; + struct amdgpu_hive_info *hive; + uint64_t src_node_id = psp->adev->gmc.xgmi.node_id; + uint64_t dst_node_id = node_info.node_id; + uint8_t dst_num_hops = node_info.num_hops; + uint8_t dst_num_links = node_info.num_links; + + hive = amdgpu_get_xgmi_hive(psp->adev); + list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) { + struct psp_xgmi_topology_info *mirror_top_info; + int j; + + if (mirror_adev->gmc.xgmi.node_id != dst_node_id) + continue; + + mirror_top_info = &mirror_adev->psp.xgmi_context.top_info; + for (j = 0; j < mirror_top_info->num_nodes; j++) { + if (mirror_top_info->nodes[j].node_id != src_node_id) + continue; + + mirror_top_info->nodes[j].num_hops = dst_num_hops; + /* + * prevent 0 num_links value re-reflection since reflection + * criteria is based on num_hops (direct or indirect). + * + */ + if (dst_num_links) + mirror_top_info->nodes[j].num_links = dst_num_links; + + break; + } + + break; + } } int psp_xgmi_get_topology_info(struct psp_context *psp, int number_devices, - struct psp_xgmi_topology_info *topology) + struct psp_xgmi_topology_info *topology, + bool get_extended_data) { struct ta_xgmi_shared_memory *xgmi_cmd; struct ta_xgmi_cmd_get_topology_info_input *topology_info_input; @@ -1104,8 +1181,9 @@ int psp_xgmi_get_topology_info(struct psp_context *psp, if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES) return -EINVAL; - xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.xgmi_shared_buf; + xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf; memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); + xgmi_cmd->flag_extend_link_record = get_extended_data; /* Fill in the shared memory with topology information as input */ topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info; @@ -1128,10 +1206,19 @@ int psp_xgmi_get_topology_info(struct psp_context *psp, topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info; topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes; for (i = 0; i < topology->num_nodes; i++) { - topology->nodes[i].node_id = topology_info_output->nodes[i].node_id; - topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops; - topology->nodes[i].is_sharing_enabled = topology_info_output->nodes[i].is_sharing_enabled; - topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine; + /* extended data will either be 0 or equal to non-extended data */ + if (topology_info_output->nodes[i].num_hops) + topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops; + + /* non-extended data gets everything here so no need to update */ + if (!get_extended_data) { + topology->nodes[i].node_id = topology_info_output->nodes[i].node_id; + topology->nodes[i].is_sharing_enabled = + topology_info_output->nodes[i].is_sharing_enabled; + topology->nodes[i].sdma_engine = + topology_info_output->nodes[i].sdma_engine; + } + } /* Invoke xgmi ta again to get the link information */ @@ -1146,9 +1233,18 @@ int psp_xgmi_get_topology_info(struct psp_context *psp, return ret; link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info; - for (i = 0; i < topology->num_nodes; i++) - topology->nodes[i].num_links = + for (i = 0; i < topology->num_nodes; i++) { + /* accumulate num_links on extended data */ + topology->nodes[i].num_links = get_extended_data ? + topology->nodes[i].num_links + + link_info_output->nodes[i].num_links : link_info_output->nodes[i].num_links; + + /* reflect the topology information for bi-directionality */ + if (psp->xgmi_context.supports_extended_data && + get_extended_data && topology->nodes[i].num_hops) + psp_xgmi_reflect_topology_info(psp, topology->nodes[i]); + } } return 0; @@ -1165,7 +1261,7 @@ int psp_xgmi_set_topology_info(struct psp_context *psp, if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES) return -EINVAL; - xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.xgmi_shared_buf; + xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf; memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info; @@ -1186,19 +1282,8 @@ int psp_xgmi_set_topology_info(struct psp_context *psp, // ras begin static int psp_ras_init_shared_buf(struct psp_context *psp) { - int ret; - - /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for ras ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->ras.ras_shared_bo, - &psp->ras.ras_shared_mc_addr, - &psp->ras.ras_shared_buf); - - return ret; + return psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context, + PSP_RAS_SHARED_MEM_SIZE); } static int psp_ras_load(struct psp_context *psp) @@ -1213,42 +1298,40 @@ static int psp_ras_load(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + psp_copy_fw(psp, psp->ras.start_addr, psp->ras.size_bytes); - psp_copy_fw(psp, psp->ta_ras_start_addr, psp->ta_ras_ucode_size); - - ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; + ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf; if (psp->adev->gmc.xgmi.connected_to_cpu) ras_cmd->ras_in_message.init_flags.poison_mode_en = 1; else ras_cmd->ras_in_message.init_flags.dgpu_mode = 1; + cmd = acquire_psp_cmd_buf(psp); + psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_ras_ucode_size, - psp->ras.ras_shared_mc_addr, + psp->ras.size_bytes, + psp->ras_context.context.mem_context.shared_mc_addr, PSP_RAS_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (!ret) { - psp->ras.session_id = cmd->resp.session_id; + psp->ras_context.context.session_id = cmd->resp.session_id; if (!ras_cmd->ras_status) - psp->ras.ras_initialized = true; + psp->ras_context.context.initialized = true; else dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status); } + release_psp_cmd_buf(psp); + if (ret || ras_cmd->ras_status) amdgpu_ras_fini(psp->adev); - kfree(cmd); - return ret; } @@ -1263,16 +1346,14 @@ static int psp_ras_unload(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->ras_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1282,7 +1363,7 @@ int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) struct ta_ras_shared_memory *ras_cmd; int ret; - ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; + ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf; /* * TODO: bypass the loading in sriov for now @@ -1290,7 +1371,7 @@ int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) if (amdgpu_sriov_vf(psp->adev)) return 0; - ret = psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id); + ret = psp_ta_invoke(psp, ta_cmd_id, psp->ras_context.context.session_id); if (amdgpu_ras_intr_triggered()) return ret; @@ -1346,10 +1427,10 @@ int psp_ras_enable_features(struct psp_context *psp, struct ta_ras_shared_memory *ras_cmd; int ret; - if (!psp->ras.ras_initialized) + if (!psp->ras_context.context.initialized) return -EINVAL; - ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; + ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf; memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory)); if (enable) @@ -1376,19 +1457,17 @@ static int psp_ras_terminate(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->ras.ras_initialized) + if (!psp->ras_context.context.initialized) return 0; ret = psp_ras_unload(psp); if (ret) return ret; - psp->ras.ras_initialized = false; + psp->ras_context.context.initialized = false; /* free ras shared memory */ - amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo, - &psp->ras.ras_shared_mc_addr, - &psp->ras.ras_shared_buf); + psp_ta_free_shared_buf(&psp->ras_context.context.mem_context); return 0; } @@ -1405,8 +1484,8 @@ static int psp_ras_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(adev)) return 0; - if (!adev->psp.ta_ras_ucode_size || - !adev->psp.ta_ras_start_addr) { + if (!adev->psp.ras.size_bytes || + !adev->psp.ras.start_addr) { dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n"); return 0; } @@ -1452,7 +1531,7 @@ static int psp_ras_initialize(struct psp_context *psp) } } - if (!psp->ras.ras_initialized) { + if (!psp->ras_context.context.initialized) { ret = psp_ras_init_shared_buf(psp); if (ret) return ret; @@ -1471,10 +1550,10 @@ int psp_ras_trigger_error(struct psp_context *psp, struct ta_ras_shared_memory *ras_cmd; int ret; - if (!psp->ras.ras_initialized) + if (!psp->ras_context.context.initialized) return -EINVAL; - ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; + ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf; memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory)); ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR; @@ -1496,19 +1575,8 @@ int psp_ras_trigger_error(struct psp_context *psp, // HDCP start static int psp_hdcp_init_shared_buf(struct psp_context *psp) { - int ret; - - /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for hdcp ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->hdcp_context.hdcp_shared_bo, - &psp->hdcp_context.hdcp_shared_mc_addr, - &psp->hdcp_context.hdcp_shared_buf); - - return ret; + return psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context, + PSP_HDCP_SHARED_MEM_SIZE); } static int psp_hdcp_load(struct psp_context *psp) @@ -1522,28 +1590,26 @@ static int psp_hdcp_load(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + psp_copy_fw(psp, psp->hdcp.start_addr, + psp->hdcp.size_bytes); - psp_copy_fw(psp, psp->ta_hdcp_start_addr, - psp->ta_hdcp_ucode_size); + cmd = acquire_psp_cmd_buf(psp); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_hdcp_ucode_size, - psp->hdcp_context.hdcp_shared_mc_addr, + psp->hdcp.size_bytes, + psp->hdcp_context.context.mem_context.shared_mc_addr, PSP_HDCP_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (!ret) { - psp->hdcp_context.hdcp_initialized = true; - psp->hdcp_context.session_id = cmd->resp.session_id; + psp->hdcp_context.context.initialized = true; + psp->hdcp_context.context.session_id = cmd->resp.session_id; mutex_init(&psp->hdcp_context.mutex); } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1557,13 +1623,13 @@ static int psp_hdcp_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->adev->psp.ta_hdcp_ucode_size || - !psp->adev->psp.ta_hdcp_start_addr) { + if (!psp->hdcp.size_bytes || + !psp->hdcp.start_addr) { dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n"); return 0; } - if (!psp->hdcp_context.hdcp_initialized) { + if (!psp->hdcp_context.context.initialized) { ret = psp_hdcp_init_shared_buf(psp); if (ret) return ret; @@ -1587,15 +1653,13 @@ static int psp_hdcp_unload(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1608,7 +1672,7 @@ int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id) if (amdgpu_sriov_vf(psp->adev)) return 0; - return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id); + return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.context.session_id); } static int psp_hdcp_terminate(struct psp_context *psp) @@ -1621,8 +1685,8 @@ static int psp_hdcp_terminate(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->hdcp_context.hdcp_initialized) { - if (psp->hdcp_context.hdcp_shared_buf) + if (!psp->hdcp_context.context.initialized) { + if (psp->hdcp_context.context.mem_context.shared_buf) goto out; else return 0; @@ -1632,13 +1696,11 @@ static int psp_hdcp_terminate(struct psp_context *psp) if (ret) return ret; - psp->hdcp_context.hdcp_initialized = false; + psp->hdcp_context.context.initialized = false; out: /* free hdcp shared memory */ - amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo, - &psp->hdcp_context.hdcp_shared_mc_addr, - &psp->hdcp_context.hdcp_shared_buf); + psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context); return 0; } @@ -1647,19 +1709,8 @@ out: // DTM start static int psp_dtm_init_shared_buf(struct psp_context *psp) { - int ret; - - /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for dtm ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->dtm_context.dtm_shared_bo, - &psp->dtm_context.dtm_shared_mc_addr, - &psp->dtm_context.dtm_shared_buf); - - return ret; + return psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context, + PSP_DTM_SHARED_MEM_SIZE); } static int psp_dtm_load(struct psp_context *psp) @@ -1673,27 +1724,25 @@ static int psp_dtm_load(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + psp_copy_fw(psp, psp->dtm.start_addr, psp->dtm.size_bytes); - psp_copy_fw(psp, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size); + cmd = acquire_psp_cmd_buf(psp); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_dtm_ucode_size, - psp->dtm_context.dtm_shared_mc_addr, + psp->dtm.size_bytes, + psp->dtm_context.context.mem_context.shared_mc_addr, PSP_DTM_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (!ret) { - psp->dtm_context.dtm_initialized = true; - psp->dtm_context.session_id = cmd->resp.session_id; + psp->dtm_context.context.initialized = true; + psp->dtm_context.context.session_id = cmd->resp.session_id; mutex_init(&psp->dtm_context.mutex); } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1708,13 +1757,13 @@ static int psp_dtm_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->adev->psp.ta_dtm_ucode_size || - !psp->adev->psp.ta_dtm_start_addr) { + if (!psp->dtm.size_bytes || + !psp->dtm.start_addr) { dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n"); return 0; } - if (!psp->dtm_context.dtm_initialized) { + if (!psp->dtm_context.context.initialized) { ret = psp_dtm_init_shared_buf(psp); if (ret) return ret; @@ -1738,15 +1787,13 @@ static int psp_dtm_unload(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->dtm_context.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->dtm_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1759,7 +1806,7 @@ int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id) if (amdgpu_sriov_vf(psp->adev)) return 0; - return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id); + return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.context.session_id); } static int psp_dtm_terminate(struct psp_context *psp) @@ -1772,8 +1819,8 @@ static int psp_dtm_terminate(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->dtm_context.dtm_initialized) { - if (psp->dtm_context.dtm_shared_buf) + if (!psp->dtm_context.context.initialized) { + if (psp->dtm_context.context.mem_context.shared_buf) goto out; else return 0; @@ -1783,13 +1830,11 @@ static int psp_dtm_terminate(struct psp_context *psp) if (ret) return ret; - psp->dtm_context.dtm_initialized = false; + psp->dtm_context.context.initialized = false; out: - /* free hdcp shared memory */ - amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo, - &psp->dtm_context.dtm_shared_mc_addr, - &psp->dtm_context.dtm_shared_buf); + /* free dtm shared memory */ + psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context); return 0; } @@ -1798,19 +1843,8 @@ out: // RAP start static int psp_rap_init_shared_buf(struct psp_context *psp) { - int ret; - - /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for rap ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAP_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->rap_context.rap_shared_bo, - &psp->rap_context.rap_shared_mc_addr, - &psp->rap_context.rap_shared_buf); - - return ret; + return psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context, + PSP_RAP_SHARED_MEM_SIZE); } static int psp_rap_load(struct psp_context *psp) @@ -1818,27 +1852,25 @@ static int psp_rap_load(struct psp_context *psp) int ret; struct psp_gfx_cmd_resp *cmd; - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + psp_copy_fw(psp, psp->rap.start_addr, psp->rap.size_bytes); - psp_copy_fw(psp, psp->ta_rap_start_addr, psp->ta_rap_ucode_size); + cmd = acquire_psp_cmd_buf(psp); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_rap_ucode_size, - psp->rap_context.rap_shared_mc_addr, + psp->rap.size_bytes, + psp->rap_context.context.mem_context.shared_mc_addr, PSP_RAP_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); if (!ret) { - psp->rap_context.rap_initialized = true; - psp->rap_context.session_id = cmd->resp.session_id; + psp->rap_context.context.initialized = true; + psp->rap_context.context.session_id = cmd->resp.session_id; mutex_init(&psp->rap_context.mutex); } - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1846,17 +1878,13 @@ static int psp_rap_load(struct psp_context *psp) static int psp_rap_unload(struct psp_context *psp) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->rap_context.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->rap_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -1872,13 +1900,13 @@ static int psp_rap_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->adev->psp.ta_rap_ucode_size || - !psp->adev->psp.ta_rap_start_addr) { + if (!psp->rap.size_bytes || + !psp->rap.start_addr) { dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n"); return 0; } - if (!psp->rap_context.rap_initialized) { + if (!psp->rap_context.context.initialized) { ret = psp_rap_init_shared_buf(psp); if (ret) return ret; @@ -1892,11 +1920,9 @@ static int psp_rap_initialize(struct psp_context *psp) if (ret || status != TA_RAP_STATUS__SUCCESS) { psp_rap_unload(psp); - amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo, - &psp->rap_context.rap_shared_mc_addr, - &psp->rap_context.rap_shared_buf); + psp_ta_free_shared_buf(&psp->rap_context.context.mem_context); - psp->rap_context.rap_initialized = false; + psp->rap_context.context.initialized = false; dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n", ret, status); @@ -1911,17 +1937,15 @@ static int psp_rap_terminate(struct psp_context *psp) { int ret; - if (!psp->rap_context.rap_initialized) + if (!psp->rap_context.context.initialized) return 0; ret = psp_rap_unload(psp); - psp->rap_context.rap_initialized = false; + psp->rap_context.context.initialized = false; /* free rap shared memory */ - amdgpu_bo_free_kernel(&psp->rap_context.rap_shared_bo, - &psp->rap_context.rap_shared_mc_addr, - &psp->rap_context.rap_shared_buf); + psp_ta_free_shared_buf(&psp->rap_context.context.mem_context); return ret; } @@ -1931,7 +1955,7 @@ int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_stat struct ta_rap_shared_memory *rap_cmd; int ret = 0; - if (!psp->rap_context.rap_initialized) + if (!psp->rap_context.context.initialized) return 0; if (ta_cmd_id != TA_CMD_RAP__INITIALIZE && @@ -1941,13 +1965,13 @@ int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_stat mutex_lock(&psp->rap_context.mutex); rap_cmd = (struct ta_rap_shared_memory *) - psp->rap_context.rap_shared_buf; + psp->rap_context.context.mem_context.shared_buf; memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory)); rap_cmd->cmd_id = ta_cmd_id; rap_cmd->validation_method_id = METHOD_A; - ret = psp_ta_invoke(psp, rap_cmd->cmd_id, psp->rap_context.session_id); + ret = psp_ta_invoke(psp, rap_cmd->cmd_id, psp->rap_context.context.session_id); if (ret) goto out_unlock; @@ -1964,67 +1988,48 @@ out_unlock: /* securedisplay start */ static int psp_securedisplay_init_shared_buf(struct psp_context *psp) { - int ret; - - /* - * Allocate 16k memory aligned to 4k from Frame Buffer (local - * physical) for sa ta <-> Driver - */ - ret = amdgpu_bo_create_kernel(psp->adev, PSP_SECUREDISPLAY_SHARED_MEM_SIZE, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, - &psp->securedisplay_context.securedisplay_shared_bo, - &psp->securedisplay_context.securedisplay_shared_mc_addr, - &psp->securedisplay_context.securedisplay_shared_buf); - - return ret; + return psp_ta_init_shared_buf( + psp, &psp->securedisplay_context.context.mem_context, + PSP_SECUREDISPLAY_SHARED_MEM_SIZE); } static int psp_securedisplay_load(struct psp_context *psp) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->ta_securedisplay_start_addr, psp->ta_securedisplay_ucode_size); + memcpy(psp->fw_pri_buf, psp->securedisplay.start_addr, psp->securedisplay.size_bytes); psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, - psp->ta_securedisplay_ucode_size, - psp->securedisplay_context.securedisplay_shared_mc_addr, + psp->securedisplay.size_bytes, + psp->securedisplay_context.context.mem_context.shared_mc_addr, PSP_SECUREDISPLAY_SHARED_MEM_SIZE); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - if (ret) - goto failed; + if (!ret) { + psp->securedisplay_context.context.initialized = true; + psp->securedisplay_context.context.session_id = cmd->resp.session_id; + mutex_init(&psp->securedisplay_context.mutex); + } - psp->securedisplay_context.securedisplay_initialized = true; - psp->securedisplay_context.session_id = cmd->resp.session_id; - mutex_init(&psp->securedisplay_context.mutex); + release_psp_cmd_buf(psp); -failed: - kfree(cmd); return ret; } static int psp_securedisplay_unload(struct psp_context *psp) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - psp_prep_ta_unload_cmd_buf(cmd, psp->securedisplay_context.session_id); + psp_prep_ta_unload_cmd_buf(cmd, psp->securedisplay_context.context.session_id); ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + release_psp_cmd_buf(psp); return ret; } @@ -2040,13 +2045,13 @@ static int psp_securedisplay_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->adev->psp.ta_securedisplay_ucode_size || - !psp->adev->psp.ta_securedisplay_start_addr) { + if (!psp->securedisplay.size_bytes || + !psp->securedisplay.start_addr) { dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n"); return 0; } - if (!psp->securedisplay_context.securedisplay_initialized) { + if (!psp->securedisplay_context.context.initialized) { ret = psp_securedisplay_init_shared_buf(psp); if (ret) return ret; @@ -2063,11 +2068,9 @@ static int psp_securedisplay_initialize(struct psp_context *psp) if (ret) { psp_securedisplay_unload(psp); - amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, - &psp->securedisplay_context.securedisplay_shared_mc_addr, - &psp->securedisplay_context.securedisplay_shared_buf); + psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context); - psp->securedisplay_context.securedisplay_initialized = false; + psp->securedisplay_context.context.initialized = false; dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n"); return -EINVAL; @@ -2092,19 +2095,17 @@ static int psp_securedisplay_terminate(struct psp_context *psp) if (amdgpu_sriov_vf(psp->adev)) return 0; - if (!psp->securedisplay_context.securedisplay_initialized) + if (!psp->securedisplay_context.context.initialized) return 0; ret = psp_securedisplay_unload(psp); if (ret) return ret; - psp->securedisplay_context.securedisplay_initialized = false; + psp->securedisplay_context.context.initialized = false; /* free securedisplay shared memory */ - amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, - &psp->securedisplay_context.securedisplay_shared_mc_addr, - &psp->securedisplay_context.securedisplay_shared_buf); + psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context); return ret; } @@ -2113,7 +2114,7 @@ int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id) { int ret; - if (!psp->securedisplay_context.securedisplay_initialized) + if (!psp->securedisplay_context.context.initialized) return -EINVAL; if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA && @@ -2122,7 +2123,7 @@ int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id) mutex_lock(&psp->securedisplay_context.mutex); - ret = psp_ta_invoke(psp, ta_cmd_id, psp->securedisplay_context.session_id); + ret = psp_ta_invoke(psp, ta_cmd_id, psp->securedisplay_context.context.session_id); mutex_unlock(&psp->securedisplay_context.mutex); @@ -2400,8 +2401,6 @@ static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, int ret; uint64_t fw_mem_mc_addr = ucode->mc_addr; - memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); - cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); @@ -2418,13 +2417,15 @@ static int psp_execute_non_psp_fw_load(struct psp_context *psp, struct amdgpu_firmware_info *ucode) { int ret = 0; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd); - if (ret) - return ret; + ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd); + if (!ret) { + ret = psp_cmd_submit_buf(psp, ucode, cmd, + psp->fence_buf_mc_addr); + } - ret = psp_cmd_submit_buf(psp, ucode, psp->cmd, - psp->fence_buf_mc_addr); + release_psp_cmd_buf(psp); return ret; } @@ -2435,7 +2436,7 @@ static int psp_load_smu_fw(struct psp_context *psp) struct amdgpu_device *adev = psp->adev; struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC]; - struct amdgpu_ras *ras = psp->ras.ras; + struct amdgpu_ras *ras = psp->ras_context.ras; if (!ucode->fw || amdgpu_sriov_vf(psp->adev)) return 0; @@ -2579,10 +2580,6 @@ static int psp_load_fw(struct amdgpu_device *adev) goto skip_memalloc; } - psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!psp->cmd) - return -ENOMEM; - if (amdgpu_sriov_vf(adev)) { ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG, AMDGPU_GEM_DOMAIN_VRAM, @@ -2644,7 +2641,7 @@ skip_memalloc: return ret; } - if (psp->adev->psp.ta_fw) { + if (psp->ta_fw) { ret = psp_ras_initialize(psp); if (ret) dev_err(psp->adev->dev, @@ -2716,7 +2713,7 @@ static int psp_hw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct psp_context *psp = &adev->psp; - if (psp->adev->psp.ta_fw) { + if (psp->ta_fw) { psp_ras_terminate(psp); psp_securedisplay_terminate(psp); psp_rap_terminate(psp); @@ -2736,9 +2733,6 @@ static int psp_hw_fini(void *handle) amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, (void **)&psp->cmd_buf_mem); - kfree(psp->cmd); - psp->cmd = NULL; - return 0; } @@ -2749,7 +2743,7 @@ static int psp_suspend(void *handle) struct psp_context *psp = &adev->psp; if (adev->gmc.xgmi.num_physical_nodes > 1 && - psp->xgmi_context.initialized == 1) { + psp->xgmi_context.context.initialized) { ret = psp_xgmi_terminate(psp); if (ret) { DRM_ERROR("Failed to terminate xgmi ta\n"); @@ -2757,7 +2751,7 @@ static int psp_suspend(void *handle) } } - if (psp->adev->psp.ta_fw) { + if (psp->ta_fw) { ret = psp_ras_terminate(psp); if (ret) { DRM_ERROR("Failed to terminate ras ta\n"); @@ -2839,7 +2833,7 @@ static int psp_resume(void *handle) } if (adev->gmc.xgmi.num_physical_nodes > 1) { - ret = psp_xgmi_initialize(psp); + ret = psp_xgmi_initialize(psp, false, true); /* Warning the XGMI seesion initialize failure * Instead of stop driver initialization */ @@ -2848,7 +2842,7 @@ static int psp_resume(void *handle) "XGMI: Failed to initialize XGMI session\n"); } - if (psp->adev->psp.ta_fw) { + if (psp->ta_fw) { ret = psp_ras_initialize(psp); if (ret) dev_err(psp->adev->dev, @@ -2902,17 +2896,15 @@ int psp_gpu_reset(struct amdgpu_device *adev) int psp_rlc_autoload_start(struct psp_context *psp) { int ret; - struct psp_gfx_cmd_resp *cmd; - - cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); - if (!cmd) - return -ENOMEM; + struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC; ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); - kfree(cmd); + + release_psp_cmd_buf(psp); + return ret; } @@ -3002,10 +2994,10 @@ int psp_init_asd_microcode(struct psp_context *psp, goto out; asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data; - adev->psp.asd_fw_version = le32_to_cpu(asd_hdr->header.ucode_version); - adev->psp.asd_feature_version = le32_to_cpu(asd_hdr->sos.fw_version); - adev->psp.asd_ucode_size = le32_to_cpu(asd_hdr->header.ucode_size_bytes); - adev->psp.asd_start_addr = (uint8_t *)asd_hdr + + adev->psp.asd.fw_version = le32_to_cpu(asd_hdr->header.ucode_version); + adev->psp.asd.feature_version = le32_to_cpu(asd_hdr->sos.fw_version); + adev->psp.asd.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes); + adev->psp.asd.start_addr = (uint8_t *)asd_hdr + le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes); return 0; out: @@ -3147,6 +3139,7 @@ static int psp_init_sos_base_fw(struct amdgpu_device *adev) adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes); adev->psp.sos.start_addr = ucode_array_start_addr + le32_to_cpu(sos_hdr->sos.offset_bytes); + adev->psp.xgmi_context.supports_extended_data = false; } else { /* Load alternate PSP SOS FW */ sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data; @@ -3161,6 +3154,7 @@ static int psp_init_sos_base_fw(struct amdgpu_device *adev) adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes); adev->psp.sos.start_addr = ucode_array_start_addr + le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes); + adev->psp.xgmi_context.supports_extended_data = true; } if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) { @@ -3290,40 +3284,40 @@ static int parse_ta_bin_descriptor(struct psp_context *psp, switch (desc->fw_type) { case TA_FW_TYPE_PSP_ASD: - psp->asd_fw_version = le32_to_cpu(desc->fw_version); - psp->asd_feature_version = le32_to_cpu(desc->fw_version); - psp->asd_ucode_size = le32_to_cpu(desc->size_bytes); - psp->asd_start_addr = ucode_start_addr; + psp->asd.fw_version = le32_to_cpu(desc->fw_version); + psp->asd.feature_version = le32_to_cpu(desc->fw_version); + psp->asd.size_bytes = le32_to_cpu(desc->size_bytes); + psp->asd.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_XGMI: - psp->ta_xgmi_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_xgmi_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_xgmi_start_addr = ucode_start_addr; + psp->xgmi.feature_version = le32_to_cpu(desc->fw_version); + psp->xgmi.size_bytes = le32_to_cpu(desc->size_bytes); + psp->xgmi.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_RAS: - psp->ta_ras_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_ras_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_ras_start_addr = ucode_start_addr; + psp->ras.feature_version = le32_to_cpu(desc->fw_version); + psp->ras.size_bytes = le32_to_cpu(desc->size_bytes); + psp->ras.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_HDCP: - psp->ta_hdcp_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_hdcp_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_hdcp_start_addr = ucode_start_addr; + psp->hdcp.feature_version = le32_to_cpu(desc->fw_version); + psp->hdcp.size_bytes = le32_to_cpu(desc->size_bytes); + psp->hdcp.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_DTM: - psp->ta_dtm_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_dtm_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_dtm_start_addr = ucode_start_addr; + psp->dtm.feature_version = le32_to_cpu(desc->fw_version); + psp->dtm.size_bytes = le32_to_cpu(desc->size_bytes); + psp->dtm.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_RAP: - psp->ta_rap_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_rap_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_rap_start_addr = ucode_start_addr; + psp->rap.feature_version = le32_to_cpu(desc->fw_version); + psp->rap.size_bytes = le32_to_cpu(desc->size_bytes); + psp->rap.start_addr = ucode_start_addr; break; case TA_FW_TYPE_PSP_SECUREDISPLAY: - psp->ta_securedisplay_ucode_version = le32_to_cpu(desc->fw_version); - psp->ta_securedisplay_ucode_size = le32_to_cpu(desc->size_bytes); - psp->ta_securedisplay_start_addr = ucode_start_addr; + psp->securedisplay.feature_version = le32_to_cpu(desc->fw_version); + psp->securedisplay.size_bytes = le32_to_cpu(desc->size_bytes); + psp->securedisplay.start_addr = ucode_start_addr; break; default: dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 4435b180ace0..8ef2d28af92a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -54,9 +54,9 @@ enum psp_bootloader_cmd { PSP_BL__LOAD_SYSDRV = 0x10000, PSP_BL__LOAD_SOSDRV = 0x20000, PSP_BL__LOAD_KEY_DATABASE = 0x80000, - PSP_BL__LOAD_SOCDRV = 0x90000, - PSP_BL__LOAD_INTFDRV = 0xA0000, - PSP_BL__LOAD_DBGDRV = 0xB0000, + PSP_BL__LOAD_SOCDRV = 0xB0000, + PSP_BL__LOAD_INTFDRV = 0xC0000, + PSP_BL__LOAD_DBGDRV = 0xD0000, PSP_BL__DRAM_LONG_TRAIN = 0x100000, PSP_BL__DRAM_SHORT_TRAIN = 0x200000, PSP_BL__LOAD_TOS_SPL_TABLE = 0x10000000, @@ -136,59 +136,32 @@ struct psp_asd_context { uint32_t session_id; }; -struct psp_xgmi_context { - uint8_t initialized; - uint32_t session_id; - struct amdgpu_bo *xgmi_shared_bo; - uint64_t xgmi_shared_mc_addr; - void *xgmi_shared_buf; - struct psp_xgmi_topology_info top_info; +struct ta_mem_context { + struct amdgpu_bo *shared_bo; + uint64_t shared_mc_addr; + void *shared_buf; }; -struct psp_ras_context { - /*ras fw*/ - bool ras_initialized; +struct ta_context { + bool initialized; uint32_t session_id; - struct amdgpu_bo *ras_shared_bo; - uint64_t ras_shared_mc_addr; - void *ras_shared_buf; - struct amdgpu_ras *ras; + struct ta_mem_context mem_context; }; -struct psp_hdcp_context { - bool hdcp_initialized; - uint32_t session_id; - struct amdgpu_bo *hdcp_shared_bo; - uint64_t hdcp_shared_mc_addr; - void *hdcp_shared_buf; - struct mutex mutex; -}; - -struct psp_dtm_context { - bool dtm_initialized; - uint32_t session_id; - struct amdgpu_bo *dtm_shared_bo; - uint64_t dtm_shared_mc_addr; - void *dtm_shared_buf; - struct mutex mutex; +struct ta_cp_context { + struct ta_context context; + struct mutex mutex; }; -struct psp_rap_context { - bool rap_initialized; - uint32_t session_id; - struct amdgpu_bo *rap_shared_bo; - uint64_t rap_shared_mc_addr; - void *rap_shared_buf; - struct mutex mutex; +struct psp_xgmi_context { + struct ta_context context; + struct psp_xgmi_topology_info top_info; + bool supports_extended_data; }; -struct psp_securedisplay_context { - bool securedisplay_initialized; - uint32_t session_id; - struct amdgpu_bo *securedisplay_shared_bo; - uint64_t securedisplay_shared_mc_addr; - void *securedisplay_shared_buf; - struct mutex mutex; +struct psp_ras_context { + struct ta_context context; + struct amdgpu_ras *ras; }; #define MEM_TRAIN_SYSTEM_SIGNATURE 0x54534942 @@ -327,11 +300,8 @@ struct psp_context uint64_t tmr_mc_addr; /* asd firmware */ - const struct firmware *asd_fw; - uint32_t asd_fw_version; - uint32_t asd_feature_version; - uint32_t asd_ucode_size; - uint8_t *asd_start_addr; + const struct firmware *asd_fw; + struct psp_bin_desc asd; /* toc firmware */ const struct firmware *toc_fw; @@ -356,36 +326,20 @@ struct psp_context /* xgmi ta firmware and buffer */ const struct firmware *ta_fw; uint32_t ta_fw_version; - uint32_t ta_xgmi_ucode_version; - uint32_t ta_xgmi_ucode_size; - uint8_t *ta_xgmi_start_addr; - uint32_t ta_ras_ucode_version; - uint32_t ta_ras_ucode_size; - uint8_t *ta_ras_start_addr; - - uint32_t ta_hdcp_ucode_version; - uint32_t ta_hdcp_ucode_size; - uint8_t *ta_hdcp_start_addr; - - uint32_t ta_dtm_ucode_version; - uint32_t ta_dtm_ucode_size; - uint8_t *ta_dtm_start_addr; - - uint32_t ta_rap_ucode_version; - uint32_t ta_rap_ucode_size; - uint8_t *ta_rap_start_addr; - - uint32_t ta_securedisplay_ucode_version; - uint32_t ta_securedisplay_ucode_size; - uint8_t *ta_securedisplay_start_addr; + struct psp_bin_desc xgmi; + struct psp_bin_desc ras; + struct psp_bin_desc hdcp; + struct psp_bin_desc dtm; + struct psp_bin_desc rap; + struct psp_bin_desc securedisplay; struct psp_asd_context asd_context; struct psp_xgmi_context xgmi_context; - struct psp_ras_context ras; - struct psp_hdcp_context hdcp_context; - struct psp_dtm_context dtm_context; - struct psp_rap_context rap_context; - struct psp_securedisplay_context securedisplay_context; + struct psp_ras_context ras_context; + struct ta_cp_context hdcp_context; + struct ta_cp_context dtm_context; + struct ta_cp_context rap_context; + struct ta_cp_context securedisplay_context; struct mutex mutex; struct psp_memory_training_context mem_train_ctx; @@ -452,14 +406,15 @@ int psp_gpu_reset(struct amdgpu_device *adev); int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx, uint64_t cmd_gpu_addr, int cmd_size); -int psp_xgmi_initialize(struct psp_context *psp); +int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta); int psp_xgmi_terminate(struct psp_context *psp); int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id); int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id); int psp_xgmi_get_topology_info(struct psp_context *psp, int number_devices, - struct psp_xgmi_topology_info *topology); + struct psp_xgmi_topology_info *topology, + bool get_extended_data); int psp_xgmi_set_topology_info(struct psp_context *psp, int number_devices, struct psp_xgmi_topology_info *topology); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c index 51909bf8798c..12010c988c8b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rap.c @@ -76,7 +76,7 @@ static ssize_t amdgpu_rap_debugfs_write(struct file *f, const char __user *buf, dev_info(adev->dev, "RAP L0 validate test success.\n"); } else { rap_shared_mem = (struct ta_rap_shared_memory *) - adev->psp.rap_context.rap_shared_buf; + adev->psp.rap_context.context.mem_context.shared_buf; rap_cmd_output = &(rap_shared_mem->rap_out_message.output); dev_info(adev->dev, "RAP test failed, the output is:\n"); @@ -119,7 +119,7 @@ void amdgpu_rap_debugfs_init(struct amdgpu_device *adev) #if defined(CONFIG_DEBUG_FS) struct drm_minor *minor = adev_to_drm(adev)->primary; - if (!adev->psp.rap_context.rap_initialized) + if (!adev->psp.rap_context.context.initialized) return; debugfs_create_file("rap_test", S_IWUSR, minor->debugfs_root, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 194f7ccfbf94..96a8fd0ca1df 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -64,7 +64,6 @@ const char *ras_block_string[] = { }; #define ras_err_str(i) (ras_error_string[ffs(i)]) -#define ras_block_str(i) (ras_block_string[i]) #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) @@ -530,7 +529,7 @@ static inline void put_obj(struct ras_manager *obj) if (obj && (--obj->use == 0)) list_del(&obj->node); if (obj && (obj->use < 0)) - DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", obj->head.name); + DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", ras_block_str(obj->head.block)); } /* make one obj and return it. */ @@ -793,7 +792,6 @@ static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, .type = default_ras_type, .sub_block_index = 0, }; - strcpy(head.name, ras_block_str(i)); if (bypass) { /* * bypass psp. vbios enable ras for us. @@ -1866,7 +1864,7 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev) static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) { struct amdgpu_ras_eeprom_control *control = - &adev->psp.ras.ras->eeprom_control; + &adev->psp.ras_context.ras->eeprom_control; struct eeprom_table_record *bps; int ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h index 4d9c63f2f377..eae604fd90b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h @@ -49,10 +49,14 @@ enum amdgpu_ras_block { AMDGPU_RAS_BLOCK__MP0, AMDGPU_RAS_BLOCK__MP1, AMDGPU_RAS_BLOCK__FUSE, + AMDGPU_RAS_BLOCK__MPIO, AMDGPU_RAS_BLOCK__LAST }; +extern const char *ras_block_string[]; + +#define ras_block_str(i) (ras_block_string[i]) #define AMDGPU_RAS_BLOCK_COUNT AMDGPU_RAS_BLOCK__LAST #define AMDGPU_RAS_BLOCK_MASK ((1ULL << AMDGPU_RAS_BLOCK_COUNT) - 1) @@ -306,7 +310,6 @@ struct ras_common_if { enum amdgpu_ras_block block; enum amdgpu_ras_error_type type; uint32_t sub_block_index; - /* block name */ char name[32]; }; @@ -418,7 +421,7 @@ struct ras_badpage { /* interfaces for IP */ struct ras_fs_if { struct ras_common_if head; - char sysfs_name[32]; + const char* sysfs_name; char debugfs_name[32]; }; @@ -470,8 +473,8 @@ struct ras_debug_if { * 8: feature disable */ -#define amdgpu_ras_get_context(adev) ((adev)->psp.ras.ras) -#define amdgpu_ras_set_context(adev, ras_con) ((adev)->psp.ras.ras = (ras_con)) +#define amdgpu_ras_get_context(adev) ((adev)->psp.ras_context.ras) +#define amdgpu_ras_set_context(adev, ras_con) ((adev)->psp.ras_context.ras = (ras_con)) /* check if ras is supported on block, say, sdma, gfx */ static inline int amdgpu_ras_is_supported(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index d2e5b2567bc1..dc44c946a244 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -27,6 +27,7 @@ #include <linux/bits.h> #include "atom.h" #include "amdgpu_eeprom.h" +#include "amdgpu_atomfirmware.h" #include <linux/debugfs.h> #include <linux/uaccess.h> @@ -113,9 +114,26 @@ static bool __get_eeprom_i2c_addr_arct(struct amdgpu_device *adev, static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, struct amdgpu_ras_eeprom_control *control) { + u8 i2c_addr; + if (!control) return false; + if (amdgpu_atomfirmware_ras_rom_addr(adev, &i2c_addr)) { + /* The address given by VBIOS is an 8-bit, wire-format + * address, i.e. the most significant byte. + * + * Normalize it to a 19-bit EEPROM address. Remove the + * device type identifier and make it a 7-bit address; + * then make it a 19-bit EEPROM address. See top of + * amdgpu_eeprom.c. + */ + i2c_addr = (i2c_addr & 0x0F) >> 1; + control->i2c_address = ((u32) i2c_addr) << 16; + + return true; + } + switch (adev->asic_type) { case CHIP_VEGA20: control->i2c_address = EEPROM_I2C_MADDR_VEGA20; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h index 59e0fefb15aa..acfa207cf970 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h @@ -54,11 +54,12 @@ static inline void amdgpu_res_first(struct ttm_resource *res, { struct drm_mm_node *node; - if (!res) { + if (!res || res->mem_type == TTM_PL_SYSTEM) { cur->start = start; cur->size = size; cur->remaining = size; cur->node = NULL; + WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT); return; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 27adffa7658d..e713d31619fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -48,6 +48,9 @@ #define AMDGPU_FENCE_FLAG_INT (1 << 1) #define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) +/* fence flag bit to indicate the face is embedded in job*/ +#define AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT (DMA_FENCE_FLAG_USER_BITS + 1) + #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched) #define AMDGPU_IB_POOL_SIZE (1024 * 1024) @@ -106,7 +109,6 @@ struct amdgpu_fence_driver { struct dma_fence **fences; }; -int amdgpu_fence_driver_init(struct amdgpu_device *adev); void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, @@ -115,10 +117,11 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, struct amdgpu_irq_src *irq_src, unsigned irq_type); +void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev); void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev); +int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev); void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev); -void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev); -int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job, unsigned flags); int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s, uint32_t timeout); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index de91d29c9d96..65debb65a5df 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -105,7 +105,6 @@ int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev, adev->sdma.ras_if->block = AMDGPU_RAS_BLOCK__SDMA; adev->sdma.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->sdma.ras_if->sub_block_index = 0; - strcpy(adev->sdma.ras_if->name, "sdma"); } fs_info.head = ih_info->head = *adev->sdma.ras_if; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c index 123453999093..cc7597a15fe9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c @@ -80,7 +80,7 @@ void psp_securedisplay_parse_resp_status(struct psp_context *psp, void psp_prep_securedisplay_cmd_buf(struct psp_context *psp, struct securedisplay_cmd **cmd, enum ta_securedisplay_command command_id) { - *cmd = (struct securedisplay_cmd *)psp->securedisplay_context.securedisplay_shared_buf; + *cmd = (struct securedisplay_cmd *)psp->securedisplay_context.context.mem_context.shared_buf; memset(*cmd, 0, sizeof(struct securedisplay_cmd)); (*cmd)->status = TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE; (*cmd)->cmd_id = command_id; @@ -170,7 +170,7 @@ void amdgpu_securedisplay_debugfs_init(struct amdgpu_device *adev) { #if defined(CONFIG_DEBUG_FS) - if (!adev->psp.securedisplay_context.securedisplay_initialized) + if (!adev->psp.securedisplay_context.context.initialized) return; debugfs_create_file("securedisplay_test", S_IWUSR, adev_to_drm(adev)->primary->debugfs_root, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 489e22190e29..1129e17e9f09 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1121,7 +1121,7 @@ static int amdgpu_ttm_tt_populate(struct ttm_device *bdev, struct amdgpu_ttm_tt *gtt = (void *)ttm; /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */ - if (gtt && gtt->userptr) { + if (gtt->userptr) { ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); if (!ttm->sg) return -ENOMEM; @@ -1148,7 +1148,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev, amdgpu_ttm_backend_unbind(bdev, ttm); - if (gtt && gtt->userptr) { + if (gtt->userptr) { amdgpu_ttm_tt_set_user_pages(ttm, NULL); kfree(ttm->sg); ttm->sg = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index 5fdeceaa979f..abd8469380e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -525,9 +525,9 @@ FW_VERSION_ATTR(rlc_srls_fw_version, 0444, gfx.rlc_srls_fw_version); FW_VERSION_ATTR(mec_fw_version, 0444, gfx.mec_fw_version); FW_VERSION_ATTR(mec2_fw_version, 0444, gfx.mec2_fw_version); FW_VERSION_ATTR(sos_fw_version, 0444, psp.sos.fw_version); -FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd_fw_version); -FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ta_ras_ucode_version); -FW_VERSION_ATTR(ta_xgmi_fw_version, 0444, psp.ta_xgmi_ucode_version); +FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd.fw_version); +FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ras.feature_version); +FW_VERSION_ATTR(ta_xgmi_fw_version, 0444, psp.xgmi.feature_version); FW_VERSION_ATTR(smc_fw_version, 0444, pm.fw_version); FW_VERSION_ATTR(sdma_fw_version, 0444, sdma.instance[0].fw_version); FW_VERSION_ATTR(sdma2_fw_version, 0444, sdma.instance[1].fw_version); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index e2e2624ac653..7c2538db3cd5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -136,21 +136,11 @@ struct psp_firmware_header_v2_0 { /* version_major=1, version_minor=0 */ struct ta_firmware_header_v1_0 { struct common_firmware_header header; - uint32_t ta_xgmi_ucode_version; - uint32_t ta_xgmi_offset_bytes; - uint32_t ta_xgmi_size_bytes; - uint32_t ta_ras_ucode_version; - uint32_t ta_ras_offset_bytes; - uint32_t ta_ras_size_bytes; - uint32_t ta_hdcp_ucode_version; - uint32_t ta_hdcp_offset_bytes; - uint32_t ta_hdcp_size_bytes; - uint32_t ta_dtm_ucode_version; - uint32_t ta_dtm_offset_bytes; - uint32_t ta_dtm_size_bytes; - uint32_t ta_securedisplay_ucode_version; - uint32_t ta_securedisplay_offset_bytes; - uint32_t ta_securedisplay_size_bytes; + struct psp_fw_legacy_bin_desc xgmi; + struct psp_fw_legacy_bin_desc ras; + struct psp_fw_legacy_bin_desc hdcp; + struct psp_fw_legacy_bin_desc dtm; + struct psp_fw_legacy_bin_desc securedisplay; }; enum ta_fw_type { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index 0c7c56a91b25..a90029ee9733 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -41,7 +41,6 @@ int amdgpu_umc_ras_late_init(struct amdgpu_device *adev) adev->umc.ras_if->block = AMDGPU_RAS_BLOCK__UMC; adev->umc.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->umc.ras_if->sub_block_index = 0; - strcpy(adev->umc.ras_if->name, "umc"); } ih_info.head = fs_info.head = *adev->umc.ras_if; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 0f576f294d8a..d451c359606a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -326,7 +326,6 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) { int i, j; - cancel_delayed_work_sync(&adev->uvd.idle_work); drm_sched_entity_destroy(&adev->uvd.entity); for (j = 0; j < adev->uvd.num_uvd_inst; ++j) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 1ae7f824adc7..8e8dee9fac9f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -218,7 +218,6 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev) if (adev->vce.vcpu_bo == NULL) return 0; - cancel_delayed_work_sync(&adev->vce.idle_work); drm_sched_entity_destroy(&adev->vce.entity); amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 6780df0fb265..008a308a4eca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -258,8 +258,6 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev) { int i, j; - cancel_delayed_work_sync(&adev->vcn.idle_work); - for (j = 0; j < adev->vcn.num_vcn_inst; ++j) { if (adev->vcn.harvest_config & (1 << j)) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 12a7cc2f01cd..ca058fbcccd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -532,9 +532,9 @@ static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); - POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, adev->psp.asd_fw_version); - POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS, adev->psp.ta_ras_ucode_version); - POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI, adev->psp.ta_xgmi_ucode_version); + POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, adev->psp.asd.fw_version); + POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS, adev->psp.ras.feature_version); + POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI, adev->psp.xgmi.feature_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC, adev->pm.fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA, adev->sdma.instance[0].fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2, adev->sdma.instance[1].fw_version); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c new file mode 100644 index 000000000000..ce982afeff91 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <drm/drm_atomic_helper.h> +#include <drm/drm_simple_kms_helper.h> +#include <drm/drm_vblank.h> + +#include "amdgpu.h" +#ifdef CONFIG_DRM_AMDGPU_SI +#include "dce_v6_0.h" +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK +#include "dce_v8_0.h" +#endif +#include "dce_v10_0.h" +#include "dce_v11_0.h" +#include "ivsrcid/ivsrcid_vislands30.h" +#include "amdgpu_vkms.h" +#include "amdgpu_display.h" + +/** + * DOC: amdgpu_vkms + * + * The amdgpu vkms interface provides a virtual KMS interface for several use + * cases: devices without display hardware, platforms where the actual display + * hardware is not useful (e.g., servers), SR-IOV virtual functions, device + * emulation/simulation, and device bring up prior to display hardware being + * usable. We previously emulated a legacy KMS interface, but there was a desire + * to move to the atomic KMS interface. The vkms driver did everything we + * needed, but we wanted KMS support natively in the driver without buffer + * sharing and the ability to support an instance of VKMS per device. We first + * looked at splitting vkms into a stub driver and a helper module that other + * drivers could use to implement a virtual display, but this strategy ended up + * being messy due to driver specific callbacks needed for buffer management. + * Ultimately, it proved easier to import the vkms code as it mostly used core + * drm helpers anyway. + */ + +static const u32 amdgpu_vkms_formats[] = { + DRM_FORMAT_XRGB8888, +}; + +static enum hrtimer_restart amdgpu_vkms_vblank_simulate(struct hrtimer *timer) +{ + struct amdgpu_vkms_output *output = container_of(timer, + struct amdgpu_vkms_output, + vblank_hrtimer); + struct drm_crtc *crtc = &output->crtc; + u64 ret_overrun; + bool ret; + + ret_overrun = hrtimer_forward_now(&output->vblank_hrtimer, + output->period_ns); + WARN_ON(ret_overrun != 1); + + ret = drm_crtc_handle_vblank(crtc); + if (!ret) + DRM_ERROR("amdgpu_vkms failure on handling vblank"); + + return HRTIMER_RESTART; +} + +static int amdgpu_vkms_enable_vblank(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + unsigned int pipe = drm_crtc_index(crtc); + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; + struct amdgpu_vkms_output *out = drm_crtc_to_amdgpu_vkms_output(crtc); + + drm_calc_timestamping_constants(crtc, &crtc->mode); + + hrtimer_init(&out->vblank_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + out->vblank_hrtimer.function = &amdgpu_vkms_vblank_simulate; + out->period_ns = ktime_set(0, vblank->framedur_ns); + hrtimer_start(&out->vblank_hrtimer, out->period_ns, HRTIMER_MODE_REL); + + return 0; +} + +static void amdgpu_vkms_disable_vblank(struct drm_crtc *crtc) +{ + struct amdgpu_vkms_output *out = drm_crtc_to_amdgpu_vkms_output(crtc); + + hrtimer_cancel(&out->vblank_hrtimer); +} + +static bool amdgpu_vkms_get_vblank_timestamp(struct drm_crtc *crtc, + int *max_error, + ktime_t *vblank_time, + bool in_vblank_irq) +{ + struct drm_device *dev = crtc->dev; + unsigned int pipe = crtc->index; + struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc); + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; + + if (!READ_ONCE(vblank->enabled)) { + *vblank_time = ktime_get(); + return true; + } + + *vblank_time = READ_ONCE(output->vblank_hrtimer.node.expires); + + if (WARN_ON(*vblank_time == vblank->time)) + return true; + + /* + * To prevent races we roll the hrtimer forward before we do any + * interrupt processing - this is how real hw works (the interrupt is + * only generated after all the vblank registers are updated) and what + * the vblank core expects. Therefore we need to always correct the + * timestampe by one frame. + */ + *vblank_time -= output->period_ns; + + return true; +} + +static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = { + .set_config = drm_atomic_helper_set_config, + .destroy = drm_crtc_cleanup, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = amdgpu_vkms_enable_vblank, + .disable_vblank = amdgpu_vkms_disable_vblank, + .get_vblank_timestamp = amdgpu_vkms_get_vblank_timestamp, +}; + +static void amdgpu_vkms_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + drm_crtc_vblank_on(crtc); +} + +static void amdgpu_vkms_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + drm_crtc_vblank_off(crtc); +} + +static void amdgpu_vkms_crtc_atomic_flush(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + if (crtc->state->event) { + spin_lock(&crtc->dev->event_lock); + + if (drm_crtc_vblank_get(crtc) != 0) + drm_crtc_send_vblank_event(crtc, crtc->state->event); + else + drm_crtc_arm_vblank_event(crtc, crtc->state->event); + + spin_unlock(&crtc->dev->event_lock); + + crtc->state->event = NULL; + } +} + +static const struct drm_crtc_helper_funcs amdgpu_vkms_crtc_helper_funcs = { + .atomic_flush = amdgpu_vkms_crtc_atomic_flush, + .atomic_enable = amdgpu_vkms_crtc_atomic_enable, + .atomic_disable = amdgpu_vkms_crtc_atomic_disable, +}; + +static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc, + struct drm_plane *primary, struct drm_plane *cursor) +{ + int ret; + + ret = drm_crtc_init_with_planes(dev, crtc, primary, cursor, + &amdgpu_vkms_crtc_funcs, NULL); + if (ret) { + DRM_ERROR("Failed to init CRTC\n"); + return ret; + } + + drm_crtc_helper_add(crtc, &amdgpu_vkms_crtc_helper_funcs); + + return ret; +} + +static const struct drm_connector_funcs amdgpu_vkms_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = drm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static int amdgpu_vkms_conn_get_modes(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode = NULL; + unsigned i; + static const struct mode_size { + int w; + int h; + } common_modes[] = { + { 640, 480}, + { 720, 480}, + { 800, 600}, + { 848, 480}, + {1024, 768}, + {1152, 768}, + {1280, 720}, + {1280, 800}, + {1280, 854}, + {1280, 960}, + {1280, 1024}, + {1440, 900}, + {1400, 1050}, + {1680, 1050}, + {1600, 1200}, + {1920, 1080}, + {1920, 1200}, + {2560, 1440}, + {4096, 3112}, + {3656, 2664}, + {3840, 2160}, + {4096, 2160}, + }; + + for (i = 0; i < ARRAY_SIZE(common_modes); i++) { + mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false); + drm_mode_probed_add(connector, mode); + } + + drm_set_preferred_mode(connector, XRES_DEF, YRES_DEF); + + return ARRAY_SIZE(common_modes); +} + +static const struct drm_connector_helper_funcs amdgpu_vkms_conn_helper_funcs = { + .get_modes = amdgpu_vkms_conn_get_modes, +}; + +static const struct drm_plane_funcs amdgpu_vkms_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = drm_plane_cleanup, + .reset = drm_atomic_helper_plane_reset, + .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +static void amdgpu_vkms_plane_atomic_update(struct drm_plane *plane, + struct drm_atomic_state *old_state) +{ + return; +} + +static int amdgpu_vkms_plane_atomic_check(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, + plane); + struct drm_crtc_state *crtc_state; + int ret; + + if (!new_plane_state->fb || WARN_ON(!new_plane_state->crtc)) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, + new_plane_state->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + false, true); + if (ret != 0) + return ret; + + /* for now primary plane must be visible and full screen */ + if (!new_plane_state->visible) + return -EINVAL; + + return 0; +} + +static int amdgpu_vkms_prepare_fb(struct drm_plane *plane, + struct drm_plane_state *new_state) +{ + struct amdgpu_framebuffer *afb; + struct drm_gem_object *obj; + struct amdgpu_device *adev; + struct amdgpu_bo *rbo; + struct list_head list; + struct ttm_validate_buffer tv; + struct ww_acquire_ctx ticket; + uint32_t domain; + int r; + + if (!new_state->fb) { + DRM_DEBUG_KMS("No FB bound\n"); + return 0; + } + afb = to_amdgpu_framebuffer(new_state->fb); + obj = new_state->fb->obj[0]; + rbo = gem_to_amdgpu_bo(obj); + adev = amdgpu_ttm_adev(rbo->tbo.bdev); + INIT_LIST_HEAD(&list); + + tv.bo = &rbo->tbo; + tv.num_shared = 1; + list_add(&tv.head, &list); + + r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL); + if (r) { + dev_err(adev->dev, "fail to reserve bo (%d)\n", r); + return r; + } + + if (plane->type != DRM_PLANE_TYPE_CURSOR) + domain = amdgpu_display_supported_domains(adev, rbo->flags); + else + domain = AMDGPU_GEM_DOMAIN_VRAM; + + r = amdgpu_bo_pin(rbo, domain); + if (unlikely(r != 0)) { + if (r != -ERESTARTSYS) + DRM_ERROR("Failed to pin framebuffer with error %d\n", r); + ttm_eu_backoff_reservation(&ticket, &list); + return r; + } + + r = amdgpu_ttm_alloc_gart(&rbo->tbo); + if (unlikely(r != 0)) { + amdgpu_bo_unpin(rbo); + ttm_eu_backoff_reservation(&ticket, &list); + DRM_ERROR("%p bind failed\n", rbo); + return r; + } + + ttm_eu_backoff_reservation(&ticket, &list); + + afb->address = amdgpu_bo_gpu_offset(rbo); + + amdgpu_bo_ref(rbo); + + return 0; +} + +static void amdgpu_vkms_cleanup_fb(struct drm_plane *plane, + struct drm_plane_state *old_state) +{ + struct amdgpu_bo *rbo; + int r; + + if (!old_state->fb) + return; + + rbo = gem_to_amdgpu_bo(old_state->fb->obj[0]); + r = amdgpu_bo_reserve(rbo, false); + if (unlikely(r)) { + DRM_ERROR("failed to reserve rbo before unpin\n"); + return; + } + + amdgpu_bo_unpin(rbo); + amdgpu_bo_unreserve(rbo); + amdgpu_bo_unref(&rbo); +} + +static const struct drm_plane_helper_funcs amdgpu_vkms_primary_helper_funcs = { + .atomic_update = amdgpu_vkms_plane_atomic_update, + .atomic_check = amdgpu_vkms_plane_atomic_check, + .prepare_fb = amdgpu_vkms_prepare_fb, + .cleanup_fb = amdgpu_vkms_cleanup_fb, +}; + +static struct drm_plane *amdgpu_vkms_plane_init(struct drm_device *dev, + enum drm_plane_type type, + int index) +{ + struct drm_plane *plane; + int ret; + + plane = kzalloc(sizeof(*plane), GFP_KERNEL); + if (!plane) + return ERR_PTR(-ENOMEM); + + ret = drm_universal_plane_init(dev, plane, 1 << index, + &amdgpu_vkms_plane_funcs, + amdgpu_vkms_formats, + ARRAY_SIZE(amdgpu_vkms_formats), + NULL, type, NULL); + if (ret) { + kfree(plane); + return ERR_PTR(ret); + } + + drm_plane_helper_add(plane, &amdgpu_vkms_primary_helper_funcs); + + return plane; +} + +int amdgpu_vkms_output_init(struct drm_device *dev, + struct amdgpu_vkms_output *output, int index) +{ + struct drm_connector *connector = &output->connector; + struct drm_encoder *encoder = &output->encoder; + struct drm_crtc *crtc = &output->crtc; + struct drm_plane *primary, *cursor = NULL; + int ret; + + primary = amdgpu_vkms_plane_init(dev, DRM_PLANE_TYPE_PRIMARY, index); + if (IS_ERR(primary)) + return PTR_ERR(primary); + + ret = amdgpu_vkms_crtc_init(dev, crtc, primary, cursor); + if (ret) + goto err_crtc; + + ret = drm_connector_init(dev, connector, &amdgpu_vkms_connector_funcs, + DRM_MODE_CONNECTOR_VIRTUAL); + if (ret) { + DRM_ERROR("Failed to init connector\n"); + goto err_connector; + } + + drm_connector_helper_add(connector, &amdgpu_vkms_conn_helper_funcs); + + ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_VIRTUAL); + if (ret) { + DRM_ERROR("Failed to init encoder\n"); + goto err_encoder; + } + encoder->possible_crtcs = 1 << index; + + ret = drm_connector_attach_encoder(connector, encoder); + if (ret) { + DRM_ERROR("Failed to attach connector to encoder\n"); + goto err_attach; + } + + drm_mode_config_reset(dev); + + return 0; + +err_attach: + drm_encoder_cleanup(encoder); + +err_encoder: + drm_connector_cleanup(connector); + +err_connector: + drm_crtc_cleanup(crtc); + +err_crtc: + drm_plane_cleanup(primary); + + return ret; +} + +const struct drm_mode_config_funcs amdgpu_vkms_mode_funcs = { + .fb_create = amdgpu_display_user_framebuffer_create, + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, +}; + +static int amdgpu_vkms_sw_init(void *handle) +{ + int r, i; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + adev_to_drm(adev)->max_vblank_count = 0; + + adev_to_drm(adev)->mode_config.funcs = &amdgpu_vkms_mode_funcs; + + adev_to_drm(adev)->mode_config.max_width = XRES_MAX; + adev_to_drm(adev)->mode_config.max_height = YRES_MAX; + + adev_to_drm(adev)->mode_config.preferred_depth = 24; + adev_to_drm(adev)->mode_config.prefer_shadow = 1; + + adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; + + r = amdgpu_display_modeset_create_props(adev); + if (r) + return r; + + adev->amdgpu_vkms_output = kcalloc(adev->mode_info.num_crtc, sizeof(struct amdgpu_vkms_output), GFP_KERNEL); + if (!adev->amdgpu_vkms_output) + return -ENOMEM; + + /* allocate crtcs, encoders, connectors */ + for (i = 0; i < adev->mode_info.num_crtc; i++) { + r = amdgpu_vkms_output_init(adev_to_drm(adev), &adev->amdgpu_vkms_output[i], i); + if (r) + return r; + } + + drm_kms_helper_poll_init(adev_to_drm(adev)); + + adev->mode_info.mode_config_initialized = true; + return 0; +} + +static int amdgpu_vkms_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int i = 0; + + for (i = 0; i < adev->mode_info.num_crtc; i++) + if (adev->mode_info.crtcs[i]) + hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer); + + kfree(adev->mode_info.bios_hardcoded_edid); + kfree(adev->amdgpu_vkms_output); + + drm_kms_helper_poll_fini(adev_to_drm(adev)); + + adev->mode_info.mode_config_initialized = false; + return 0; +} + +static int amdgpu_vkms_hw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + switch (adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_SI + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + dce_v6_0_disable_dce(adev); + break; +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + case CHIP_HAWAII: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: + dce_v8_0_disable_dce(adev); + break; +#endif + case CHIP_FIJI: + case CHIP_TONGA: + dce_v10_0_disable_dce(adev); + break; + case CHIP_CARRIZO: + case CHIP_STONEY: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_VEGAM: + dce_v11_0_disable_dce(adev); + break; + case CHIP_TOPAZ: +#ifdef CONFIG_DRM_AMDGPU_SI + case CHIP_HAINAN: +#endif + /* no DCE */ + break; + default: + break; + } + return 0; +} + +static int amdgpu_vkms_hw_fini(void *handle) +{ + return 0; +} + +static int amdgpu_vkms_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r; + + r = drm_mode_config_helper_suspend(adev_to_drm(adev)); + if (r) + return r; + return amdgpu_vkms_hw_fini(handle); +} + +static int amdgpu_vkms_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r; + + r = amdgpu_vkms_hw_init(handle); + if (r) + return r; + return drm_mode_config_helper_resume(adev_to_drm(adev)); +} + +static bool amdgpu_vkms_is_idle(void *handle) +{ + return true; +} + +static int amdgpu_vkms_wait_for_idle(void *handle) +{ + return 0; +} + +static int amdgpu_vkms_soft_reset(void *handle) +{ + return 0; +} + +static int amdgpu_vkms_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int amdgpu_vkms_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +static const struct amd_ip_funcs amdgpu_vkms_ip_funcs = { + .name = "amdgpu_vkms", + .early_init = NULL, + .late_init = NULL, + .sw_init = amdgpu_vkms_sw_init, + .sw_fini = amdgpu_vkms_sw_fini, + .hw_init = amdgpu_vkms_hw_init, + .hw_fini = amdgpu_vkms_hw_fini, + .suspend = amdgpu_vkms_suspend, + .resume = amdgpu_vkms_resume, + .is_idle = amdgpu_vkms_is_idle, + .wait_for_idle = amdgpu_vkms_wait_for_idle, + .soft_reset = amdgpu_vkms_soft_reset, + .set_clockgating_state = amdgpu_vkms_set_clockgating_state, + .set_powergating_state = amdgpu_vkms_set_powergating_state, +}; + +const struct amdgpu_ip_block_version amdgpu_vkms_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_DCE, + .major = 1, + .minor = 0, + .rev = 0, + .funcs = &amdgpu_vkms_ip_funcs, +}; + diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.h new file mode 100644 index 000000000000..97f1b79c0724 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _AMDGPU_VKMS_H_ +#define _AMDGPU_VKMS_H_ + +#define XRES_DEF 1024 +#define YRES_DEF 768 + +#define XRES_MAX 16384 +#define YRES_MAX 16384 + +#define drm_crtc_to_amdgpu_vkms_output(target) \ + container_of(target, struct amdgpu_vkms_output, crtc) + +extern const struct amdgpu_ip_block_version amdgpu_vkms_ip_block; + +struct amdgpu_vkms_output { + struct drm_crtc crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct hrtimer vblank_hrtimer; + ktime_t period_ns; + struct drm_pending_vblank_event *event; +}; + +#endif /* _AMDGPU_VKMS_H_ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 6362e861a3f5..6b15cad78de9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -926,7 +926,7 @@ static int amdgpu_vm_pt_create(struct amdgpu_device *adev, bp.size = amdgpu_vm_bo_size(adev, level); bp.byte_align = AMDGPU_GPU_PAGE_SIZE; bp.domain = AMDGPU_GEM_DOMAIN_VRAM; - bp.domain = amdgpu_bo_get_preferred_pin_domain(adev, bp.domain); + bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_CPU_GTT_USWC; @@ -1218,7 +1218,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); if (vm_flush_needed || pasid_mapping_needed) { - r = amdgpu_fence_emit(ring, &fence, 0); + r = amdgpu_fence_emit(ring, &fence, NULL, 0); if (r) return r; } @@ -1798,7 +1798,7 @@ int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, r = vm->update_funcs->commit(¶ms, fence); if (table_freed) - *table_freed = params.table_freed; + *table_freed = *table_freed || params.table_freed; error_unlock: amdgpu_vm_eviction_unlock(vm); @@ -1856,6 +1856,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, * @adev: amdgpu_device pointer * @bo_va: requested BO and VM object * @clear: if true clear the entries + * @table_freed: return true if page table is freed * * Fill in the page table entries for @bo_va. * @@ -1863,7 +1864,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, * 0 for success, -EINVAL for failure. */ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, - bool clear) + bool clear, bool *table_freed) { struct amdgpu_bo *bo = bo_va->base.bo; struct amdgpu_vm *vm = bo_va->base.vm; @@ -1942,7 +1943,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, resv, mapping->start, mapping->last, update_flags, mapping->offset, mem, - pages_addr, last_update, NULL); + pages_addr, last_update, table_freed); if (r) return r; } @@ -2194,7 +2195,7 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { /* Per VM BOs never need to bo cleared in the page tables */ - r = amdgpu_vm_bo_update(adev, bo_va, false); + r = amdgpu_vm_bo_update(adev, bo_va, false, NULL); if (r) return r; } @@ -2213,7 +2214,7 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, else clear = true; - r = amdgpu_vm_bo_update(adev, bo_va, clear); + r = amdgpu_vm_bo_update(adev, bo_va, clear, NULL); if (r) return r; @@ -3344,12 +3345,13 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) * @adev: amdgpu device pointer * @pasid: PASID of the VM * @addr: Address of the fault + * @write_fault: true is write fault, false is read fault * * Try to gracefully handle a VM fault. Return true if the fault was handled and * shouldn't be reported any more. */ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, - uint64_t addr) + uint64_t addr, bool write_fault) { bool is_compute_context = false; struct amdgpu_bo *root; @@ -3374,7 +3376,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, addr /= AMDGPU_GPU_PAGE_SIZE; if (is_compute_context && - !svm_range_restore_pages(adev, pasid, addr)) { + !svm_range_restore_pages(adev, pasid, addr, write_fault)) { amdgpu_bo_unref(&root); return true; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index ee0028ec2c4f..85fcfb8c5efd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -408,7 +408,7 @@ int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, struct dma_fence **fence, bool *free_table); int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, - bool clear); + bool clear, bool *table_freed); bool amdgpu_vm_evictable(struct amdgpu_bo *bo); void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, struct amdgpu_bo *bo, bool evicted); @@ -448,7 +448,7 @@ void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev); void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid, struct amdgpu_task_info *task_info); bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, - uint64_t addr); + uint64_t addr, bool write_fault); void amdgpu_vm_set_task_info(struct amdgpu_vm *vm); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 2fd77c36a1ff..7b2b0980ec41 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -361,7 +361,7 @@ static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem, * @man: TTM memory type manager * @tbo: TTM BO we need this range for * @place: placement flags and restrictions - * @mem: the resulting mem object + * @res: the resulting mem object * * Allocate VRAM for the given BO. */ @@ -487,7 +487,7 @@ error_sub: * amdgpu_vram_mgr_del - free ranges * * @man: TTM memory type manager - * @mem: TTM memory object + * @res: TTM memory object * * Free the allocated VRAM again. */ @@ -522,7 +522,7 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table * * @adev: amdgpu device pointer - * @mem: TTM memory object + * @res: TTM memory object * @offset: byte offset from the base of VRAM BO * @length: number of bytes to export in sg_table * @dev: the other device diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index 258cf86b32f6..978ac927ac11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -32,6 +32,10 @@ #include "wafl/wafl2_4_0_0_smn.h" #include "wafl/wafl2_4_0_0_sh_mask.h" +#define smnPCS_XGMI23_PCS_ERROR_STATUS 0x11a01210 +#define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c +#define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210 + static DEFINE_MUTEX(xgmi_mutex); #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4 @@ -63,6 +67,33 @@ static const int wafl_pcs_err_status_reg_arct[] = { smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000, }; +static const int xgmi23_pcs_err_status_reg_aldebaran[] = { + smnPCS_XGMI23_PCS_ERROR_STATUS, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x100000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x200000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x300000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x400000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x500000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x600000, + smnPCS_XGMI23_PCS_ERROR_STATUS + 0x700000 +}; + +static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = { + smnPCS_XGMI3X16_PCS_ERROR_STATUS, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000, + smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000 +}; + +static const int walf_pcs_err_status_reg_aldebaran[] = { + smnPCS_GOPX1_PCS_ERROR_STATUS, + smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000 +}; + static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = { {"XGMI PCS DataLossErr", SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)}, @@ -498,6 +529,32 @@ int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev, return -EINVAL; } +/* + * Devices that support extended data require the entire hive to initialize with + * the shared memory buffer flag set. + * + * Hive locks and conditions apply - see amdgpu_xgmi_add_device + */ +static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive, + bool set_extended_data) +{ + struct amdgpu_device *tmp_adev; + int ret; + + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false); + if (ret) { + dev_err(tmp_adev->dev, + "XGMI: Failed to initialize xgmi session for data partition %i\n", + set_extended_data); + return ret; + } + + } + + return 0; +} + int amdgpu_xgmi_add_device(struct amdgpu_device *adev) { struct psp_xgmi_topology_info *top_info; @@ -512,7 +569,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) if (!adev->gmc.xgmi.pending_reset && amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) { - ret = psp_xgmi_initialize(&adev->psp); + ret = psp_xgmi_initialize(&adev->psp, false, true); if (ret) { dev_err(adev->dev, "XGMI: Failed to initialize xgmi session\n"); @@ -575,7 +632,7 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) /* get latest topology info for each device from psp */ list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, - &tmp_adev->psp.xgmi_context.top_info); + &tmp_adev->psp.xgmi_context.top_info, false); if (ret) { dev_err(tmp_adev->dev, "XGMI: Get topology failure on device %llx, hive %llx, ret %d", @@ -585,6 +642,34 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) goto exit_unlock; } } + + /* get topology again for hives that support extended data */ + if (adev->psp.xgmi_context.supports_extended_data) { + + /* initialize the hive to get extended data. */ + ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true); + if (ret) + goto exit_unlock; + + /* get the extended data. */ + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, + &tmp_adev->psp.xgmi_context.top_info, true); + if (ret) { + dev_err(tmp_adev->dev, + "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d", + tmp_adev->gmc.xgmi.node_id, + tmp_adev->gmc.xgmi.hive_id, ret); + goto exit_unlock; + } + } + + /* initialize the hive to get non-extended data for the next round. */ + ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false); + if (ret) + goto exit_unlock; + + } } if (!ret && !adev->gmc.xgmi.pending_reset) @@ -663,7 +748,6 @@ static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev) adev->gmc.xgmi.ras_if->block = AMDGPU_RAS_BLOCK__XGMI_WAFL; adev->gmc.xgmi.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->gmc.xgmi.ras_if->sub_block_index = 0; - strcpy(adev->gmc.xgmi.ras_if->name, "xgmi_wafl"); } ih_info.head = fs_info.head = *adev->gmc.xgmi.ras_if; r = amdgpu_ras_late_init(adev, adev->gmc.xgmi.ras_if, @@ -718,6 +802,17 @@ static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev) pcs_clear_status(adev, xgmi_pcs_err_status_reg_vg20[i]); break; + case CHIP_ALDEBARAN: + for (i = 0; i < ARRAY_SIZE(xgmi23_pcs_err_status_reg_aldebaran); i++) + pcs_clear_status(adev, + xgmi23_pcs_err_status_reg_aldebaran[i]); + for (i = 0; i < ARRAY_SIZE(xgmi23_pcs_err_status_reg_aldebaran); i++) + pcs_clear_status(adev, + xgmi23_pcs_err_status_reg_aldebaran[i]); + for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) + pcs_clear_status(adev, + walf_pcs_err_status_reg_aldebaran[i]); + break; default: break; } @@ -795,7 +890,6 @@ static int amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev, } break; case CHIP_VEGA20: - default: /* check xgmi pcs error */ for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) { data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]); @@ -811,6 +905,32 @@ static int amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev, data, &ue_cnt, &ce_cnt, false); } break; + case CHIP_ALDEBARAN: + /* check xgmi23 pcs error */ + for (i = 0; i < ARRAY_SIZE(xgmi23_pcs_err_status_reg_aldebaran); i++) { + data = RREG32_PCIE(xgmi23_pcs_err_status_reg_aldebaran[i]); + if (data) + amdgpu_xgmi_query_pcs_error_status(adev, + data, &ue_cnt, &ce_cnt, true); + } + /* check xgmi3x16 pcs error */ + for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) { + data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]); + if (data) + amdgpu_xgmi_query_pcs_error_status(adev, + data, &ue_cnt, &ce_cnt, true); + } + /* check wafl pcs error */ + for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) { + data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]); + if (data) + amdgpu_xgmi_query_pcs_error_status(adev, + data, &ue_cnt, &ce_cnt, false); + } + break; + default: + dev_warn(adev->dev, "XGMI RAS error query not supported"); + break; } adev->gmc.xgmi.ras_funcs->reset_ras_error_count(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c index 159a2a4385a1..afad094f84c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c @@ -851,7 +851,7 @@ void amdgpu_atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode pll->reference_div = amdgpu_crtc->pll_reference_div; pll->post_div = amdgpu_crtc->pll_post_div; - amdgpu_pll_compute(pll, amdgpu_crtc->adjusted_clock, &pll_clock, + amdgpu_pll_compute(adev, pll, amdgpu_crtc->adjusted_clock, &pll_clock, &fb_div, &frac_fb_div, &ref_div, &post_div); amdgpu_atombios_crtc_program_ss(adev, ATOM_DISABLE, amdgpu_crtc->pll_id, diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index c0fcc41ee574..54f28c075f21 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -70,7 +70,7 @@ #include "amdgpu_dm.h" #include "amdgpu_amdkfd.h" -#include "dce_virtual.h" +#include "amdgpu_vkms.h" static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] = { @@ -2259,7 +2259,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2277,7 +2277,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2295,7 +2295,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); amdgpu_device_ip_block_add(adev, &kv_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2315,7 +2315,7 @@ int cik_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); amdgpu_device_ip_block_add(adev, &kv_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c deleted file mode 100644 index 7e0d8c092c7e..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c +++ /dev/null @@ -1,780 +0,0 @@ -/* - * Copyright 2014 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include <drm/drm_vblank.h> - -#include "amdgpu.h" -#include "amdgpu_pm.h" -#include "amdgpu_i2c.h" -#include "atom.h" -#include "amdgpu_pll.h" -#include "amdgpu_connectors.h" -#ifdef CONFIG_DRM_AMDGPU_SI -#include "dce_v6_0.h" -#endif -#ifdef CONFIG_DRM_AMDGPU_CIK -#include "dce_v8_0.h" -#endif -#include "dce_v10_0.h" -#include "dce_v11_0.h" -#include "dce_virtual.h" -#include "ivsrcid/ivsrcid_vislands30.h" -#include "amdgpu_display.h" - -#define DCE_VIRTUAL_VBLANK_PERIOD 16666666 - - -static void dce_virtual_set_display_funcs(struct amdgpu_device *adev); -static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev); -static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev, - int index); -static int dce_virtual_pageflip(struct amdgpu_device *adev, - unsigned crtc_id); -static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vblank_timer); -static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev, - int crtc, - enum amdgpu_interrupt_state state); - -static u32 dce_virtual_vblank_get_counter(struct amdgpu_device *adev, int crtc) -{ - return 0; -} - -static void dce_virtual_page_flip(struct amdgpu_device *adev, - int crtc_id, u64 crtc_base, bool async) -{ - return; -} - -static int dce_virtual_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc, - u32 *vbl, u32 *position) -{ - *vbl = 0; - *position = 0; - - return -EINVAL; -} - -static bool dce_virtual_hpd_sense(struct amdgpu_device *adev, - enum amdgpu_hpd_id hpd) -{ - return true; -} - -static void dce_virtual_hpd_set_polarity(struct amdgpu_device *adev, - enum amdgpu_hpd_id hpd) -{ - return; -} - -static u32 dce_virtual_hpd_get_gpio_reg(struct amdgpu_device *adev) -{ - return 0; -} - -/** - * dce_virtual_bandwidth_update - program display watermarks - * - * @adev: amdgpu_device pointer - * - * Calculate and program the display watermarks and line - * buffer allocation (CIK). - */ -static void dce_virtual_bandwidth_update(struct amdgpu_device *adev) -{ - return; -} - -static int dce_virtual_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, - u16 *green, u16 *blue, uint32_t size, - struct drm_modeset_acquire_ctx *ctx) -{ - return 0; -} - -static void dce_virtual_crtc_destroy(struct drm_crtc *crtc) -{ - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - drm_crtc_cleanup(crtc); - kfree(amdgpu_crtc); -} - -static const struct drm_crtc_funcs dce_virtual_crtc_funcs = { - .cursor_set2 = NULL, - .cursor_move = NULL, - .gamma_set = dce_virtual_crtc_gamma_set, - .set_config = amdgpu_display_crtc_set_config, - .destroy = dce_virtual_crtc_destroy, - .page_flip_target = amdgpu_display_crtc_page_flip_target, - .get_vblank_counter = amdgpu_get_vblank_counter_kms, - .enable_vblank = amdgpu_enable_vblank_kms, - .disable_vblank = amdgpu_disable_vblank_kms, - .get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp, -}; - -static void dce_virtual_crtc_dpms(struct drm_crtc *crtc, int mode) -{ - struct drm_device *dev = crtc->dev; - struct amdgpu_device *adev = drm_to_adev(dev); - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - unsigned type; - - switch (mode) { - case DRM_MODE_DPMS_ON: - amdgpu_crtc->enabled = true; - /* Make sure VBLANK interrupts are still enabled */ - type = amdgpu_display_crtc_idx_to_irq_type(adev, - amdgpu_crtc->crtc_id); - amdgpu_irq_update(adev, &adev->crtc_irq, type); - drm_crtc_vblank_on(crtc); - break; - case DRM_MODE_DPMS_STANDBY: - case DRM_MODE_DPMS_SUSPEND: - case DRM_MODE_DPMS_OFF: - drm_crtc_vblank_off(crtc); - amdgpu_crtc->enabled = false; - break; - } -} - - -static void dce_virtual_crtc_prepare(struct drm_crtc *crtc) -{ - dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_OFF); -} - -static void dce_virtual_crtc_commit(struct drm_crtc *crtc) -{ - dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_ON); -} - -static void dce_virtual_crtc_disable(struct drm_crtc *crtc) -{ - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - struct drm_device *dev = crtc->dev; - - if (dev->num_crtcs) - drm_crtc_vblank_off(crtc); - - amdgpu_crtc->enabled = false; - amdgpu_crtc->pll_id = ATOM_PPLL_INVALID; - amdgpu_crtc->encoder = NULL; - amdgpu_crtc->connector = NULL; -} - -static int dce_virtual_crtc_mode_set(struct drm_crtc *crtc, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode, - int x, int y, struct drm_framebuffer *old_fb) -{ - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - /* update the hw version fpr dpm */ - amdgpu_crtc->hw_mode = *adjusted_mode; - - return 0; -} - -static bool dce_virtual_crtc_mode_fixup(struct drm_crtc *crtc, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - return true; -} - - -static int dce_virtual_crtc_set_base(struct drm_crtc *crtc, int x, int y, - struct drm_framebuffer *old_fb) -{ - return 0; -} - -static int dce_virtual_crtc_set_base_atomic(struct drm_crtc *crtc, - struct drm_framebuffer *fb, - int x, int y, enum mode_set_atomic state) -{ - return 0; -} - -static const struct drm_crtc_helper_funcs dce_virtual_crtc_helper_funcs = { - .dpms = dce_virtual_crtc_dpms, - .mode_fixup = dce_virtual_crtc_mode_fixup, - .mode_set = dce_virtual_crtc_mode_set, - .mode_set_base = dce_virtual_crtc_set_base, - .mode_set_base_atomic = dce_virtual_crtc_set_base_atomic, - .prepare = dce_virtual_crtc_prepare, - .commit = dce_virtual_crtc_commit, - .disable = dce_virtual_crtc_disable, - .get_scanout_position = amdgpu_crtc_get_scanout_position, -}; - -static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index) -{ - struct amdgpu_crtc *amdgpu_crtc; - - amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) + - (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL); - if (amdgpu_crtc == NULL) - return -ENOMEM; - - drm_crtc_init(adev_to_drm(adev), &amdgpu_crtc->base, &dce_virtual_crtc_funcs); - - drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256); - amdgpu_crtc->crtc_id = index; - adev->mode_info.crtcs[index] = amdgpu_crtc; - - amdgpu_crtc->pll_id = ATOM_PPLL_INVALID; - amdgpu_crtc->encoder = NULL; - amdgpu_crtc->connector = NULL; - amdgpu_crtc->vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE; - drm_crtc_helper_add(&amdgpu_crtc->base, &dce_virtual_crtc_helper_funcs); - - hrtimer_init(&amdgpu_crtc->vblank_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer_set_expires(&amdgpu_crtc->vblank_timer, DCE_VIRTUAL_VBLANK_PERIOD); - amdgpu_crtc->vblank_timer.function = dce_virtual_vblank_timer_handle; - hrtimer_start(&amdgpu_crtc->vblank_timer, - DCE_VIRTUAL_VBLANK_PERIOD, HRTIMER_MODE_REL); - return 0; -} - -static int dce_virtual_early_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - dce_virtual_set_display_funcs(adev); - dce_virtual_set_irq_funcs(adev); - - adev->mode_info.num_hpd = 1; - adev->mode_info.num_dig = 1; - return 0; -} - -static struct drm_encoder * -dce_virtual_encoder(struct drm_connector *connector) -{ - struct drm_encoder *encoder; - - drm_connector_for_each_possible_encoder(connector, encoder) { - if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL) - return encoder; - } - - /* pick the first one */ - drm_connector_for_each_possible_encoder(connector, encoder) - return encoder; - - return NULL; -} - -static int dce_virtual_get_modes(struct drm_connector *connector) -{ - struct drm_device *dev = connector->dev; - struct drm_display_mode *mode = NULL; - unsigned i; - static const struct mode_size { - int w; - int h; - } common_modes[] = { - { 640, 480}, - { 720, 480}, - { 800, 600}, - { 848, 480}, - {1024, 768}, - {1152, 768}, - {1280, 720}, - {1280, 800}, - {1280, 854}, - {1280, 960}, - {1280, 1024}, - {1440, 900}, - {1400, 1050}, - {1680, 1050}, - {1600, 1200}, - {1920, 1080}, - {1920, 1200}, - {2560, 1440}, - {4096, 3112}, - {3656, 2664}, - {3840, 2160}, - {4096, 2160}, - }; - - for (i = 0; i < ARRAY_SIZE(common_modes); i++) { - mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false); - drm_mode_probed_add(connector, mode); - } - - return 0; -} - -static enum drm_mode_status dce_virtual_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - return MODE_OK; -} - -static int -dce_virtual_dpms(struct drm_connector *connector, int mode) -{ - return 0; -} - -static int -dce_virtual_set_property(struct drm_connector *connector, - struct drm_property *property, - uint64_t val) -{ - return 0; -} - -static void dce_virtual_destroy(struct drm_connector *connector) -{ - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - kfree(connector); -} - -static void dce_virtual_force(struct drm_connector *connector) -{ - return; -} - -static const struct drm_connector_helper_funcs dce_virtual_connector_helper_funcs = { - .get_modes = dce_virtual_get_modes, - .mode_valid = dce_virtual_mode_valid, - .best_encoder = dce_virtual_encoder, -}; - -static const struct drm_connector_funcs dce_virtual_connector_funcs = { - .dpms = dce_virtual_dpms, - .fill_modes = drm_helper_probe_single_connector_modes, - .set_property = dce_virtual_set_property, - .destroy = dce_virtual_destroy, - .force = dce_virtual_force, -}; - -static int dce_virtual_sw_init(void *handle) -{ - int r, i; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SMU_DISP_TIMER2_TRIGGER, &adev->crtc_irq); - if (r) - return r; - - adev_to_drm(adev)->max_vblank_count = 0; - - adev_to_drm(adev)->mode_config.funcs = &amdgpu_mode_funcs; - - adev_to_drm(adev)->mode_config.max_width = 16384; - adev_to_drm(adev)->mode_config.max_height = 16384; - - adev_to_drm(adev)->mode_config.preferred_depth = 24; - adev_to_drm(adev)->mode_config.prefer_shadow = 1; - - adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; - - r = amdgpu_display_modeset_create_props(adev); - if (r) - return r; - - adev_to_drm(adev)->mode_config.max_width = 16384; - adev_to_drm(adev)->mode_config.max_height = 16384; - - /* allocate crtcs, encoders, connectors */ - for (i = 0; i < adev->mode_info.num_crtc; i++) { - r = dce_virtual_crtc_init(adev, i); - if (r) - return r; - r = dce_virtual_connector_encoder_init(adev, i); - if (r) - return r; - } - - drm_kms_helper_poll_init(adev_to_drm(adev)); - - adev->mode_info.mode_config_initialized = true; - return 0; -} - -static int dce_virtual_sw_fini(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int i = 0; - - for (i = 0; i < adev->mode_info.num_crtc; i++) - if (adev->mode_info.crtcs[i]) - hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer); - - kfree(adev->mode_info.bios_hardcoded_edid); - - drm_kms_helper_poll_fini(adev_to_drm(adev)); - - drm_mode_config_cleanup(adev_to_drm(adev)); - /* clear crtcs pointer to avoid dce irq finish routine access freed data */ - memset(adev->mode_info.crtcs, 0, sizeof(adev->mode_info.crtcs[0]) * AMDGPU_MAX_CRTCS); - adev->mode_info.mode_config_initialized = false; - return 0; -} - -static int dce_virtual_hw_init(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - switch (adev->asic_type) { -#ifdef CONFIG_DRM_AMDGPU_SI - case CHIP_TAHITI: - case CHIP_PITCAIRN: - case CHIP_VERDE: - case CHIP_OLAND: - dce_v6_0_disable_dce(adev); - break; -#endif -#ifdef CONFIG_DRM_AMDGPU_CIK - case CHIP_BONAIRE: - case CHIP_HAWAII: - case CHIP_KAVERI: - case CHIP_KABINI: - case CHIP_MULLINS: - dce_v8_0_disable_dce(adev); - break; -#endif - case CHIP_FIJI: - case CHIP_TONGA: - dce_v10_0_disable_dce(adev); - break; - case CHIP_CARRIZO: - case CHIP_STONEY: - case CHIP_POLARIS10: - case CHIP_POLARIS11: - case CHIP_VEGAM: - dce_v11_0_disable_dce(adev); - break; - case CHIP_TOPAZ: -#ifdef CONFIG_DRM_AMDGPU_SI - case CHIP_HAINAN: -#endif - /* no DCE */ - break; - default: - break; - } - return 0; -} - -static int dce_virtual_hw_fini(void *handle) -{ - return 0; -} - -static int dce_virtual_suspend(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = amdgpu_display_suspend_helper(adev); - if (r) - return r; - return dce_virtual_hw_fini(handle); -} - -static int dce_virtual_resume(void *handle) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - int r; - - r = dce_virtual_hw_init(handle); - if (r) - return r; - return amdgpu_display_resume_helper(adev); -} - -static bool dce_virtual_is_idle(void *handle) -{ - return true; -} - -static int dce_virtual_wait_for_idle(void *handle) -{ - return 0; -} - -static int dce_virtual_soft_reset(void *handle) -{ - return 0; -} - -static int dce_virtual_set_clockgating_state(void *handle, - enum amd_clockgating_state state) -{ - return 0; -} - -static int dce_virtual_set_powergating_state(void *handle, - enum amd_powergating_state state) -{ - return 0; -} - -static const struct amd_ip_funcs dce_virtual_ip_funcs = { - .name = "dce_virtual", - .early_init = dce_virtual_early_init, - .late_init = NULL, - .sw_init = dce_virtual_sw_init, - .sw_fini = dce_virtual_sw_fini, - .hw_init = dce_virtual_hw_init, - .hw_fini = dce_virtual_hw_fini, - .suspend = dce_virtual_suspend, - .resume = dce_virtual_resume, - .is_idle = dce_virtual_is_idle, - .wait_for_idle = dce_virtual_wait_for_idle, - .soft_reset = dce_virtual_soft_reset, - .set_clockgating_state = dce_virtual_set_clockgating_state, - .set_powergating_state = dce_virtual_set_powergating_state, -}; - -/* these are handled by the primary encoders */ -static void dce_virtual_encoder_prepare(struct drm_encoder *encoder) -{ - return; -} - -static void dce_virtual_encoder_commit(struct drm_encoder *encoder) -{ - return; -} - -static void -dce_virtual_encoder_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - return; -} - -static void dce_virtual_encoder_disable(struct drm_encoder *encoder) -{ - return; -} - -static void -dce_virtual_encoder_dpms(struct drm_encoder *encoder, int mode) -{ - return; -} - -static bool dce_virtual_encoder_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - return true; -} - -static const struct drm_encoder_helper_funcs dce_virtual_encoder_helper_funcs = { - .dpms = dce_virtual_encoder_dpms, - .mode_fixup = dce_virtual_encoder_mode_fixup, - .prepare = dce_virtual_encoder_prepare, - .mode_set = dce_virtual_encoder_mode_set, - .commit = dce_virtual_encoder_commit, - .disable = dce_virtual_encoder_disable, -}; - -static void dce_virtual_encoder_destroy(struct drm_encoder *encoder) -{ - drm_encoder_cleanup(encoder); - kfree(encoder); -} - -static const struct drm_encoder_funcs dce_virtual_encoder_funcs = { - .destroy = dce_virtual_encoder_destroy, -}; - -static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev, - int index) -{ - struct drm_encoder *encoder; - struct drm_connector *connector; - - /* add a new encoder */ - encoder = kzalloc(sizeof(struct drm_encoder), GFP_KERNEL); - if (!encoder) - return -ENOMEM; - encoder->possible_crtcs = 1 << index; - drm_encoder_init(adev_to_drm(adev), encoder, &dce_virtual_encoder_funcs, - DRM_MODE_ENCODER_VIRTUAL, NULL); - drm_encoder_helper_add(encoder, &dce_virtual_encoder_helper_funcs); - - connector = kzalloc(sizeof(struct drm_connector), GFP_KERNEL); - if (!connector) { - kfree(encoder); - return -ENOMEM; - } - - /* add a new connector */ - drm_connector_init(adev_to_drm(adev), connector, &dce_virtual_connector_funcs, - DRM_MODE_CONNECTOR_VIRTUAL); - drm_connector_helper_add(connector, &dce_virtual_connector_helper_funcs); - connector->display_info.subpixel_order = SubPixelHorizontalRGB; - connector->interlace_allowed = false; - connector->doublescan_allowed = false; - - /* link them */ - drm_connector_attach_encoder(connector, encoder); - - return 0; -} - -static const struct amdgpu_display_funcs dce_virtual_display_funcs = { - .bandwidth_update = &dce_virtual_bandwidth_update, - .vblank_get_counter = &dce_virtual_vblank_get_counter, - .backlight_set_level = NULL, - .backlight_get_level = NULL, - .hpd_sense = &dce_virtual_hpd_sense, - .hpd_set_polarity = &dce_virtual_hpd_set_polarity, - .hpd_get_gpio_reg = &dce_virtual_hpd_get_gpio_reg, - .page_flip = &dce_virtual_page_flip, - .page_flip_get_scanoutpos = &dce_virtual_crtc_get_scanoutpos, - .add_encoder = NULL, - .add_connector = NULL, -}; - -static void dce_virtual_set_display_funcs(struct amdgpu_device *adev) -{ - adev->mode_info.funcs = &dce_virtual_display_funcs; -} - -static int dce_virtual_pageflip(struct amdgpu_device *adev, - unsigned crtc_id) -{ - unsigned long flags; - struct amdgpu_crtc *amdgpu_crtc; - struct amdgpu_flip_work *works; - - amdgpu_crtc = adev->mode_info.crtcs[crtc_id]; - - if (crtc_id >= adev->mode_info.num_crtc) { - DRM_ERROR("invalid pageflip crtc %d\n", crtc_id); - return -EINVAL; - } - - /* IRQ could occur when in initial stage */ - if (amdgpu_crtc == NULL) - return 0; - - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - works = amdgpu_crtc->pflip_works; - if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { - DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != " - "AMDGPU_FLIP_SUBMITTED(%d)\n", - amdgpu_crtc->pflip_status, - AMDGPU_FLIP_SUBMITTED); - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - return 0; - } - - /* page flip completed. clean up */ - amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; - amdgpu_crtc->pflip_works = NULL; - - /* wakeup usersapce */ - if (works->event) - drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event); - - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - - drm_crtc_vblank_put(&amdgpu_crtc->base); - amdgpu_bo_unref(&works->old_abo); - kfree(works->shared); - kfree(works); - - return 0; -} - -static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vblank_timer) -{ - struct amdgpu_crtc *amdgpu_crtc = container_of(vblank_timer, - struct amdgpu_crtc, vblank_timer); - struct drm_device *ddev = amdgpu_crtc->base.dev; - struct amdgpu_device *adev = drm_to_adev(ddev); - struct amdgpu_irq_src *source = adev->irq.client[AMDGPU_IRQ_CLIENTID_LEGACY].sources - [VISLANDS30_IV_SRCID_SMU_DISP_TIMER2_TRIGGER]; - int irq_type = amdgpu_display_crtc_idx_to_irq_type(adev, - amdgpu_crtc->crtc_id); - - if (amdgpu_irq_enabled(adev, source, irq_type)) { - drm_handle_vblank(ddev, amdgpu_crtc->crtc_id); - dce_virtual_pageflip(adev, amdgpu_crtc->crtc_id); - } - hrtimer_start(vblank_timer, DCE_VIRTUAL_VBLANK_PERIOD, - HRTIMER_MODE_REL); - - return HRTIMER_NORESTART; -} - -static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev, - int crtc, - enum amdgpu_interrupt_state state) -{ - if (crtc >= adev->mode_info.num_crtc || !adev->mode_info.crtcs[crtc]) { - DRM_DEBUG("invalid crtc %d\n", crtc); - return; - } - - adev->mode_info.crtcs[crtc]->vsync_timer_enabled = state; - DRM_DEBUG("[FM]set crtc %d vblank interrupt state %d\n", crtc, state); -} - - -static int dce_virtual_set_crtc_irq_state(struct amdgpu_device *adev, - struct amdgpu_irq_src *source, - unsigned type, - enum amdgpu_interrupt_state state) -{ - if (type > AMDGPU_CRTC_IRQ_VBLANK6) - return -EINVAL; - - dce_virtual_set_crtc_vblank_interrupt_state(adev, type, state); - - return 0; -} - -static const struct amdgpu_irq_src_funcs dce_virtual_crtc_irq_funcs = { - .set = dce_virtual_set_crtc_irq_state, - .process = NULL, -}; - -static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev) -{ - adev->crtc_irq.num_types = adev->mode_info.num_crtc; - adev->crtc_irq.funcs = &dce_virtual_crtc_irq_funcs; -} - -const struct amdgpu_ip_block_version dce_virtual_ip_block = -{ - .type = AMD_IP_BLOCK_TYPE_DCE, - .major = 1, - .minor = 0, - .rev = 0, - .funcs = &dce_virtual_ip_funcs, -}; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 24b0195fe665..16dbe593cba2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4621,6 +4621,7 @@ static void gfx_v10_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); + dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); } static void gfx_v10_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index 3a8d52a54873..6a8dadea40f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -3027,6 +3027,7 @@ static void gfx_v6_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, u dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); + dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); } static void gfx_v6_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 685212c3ddae..37b4a3db6360 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -4198,6 +4198,7 @@ static void gfx_v7_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, u dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); + dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); } static void gfx_v7_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index c26e06059466..e0302c23e9a7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -5279,6 +5279,7 @@ static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, u dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); + dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); } static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 03acc777adf7..603c259b073b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1295,6 +1295,16 @@ static bool is_raven_kicker(struct amdgpu_device *adev) return false; } +static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) +{ + if ((adev->asic_type == CHIP_RENOIR) && + (adev->gfx.me_fw_version >= 0x000000a5) && + (adev->gfx.me_feature_version >= 52)) + return true; + else + return false; +} + static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) { if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) @@ -2080,6 +2090,7 @@ static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, u dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); + dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); } static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, @@ -3675,7 +3686,16 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) if (ring->use_doorbell) { WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, (adev->doorbell_index.kiq * 2) << 2); - WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, + /* If GC has entered CGPG, ringing doorbell > first page + * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to + * workaround this issue. And this change has to align with firmware + * update. + */ + if (check_if_enlarge_doorbell_range(adev)) + WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, + (adev->doorbell.size - 4)); + else + WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, (adev->doorbell_index.userqueue_end * 2) << 2); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c index 1769c4cba2ad..00a2b36a24b3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c @@ -85,7 +85,7 @@ static const struct soc15_reg_golden golden_settings_gc_9_4_2_alde[] = { SOC15_REG_GOLDEN_VALUE(GC, 0, regTCI_CNTL_3, 0xff, 0x20), }; -/** +/* * This shader is used to clear VGPRS and LDS, and also write the input * pattern into the write back buffer, which will be used by driver to * check whether all SIMDs have been covered. @@ -206,7 +206,7 @@ const struct soc15_reg_entry vgpr_init_regs_aldebaran[] = { { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, }; -/** +/* * The below shaders are used to clear SGPRS, and also write the input * pattern into the write back buffer. The first two dispatch should be * scheduled simultaneously which make sure that all SGPRS could be @@ -302,7 +302,7 @@ const struct soc15_reg_entry sgpr96_init_regs_aldebaran[] = { { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, }; -/** +/* * This shader is used to clear the uninitiated sgprs after the above * two dispatches, because of hardware feature, dispath 0 couldn't clear * top hole sgprs. Therefore need 4 waves per SIMD to cover these sgprs diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c index 8fca72ebd11c..497b86c376c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c @@ -75,9 +75,8 @@ int gfxhub_v1_1_get_xgmi_info(struct amdgpu_device *adev) max_physical_node_id = 7; break; case CHIP_ALDEBARAN: - /* just using duplicates for Aldebaran support, revisit later */ - max_num_physical_nodes = 8; - max_physical_node_id = 7; + max_num_physical_nodes = 16; + max_physical_node_id = 15; break; default: return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 24b781e90bef..41c3a0d70b7c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -93,6 +93,7 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { bool retry_fault = !!(entry->src_data[1] & 0x80); + bool write_fault = !!(entry->src_data[1] & 0x20); struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; struct amdgpu_task_info task_info; uint32_t status = 0; @@ -121,7 +122,7 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, /* Try to handle the recoverable page faults by filling page * tables */ - if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) + if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault)) return 1; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 7cf653f9e9a7..d90c16a6b2b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -55,6 +55,7 @@ #include "umc_v6_0.h" #include "umc_v6_7.h" #include "hdp_v4_0.h" +#include "mca_v3_0.h" #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" @@ -506,6 +507,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { bool retry_fault = !!(entry->src_data[1] & 0x80); + bool write_fault = !!(entry->src_data[1] & 0x20); uint32_t status = 0, cid = 0, rw = 0; struct amdgpu_task_info task_info; struct amdgpu_vmhub *hub; @@ -536,7 +538,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, /* Try to handle the recoverable page faults by filling page * tables */ - if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) + if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault)) return 1; } @@ -1171,8 +1173,8 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) break; case CHIP_ALDEBARAN: adev->umc.max_ras_err_cnt_per_query = UMC_V6_7_TOTAL_CHANNEL_NUM; - adev->umc.channel_inst_num = UMC_V6_7_UMC_INSTANCE_NUM; - adev->umc.umc_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM; + adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM; + adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM; adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET; if (!adev->gmc.xgmi.connected_to_cpu) adev->umc.ras_funcs = &umc_v6_7_ras_funcs; @@ -1229,6 +1231,18 @@ static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev) adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs; } +static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev) +{ + switch (adev->asic_type) { + case CHIP_ALDEBARAN: + if (!adev->gmc.xgmi.connected_to_cpu) + adev->mca.funcs = &mca_v3_0_funcs; + break; + default: + break; + } +} + static int gmc_v9_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -1250,6 +1264,7 @@ static int gmc_v9_0_early_init(void *handle) gmc_v9_0_set_mmhub_ras_funcs(adev); gmc_v9_0_set_gfxhub_funcs(adev); gmc_v9_0_set_hdp_ras_funcs(adev); + gmc_v9_0_set_mca_funcs(adev); adev->gmc.shared_aperture_start = 0x2000000000000000ULL; adev->gmc.shared_aperture_end = @@ -1461,6 +1476,8 @@ static int gmc_v9_0_sw_init(void *handle) adev->gfxhub.funcs->init(adev); adev->mmhub.funcs->init(adev); + if (adev->mca.funcs) + adev->mca.funcs->init(adev); spin_lock_init(&adev->gmc.invalidate_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c new file mode 100644 index 000000000000..058b65730a84 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c @@ -0,0 +1,125 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu_ras.h" +#include "amdgpu.h" +#include "amdgpu_mca.h" + +#define smnMCMP0_STATUST0 0x03830408 +#define smnMCMP1_STATUST0 0x03b30408 +#define smnMCMPIO_STATUST0 0x0c930408 + + +static void mca_v3_0_mp0_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status) +{ + amdgpu_mca_query_ras_error_count(adev, + smnMCMP0_STATUST0, + ras_error_status); +} + +static int mca_v3_0_mp0_ras_late_init(struct amdgpu_device *adev) +{ + return amdgpu_mca_ras_late_init(adev, &adev->mca.mp0); +} + +static void mca_v3_0_mp0_ras_fini(struct amdgpu_device *adev) +{ + amdgpu_mca_ras_fini(adev, &adev->mca.mp0); +} + +const struct amdgpu_mca_ras_funcs mca_v3_0_mp0_ras_funcs = { + .ras_late_init = mca_v3_0_mp0_ras_late_init, + .ras_fini = mca_v3_0_mp0_ras_fini, + .query_ras_error_count = mca_v3_0_mp0_query_ras_error_count, + .query_ras_error_address = NULL, + .ras_block = AMDGPU_RAS_BLOCK__MP0, + .sysfs_name = "mp0_err_count", +}; + +static void mca_v3_0_mp1_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status) +{ + amdgpu_mca_query_ras_error_count(adev, + smnMCMP1_STATUST0, + ras_error_status); +} + +static int mca_v3_0_mp1_ras_late_init(struct amdgpu_device *adev) +{ + return amdgpu_mca_ras_late_init(adev, &adev->mca.mp1); +} + +static void mca_v3_0_mp1_ras_fini(struct amdgpu_device *adev) +{ + amdgpu_mca_ras_fini(adev, &adev->mca.mp1); +} + +const struct amdgpu_mca_ras_funcs mca_v3_0_mp1_ras_funcs = { + .ras_late_init = mca_v3_0_mp1_ras_late_init, + .ras_fini = mca_v3_0_mp1_ras_fini, + .query_ras_error_count = mca_v3_0_mp1_query_ras_error_count, + .query_ras_error_address = NULL, + .ras_block = AMDGPU_RAS_BLOCK__MP1, + .sysfs_name = "mp1_err_count", +}; + +static void mca_v3_0_mpio_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status) +{ + amdgpu_mca_query_ras_error_count(adev, + smnMCMPIO_STATUST0, + ras_error_status); +} + +static int mca_v3_0_mpio_ras_late_init(struct amdgpu_device *adev) +{ + return amdgpu_mca_ras_late_init(adev, &adev->mca.mpio); +} + +static void mca_v3_0_mpio_ras_fini(struct amdgpu_device *adev) +{ + amdgpu_mca_ras_fini(adev, &adev->mca.mpio); +} + +const struct amdgpu_mca_ras_funcs mca_v3_0_mpio_ras_funcs = { + .ras_late_init = mca_v3_0_mpio_ras_late_init, + .ras_fini = mca_v3_0_mpio_ras_fini, + .query_ras_error_count = mca_v3_0_mpio_query_ras_error_count, + .query_ras_error_address = NULL, + .ras_block = AMDGPU_RAS_BLOCK__MPIO, + .sysfs_name = "mpio_err_count", +}; + + +static void mca_v3_0_init(struct amdgpu_device *adev) +{ + struct amdgpu_mca *mca = &adev->mca; + + mca->mp0.ras_funcs = &mca_v3_0_mp0_ras_funcs; + mca->mp1.ras_funcs = &mca_v3_0_mp1_ras_funcs; + mca->mpio.ras_funcs = &mca_v3_0_mpio_ras_funcs; +} + +const struct amdgpu_mca_funcs mca_v3_0_funcs = { + .init = mca_v3_0_init, +};
\ No newline at end of file diff --git a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h new file mode 100644 index 000000000000..b899b86194c2 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __MCA_V3_0_H__ +#define __MCA_V3_0_H__ + +extern const struct amdgpu_mca_funcs mca_v3_0_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h b/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h index 20958639b601..2cdab8062c86 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h +++ b/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h @@ -24,9 +24,7 @@ #ifndef __MMSCH_V1_0_H__ #define __MMSCH_V1_0_H__ -#define MMSCH_VERSION_MAJOR 1 -#define MMSCH_VERSION_MINOR 0 -#define MMSCH_VERSION (MMSCH_VERSION_MAJOR << 16 | MMSCH_VERSION_MINOR) +#define MMSCH_VERSION 0x1 enum mmsch_v1_0_command_type { MMSCH_COMMAND__DIRECT_REG_WRITE = 0, diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index ff2307d7ee0f..23b066bcffb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -258,6 +258,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); + xgpu_ai_mailbox_trans_msg(adev, IDH_READY_TO_RESET, 0, 0, 0); + do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) goto flr_done; diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h index 50572635d0f8..bd3b23171579 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h @@ -37,6 +37,7 @@ enum idh_request { IDH_REQ_GPU_RESET_ACCESS, IDH_LOG_VF_ERROR = 200, + IDH_READY_TO_RESET = 201, }; enum idh_event { diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index 9f7aac435d69..a35e6d87e537 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -96,7 +96,11 @@ static int xgpu_nv_poll_ack(struct amdgpu_device *adev) static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event) { - int r, timeout = NV_MAILBOX_POLL_MSG_TIMEDOUT; + int r; + uint64_t timeout, now; + + now = (uint64_t)ktime_to_ms(ktime_get()); + timeout = now + NV_MAILBOX_POLL_MSG_TIMEDOUT; do { r = xgpu_nv_mailbox_rcv_msg(adev, event); @@ -104,8 +108,8 @@ static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event) return 0; msleep(10); - timeout -= 10; - } while (timeout > 1); + now = (uint64_t)ktime_to_ms(ktime_get()); + } while (timeout > now); return -ETIME; @@ -149,9 +153,10 @@ static void xgpu_nv_mailbox_trans_msg (struct amdgpu_device *adev, static int xgpu_nv_send_access_requests(struct amdgpu_device *adev, enum idh_request req) { - int r; + int r, retry = 1; enum idh_event event = -1; +send_request: xgpu_nv_mailbox_trans_msg(adev, req, 0, 0, 0); switch (req) { @@ -170,6 +175,9 @@ static int xgpu_nv_send_access_requests(struct amdgpu_device *adev, if (event != -1) { r = xgpu_nv_poll_msg(adev, event); if (r) { + if (retry++ < 2) + goto send_request; + if (req != IDH_REQ_GPU_INIT_DATA) { pr_err("Doesn't get msg:%d from pf, error=%d\n", event, r); return r; @@ -279,6 +287,8 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); + xgpu_nv_mailbox_trans_msg(adev, IDH_READY_TO_RESET, 0, 0, 0); + do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) goto flr_done; diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h index 9f5808616174..73887b0aa1d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h @@ -37,7 +37,8 @@ enum idh_request { IDH_REQ_GPU_RESET_ACCESS, IDH_REQ_GPU_INIT_DATA, - IDH_LOG_VF_ERROR = 200, + IDH_LOG_VF_ERROR = 200, + IDH_READY_TO_RESET = 201, }; enum idh_event { diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index cef929746739..f50045cebd44 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -85,6 +85,14 @@ #define mmRCC_DEV0_EPF0_STRAP0_ALDE 0x0015 #define mmRCC_DEV0_EPF0_STRAP0_ALDE_BASE_IDX 2 +#define mmBIF_DOORBELL_INT_CNTL_ALDE 0x00fe +#define mmBIF_DOORBELL_INT_CNTL_ALDE_BASE_IDX 2 +#define BIF_DOORBELL_INT_CNTL_ALDE__DOORBELL_INTERRUPT_DISABLE__SHIFT 0x18 +#define BIF_DOORBELL_INT_CNTL_ALDE__DOORBELL_INTERRUPT_DISABLE_MASK 0x01000000L + +#define mmBIF_INTR_CNTL_ALDE 0x0101 +#define mmBIF_INTR_CNTL_ALDE_BASE_IDX 2 + static void nbio_v7_4_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status); @@ -346,14 +354,21 @@ static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device struct ras_err_data err_data = {0, 0, 0, NULL}; struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); - bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); + if (adev->asic_type == CHIP_ALDEBARAN) + bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE); + else + bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); + if (REG_GET_FIELD(bif_doorbell_intr_cntl, BIF_DOORBELL_INT_CNTL, RAS_CNTLR_INTERRUPT_STATUS)) { /* driver has to clear the interrupt status when bif ring is disabled */ bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, BIF_DOORBELL_INT_CNTL, RAS_CNTLR_INTERRUPT_CLEAR, 1); - WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); + if (adev->asic_type == CHIP_ALDEBARAN) + WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE, bif_doorbell_intr_cntl); + else + WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); if (!ras->disable_ras_err_cnt_harvest) { /* @@ -372,13 +387,13 @@ static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device "errors detected in %s block, " "no user action is needed.\n", obj->err_data.ce_count, - adev->nbio.ras_if->name); + ras_block_str(adev->nbio.ras_if->block)); if (err_data.ue_count) dev_info(adev->dev, "%ld uncorrectable hardware " "errors detected in %s block\n", obj->err_data.ue_count, - adev->nbio.ras_if->name); + ras_block_str(adev->nbio.ras_if->block)); } dev_info(adev->dev, "RAS controller interrupt triggered " @@ -395,14 +410,22 @@ static void nbio_v7_4_handle_ras_err_event_athub_intr_no_bifring(struct amdgpu_d { uint32_t bif_doorbell_intr_cntl; - bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); + if (adev->asic_type == CHIP_ALDEBARAN) + bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE); + else + bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); + if (REG_GET_FIELD(bif_doorbell_intr_cntl, BIF_DOORBELL_INT_CNTL, RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS)) { /* driver has to clear the interrupt status when bif ring is disabled */ bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, BIF_DOORBELL_INT_CNTL, RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR, 1); - WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); + + if (adev->asic_type == CHIP_ALDEBARAN) + WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE, bif_doorbell_intr_cntl); + else + WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); amdgpu_ras_global_ras_isr(adev); } @@ -420,14 +443,23 @@ static int nbio_v7_4_set_ras_controller_irq_state(struct amdgpu_device *adev, */ uint32_t bif_intr_cntl; - bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL); + if (adev->asic_type == CHIP_ALDEBARAN) + bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE); + else + bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL); + if (state == AMDGPU_IRQ_STATE_ENABLE) { /* set interrupt vector select bit to 0 to select * vetcor 1 for bare metal case */ bif_intr_cntl = REG_SET_FIELD(bif_intr_cntl, BIF_INTR_CNTL, RAS_INTR_VEC_SEL, 0); - WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl); + + if (adev->asic_type == CHIP_ALDEBARAN) + WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE, bif_intr_cntl); + else + WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl); + } return 0; @@ -456,14 +488,22 @@ static int nbio_v7_4_set_ras_err_event_athub_irq_state(struct amdgpu_device *ade */ uint32_t bif_intr_cntl; - bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL); + if (adev->asic_type == CHIP_ALDEBARAN) + bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE); + else + bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL); + if (state == AMDGPU_IRQ_STATE_ENABLE) { /* set interrupt vector select bit to 0 to select * vetcor 1 for bare metal case */ bif_intr_cntl = REG_SET_FIELD(bif_intr_cntl, BIF_INTR_CNTL, RAS_INTR_VEC_SEL, 0); - WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl); + + if (adev->asic_type == CHIP_ALDEBARAN) + WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE, bif_intr_cntl); + else + WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl); } return 0; @@ -572,7 +612,11 @@ static void nbio_v7_4_query_ras_error_count(struct amdgpu_device *adev, static void nbio_v7_4_enable_doorbell_interrupt(struct amdgpu_device *adev, bool enable) { - WREG32_FIELD15(NBIO, 0, BIF_DOORBELL_INT_CNTL, + if (adev->asic_type == CHIP_ALDEBARAN) + WREG32_FIELD15(NBIO, 0, BIF_DOORBELL_INT_CNTL_ALDE, + DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1); + else + WREG32_FIELD15(NBIO, 0, BIF_DOORBELL_INT_CNTL, DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1); } diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 436fb13e32f0..ff80786e3918 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -58,7 +58,7 @@ #include "jpeg_v2_0.h" #include "vcn_v3_0.h" #include "jpeg_v3_0.h" -#include "dce_virtual.h" +#include "amdgpu_vkms.h" #include "mes_v10_1.h" #include "mxgpu_nv.h" #include "smuio_v11_0.h" @@ -721,7 +721,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) !amdgpu_sriov_vf(adev)) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -749,7 +749,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -779,7 +779,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) is_support_sw_smu(adev)) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -802,7 +802,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) is_support_sw_smu(adev)) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -823,7 +823,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -843,7 +843,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) is_support_sw_smu(adev)) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -865,7 +865,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v5_2_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -883,11 +883,11 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v13_0_ip_block); amdgpu_device_ip_block_add(adev, &smu_v13_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v5_2_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -905,7 +905,7 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); } if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v5_0_ip_block); break; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c index 4b1cc5e9ee92..5872d68ed13d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c @@ -84,29 +84,29 @@ static int psp_v10_0_init_microcode(struct psp_context *psp) ta_hdr = (const struct ta_firmware_header_v1_0 *) adev->psp.ta_fw->data; - adev->psp.ta_hdcp_ucode_version = - le32_to_cpu(ta_hdr->ta_hdcp_ucode_version); - adev->psp.ta_hdcp_ucode_size = - le32_to_cpu(ta_hdr->ta_hdcp_size_bytes); - adev->psp.ta_hdcp_start_addr = + adev->psp.hdcp.feature_version = + le32_to_cpu(ta_hdr->hdcp.fw_version); + adev->psp.hdcp.size_bytes = + le32_to_cpu(ta_hdr->hdcp.size_bytes); + adev->psp.hdcp.start_addr = (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); - adev->psp.ta_dtm_ucode_version = - le32_to_cpu(ta_hdr->ta_dtm_ucode_version); - adev->psp.ta_dtm_ucode_size = - le32_to_cpu(ta_hdr->ta_dtm_size_bytes); - adev->psp.ta_dtm_start_addr = - (uint8_t *)adev->psp.ta_hdcp_start_addr + - le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); - - adev->psp.ta_securedisplay_ucode_version = - le32_to_cpu(ta_hdr->ta_securedisplay_ucode_version); - adev->psp.ta_securedisplay_ucode_size = - le32_to_cpu(ta_hdr->ta_securedisplay_size_bytes); - adev->psp.ta_securedisplay_start_addr = - (uint8_t *)adev->psp.ta_hdcp_start_addr + - le32_to_cpu(ta_hdr->ta_securedisplay_offset_bytes); + adev->psp.dtm.feature_version = + le32_to_cpu(ta_hdr->dtm.fw_version); + adev->psp.dtm.size_bytes = + le32_to_cpu(ta_hdr->dtm.size_bytes); + adev->psp.dtm.start_addr = + (uint8_t *)adev->psp.hdcp.start_addr + + le32_to_cpu(ta_hdr->dtm.offset_bytes); + + adev->psp.securedisplay.feature_version = + le32_to_cpu(ta_hdr->securedisplay.fw_version); + adev->psp.securedisplay.size_bytes = + le32_to_cpu(ta_hdr->securedisplay.size_bytes); + adev->psp.securedisplay.start_addr = + (uint8_t *)adev->psp.hdcp.start_addr + + le32_to_cpu(ta_hdr->securedisplay.offset_bytes); adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index 8862684f8b43..29bf9f09944b 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -151,15 +151,15 @@ static int psp_v11_0_init_microcode(struct psp_context *psp) goto out2; ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data; - adev->psp.ta_xgmi_ucode_version = le32_to_cpu(ta_hdr->ta_xgmi_ucode_version); - adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes); - adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr + + adev->psp.xgmi.feature_version = le32_to_cpu(ta_hdr->xgmi.fw_version); + adev->psp.xgmi.size_bytes = le32_to_cpu(ta_hdr->xgmi.size_bytes); + adev->psp.xgmi.start_addr = (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); - adev->psp.ta_ras_ucode_version = le32_to_cpu(ta_hdr->ta_ras_ucode_version); - adev->psp.ta_ras_ucode_size = le32_to_cpu(ta_hdr->ta_ras_size_bytes); - adev->psp.ta_ras_start_addr = (uint8_t *)adev->psp.ta_xgmi_start_addr + - le32_to_cpu(ta_hdr->ta_ras_offset_bytes); + adev->psp.ras.feature_version = le32_to_cpu(ta_hdr->ras.fw_version); + adev->psp.ras.size_bytes = le32_to_cpu(ta_hdr->ras.size_bytes); + adev->psp.ras.start_addr = (uint8_t *)adev->psp.xgmi.start_addr + + le32_to_cpu(ta_hdr->ras.offset_bytes); } break; case CHIP_NAVI10: @@ -186,17 +186,17 @@ static int psp_v11_0_init_microcode(struct psp_context *psp) goto out2; ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data; - adev->psp.ta_hdcp_ucode_version = le32_to_cpu(ta_hdr->ta_hdcp_ucode_version); - adev->psp.ta_hdcp_ucode_size = le32_to_cpu(ta_hdr->ta_hdcp_size_bytes); - adev->psp.ta_hdcp_start_addr = (uint8_t *)ta_hdr + + adev->psp.hdcp.feature_version = le32_to_cpu(ta_hdr->hdcp.fw_version); + adev->psp.hdcp.size_bytes = le32_to_cpu(ta_hdr->hdcp.size_bytes); + adev->psp.hdcp.start_addr = (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); - adev->psp.ta_dtm_ucode_version = le32_to_cpu(ta_hdr->ta_dtm_ucode_version); - adev->psp.ta_dtm_ucode_size = le32_to_cpu(ta_hdr->ta_dtm_size_bytes); - adev->psp.ta_dtm_start_addr = (uint8_t *)adev->psp.ta_hdcp_start_addr + - le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); + adev->psp.dtm.feature_version = le32_to_cpu(ta_hdr->dtm.fw_version); + adev->psp.dtm.size_bytes = le32_to_cpu(ta_hdr->dtm.size_bytes); + adev->psp.dtm.start_addr = (uint8_t *)adev->psp.hdcp.start_addr + + le32_to_cpu(ta_hdr->dtm.offset_bytes); } break; case CHIP_SIENNA_CICHLID: diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index 0c908d4566e8..cc649406234b 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -84,23 +84,23 @@ static int psp_v12_0_init_microcode(struct psp_context *psp) ta_hdr = (const struct ta_firmware_header_v1_0 *) adev->psp.ta_fw->data; - adev->psp.ta_hdcp_ucode_version = - le32_to_cpu(ta_hdr->ta_hdcp_ucode_version); - adev->psp.ta_hdcp_ucode_size = - le32_to_cpu(ta_hdr->ta_hdcp_size_bytes); - adev->psp.ta_hdcp_start_addr = + adev->psp.hdcp.feature_version = + le32_to_cpu(ta_hdr->hdcp.fw_version); + adev->psp.hdcp.size_bytes = + le32_to_cpu(ta_hdr->hdcp.size_bytes); + adev->psp.hdcp.start_addr = (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); - adev->psp.ta_dtm_ucode_version = - le32_to_cpu(ta_hdr->ta_dtm_ucode_version); - adev->psp.ta_dtm_ucode_size = - le32_to_cpu(ta_hdr->ta_dtm_size_bytes); - adev->psp.ta_dtm_start_addr = - (uint8_t *)adev->psp.ta_hdcp_start_addr + - le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); + adev->psp.dtm.feature_version = + le32_to_cpu(ta_hdr->dtm.fw_version); + adev->psp.dtm.size_bytes = + le32_to_cpu(ta_hdr->dtm.size_bytes); + adev->psp.dtm.start_addr = + (uint8_t *)adev->psp.hdcp.start_addr + + le32_to_cpu(ta_hdr->dtm.offset_bytes); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 7cbc2bb03bc6..e6d2f74a7976 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -44,7 +44,7 @@ #include "dce_v6_0.h" #include "si.h" #include "uvd_v3_1.h" -#include "dce_virtual.h" +#include "amdgpu_vkms.h" #include "gca/gfx_6_0_d.h" #include "oss/oss_1_0_d.h" #include "oss/oss_1_0_sh_mask.h" @@ -2759,7 +2759,7 @@ int si_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &si_dma_ip_block); amdgpu_device_ip_block_add(adev, &si_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2777,7 +2777,7 @@ int si_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &si_dma_ip_block); amdgpu_device_ip_block_add(adev, &si_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2795,7 +2795,7 @@ int si_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &si_dma_ip_block); amdgpu_device_ip_block_add(adev, &si_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); break; default: BUG(); diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index a5e085e570f7..0fc97c364fd7 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -74,7 +74,7 @@ #include "smuio_v9_0.h" #include "smuio_v11_0.h" #include "smuio_v13_0.h" -#include "dce_virtual.h" +#include "amdgpu_vkms.h" #include "mxgpu_ai.h" #include "amdgpu_ras.h" #include "amdgpu_xgmi.h" @@ -843,7 +843,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); } if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -863,7 +863,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -885,7 +885,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) } if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); @@ -909,7 +909,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -1353,8 +1353,6 @@ static int soc15_common_early_init(void *handle) adev->asic_funcs = &vega20_asic_funcs; adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS | - AMD_CG_SUPPORT_GFX_CGCG | - AMD_CG_SUPPORT_GFX_CGLS | AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_SDMA_MGCG | diff --git a/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h b/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h index cce7127afeaa..da815a93d46e 100644 --- a/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h +++ b/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h @@ -134,7 +134,8 @@ struct ta_xgmi_shared_memory { uint32_t cmd_id; uint32_t resp_id; enum ta_xgmi_status xgmi_status; - uint32_t reserved; + uint8_t flag_extend_link_record; + uint8_t reserved0[3]; union ta_xgmi_cmd_input xgmi_in_message; union ta_xgmi_cmd_output xgmi_out_message; }; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c index 7da12110425c..bb30336b1e8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c @@ -30,17 +30,17 @@ const uint32_t umc_v6_7_channel_idx_tbl_second[UMC_V6_7_UMC_INSTANCE_NUM][UMC_V6_7_CHANNEL_INSTANCE_NUM] = { - {28, 12, 6, 22}, {19, 3, 9, 25}, - {20, 4, 30, 14}, {11, 27, 1, 17}, - {24, 8, 2, 18}, {15, 31, 5, 21}, - {16, 0, 26, 10}, {7, 23, 29, 13} + {28, 20, 24, 16, 12, 4, 8, 0}, + {6, 30, 2, 26, 22, 14, 18, 10}, + {19, 11, 15, 7, 3, 27, 31, 23}, + {9, 1, 5, 29, 25, 17, 21, 13} }; const uint32_t umc_v6_7_channel_idx_tbl_first[UMC_V6_7_UMC_INSTANCE_NUM][UMC_V6_7_CHANNEL_INSTANCE_NUM] = { - {19, 3, 9, 25}, {28, 12, 6, 22}, - {11, 27, 1, 17}, {20, 4, 30, 14}, - {15, 31, 5, 21}, {24, 8, 2, 18}, - {7, 23, 29, 13}, {16, 0, 26, 10} + {19, 11, 15, 7, 3, 27, 31, 23}, + {9, 1, 5, 29, 25, 17, 21, 13}, + {28, 20, 24, 16, 12, 4, 8, 0}, + {6, 30, 2, 26, 22, 14, 18, 10}, }; static inline uint32_t get_umc_v6_7_reg_offset(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h index 81b8f1844091..57f2557e7aca 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h @@ -36,9 +36,9 @@ #define UMC_V6_7_INST_DIST 0x40000 /* number of umc channel instance with memory map register access */ -#define UMC_V6_7_CHANNEL_INSTANCE_NUM 4 +#define UMC_V6_7_UMC_INSTANCE_NUM 4 /* number of umc instance with memory map register access */ -#define UMC_V6_7_UMC_INSTANCE_NUM 8 +#define UMC_V6_7_CHANNEL_INSTANCE_NUM 8 /* total channel instances in one umc block */ #define UMC_V6_7_TOTAL_CHANNEL_NUM (UMC_V6_7_CHANNEL_INSTANCE_NUM * UMC_V6_7_UMC_INSTANCE_NUM) /* UMC regiser per channel offset */ diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c index 6c0e91495365..7232241e3bfb 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c @@ -698,6 +698,30 @@ static int uvd_v3_1_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->uvd.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + if (RREG32(mmUVD_STATUS) != 0) uvd_v3_1_stop(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index a301518e4957..52d6de969f46 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -212,6 +212,30 @@ static int uvd_v4_2_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->uvd.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + if (RREG32(mmUVD_STATUS) != 0) uvd_v4_2_stop(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index a4d5bd21c83c..db6d06758e4d 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -210,6 +210,30 @@ static int uvd_v5_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->uvd.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + if (RREG32(mmUVD_STATUS) != 0) uvd_v5_0_stop(adev); @@ -224,7 +248,6 @@ static int uvd_v5_0_suspend(void *handle) r = uvd_v5_0_hw_fini(adev); if (r) return r; - uvd_v5_0_set_clockgating_state(adev, AMD_CG_STATE_GATE); return amdgpu_uvd_suspend(adev); } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index cf3803f8f075..bc571833632e 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -543,6 +543,30 @@ static int uvd_v6_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->uvd.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + if (RREG32(mmUVD_STATUS) != 0) uvd_v6_0_stop(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index 939bcfa2a4ec..b6e82d75561f 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c @@ -606,6 +606,30 @@ static int uvd_v7_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->uvd.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + if (!amdgpu_sriov_vf(adev)) uvd_v7_0_stop(adev); else { diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c index c7d28c169be5..b70c17f0c52e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c @@ -477,6 +477,31 @@ static int vce_v2_0_hw_init(void *handle) static int vce_v2_0_hw_fini(void *handle) { + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->vce.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_vce(adev, false); + } else { + amdgpu_asic_set_vce_clocks(adev, 0, 0); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_GATE); + } + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 3b82fb289ef6..9de66893ccd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -490,6 +490,29 @@ static int vce_v3_0_hw_fini(void *handle) int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->vce.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_vce(adev, false); + } else { + amdgpu_asic_set_vce_clocks(adev, 0, 0); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_GATE); + } + r = vce_v3_0_wait_for_idle(handle); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c index 90910d19db12..fec902b800c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c @@ -542,6 +542,29 @@ static int vce_v4_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + /* + * Proper cleanups before halting the HW engine: + * - cancel the delayed idle work + * - enable powergating + * - enable clockgating + * - disable dpm + * + * TODO: to align with the VCN implementation, move the + * jobs for clockgating/powergating/dpm setting to + * ->set_powergating_state(). + */ + cancel_delayed_work_sync(&adev->vce.idle_work); + + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_vce(adev, false); + } else { + amdgpu_asic_set_vce_clocks(adev, 0, 0); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_GATE); + } + if (!amdgpu_sriov_vf(adev)) { /* vce_v4_0_wait_for_idle(handle); */ vce_v4_0_stop(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index 2f017560948e..3d18aab88b4e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -90,9 +90,7 @@ static int vcn_v3_0_early_init(void *handle) int i; if (amdgpu_sriov_vf(adev)) { - for (i = 0; i < VCN_INSTANCES_SIENNA_CICHLID; i++) - if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) - adev->vcn.num_vcn_inst++; + adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID; adev->vcn.harvest_config = 0; adev->vcn.num_enc_rings = 1; @@ -153,8 +151,7 @@ static int vcn_v3_0_sw_init(void *handle) adev->firmware.fw_size += ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE); - if ((adev->vcn.num_vcn_inst == VCN_INSTANCES_SIENNA_CICHLID) || - (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)) { + if (adev->vcn.num_vcn_inst == VCN_INSTANCES_SIENNA_CICHLID) { adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].ucode_id = AMDGPU_UCODE_ID_VCN1; adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].fw = adev->vcn.fw; adev->firmware.fw_size += @@ -328,18 +325,28 @@ static int vcn_v3_0_hw_init(void *handle) continue; ring = &adev->vcn.inst[i].ring_dec; - ring->wptr = 0; - ring->wptr_old = 0; - vcn_v3_0_dec_ring_set_wptr(ring); - ring->sched.ready = true; - - for (j = 0; j < adev->vcn.num_enc_rings; ++j) { - ring = &adev->vcn.inst[i].ring_enc[j]; + if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) { + ring->sched.ready = false; + dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name); + } else { ring->wptr = 0; ring->wptr_old = 0; - vcn_v3_0_enc_ring_set_wptr(ring); + vcn_v3_0_dec_ring_set_wptr(ring); ring->sched.ready = true; } + + for (j = 0; j < adev->vcn.num_enc_rings; ++j) { + ring = &adev->vcn.inst[i].ring_enc[j]; + if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) { + ring->sched.ready = false; + dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name); + } else { + ring->wptr = 0; + ring->wptr_old = 0; + vcn_v3_0_enc_ring_set_wptr(ring); + ring->sched.ready = true; + } + } } } else { for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 3d21c0799037..fe9a7cc8d9eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -77,7 +77,7 @@ #if defined(CONFIG_DRM_AMD_ACP) #include "amdgpu_acp.h" #endif -#include "dce_virtual.h" +#include "amdgpu_vkms.h" #include "mxgpu_vi.h" #include "amdgpu_dm.h" @@ -2102,7 +2102,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v2_4_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); break; case CHIP_FIJI: amdgpu_device_ip_block_add(adev, &vi_common_ip_block); @@ -2112,7 +2112,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2132,7 +2132,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2155,7 +2155,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v3_1_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2173,7 +2173,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); @@ -2194,7 +2194,7 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block); amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); if (adev->enable_virtual_display) - amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); + amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block); #if defined(CONFIG_DRM_AMD_DC) else if (amdgpu_device_has_dc_support(adev)) amdgpu_device_ip_block_add(adev, &dm_ip_block); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index e48acdd03c1a..86afd37b098d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1393,6 +1393,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, long err = 0; int i; uint32_t *devices_arr = NULL; + bool table_freed = false; dev = kfd_device_by_id(GET_GPU_ID(args->handle)); if (!dev) @@ -1450,7 +1451,8 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, goto get_mem_obj_from_handle_failed; } err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu( - peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv); + peer->kgd, (struct kgd_mem *)mem, + peer_pdd->drm_priv, &table_freed); if (err) { pr_err("Failed to map to gpu %d/%d\n", i, args->n_devices); @@ -1468,16 +1470,17 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, } /* Flush TLBs after waiting for the page table updates to complete */ - for (i = 0; i < args->n_devices; i++) { - peer = kfd_device_by_id(devices_arr[i]); - if (WARN_ON_ONCE(!peer)) - continue; - peer_pdd = kfd_get_process_device_data(peer, p); - if (WARN_ON_ONCE(!peer_pdd)) - continue; - kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY); + if (table_freed) { + for (i = 0; i < args->n_devices; i++) { + peer = kfd_device_by_id(devices_arr[i]); + if (WARN_ON_ONCE(!peer)) + continue; + peer_pdd = kfd_get_process_device_data(peer, p); + if (WARN_ON_ONCE(!peer_pdd)) + continue; + kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY); + } } - kfree(devices_arr); return err; @@ -1565,10 +1568,29 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep, } args->n_success = i+1; } - kfree(devices_arr); - mutex_unlock(&p->mutex); + if (dev->device_info->asic_family == CHIP_ALDEBARAN) { + err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, + (struct kgd_mem *) mem, true); + if (err) { + pr_debug("Sync memory failed, wait interrupted by user signal\n"); + goto sync_memory_failed; + } + + /* Flush TLBs after waiting for the page table updates to complete */ + for (i = 0; i < args->n_devices; i++) { + peer = kfd_device_by_id(devices_arr[i]); + if (WARN_ON_ONCE(!peer)) + continue; + peer_pdd = kfd_get_process_device_data(peer, p); + if (WARN_ON_ONCE(!peer_pdd)) + continue; + kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT); + } + } + kfree(devices_arr); + return 0; bind_process_to_device_failed: @@ -1576,6 +1598,7 @@ get_mem_obj_from_handle_failed: unmap_memory_from_gpu_failed: mutex_unlock(&p->mutex); copy_from_user_failed: +sync_memory_failed: kfree(devices_arr); return err; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index b551dd675085..16a57b70cc1a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -91,6 +91,7 @@ static const struct kfd2kgd_calls *kfd2kgd_funcs[] = { static const struct kfd_device_info kaveri_device_info = { .asic_family = CHIP_KAVERI, .asic_name = "kaveri", + .gfx_target_version = 70000, .max_pasid_bits = 16, /* max num of queues for KV.TODO should be a dynamic value */ .max_no_of_hqd = 24, @@ -110,6 +111,7 @@ static const struct kfd_device_info kaveri_device_info = { static const struct kfd_device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, .asic_name = "carrizo", + .gfx_target_version = 80001, .max_pasid_bits = 16, /* max num of queues for CZ.TODO should be a dynamic value */ .max_no_of_hqd = 24, @@ -130,6 +132,7 @@ static const struct kfd_device_info carrizo_device_info = { static const struct kfd_device_info raven_device_info = { .asic_family = CHIP_RAVEN, .asic_name = "raven", + .gfx_target_version = 90002, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -148,6 +151,7 @@ static const struct kfd_device_info raven_device_info = { static const struct kfd_device_info hawaii_device_info = { .asic_family = CHIP_HAWAII, .asic_name = "hawaii", + .gfx_target_version = 70001, .max_pasid_bits = 16, /* max num of queues for KV.TODO should be a dynamic value */ .max_no_of_hqd = 24, @@ -167,6 +171,7 @@ static const struct kfd_device_info hawaii_device_info = { static const struct kfd_device_info tonga_device_info = { .asic_family = CHIP_TONGA, .asic_name = "tonga", + .gfx_target_version = 80002, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -185,6 +190,7 @@ static const struct kfd_device_info tonga_device_info = { static const struct kfd_device_info fiji_device_info = { .asic_family = CHIP_FIJI, .asic_name = "fiji", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -203,6 +209,7 @@ static const struct kfd_device_info fiji_device_info = { static const struct kfd_device_info fiji_vf_device_info = { .asic_family = CHIP_FIJI, .asic_name = "fiji", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -222,6 +229,7 @@ static const struct kfd_device_info fiji_vf_device_info = { static const struct kfd_device_info polaris10_device_info = { .asic_family = CHIP_POLARIS10, .asic_name = "polaris10", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -240,6 +248,7 @@ static const struct kfd_device_info polaris10_device_info = { static const struct kfd_device_info polaris10_vf_device_info = { .asic_family = CHIP_POLARIS10, .asic_name = "polaris10", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -258,6 +267,7 @@ static const struct kfd_device_info polaris10_vf_device_info = { static const struct kfd_device_info polaris11_device_info = { .asic_family = CHIP_POLARIS11, .asic_name = "polaris11", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -276,6 +286,7 @@ static const struct kfd_device_info polaris11_device_info = { static const struct kfd_device_info polaris12_device_info = { .asic_family = CHIP_POLARIS12, .asic_name = "polaris12", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -294,6 +305,7 @@ static const struct kfd_device_info polaris12_device_info = { static const struct kfd_device_info vegam_device_info = { .asic_family = CHIP_VEGAM, .asic_name = "vegam", + .gfx_target_version = 80003, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 4, @@ -312,6 +324,7 @@ static const struct kfd_device_info vegam_device_info = { static const struct kfd_device_info vega10_device_info = { .asic_family = CHIP_VEGA10, .asic_name = "vega10", + .gfx_target_version = 90000, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -330,6 +343,7 @@ static const struct kfd_device_info vega10_device_info = { static const struct kfd_device_info vega10_vf_device_info = { .asic_family = CHIP_VEGA10, .asic_name = "vega10", + .gfx_target_version = 90000, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -348,6 +362,7 @@ static const struct kfd_device_info vega10_vf_device_info = { static const struct kfd_device_info vega12_device_info = { .asic_family = CHIP_VEGA12, .asic_name = "vega12", + .gfx_target_version = 90004, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -366,6 +381,7 @@ static const struct kfd_device_info vega12_device_info = { static const struct kfd_device_info vega20_device_info = { .asic_family = CHIP_VEGA20, .asic_name = "vega20", + .gfx_target_version = 90006, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -384,6 +400,7 @@ static const struct kfd_device_info vega20_device_info = { static const struct kfd_device_info arcturus_device_info = { .asic_family = CHIP_ARCTURUS, .asic_name = "arcturus", + .gfx_target_version = 90008, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -402,6 +419,7 @@ static const struct kfd_device_info arcturus_device_info = { static const struct kfd_device_info aldebaran_device_info = { .asic_family = CHIP_ALDEBARAN, .asic_name = "aldebaran", + .gfx_target_version = 90010, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -420,6 +438,7 @@ static const struct kfd_device_info aldebaran_device_info = { static const struct kfd_device_info renoir_device_info = { .asic_family = CHIP_RENOIR, .asic_name = "renoir", + .gfx_target_version = 90002, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -438,6 +457,7 @@ static const struct kfd_device_info renoir_device_info = { static const struct kfd_device_info navi10_device_info = { .asic_family = CHIP_NAVI10, .asic_name = "navi10", + .gfx_target_version = 100100, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -456,6 +476,7 @@ static const struct kfd_device_info navi10_device_info = { static const struct kfd_device_info navi12_device_info = { .asic_family = CHIP_NAVI12, .asic_name = "navi12", + .gfx_target_version = 100101, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -474,6 +495,7 @@ static const struct kfd_device_info navi12_device_info = { static const struct kfd_device_info navi14_device_info = { .asic_family = CHIP_NAVI14, .asic_name = "navi14", + .gfx_target_version = 100102, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -492,6 +514,7 @@ static const struct kfd_device_info navi14_device_info = { static const struct kfd_device_info sienna_cichlid_device_info = { .asic_family = CHIP_SIENNA_CICHLID, .asic_name = "sienna_cichlid", + .gfx_target_version = 100300, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -510,6 +533,7 @@ static const struct kfd_device_info sienna_cichlid_device_info = { static const struct kfd_device_info navy_flounder_device_info = { .asic_family = CHIP_NAVY_FLOUNDER, .asic_name = "navy_flounder", + .gfx_target_version = 100301, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -528,6 +552,7 @@ static const struct kfd_device_info navy_flounder_device_info = { static const struct kfd_device_info vangogh_device_info = { .asic_family = CHIP_VANGOGH, .asic_name = "vangogh", + .gfx_target_version = 100303, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -546,6 +571,7 @@ static const struct kfd_device_info vangogh_device_info = { static const struct kfd_device_info dimgrey_cavefish_device_info = { .asic_family = CHIP_DIMGREY_CAVEFISH, .asic_name = "dimgrey_cavefish", + .gfx_target_version = 100302, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -564,6 +590,7 @@ static const struct kfd_device_info dimgrey_cavefish_device_info = { static const struct kfd_device_info beige_goby_device_info = { .asic_family = CHIP_BEIGE_GOBY, .asic_name = "beige_goby", + .gfx_target_version = 100304, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -582,6 +609,7 @@ static const struct kfd_device_info beige_goby_device_info = { static const struct kfd_device_info yellow_carp_device_info = { .asic_family = CHIP_YELLOW_CARP, .asic_name = "yellow_carp", + .gfx_target_version = 100305, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, @@ -600,6 +628,7 @@ static const struct kfd_device_info yellow_carp_device_info = { static const struct kfd_device_info cyan_skillfish_device_info = { .asic_family = CHIP_CYAN_SKILLFISH, .asic_name = "cyan_skillfish", + .gfx_target_version = 100103, .max_pasid_bits = 16, .max_no_of_hqd = 24, .doorbell_size = 8, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index a972ef5eae68..f8fce9d05f50 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -211,6 +211,15 @@ static void deallocate_doorbell(struct qcm_process_device *qpd, WARN_ON(!old); } +static void program_trap_handler_settings(struct device_queue_manager *dqm, + struct qcm_process_device *qpd) +{ + if (dqm->dev->kfd2kgd->program_trap_handler_settings) + dqm->dev->kfd2kgd->program_trap_handler_settings( + dqm->dev->kgd, qpd->vmid, + qpd->tba_addr, qpd->tma_addr); +} + static int allocate_vmid(struct device_queue_manager *dqm, struct qcm_process_device *qpd, struct queue *q) @@ -241,6 +250,10 @@ static int allocate_vmid(struct device_queue_manager *dqm, program_sh_mem_settings(dqm, qpd); + if (dqm->dev->device_info->asic_family >= CHIP_VEGA10 && + dqm->dev->cwsr_enabled) + program_trap_handler_settings(dqm, qpd); + /* qpd->page_table_base is set earlier when register_process() * is called, i.e. when the first queue is created. */ @@ -582,7 +595,9 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) } retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, - KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, + (dqm->dev->cwsr_enabled? + KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: + KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); if (retval) { pr_err("destroy mqd failed\n"); @@ -675,7 +690,9 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, continue; retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, - KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, + (dqm->dev->cwsr_enabled? + KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: + KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); if (retval && !ret) /* Return the first error, but keep going to diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c index 88813dad731f..c021519af810 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c @@ -98,36 +98,78 @@ void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm, uint32_t *se_mask) { struct kfd_cu_info cu_info; - uint32_t cu_per_se[KFD_MAX_NUM_SE] = {0}; - int i, se, sh, cu = 0; - + uint32_t cu_per_sh[KFD_MAX_NUM_SE][KFD_MAX_NUM_SH_PER_SE] = {0}; + int i, se, sh, cu; amdgpu_amdkfd_get_cu_info(mm->dev->kgd, &cu_info); if (cu_mask_count > cu_info.cu_active_number) cu_mask_count = cu_info.cu_active_number; + /* Exceeding these bounds corrupts the stack and indicates a coding error. + * Returning with no CU's enabled will hang the queue, which should be + * attention grabbing. + */ + if (cu_info.num_shader_engines > KFD_MAX_NUM_SE) { + pr_err("Exceeded KFD_MAX_NUM_SE, chip reports %d\n", cu_info.num_shader_engines); + return; + } + if (cu_info.num_shader_arrays_per_engine > KFD_MAX_NUM_SH_PER_SE) { + pr_err("Exceeded KFD_MAX_NUM_SH, chip reports %d\n", + cu_info.num_shader_arrays_per_engine * cu_info.num_shader_engines); + return; + } + /* Count active CUs per SH. + * + * Some CUs in an SH may be disabled. HW expects disabled CUs to be + * represented in the high bits of each SH's enable mask (the upper and lower + * 16 bits of se_mask) and will take care of the actual distribution of + * disabled CUs within each SH automatically. + * Each half of se_mask must be filled only on bits 0-cu_per_sh[se][sh]-1. + * + * See note on Arcturus cu_bitmap layout in gfx_v9_0_get_cu_info. + */ for (se = 0; se < cu_info.num_shader_engines; se++) for (sh = 0; sh < cu_info.num_shader_arrays_per_engine; sh++) - cu_per_se[se] += hweight32(cu_info.cu_bitmap[se % 4][sh + (se / 4)]); - - /* Symmetrically map cu_mask to all SEs: - * cu_mask[0] bit0 -> se_mask[0] bit0; - * cu_mask[0] bit1 -> se_mask[1] bit0; - * ... (if # SE is 4) - * cu_mask[0] bit4 -> se_mask[0] bit1; + cu_per_sh[se][sh] = hweight32(cu_info.cu_bitmap[se % 4][sh + (se / 4)]); + + /* Symmetrically map cu_mask to all SEs & SHs: + * se_mask programs up to 2 SH in the upper and lower 16 bits. + * + * Examples + * Assuming 1 SH/SE, 4 SEs: + * cu_mask[0] bit0 -> se_mask[0] bit0 + * cu_mask[0] bit1 -> se_mask[1] bit0 + * ... + * cu_mask[0] bit4 -> se_mask[0] bit1 + * ... + * + * Assuming 2 SH/SE, 4 SEs + * cu_mask[0] bit0 -> se_mask[0] bit0 (SE0,SH0,CU0) + * cu_mask[0] bit1 -> se_mask[1] bit0 (SE1,SH0,CU0) + * ... + * cu_mask[0] bit4 -> se_mask[0] bit16 (SE0,SH1,CU0) + * cu_mask[0] bit5 -> se_mask[1] bit16 (SE1,SH1,CU0) + * ... + * cu_mask[0] bit8 -> se_mask[0] bit1 (SE0,SH0,CU1) * ... + * + * First ensure all CUs are disabled, then enable user specified CUs. */ - se = 0; - for (i = 0; i < cu_mask_count; i++) { - if (cu_mask[i / 32] & (1 << (i % 32))) - se_mask[se] |= 1 << cu; - - do { - se++; - if (se == cu_info.num_shader_engines) { - se = 0; - cu++; + for (i = 0; i < cu_info.num_shader_engines; i++) + se_mask[i] = 0; + + i = 0; + for (cu = 0; cu < 16; cu++) { + for (sh = 0; sh < cu_info.num_shader_arrays_per_engine; sh++) { + for (se = 0; se < cu_info.num_shader_engines; se++) { + if (cu_per_sh[se][sh] > cu) { + if (cu_mask[i / 32] & (1 << (i % 32))) + se_mask[se] |= 1 << (cu + sh * 16); + i++; + if (i == cu_mask_count) + return; + } } - } while (cu >= cu_per_se[se] && cu < 32); + } } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h index b5e2ea7550d4..6e6918ccedfd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h @@ -27,6 +27,7 @@ #include "kfd_priv.h" #define KFD_MAX_NUM_SE 8 +#define KFD_MAX_NUM_SH_PER_SE 2 /** * struct mqd_manager diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 8a5dfda224bf..ab83b0de6b22 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -196,6 +196,7 @@ struct kfd_event_interrupt_class { struct kfd_device_info { enum amd_asic_type asic_family; const char *asic_name; + uint32_t gfx_target_version; const struct kfd_event_interrupt_class *event_interrupt_class; unsigned int max_pasid_bits; unsigned int max_no_of_hqd; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 8a2c6fc438c0..21ec8a18cad2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -714,7 +714,8 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, if (err) goto err_alloc_mem; - err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->drm_priv); + err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, + pdd->drm_priv, NULL); if (err) goto err_map_mem; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index c7b364e4a287..9fc8021bb0ab 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -120,6 +120,7 @@ static void svm_range_remove_notifier(struct svm_range *prange) static int svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange, + unsigned long offset, unsigned long npages, unsigned long *hmm_pfns, uint32_t gpuidx) { enum dma_data_direction dir = DMA_BIDIRECTIONAL; @@ -136,7 +137,8 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange, prange->dma_addr[gpuidx] = addr; } - for (i = 0; i < prange->npages; i++) { + addr += offset; + for (i = 0; i < npages; i++) { if (WARN_ONCE(addr[i] && !dma_mapping_error(dev, addr[i]), "leaking dma mapping\n")) dma_unmap_page(dev, addr[i], PAGE_SIZE, dir); @@ -167,6 +169,7 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange, static int svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap, + unsigned long offset, unsigned long npages, unsigned long *hmm_pfns) { struct kfd_process *p; @@ -187,7 +190,8 @@ svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap, } adev = (struct amdgpu_device *)pdd->dev->kgd; - r = svm_range_dma_map_dev(adev, prange, hmm_pfns, gpuidx); + r = svm_range_dma_map_dev(adev, prange, offset, npages, + hmm_pfns, gpuidx); if (r) break; } @@ -1088,11 +1092,6 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange, pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; pte_flags |= amdgpu_gem_va_map_flags(adev, mapping_flags); - - pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d PTE 0x%llx mapping 0x%x\n", - prange->svms, prange->start, prange->last, - (domain == SVM_RANGE_VRAM_DOMAIN) ? 1:0, pte_flags, mapping_flags); - return pte_flags; } @@ -1156,7 +1155,8 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, static int svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, - struct svm_range *prange, dma_addr_t *dma_addr, + struct svm_range *prange, unsigned long offset, + unsigned long npages, bool readonly, dma_addr_t *dma_addr, struct amdgpu_device *bo_adev, struct dma_fence **fence) { struct amdgpu_bo_va bo_va; @@ -1167,14 +1167,15 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, int r = 0; int64_t i; - pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start, - prange->last); + last_start = prange->start + offset; + + pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms, + last_start, last_start + npages - 1, readonly); if (prange->svm_bo && prange->ttm_res) bo_va.is_xgmi = amdgpu_xgmi_same_hive(adev, bo_adev); - last_start = prange->start; - for (i = 0; i < prange->npages; i++) { + for (i = offset; i < offset + npages; i++) { last_domain = dma_addr[i] & SVM_RANGE_VRAM_DOMAIN; dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN; if ((prange->start + i) < prange->last && @@ -1183,13 +1184,21 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n", last_start, prange->start + i, last_domain ? "GPU" : "CPU"); + pte_flags = svm_range_get_pte_flags(adev, prange, last_domain); - r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, NULL, - last_start, + if (readonly) + pte_flags &= ~AMDGPU_PTE_WRITEABLE; + + pr_debug("svms 0x%p map [0x%lx 0x%llx] vram %d PTE 0x%llx\n", + prange->svms, last_start, prange->start + i, + (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0, + pte_flags); + + r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, + NULL, last_start, prange->start + i, pte_flags, last_start - prange->start, - NULL, - dma_addr, + NULL, dma_addr, &vm->last_update, &table_freed); if (r) { @@ -1220,8 +1229,10 @@ out: return r; } -static int svm_range_map_to_gpus(struct svm_range *prange, - unsigned long *bitmap, bool wait) +static int +svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset, + unsigned long npages, bool readonly, + unsigned long *bitmap, bool wait) { struct kfd_process_device *pdd; struct amdgpu_device *bo_adev; @@ -1257,7 +1268,8 @@ static int svm_range_map_to_gpus(struct svm_range *prange, } r = svm_range_map_to_gpu(adev, drm_priv_to_vm(pdd->drm_priv), - prange, prange->dma_addr[gpuidx], + prange, offset, npages, readonly, + prange->dma_addr[gpuidx], bo_adev, wait ? &fence : NULL); if (r) break; @@ -1390,7 +1402,7 @@ static int svm_range_validate_and_map(struct mm_struct *mm, int32_t gpuidx, bool intr, bool wait) { struct svm_validate_context ctx; - struct hmm_range *hmm_range; + unsigned long start, end, addr; struct kfd_process *p; void *owner; int32_t idx; @@ -1448,40 +1460,66 @@ static int svm_range_validate_and_map(struct mm_struct *mm, break; } } - r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL, - prange->start << PAGE_SHIFT, - prange->npages, &hmm_range, - false, true, owner); - if (r) { - pr_debug("failed %d to get svm range pages\n", r); - goto unreserve_out; - } - r = svm_range_dma_map(prange, ctx.bitmap, - hmm_range->hmm_pfns); - if (r) { - pr_debug("failed %d to dma map range\n", r); - goto unreserve_out; - } + start = prange->start << PAGE_SHIFT; + end = (prange->last + 1) << PAGE_SHIFT; + for (addr = start; addr < end && !r; ) { + struct hmm_range *hmm_range; + struct vm_area_struct *vma; + unsigned long next; + unsigned long offset; + unsigned long npages; + bool readonly; - prange->validated_once = true; + vma = find_vma(mm, addr); + if (!vma || addr < vma->vm_start) { + r = -EFAULT; + goto unreserve_out; + } + readonly = !(vma->vm_flags & VM_WRITE); - svm_range_lock(prange); - if (amdgpu_hmm_range_get_pages_done(hmm_range)) { - pr_debug("hmm update the range, need validate again\n"); - r = -EAGAIN; - goto unlock_out; - } - if (!list_empty(&prange->child_list)) { - pr_debug("range split by unmap in parallel, validate again\n"); - r = -EAGAIN; - goto unlock_out; - } + next = min(vma->vm_end, end); + npages = (next - addr) >> PAGE_SHIFT; + r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL, + addr, npages, &hmm_range, + readonly, true, owner); + if (r) { + pr_debug("failed %d to get svm range pages\n", r); + goto unreserve_out; + } + + offset = (addr - start) >> PAGE_SHIFT; + r = svm_range_dma_map(prange, ctx.bitmap, offset, npages, + hmm_range->hmm_pfns); + if (r) { + pr_debug("failed %d to dma map range\n", r); + goto unreserve_out; + } + + svm_range_lock(prange); + if (amdgpu_hmm_range_get_pages_done(hmm_range)) { + pr_debug("hmm update the range, need validate again\n"); + r = -EAGAIN; + goto unlock_out; + } + if (!list_empty(&prange->child_list)) { + pr_debug("range split by unmap in parallel, validate again\n"); + r = -EAGAIN; + goto unlock_out; + } - r = svm_range_map_to_gpus(prange, ctx.bitmap, wait); + r = svm_range_map_to_gpus(prange, offset, npages, readonly, + ctx.bitmap, wait); unlock_out: - svm_range_unlock(prange); + svm_range_unlock(prange); + + addr = next; + } + + if (addr == end) + prange->validated_once = true; + unreserve_out: svm_range_unreserve_bos(&ctx); @@ -2400,9 +2438,29 @@ svm_range_count_fault(struct amdgpu_device *adev, struct kfd_process *p, WRITE_ONCE(pdd->faults, pdd->faults + 1); } +static bool +svm_fault_allowed(struct mm_struct *mm, uint64_t addr, bool write_fault) +{ + unsigned long requested = VM_READ; + struct vm_area_struct *vma; + + if (write_fault) + requested |= VM_WRITE; + + vma = find_vma(mm, addr << PAGE_SHIFT); + if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) { + pr_debug("address 0x%llx VMA is removed\n", addr); + return true; + } + + pr_debug("requested 0x%lx, vma permission flags 0x%lx\n", requested, + vma->vm_flags); + return (vma->vm_flags & requested) == requested; +} + int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, - uint64_t addr) + uint64_t addr, bool write_fault) { struct mm_struct *mm = NULL; struct svm_range_list *svms; @@ -2426,7 +2484,8 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, } if (!p->xnack_enabled) { pr_debug("XNACK not enabled for pasid 0x%x\n", pasid); - return -EFAULT; + r = -EFAULT; + goto out; } svms = &p->svms; @@ -2484,6 +2543,13 @@ retry_write_locked: goto out_unlock_range; } + if (!svm_fault_allowed(mm, addr, write_fault)) { + pr_debug("fault addr 0x%llx no %s permission\n", addr, + write_fault ? "write" : "read"); + r = -EPERM; + goto out_unlock_range; + } + best_loc = svm_range_best_restore_location(prange, adev, &gpuidx); if (best_loc == -1) { pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n", @@ -2675,22 +2741,26 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, return 0; } -/* svm_range_best_prefetch_location - decide the best prefetch location +/** + * svm_range_best_prefetch_location - decide the best prefetch location * @prange: svm range structure * * For xnack off: - * If range map to single GPU, the best acutal location is prefetch loc, which + * If range map to single GPU, the best prefetch location is prefetch_loc, which * can be CPU or GPU. * - * If range map to multiple GPUs, only if mGPU connection on xgmi same hive, - * the best actual location could be prefetch_loc GPU. If mGPU connection on - * PCIe, the best actual location is always CPU, because GPU cannot access vram - * of other GPUs, assuming PCIe small bar (large bar support is not upstream). + * If range is ACCESS or ACCESS_IN_PLACE by mGPUs, only if mGPU connection on + * XGMI same hive, the best prefetch location is prefetch_loc GPU, othervise + * the best prefetch location is always CPU, because GPU can not have coherent + * mapping VRAM of other GPUs even with large-BAR PCIe connection. * * For xnack on: - * The best actual location is prefetch location. If mGPU connection on xgmi - * same hive, range map to multiple GPUs. Otherwise, the range only map to - * actual location GPU. Other GPU access vm fault will trigger migration. + * If range is not ACCESS_IN_PLACE by mGPUs, the best prefetch location is + * prefetch_loc, other GPU access will generate vm fault and trigger migration. + * + * If range is ACCESS_IN_PLACE by mGPUs, only if mGPU connection on XGMI same + * hive, the best prefetch location is prefetch_loc GPU, otherwise the best + * prefetch location is always CPU. * * Context: Process context * @@ -2710,11 +2780,6 @@ svm_range_best_prefetch_location(struct svm_range *prange) p = container_of(prange->svms, struct kfd_process, svms); - /* xnack on */ - if (p->xnack_enabled) - goto out; - - /* xnack off */ if (!best_loc || best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED) goto out; @@ -2724,8 +2789,12 @@ svm_range_best_prefetch_location(struct svm_range *prange) best_loc = 0; goto out; } - bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, - MAX_GPU_INSTANCE); + + if (p->xnack_enabled) + bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); + else + bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, + MAX_GPU_INSTANCE); for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { pdd = kfd_process_device_from_gpuidx(p, gpuidx); @@ -3019,13 +3088,22 @@ svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, struct svm_range *prange; uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED; uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED; - uint32_t flags = 0xffffffff; + uint32_t flags_and = 0xffffffff; + uint32_t flags_or = 0; int gpuidx; uint32_t i; pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start, start + size - 1, nattr); + /* Flush pending deferred work to avoid racing with deferred actions from + * previous memory map changes (e.g. munmap). Concurrent memory map changes + * can still race with get_attr because we don't hold the mmap lock. But that + * would be a race condition in the application anyway, and undefined + * behaviour is acceptable in that case. + */ + flush_work(&p->svms.deferred_list_work); + mmap_read_lock(mm); if (!svm_range_is_valid(mm, start, size)) { pr_debug("invalid range\n"); @@ -3046,12 +3124,12 @@ svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, get_accessible = true; break; case KFD_IOCTL_SVM_ATTR_SET_FLAGS: + case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: get_flags = true; break; case KFD_IOCTL_SVM_ATTR_GRANULARITY: get_granularity = true; break; - case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: case KFD_IOCTL_SVM_ATTR_NO_ACCESS: fallthrough; @@ -3069,7 +3147,8 @@ svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, if (!node) { pr_debug("range attrs not found return default values\n"); svm_range_set_default_attributes(&location, &prefetch_loc, - &granularity, &flags); + &granularity, &flags_and); + flags_or = flags_and; if (p->xnack_enabled) bitmap_copy(bitmap_access, svms->bitmap_supported, MAX_GPU_INSTANCE); @@ -3115,8 +3194,10 @@ svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, bitmap_and(bitmap_aip, bitmap_aip, prange->bitmap_aip, MAX_GPU_INSTANCE); } - if (get_flags) - flags &= prange->flags; + if (get_flags) { + flags_and &= prange->flags; + flags_or |= prange->flags; + } if (get_granularity && prange->granularity < granularity) granularity = prange->granularity; @@ -3150,7 +3231,10 @@ fill_values: attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS; break; case KFD_IOCTL_SVM_ATTR_SET_FLAGS: - attrs[i].value = flags; + attrs[i].value = flags_and; + break; + case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: + attrs[i].value = ~flags_or; break; case KFD_IOCTL_SVM_ATTR_GRANULARITY: attrs[i].value = (uint32_t)granularity; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 3fc1fd8b4fbc..c6ec55354c7b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -175,7 +175,7 @@ int svm_range_split_by_granularity(struct kfd_process *p, struct mm_struct *mm, unsigned long addr, struct svm_range *parent, struct svm_range *prange); int svm_range_restore_pages(struct amdgpu_device *adev, - unsigned int pasid, uint64_t addr); + unsigned int pasid, uint64_t addr, bool write_fault); int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence); void svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, struct mm_struct *mm, @@ -209,7 +209,8 @@ static inline void svm_range_list_fini(struct kfd_process *p) } static inline int svm_range_restore_pages(struct amdgpu_device *adev, - unsigned int pasid, uint64_t addr) + unsigned int pasid, uint64_t addr, + bool write_fault) { return -EFAULT; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index ef992dd2da3a..98cca5f2b27f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -478,6 +478,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.simd_per_cu); sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu", dev->node_props.max_slots_scratch_cu); + sysfs_show_32bit_prop(buffer, offs, "gfx_target_version", + dev->node_props.gfx_target_version); sysfs_show_32bit_prop(buffer, offs, "vendor_id", dev->node_props.vendor_id); sysfs_show_32bit_prop(buffer, offs, "device_id", @@ -1360,6 +1362,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu) dev->node_props.simd_arrays_per_engine = cu_info.num_shader_arrays_per_engine; + dev->node_props.gfx_target_version = gpu->device_info->gfx_target_version; dev->node_props.vendor_id = gpu->pdev->vendor; dev->node_props.device_id = gpu->pdev->device; dev->node_props.capability |= diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 8b48c6692007..a8db017c9b8e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -78,6 +78,7 @@ struct kfd_node_properties { uint32_t simd_per_cu; uint32_t max_slots_scratch_cu; uint32_t engine_id; + uint32_t gfx_target_version; uint32_t vendor_id; uint32_t device_id; uint32_t location_id; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile index 91fb72c96545..718e123a3230 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile @@ -27,6 +27,10 @@ AMDGPUDM = amdgpu_dm.o amdgpu_dm_irq.o amdgpu_dm_mst_types.o amdgpu_dm_color.o +ifdef CONFIG_DRM_AMD_DC_DCN +AMDGPUDM += dc_fpu.o +endif + ifneq ($(CONFIG_DRM_AMD_DC),) AMDGPUDM += amdgpu_dm_services.o amdgpu_dm_helpers.o amdgpu_dm_pp_smu.o amdgpu_dm_psr.o endif diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 986c9d29d686..9b1fc54555ee 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1044,10 +1044,10 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_ } #endif #if defined(CONFIG_DRM_AMD_DC_DCN) -static void event_mall_stutter(struct work_struct *work) +static void vblank_control_worker(struct work_struct *work) { - - struct vblank_workqueue *vblank_work = container_of(work, struct vblank_workqueue, mall_work); + struct vblank_control_work *vblank_work = + container_of(work, struct vblank_control_work, work); struct amdgpu_display_manager *dm = vblank_work->dm; mutex_lock(&dm->dc_lock); @@ -1061,27 +1061,25 @@ static void event_mall_stutter(struct work_struct *work) DRM_DEBUG_KMS("Allow idle optimizations (MALL): %d\n", dm->active_vblank_irq_count == 0); - mutex_unlock(&dm->dc_lock); -} - -static struct vblank_workqueue *vblank_create_workqueue(struct amdgpu_device *adev, struct dc *dc) -{ - - int max_caps = dc->caps.max_links; - struct vblank_workqueue *vblank_work; - int i = 0; - - vblank_work = kcalloc(max_caps, sizeof(*vblank_work), GFP_KERNEL); - if (ZERO_OR_NULL_PTR(vblank_work)) { - kfree(vblank_work); - return NULL; + /* Control PSR based on vblank requirements from OS */ + if (vblank_work->stream && vblank_work->stream->link) { + if (vblank_work->enable) { + if (vblank_work->stream->link->psr_settings.psr_allow_active) + amdgpu_dm_psr_disable(vblank_work->stream); + } else if (vblank_work->stream->link->psr_settings.psr_feature_enabled && + !vblank_work->stream->link->psr_settings.psr_allow_active && + vblank_work->acrtc->dm_irq_params.allow_psr_entry) { + amdgpu_dm_psr_enable(vblank_work->stream); + } } - for (i = 0; i < max_caps; i++) - INIT_WORK(&vblank_work[i].mall_work, event_mall_stutter); + mutex_unlock(&dm->dc_lock); + + dc_stream_release(vblank_work->stream); - return vblank_work; + kfree(vblank_work); } + #endif static int amdgpu_dm_init(struct amdgpu_device *adev) { @@ -1202,7 +1200,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) dc_hardware_init(adev->dm.dc); #if defined(CONFIG_DRM_AMD_DC_DCN) - if (adev->apu_flags) { + if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) { struct dc_phy_addr_space_config pa_config; mmhub_read_system_context(adev, &pa_config); @@ -1224,12 +1222,10 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) #if defined(CONFIG_DRM_AMD_DC_DCN) if (adev->dm.dc->caps.max_links > 0) { - adev->dm.vblank_workqueue = vblank_create_workqueue(adev, adev->dm.dc); - - if (!adev->dm.vblank_workqueue) + adev->dm.vblank_control_workqueue = + create_singlethread_workqueue("dm_vblank_control_workqueue"); + if (!adev->dm.vblank_control_workqueue) DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); - else - DRM_DEBUG_DRIVER("amdgpu: vblank_workqueue init done %p.\n", adev->dm.vblank_workqueue); } #endif @@ -1302,6 +1298,13 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) { int i; +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (adev->dm.vblank_control_workqueue) { + destroy_workqueue(adev->dm.vblank_control_workqueue); + adev->dm.vblank_control_workqueue = NULL; + } +#endif + for (i = 0; i < adev->dm.display_indexes_num; i++) { drm_encoder_cleanup(&adev->dm.mst_encoders[i].base); } @@ -1325,14 +1328,6 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) dc_deinit_callbacks(adev->dm.dc); #endif -#if defined(CONFIG_DRM_AMD_DC_DCN) - if (adev->dm.vblank_workqueue) { - adev->dm.vblank_workqueue->dm = NULL; - kfree(adev->dm.vblank_workqueue); - adev->dm.vblank_workqueue = NULL; - } -#endif - dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv); if (dc_enable_dmub_notifications(adev->dm.dc)) { @@ -1548,6 +1543,7 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev) } hdr = (const struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data; + adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version); if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id = @@ -1561,7 +1557,6 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev) adev->dm.dmcub_fw_version); } - adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version); adev->dm.dmub_srv = kzalloc(sizeof(*adev->dm.dmub_srv), GFP_KERNEL); dmub_srv = adev->dm.dmub_srv; @@ -6004,7 +5999,7 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state); #if defined(CONFIG_DRM_AMD_DC_DCN) struct amdgpu_display_manager *dm = &adev->dm; - unsigned long flags; + struct vblank_control_work *work; #endif int rc = 0; @@ -6029,12 +6024,21 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) return 0; #if defined(CONFIG_DRM_AMD_DC_DCN) - spin_lock_irqsave(&dm->vblank_lock, flags); - dm->vblank_workqueue->dm = dm; - dm->vblank_workqueue->otg_inst = acrtc->otg_inst; - dm->vblank_workqueue->enable = enable; - spin_unlock_irqrestore(&dm->vblank_lock, flags); - schedule_work(&dm->vblank_workqueue->mall_work); + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return -ENOMEM; + + INIT_WORK(&work->work, vblank_control_worker); + work->dm = dm; + work->acrtc = acrtc; + work->enable = enable; + + if (acrtc_state->stream) { + dc_stream_retain(acrtc_state->stream); + work->stream = acrtc_state->stream; + } + + queue_work(dm->vblank_control_workqueue, &work->work); #endif return 0; @@ -8639,6 +8643,14 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, /* Update the planes if changed or disable if we don't have any. */ if ((planes_count || acrtc_state->active_planes == 0) && acrtc_state->stream) { +#if defined(CONFIG_DRM_AMD_DC_DCN) + /* + * If PSR or idle optimizations are enabled then flush out + * any pending work before hardware programming. + */ + flush_workqueue(dm->vblank_control_workqueue); +#endif + bundle->stream_update.stream = acrtc_state->stream; if (new_pcrtc_state->mode_changed) { bundle->stream_update.src = acrtc_state->stream->src; @@ -8707,16 +8719,20 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, acrtc_state->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED && !acrtc_state->stream->link->psr_settings.psr_feature_enabled) amdgpu_dm_link_setup_psr(acrtc_state->stream); - else if ((acrtc_state->update_type == UPDATE_TYPE_FAST) && - acrtc_state->stream->link->psr_settings.psr_feature_enabled && - !acrtc_state->stream->link->psr_settings.psr_allow_active) { - struct amdgpu_dm_connector *aconn = (struct amdgpu_dm_connector *) - acrtc_state->stream->dm_stream_context; + + /* Decrement skip count when PSR is enabled and we're doing fast updates. */ + if (acrtc_state->update_type == UPDATE_TYPE_FAST && + acrtc_state->stream->link->psr_settings.psr_feature_enabled) { + struct amdgpu_dm_connector *aconn = + (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; if (aconn->psr_skip_count > 0) aconn->psr_skip_count--; - else - amdgpu_dm_psr_enable(acrtc_state->stream); + + /* Allow PSR when skip count is 0. */ + acrtc_attach->dm_irq_params.allow_psr_entry = !aconn->psr_skip_count; + } else { + acrtc_attach->dm_irq_params.allow_psr_entry = false; } mutex_unlock(&dm->dc_lock); @@ -8965,8 +8981,12 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) if (dc_state) { /* if there mode set or reset, disable eDP PSR */ - if (mode_set_reset_required) + if (mode_set_reset_required) { +#if defined(CONFIG_DRM_AMD_DC_DCN) + flush_workqueue(dm->vblank_control_workqueue); +#endif amdgpu_dm_psr_disable_all(dm); + } dm_enable_per_frame_crtc_master_sync(dc_state); mutex_lock(&dm->dc_lock); @@ -9620,7 +9640,12 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm, } else if (amdgpu_freesync_vid_mode && aconnector && is_freesync_video_mode(&new_crtc_state->mode, aconnector)) { - set_freesync_fixed_config(dm_new_crtc_state); + struct drm_display_mode *high_mode; + + high_mode = get_highest_refresh_rate_mode(aconnector, false); + if (!drm_mode_equal(&new_crtc_state->mode, high_mode)) { + set_freesync_fixed_config(dm_new_crtc_state); + } } ret = dm_atomic_get_state(state, &dm_state); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index ab1670b16b02..d1d353a7c77d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -60,6 +60,7 @@ enum aux_return_code_type; /* Forward declarations */ struct amdgpu_device; +struct amdgpu_crtc; struct drm_device; struct dc; struct amdgpu_bo; @@ -86,16 +87,18 @@ struct dm_compressor_info { }; /** - * struct vblank_workqueue - Works to be executed in a separate thread during vblank - * @mall_work: work for mall stutter + * struct vblank_control_work - Work data for vblank control + * @work: Kernel work data for the work event * @dm: amdgpu display manager device - * @otg_inst: otg instance of which vblank is being set - * @enable: true if enable vblank + * @acrtc: amdgpu CRTC instance for which the event has occurred + * @stream: DC stream for which the event has occurred + * @enable: true if enabling vblank */ -struct vblank_workqueue { - struct work_struct mall_work; +struct vblank_control_work { + struct work_struct work; struct amdgpu_display_manager *dm; - int otg_inst; + struct amdgpu_crtc *acrtc; + struct dc_stream_state *stream; bool enable; }; @@ -380,11 +383,11 @@ struct amdgpu_display_manager { #if defined(CONFIG_DRM_AMD_DC_DCN) /** - * @vblank_workqueue: + * @vblank_control_workqueue: * - * amdgpu workqueue during vblank + * Deferred work for vblank control events. */ - struct vblank_workqueue *vblank_workqueue; + struct workqueue_struct *vblank_control_workqueue; #endif struct drm_atomic_state *cached_state; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 8e39e9245d06..c5f1dc3b5961 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -79,12 +79,12 @@ static uint8_t *psp_get_srm(struct psp_context *psp, uint32_t *srm_version, uint struct ta_hdcp_shared_memory *hdcp_cmd; - if (!psp->hdcp_context.hdcp_initialized) { + if (!psp->hdcp_context.context.initialized) { DRM_WARN("Failed to get hdcp srm. HDCP TA is not initialized."); return NULL; } - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->cmd_id = TA_HDCP_COMMAND__HDCP_GET_SRM; @@ -105,12 +105,12 @@ static int psp_set_srm(struct psp_context *psp, uint8_t *srm, uint32_t srm_size, struct ta_hdcp_shared_memory *hdcp_cmd; - if (!psp->hdcp_context.hdcp_initialized) { + if (!psp->hdcp_context.context.initialized) { DRM_WARN("Failed to get hdcp srm. HDCP TA is not initialized."); return -EINVAL; } - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); memcpy(hdcp_cmd->in_msg.hdcp_set_srm.srm_buf, srm, srm_size); @@ -414,12 +414,12 @@ static bool enable_assr(void *handle, struct dc_link *link) struct ta_dtm_shared_memory *dtm_cmd; bool res = true; - if (!psp->dtm_context.dtm_initialized) { + if (!psp->dtm_context.context.initialized) { DRM_INFO("Failed to enable ASSR, DTM TA is not initialized."); return false; } - dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; mutex_lock(&psp->dtm_context.mutex); memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index 40f617bbb86f..4aba0e8c84f8 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -584,7 +584,7 @@ static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev, handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list); /*allocate a new amdgpu_dm_irq_handler_data*/ - handler_data_add = kzalloc(sizeof(*handler_data), GFP_KERNEL); + handler_data_add = kzalloc(sizeof(*handler_data), GFP_ATOMIC); if (!handler_data_add) { DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); return; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h index f3b93ba69a27..79b5f9999fec 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h @@ -33,6 +33,7 @@ struct dm_irq_params { struct mod_vrr_params vrr_params; struct dc_stream_state *stream; int active_planes; + bool allow_psr_entry; struct mod_freesync_config freesync_config; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 5568d4e518e6..1bcba6943fd7 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -213,6 +213,29 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector) drm_connector_update_edid_property( &aconnector->base, NULL); + + DRM_DEBUG_KMS("Can't get EDID of %s. Add default remote sink.", connector->name); + if (!aconnector->dc_sink) { + struct dc_sink *dc_sink; + struct dc_sink_init_data init_params = { + .link = aconnector->dc_link, + .sink_signal = SIGNAL_TYPE_DISPLAY_PORT_MST }; + + dc_sink = dc_link_add_remote_sink( + aconnector->dc_link, + NULL, + 0, + &init_params); + + if (!dc_sink) { + DRM_ERROR("Unable to add a remote sink\n"); + return 0; + } + + dc_sink->priv = aconnector; + aconnector->dc_sink = dc_sink; + } + return ret; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h index 46a33f64cf8e..fdcaea22b456 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h @@ -637,6 +637,30 @@ TRACE_EVENT(amdgpu_refresh_rate_track, __entry->refresh_rate_ns) ); +TRACE_EVENT(dcn_fpu, + TP_PROTO(bool begin, const char *function, const int line, const int recursion_depth), + TP_ARGS(begin, function, line, recursion_depth), + + TP_STRUCT__entry( + __field(bool, begin) + __field(const char *, function) + __field(int, line) + __field(int, recursion_depth) + ), + TP_fast_assign( + __entry->begin = begin; + __entry->function = function; + __entry->line = line; + __entry->recursion_depth = recursion_depth; + ), + TP_printk("%s: recursion_depth: %d: %s()+%d:", + __entry->begin ? "begin" : "end", + __entry->recursion_depth, + __entry->function, + __entry->line + ) +); + #endif /* _AMDGPU_DM_TRACE_H_ */ #undef TRACE_INCLUDE_PATH diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c new file mode 100644 index 000000000000..c9f47d167472 --- /dev/null +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dc_trace.h" + +#if defined(CONFIG_X86) +#include <asm/fpu/api.h> +#elif defined(CONFIG_PPC64) +#include <asm/switch_to.h> +#include <asm/cputable.h> +#endif + +/** + * DOC: DC FPU manipulation overview + * + * DC core uses FPU operations in multiple parts of the code, which requires a + * more specialized way to manage these areas' entrance. To fulfill this + * requirement, we created some wrapper functions that encapsulate + * kernel_fpu_begin/end to better fit our need in the display component. In + * summary, in this file, you can find functions related to FPU operation + * management. + */ + +static DEFINE_PER_CPU(int, fpu_recursion_depth); + +/** + * dc_assert_fp_enabled - Check if FPU protection is enabled + * + * This function tells if the code is already under FPU protection or not. A + * function that works as an API for a set of FPU operations can use this + * function for checking if the caller invoked it after DC_FP_START(). For + * example, take a look at dcn2x.c file. + */ +inline void dc_assert_fp_enabled(void) +{ + int *pcpu, depth = 0; + + pcpu = get_cpu_ptr(&fpu_recursion_depth); + depth = *pcpu; + put_cpu_ptr(&fpu_recursion_depth); + + ASSERT(depth > 1); +} + +/** + * dc_fpu_begin - Enables FPU protection + * @function_name: A string containing the function name for debug purposes + * (usually __func__) + * + * @line: A line number where DC_FP_START was invoked for debug purpose + * (usually __LINE__) + * + * This function is responsible for managing the use of kernel_fpu_begin() with + * the advantage of providing an event trace for debugging. + * + * Note: Do not call this function directly; always use DC_FP_START(). + */ +void dc_fpu_begin(const char *function_name, const int line) +{ + int *pcpu; + + pcpu = get_cpu_ptr(&fpu_recursion_depth); + *pcpu += 1; + + if (*pcpu == 1) { +#if defined(CONFIG_X86) + kernel_fpu_begin(); +#elif defined(CONFIG_PPC64) + if (cpu_has_feature(CPU_FTR_VSX_COMP)) { + preempt_disable(); + enable_kernel_vsx(); + } else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) { + preempt_disable(); + enable_kernel_altivec(); + } else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) { + preempt_disable(); + enable_kernel_fp(); + } +#endif + } + + TRACE_DCN_FPU(true, function_name, line, *pcpu); + put_cpu_ptr(&fpu_recursion_depth); +} + +/** + * dc_fpu_end - Disable FPU protection + * @function_name: A string containing the function name for debug purposes + * @line: A-line number where DC_FP_END was invoked for debug purpose + * + * This function is responsible for managing the use of kernel_fpu_end() with + * the advantage of providing an event trace for debugging. + * + * Note: Do not call this function directly; always use DC_FP_END(). + */ +void dc_fpu_end(const char *function_name, const int line) +{ + int *pcpu; + + pcpu = get_cpu_ptr(&fpu_recursion_depth); + *pcpu -= 1; + if (*pcpu <= 0) { +#if defined(CONFIG_X86) + kernel_fpu_end(); +#elif defined(CONFIG_PPC64) + if (cpu_has_feature(CPU_FTR_VSX_COMP)) { + disable_kernel_vsx(); + preempt_enable(); + } else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) { + disable_kernel_altivec(); + preempt_enable(); + } else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) { + disable_kernel_fp(); + preempt_enable(); + } +#endif + } + + TRACE_DCN_FPU(false, function_name, line, *pcpu); + put_cpu_ptr(&fpu_recursion_depth); +} diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h new file mode 100644 index 000000000000..b8275b397920 --- /dev/null +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __DC_FPU_H__ +#define __DC_FPU_H__ + +void dc_assert_fp_enabled(void); +void dc_fpu_begin(const char *function_name, const int line); +void dc_fpu_end(const char *function_name, const int line); + +#endif /* __DC_FPU_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c index c6f494f0dcea..6185f9475fa2 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c @@ -66,9 +66,11 @@ int rn_get_active_display_cnt_wa( for (i = 0; i < context->stream_count; i++) { const struct dc_stream_state *stream = context->streams[i]; + /* Extend the WA to DP for Linux*/ if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A || stream->signal == SIGNAL_TYPE_DVI_SINGLE_LINK || - stream->signal == SIGNAL_TYPE_DVI_DUAL_LINK) + stream->signal == SIGNAL_TYPE_DVI_DUAL_LINK || + stream->signal == SIGNAL_TYPE_DISPLAY_PORT) tmds_present = true; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 605e297b7a59..c798c65d4276 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1481,6 +1481,22 @@ bool dc_validate_seamless_boot_timing(const struct dc *dc, return true; } +static inline bool should_update_pipe_for_stream( + struct dc_state *context, + struct pipe_ctx *pipe_ctx, + struct dc_stream_state *stream) +{ + return (pipe_ctx->stream && pipe_ctx->stream == stream); +} + +static inline bool should_update_pipe_for_plane( + struct dc_state *context, + struct pipe_ctx *pipe_ctx, + struct dc_plane_state *plane_state) +{ + return (pipe_ctx->plane_state == plane_state); +} + void dc_enable_stereo( struct dc *dc, struct dc_state *context, @@ -1491,12 +1507,15 @@ void dc_enable_stereo( struct pipe_ctx *pipe; for (i = 0; i < MAX_PIPES; i++) { - if (context != NULL) + if (context != NULL) { pipe = &context->res_ctx.pipe_ctx[i]; - else + } else { + context = dc->current_state; pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - for (j = 0 ; pipe && j < stream_count; j++) { - if (streams[j] && streams[j] == pipe->stream && + } + + for (j = 0; pipe && j < stream_count; j++) { + if (should_update_pipe_for_stream(context, pipe, streams[j]) && dc->hwss.setup_stereo) dc->hwss.setup_stereo(pipe, dc); } @@ -1530,6 +1549,12 @@ void dc_z10_restore(struct dc *dc) if (dc->hwss.z10_restore) dc->hwss.z10_restore(dc); } + +void dc_z10_save_init(struct dc *dc) +{ + if (dc->hwss.z10_save_init) + dc->hwss.z10_save_init(dc); +} #endif /* * Applies given context to HW and copy it into current context. @@ -2623,6 +2648,7 @@ static void commit_planes_for_stream(struct dc *dc, { int i, j; struct pipe_ctx *top_pipe_to_program = NULL; + bool should_lock_all_pipes = (update_type != UPDATE_TYPE_FAST); #if defined(CONFIG_DRM_AMD_DC_DCN) dc_z10_restore(dc); @@ -2694,7 +2720,7 @@ static void commit_planes_for_stream(struct dc *dc, top_pipe_to_program->stream_res.tg); } - if ((update_type != UPDATE_TYPE_FAST) && dc->hwss.interdependent_update_lock) + if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) dc->hwss.interdependent_update_lock(dc, context, true); else /* Lock the top pipe while updating plane addrs, since freesync requires @@ -2717,7 +2743,7 @@ static void commit_planes_for_stream(struct dc *dc, if (dc->hwss.program_front_end_for_ctx) dc->hwss.program_front_end_for_ctx(dc, context); - if ((update_type != UPDATE_TYPE_FAST) && dc->hwss.interdependent_update_lock) + if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) dc->hwss.interdependent_update_lock(dc, context, false); else dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false); @@ -2733,14 +2759,14 @@ static void commit_planes_for_stream(struct dc *dc, struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j]; if (!pipe_ctx->plane_state) continue; - if (pipe_ctx->plane_state != plane_state) + if (should_update_pipe_for_plane(context, pipe_ctx, plane_state)) continue; - plane_state->triplebuffer_flips = false; + pipe_ctx->plane_state->triplebuffer_flips = false; if (update_type == UPDATE_TYPE_FAST && dc->hwss.program_triplebuffer != NULL && - !plane_state->flip_immediate && dc->debug.enable_tri_buf) { + !pipe_ctx->plane_state->flip_immediate && dc->debug.enable_tri_buf) { /*triple buffer for VUpdate only*/ - plane_state->triplebuffer_flips = true; + pipe_ctx->plane_state->triplebuffer_flips = true; } } if (update_type == UPDATE_TYPE_FULL) { @@ -2756,8 +2782,7 @@ static void commit_planes_for_stream(struct dc *dc, if (!pipe_ctx->top_pipe && !pipe_ctx->prev_odm_pipe && - pipe_ctx->stream && - pipe_ctx->stream == stream) { + should_update_pipe_for_stream(context, pipe_ctx, stream)) { struct dc_stream_status *stream_status = NULL; if (!pipe_ctx->plane_state) @@ -2810,15 +2835,15 @@ static void commit_planes_for_stream(struct dc *dc, for (j = 0; j < dc->res_pool->pipe_count; j++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j]; - if (pipe_ctx->stream != stream) + if (!should_update_pipe_for_stream(context, pipe_ctx, stream)) continue; - if (pipe_ctx->plane_state != plane_state) + if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state)) continue; // GSL has to be used for flip immediate dc->hwss.set_flip_control_gsl(pipe_ctx, - plane_state->flip_immediate); + pipe_ctx->plane_state->flip_immediate); } } @@ -2829,25 +2854,26 @@ static void commit_planes_for_stream(struct dc *dc, for (j = 0; j < dc->res_pool->pipe_count; j++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j]; - if (pipe_ctx->stream != stream) + if (!should_update_pipe_for_stream(context, pipe_ctx, stream)) continue; - if (pipe_ctx->plane_state != plane_state) + if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state)) continue; + /*program triple buffer after lock based on flip type*/ if (dc->hwss.program_triplebuffer != NULL && dc->debug.enable_tri_buf) { /*only enable triplebuffer for fast_update*/ dc->hwss.program_triplebuffer( - dc, pipe_ctx, plane_state->triplebuffer_flips); + dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips); } - if (srf_updates[i].flip_addr) + if (pipe_ctx->plane_state->update_flags.bits.addr_update) dc->hwss.update_plane_addr(dc, pipe_ctx); } } } - if ((update_type != UPDATE_TYPE_FAST) && dc->hwss.interdependent_update_lock) + if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) dc->hwss.interdependent_update_lock(dc, context, false); else dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false); @@ -2891,7 +2917,7 @@ static void commit_planes_for_stream(struct dc *dc, continue; if (pipe_ctx->bottom_pipe || pipe_ctx->next_odm_pipe || - !pipe_ctx->stream || pipe_ctx->stream != stream || + !pipe_ctx->stream || !should_update_pipe_for_stream(context, pipe_ctx, stream) || !pipe_ctx->plane_state->update_flags.bits.addr_update || pipe_ctx->plane_state->skip_manual_trigger) continue; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index f56e061d35bc..330edd666b7d 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -1561,7 +1561,7 @@ bool dc_link_dp_perform_link_training_skip_aux( struct dc_link *link, const struct dc_link_settings *link_setting) { - struct link_training_settings lt_settings; + struct link_training_settings lt_settings = {0}; dp_decide_training_settings( link, @@ -1707,7 +1707,7 @@ enum link_training_result dc_link_dp_perform_link_training( bool skip_video_pattern) { enum link_training_result status = LINK_TRAINING_SUCCESS; - struct link_training_settings lt_settings; + struct link_training_settings lt_settings = {0}; enum dp_link_encoding encoding = dp_get_link_encoding_format(link_settings); @@ -1923,7 +1923,7 @@ enum link_training_result dc_link_dp_sync_lt_attempt( struct dc_link_settings *link_settings, struct dc_link_training_overrides *lt_overrides) { - struct link_training_settings lt_settings; + struct link_training_settings lt_settings = {0}; enum link_training_result lt_status = LINK_TRAINING_SUCCESS; enum dp_panel_mode panel_mode = DP_PANEL_MODE_DEFAULT; enum clock_source_id dp_cs_id = CLOCK_SOURCE_ID_EXTERNAL; @@ -3595,29 +3595,12 @@ static bool dpcd_read_sink_ext_caps(struct dc_link *link) bool dp_retrieve_lttpr_cap(struct dc_link *link) { uint8_t lttpr_dpcd_data[6]; - bool vbios_lttpr_enable = false; - bool vbios_lttpr_interop = false; - struct dc_bios *bios = link->dc->ctx->dc_bios; + bool vbios_lttpr_enable = link->dc->caps.vbios_lttpr_enable; + bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware; enum dc_status status = DC_ERROR_UNEXPECTED; bool is_lttpr_present = false; memset(lttpr_dpcd_data, '\0', sizeof(lttpr_dpcd_data)); - /* Query BIOS to determine if LTTPR functionality is forced on by system */ - if (bios->funcs->get_lttpr_caps) { - enum bp_result bp_query_result; - uint8_t is_vbios_lttpr_enable = 0; - - bp_query_result = bios->funcs->get_lttpr_caps(bios, &is_vbios_lttpr_enable); - vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable; - } - - if (bios->funcs->get_lttpr_interop) { - enum bp_result bp_query_result; - uint8_t is_vbios_interop_enabled = 0; - - bp_query_result = bios->funcs->get_lttpr_interop(bios, &is_vbios_interop_enabled); - vbios_lttpr_interop = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled; - } /* * Logic to determine LTTPR mode diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c index 1a89d565c92e..de80a9ea4cfa 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c @@ -305,7 +305,7 @@ struct link_encoder *link_enc_cfg_get_next_avail_link_enc( const struct dc_state *state) { struct link_encoder *link_enc = NULL; - enum engine_id eng_id = ENGINE_ID_UNKNOWN; + enum engine_id eng_id; eng_id = find_first_avail_link_enc(dc->ctx, state); if (eng_id != ENGINE_ID_UNKNOWN) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 1596f6b7fed7..a60396d5be44 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1030,7 +1030,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) /* Timing borders are part of vactive that we are also supposed to skip in addition * to any stream dst offset. Since dm logic assumes dst is in addressable - * space we need to add the the left and top borders to dst offsets temporarily. + * space we need to add the left and top borders to dst offsets temporarily. * TODO: fix in DM, stream dst is supposed to be in vactive */ pipe_ctx->stream->dst.x += timing->h_border_left; @@ -1051,6 +1051,11 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) /* depends on scaling ratios and recout, does not calculate offset yet */ calculate_viewport_size(pipe_ctx); + /* Stopgap for validation of ODM + MPO on one side of screen case */ + if (pipe_ctx->plane_res.scl_data.viewport.height < 1 || + pipe_ctx->plane_res.scl_data.viewport.width < 1) + return false; + /* * LB calculations depend on vp size, h/v_active and scaling ratios * Setting line buffer pixel depth to 24bpp yields banding diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 327fd1909c51..f0f54f4d3d9b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -246,6 +246,40 @@ struct dc_stream_status *dc_stream_get_status( return dc_stream_get_status_from_state(dc->current_state, stream); } +static void program_cursor_attributes( + struct dc *dc, + struct dc_stream_state *stream, + const struct dc_cursor_attributes *attributes) +{ + int i; + struct resource_context *res_ctx; + struct pipe_ctx *pipe_to_program = NULL; + + if (!stream) + return; + + res_ctx = &dc->current_state->res_ctx; + + for (i = 0; i < MAX_PIPES; i++) { + struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i]; + + if (pipe_ctx->stream != stream) + continue; + + if (!pipe_to_program) { + pipe_to_program = pipe_ctx; + dc->hwss.cursor_lock(dc, pipe_to_program, true); + } + + dc->hwss.set_cursor_attribute(pipe_ctx); + if (dc->hwss.set_cursor_sdr_white_level) + dc->hwss.set_cursor_sdr_white_level(pipe_ctx); + } + + if (pipe_to_program) + dc->hwss.cursor_lock(dc, pipe_to_program, false); +} + #ifndef TRIM_FSFT /* * dc_optimize_timing_for_fsft() - dc to optimize timing @@ -270,10 +304,7 @@ bool dc_stream_set_cursor_attributes( struct dc_stream_state *stream, const struct dc_cursor_attributes *attributes) { - int i; struct dc *dc; - struct resource_context *res_ctx; - struct pipe_ctx *pipe_to_program = NULL; #if defined(CONFIG_DRM_AMD_DC_DCN) bool reset_idle_optimizations = false; #endif @@ -293,7 +324,6 @@ bool dc_stream_set_cursor_attributes( } dc = stream->ctx->dc; - res_ctx = &dc->current_state->res_ctx; stream->cursor_attributes = *attributes; #if defined(CONFIG_DRM_AMD_DC_DCN) @@ -305,11 +335,39 @@ bool dc_stream_set_cursor_attributes( } #endif + program_cursor_attributes(dc, stream, attributes); + +#if defined(CONFIG_DRM_AMD_DC_DCN) + /* re-enable idle optimizations if necessary */ + if (reset_idle_optimizations) + dc_allow_idle_optimizations(dc, true); + +#endif + return true; +} + +static void program_cursor_position( + struct dc *dc, + struct dc_stream_state *stream, + const struct dc_cursor_position *position) +{ + int i; + struct resource_context *res_ctx; + struct pipe_ctx *pipe_to_program = NULL; + + if (!stream) + return; + + res_ctx = &dc->current_state->res_ctx; for (i = 0; i < MAX_PIPES; i++) { struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i]; - if (pipe_ctx->stream != stream) + if (pipe_ctx->stream != stream || + (!pipe_ctx->plane_res.mi && !pipe_ctx->plane_res.hubp) || + !pipe_ctx->plane_state || + (!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp) || + (!pipe_ctx->plane_res.ipp && !pipe_ctx->plane_res.dpp)) continue; if (!pipe_to_program) { @@ -317,31 +375,18 @@ bool dc_stream_set_cursor_attributes( dc->hwss.cursor_lock(dc, pipe_to_program, true); } - dc->hwss.set_cursor_attribute(pipe_ctx); - if (dc->hwss.set_cursor_sdr_white_level) - dc->hwss.set_cursor_sdr_white_level(pipe_ctx); + dc->hwss.set_cursor_position(pipe_ctx); } if (pipe_to_program) dc->hwss.cursor_lock(dc, pipe_to_program, false); - -#if defined(CONFIG_DRM_AMD_DC_DCN) - /* re-enable idle optimizations if necessary */ - if (reset_idle_optimizations) - dc_allow_idle_optimizations(dc, true); - -#endif - return true; } bool dc_stream_set_cursor_position( struct dc_stream_state *stream, const struct dc_cursor_position *position) { - int i; struct dc *dc; - struct resource_context *res_ctx; - struct pipe_ctx *pipe_to_program = NULL; #if defined(CONFIG_DRM_AMD_DC_DCN) bool reset_idle_optimizations = false; #endif @@ -357,7 +402,6 @@ bool dc_stream_set_cursor_position( } dc = stream->ctx->dc; - res_ctx = &dc->current_state->res_ctx; #if defined(CONFIG_DRM_AMD_DC_DCN) dc_z10_restore(dc); @@ -370,27 +414,7 @@ bool dc_stream_set_cursor_position( #endif stream->cursor_position = *position; - for (i = 0; i < MAX_PIPES; i++) { - struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i]; - - if (pipe_ctx->stream != stream || - (!pipe_ctx->plane_res.mi && !pipe_ctx->plane_res.hubp) || - !pipe_ctx->plane_state || - (!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp) || - (!pipe_ctx->plane_res.ipp && !pipe_ctx->plane_res.dpp)) - continue; - - if (!pipe_to_program) { - pipe_to_program = pipe_ctx; - dc->hwss.cursor_lock(dc, pipe_to_program, true); - } - - dc->hwss.set_cursor_position(pipe_ctx); - } - - if (pipe_to_program) - dc->hwss.cursor_lock(dc, pipe_to_program, false); - + program_cursor_position(dc, stream, position); #if defined(CONFIG_DRM_AMD_DC_DCN) /* re-enable idle optimizations if necessary */ if (reset_idle_optimizations) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c index f2b39ec35c89..cde8ed2560b3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c @@ -47,6 +47,9 @@ int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_c */ memcpy(&dc->vm_pa_config, pa_config, sizeof(struct dc_phy_addr_space_config)); dc->vm_pa_config.valid = true; +#if defined(CONFIG_DRM_AMD_DC_DCN) + dc_z10_save_init(dc); +#endif } return num_vmids; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index a948f4f48935..3ab52d9a82cf 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -45,7 +45,7 @@ /* forward declaration */ struct aux_payload; -#define DC_VER "3.2.146" +#define DC_VER "3.2.149" #define MAX_SURFACES 3 #define MAX_PLANES 6 @@ -183,6 +183,8 @@ struct dc_caps { unsigned int cursor_cache_size; struct dc_plane_cap planes[MAX_PLANES]; struct dc_color_caps color; + bool vbios_lttpr_aware; + bool vbios_lttpr_enable; }; struct dc_bug_wa { @@ -1336,6 +1338,7 @@ void dc_hardware_release(struct dc *dc); bool dc_set_psr_allow_active(struct dc *dc, bool enable); #if defined(CONFIG_DRM_AMD_DC_DCN) void dc_z10_restore(struct dc *dc); +void dc_z10_save_init(struct dc *dc); #endif bool dc_enable_dmub_notifications(struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c index a612ba6dc389..ab6bc5d79012 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c @@ -28,9 +28,9 @@ */ #include <linux/delay.h> +#include <linux/stdarg.h> #include "dm_services.h" -#include <stdarg.h> #include "dc.h" #include "dc_dmub_srv.h" diff --git a/drivers/gpu/drm/amd/display/dc/dc_trace.h b/drivers/gpu/drm/amd/display/dc/dc_trace.h index d2615357269b..c711797e5c9e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_trace.h +++ b/drivers/gpu/drm/amd/display/dc/dc_trace.h @@ -37,3 +37,6 @@ #define TRACE_DCN_CLOCK_STATE(dcn_clocks) \ trace_amdgpu_dm_dc_clocks_state(dcn_clocks) + +#define TRACE_DCN_FPU(begin, function, line, ref_count) \ + trace_dcn_fpu(begin, function, line, ref_count) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c index 058a9356a39a..e14f99b4b0c3 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c @@ -42,6 +42,11 @@ #define DC_LOGGER \ engine->ctx->logger +#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */ +#define IS_DC_I2CAUX_LOGGING_ENABLED() (false) +#define LOG_FLAG_Error_I2cAux LOG_ERROR +#define LOG_FLAG_I2cAux_DceAux LOG_I2C_AUX + #include "reg_helper.h" #undef FN @@ -623,6 +628,58 @@ int dce_aux_transfer_dmub_raw(struct ddc_service *ddc, #define AUX_MAX_INVALID_REPLY_RETRIES 2 #define AUX_MAX_TIMEOUT_RETRIES 3 +static void dce_aux_log_payload(const char *payload_name, + unsigned char *payload, uint32_t length, uint32_t max_length_to_log) +{ + if (!IS_DC_I2CAUX_LOGGING_ENABLED()) + return; + + if (payload && length) { + char hex_str[128] = {0}; + char *hex_str_ptr = &hex_str[0]; + uint32_t hex_str_remaining = sizeof(hex_str); + unsigned char *payload_ptr = payload; + unsigned char *payload_max_to_log_ptr = payload_ptr + min(max_length_to_log, length); + unsigned int count; + char *padding = ""; + + while (payload_ptr < payload_max_to_log_ptr) { + count = snprintf_count(hex_str_ptr, hex_str_remaining, "%s%02X", padding, *payload_ptr); + padding = " "; + hex_str_remaining -= count; + hex_str_ptr += count; + payload_ptr++; + } + + count = snprintf_count(hex_str_ptr, hex_str_remaining, " "); + hex_str_remaining -= count; + hex_str_ptr += count; + + payload_ptr = payload; + while (payload_ptr < payload_max_to_log_ptr) { + count = snprintf_count(hex_str_ptr, hex_str_remaining, "%c", + *payload_ptr >= ' ' ? *payload_ptr : '.'); + hex_str_remaining -= count; + hex_str_ptr += count; + payload_ptr++; + } + + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_log_payload: %s: length=%u: data: %s%s", + payload_name, + length, + hex_str, + (length > max_length_to_log ? " (...)" : " ")); + } else { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_log_payload: %s: length=%u: data: <empty payload>", + payload_name, + length); + } +} + bool dce_aux_transfer_with_retries(struct ddc_service *ddc, struct aux_payload *payload) { @@ -648,7 +705,34 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, } for (i = 0; i < AUX_MAX_RETRIES; i++) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: link_index=%u: START: retry %d of %d: address=0x%04x length=%u write=%d mot=%d", + ddc && ddc->link ? ddc->link->link_index : UINT_MAX, + i + 1, + (int)AUX_MAX_RETRIES, + payload->address, + payload->length, + (unsigned int) payload->write, + (unsigned int) payload->mot); + if (payload->write) + dce_aux_log_payload(" write", payload->data, payload->length, 16); ret = dce_aux_transfer_raw(ddc, payload, &operation_result); + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: link_index=%u: END: retry %d of %d: address=0x%04x length=%u write=%d mot=%d: ret=%d operation_result=%d payload->reply=%u", + ddc && ddc->link ? ddc->link->link_index : UINT_MAX, + i + 1, + (int)AUX_MAX_RETRIES, + payload->address, + payload->length, + (unsigned int) payload->write, + (unsigned int) payload->mot, + ret, + (int)operation_result, + (unsigned int) *payload->reply); + if (!payload->write) + dce_aux_log_payload(" read", payload->data, ret > 0 ? ret : 0, 16); switch (operation_result) { case AUX_RET_SUCCESS: @@ -657,30 +741,64 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, switch (*payload->reply) { case AUX_TRANSACTION_REPLY_AUX_ACK: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_AUX_ACK"); if (!payload->write && payload->length != ret) { - if (++aux_ack_retries >= AUX_MAX_RETRIES) + if (++aux_ack_retries >= AUX_MAX_RETRIES) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_ack_retries=%d >= AUX_MAX_RETRIES=%d", + aux_defer_retries, + AUX_MAX_RETRIES); goto fail; - else + } else { udelay(300); + } } else return true; break; case AUX_TRANSACTION_REPLY_AUX_DEFER: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_AUX_DEFER"); + /* polling_timeout_period is in us */ defer_time_in_ms += aux110->polling_timeout_period / 1000; ++aux_defer_retries; fallthrough; case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER: + if (*payload->reply == AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER) + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER"); + retry_on_defer = true; fallthrough; case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK: + if (*payload->reply == AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK) + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK"); + if (aux_defer_retries >= AUX_MIN_DEFER_RETRIES && defer_time_in_ms >= AUX_MAX_DEFER_TIMEOUT_MS) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_defer_retries=%d >= AUX_MIN_DEFER_RETRIES=%d && defer_time_in_ms=%d >= AUX_MAX_DEFER_TIMEOUT_MS=%d", + aux_defer_retries, + AUX_MIN_DEFER_RETRIES, + defer_time_in_ms, + AUX_MAX_DEFER_TIMEOUT_MS); goto fail; } else { if ((*payload->reply == AUX_TRANSACTION_REPLY_AUX_DEFER) || (*payload->reply == AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER)) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: payload->defer_delay=%u", + payload->defer_delay); if (payload->defer_delay > 1) { msleep(payload->defer_delay); defer_time_in_ms += payload->defer_delay; @@ -693,37 +811,86 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, break; case AUX_TRANSACTION_REPLY_I2C_DEFER: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_I2C_DEFER"); + aux_defer_retries = 0; - if (++aux_i2c_defer_retries >= AUX_MAX_I2C_DEFER_RETRIES) + if (++aux_i2c_defer_retries >= AUX_MAX_I2C_DEFER_RETRIES) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_i2c_defer_retries=%d >= AUX_MAX_I2C_DEFER_RETRIES=%d", + aux_i2c_defer_retries, + AUX_MAX_I2C_DEFER_RETRIES); goto fail; + } break; case AUX_TRANSACTION_REPLY_AUX_NACK: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_AUX_NACK"); + goto fail; + case AUX_TRANSACTION_REPLY_HPD_DISCON: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_HPD_DISCON"); + goto fail; + default: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: AUX_RET_SUCCESS: FAILURE: AUX_TRANSACTION_REPLY_* unknown, default case."); goto fail; } break; case AUX_RET_ERROR_INVALID_REPLY: - if (++aux_invalid_reply_retries >= AUX_MAX_INVALID_REPLY_RETRIES) + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_ERROR_INVALID_REPLY"); + if (++aux_invalid_reply_retries >= AUX_MAX_INVALID_REPLY_RETRIES) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_invalid_reply_retries=%d >= AUX_MAX_INVALID_REPLY_RETRIES=%d", + aux_invalid_reply_retries, + AUX_MAX_INVALID_REPLY_RETRIES); goto fail; - else + } else udelay(400); break; case AUX_RET_ERROR_TIMEOUT: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: AUX_RET_ERROR_TIMEOUT"); // Check whether a DEFER had occurred before the timeout. // If so, treat timeout as a DEFER. if (retry_on_defer) { - if (++aux_defer_retries >= AUX_MIN_DEFER_RETRIES) + if (++aux_defer_retries >= AUX_MIN_DEFER_RETRIES) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_defer_retries=%d >= AUX_MIN_DEFER_RETRIES=%d", + aux_defer_retries, + AUX_MIN_DEFER_RETRIES); goto fail; - else if (payload->defer_delay > 0) + } else if (payload->defer_delay > 0) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: payload->defer_delay=%u", + payload->defer_delay); msleep(payload->defer_delay); + } } else { - if (++aux_timeout_retries >= AUX_MAX_TIMEOUT_RETRIES) + if (++aux_timeout_retries >= AUX_MAX_TIMEOUT_RETRIES) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE: aux_timeout_retries=%d >= AUX_MAX_TIMEOUT_RETRIES=%d", + aux_timeout_retries, + AUX_MAX_TIMEOUT_RETRIES); goto fail; - else { + } else { /* * DP 1.4, 2.8.2: AUX Transaction Response/Reply Timeouts * According to the DP spec there should be 3 retries total @@ -738,11 +905,18 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, case AUX_RET_ERROR_ENGINE_ACQUIRE: case AUX_RET_ERROR_UNKNOWN: default: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + LOG_FLAG_I2cAux_DceAux, + "dce_aux_transfer_with_retries: Failure: operation_result=%d", + (int)operation_result); goto fail; } } fail: + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + LOG_FLAG_Error_I2cAux, + "dce_aux_transfer_with_retries: FAILURE"); if (!payload_reply) payload->reply = NULL; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index 1ca8b1d94bc2..aa8403bc4c83 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -29,7 +29,7 @@ #include "dmub/dmub_srv.h" #include "core_types.h" -#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */ +#define DC_TRACE_LEVEL_MESSAGE(...) do {} while (0) /* do nothing */ #define MAX_PIPES 6 diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index 35af0401f256..df8a7718a85f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -3641,13 +3641,12 @@ enum dc_status dcn10_set_clock(struct dc *dc, struct dc_clock_config clock_cfg = {0}; struct dc_clocks *current_clocks = &context->bw_ctx.bw.dcn.clk; - if (dc->clk_mgr && dc->clk_mgr->funcs->get_clock) - dc->clk_mgr->funcs->get_clock(dc->clk_mgr, - context, clock_type, &clock_cfg); - - if (!dc->clk_mgr->funcs->get_clock) + if (!dc->clk_mgr || !dc->clk_mgr->funcs->get_clock) return DC_FAIL_UNSUPPORTED_1; + dc->clk_mgr->funcs->get_clock(dc->clk_mgr, + context, clock_type, &clock_cfg); + if (clk_khz > clock_cfg.max_clock_khz) return DC_FAIL_CLK_EXCEED_MAX; @@ -3665,7 +3664,7 @@ enum dc_status dcn10_set_clock(struct dc *dc, else return DC_ERROR_UNEXPECTED; - if (dc->clk_mgr && dc->clk_mgr->funcs->update_clocks) + if (dc->clk_mgr->funcs->update_clocks) dc->clk_mgr->funcs->update_clocks(dc->clk_mgr, context, true); return DC_OK; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c index f1a08a7736ac..cf364ae93138 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c @@ -522,16 +522,21 @@ void enc1_stream_encoder_hdmi_set_stream_attribute( switch (crtc_timing->display_color_depth) { case COLOR_DEPTH_888: REG_UPDATE(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 0); + DC_LOG_DEBUG("HDMI source set to 24BPP deep color depth\n"); break; case COLOR_DEPTH_101010: if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) { REG_UPDATE_2(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 1, HDMI_DEEP_COLOR_ENABLE, 0); + DC_LOG_DEBUG("HDMI source 30BPP deep color depth" \ + "disabled for YCBCR422 pixel encoding\n"); } else { REG_UPDATE_2(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 1, HDMI_DEEP_COLOR_ENABLE, 1); + DC_LOG_DEBUG("HDMI source 30BPP deep color depth" \ + "enabled for YCBCR422 non-pixel encoding\n"); } break; case COLOR_DEPTH_121212: @@ -539,16 +544,22 @@ void enc1_stream_encoder_hdmi_set_stream_attribute( REG_UPDATE_2(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 2, HDMI_DEEP_COLOR_ENABLE, 0); + DC_LOG_DEBUG("HDMI source 36BPP deep color depth" \ + "disabled for YCBCR422 pixel encoding\n"); } else { REG_UPDATE_2(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 2, HDMI_DEEP_COLOR_ENABLE, 1); + DC_LOG_DEBUG("HDMI source 36BPP deep color depth" \ + "enabled for non-pixel YCBCR422 encoding\n"); } break; case COLOR_DEPTH_161616: REG_UPDATE_2(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 3, HDMI_DEEP_COLOR_ENABLE, 1); + DC_LOG_DEBUG("HDMI source deep color depth enabled in" \ + "reserved mode\n"); break; default: break; diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 5c2853654cca..a47ba1d45be9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1723,13 +1723,15 @@ void dcn20_program_front_end_for_ctx( pipe = pipe->bottom_pipe; } - /* Program secondary blending tree and writeback pipes */ - pipe = &context->res_ctx.pipe_ctx[i]; - if (!pipe->prev_odm_pipe && pipe->stream->num_wb_info > 0 - && (pipe->update_flags.raw || pipe->plane_state->update_flags.raw || pipe->stream->update_flags.raw) - && hws->funcs.program_all_writeback_pipes_in_tree) - hws->funcs.program_all_writeback_pipes_in_tree(dc, pipe->stream, context); } + /* Program secondary blending tree and writeback pipes */ + pipe = &context->res_ctx.pipe_ctx[i]; + if (!pipe->top_pipe && !pipe->prev_odm_pipe + && pipe->stream && pipe->stream->num_wb_info > 0 + && (pipe->update_flags.raw || (pipe->plane_state && pipe->plane_state->update_flags.raw) + || pipe->stream->update_flags.raw) + && hws->funcs.program_all_writeback_pipes_in_tree) + hws->funcs.program_all_writeback_pipes_in_tree(dc, pipe->stream, context); } } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c index 7fa9fc656b0c..f6e747f25ebe 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c @@ -464,7 +464,7 @@ void optc2_lock_doublebuffer_enable(struct timing_generator *optc) REG_UPDATE_2(OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_X, - h_blank_start - 200 - 1, + (h_blank_start - 200 - 1) / optc1->opp_count, MASTER_UPDATE_LOCK_DB_Y, v_blank_start - 1); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index 0b1cd1dbed8b..e3e01b17c164 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -35,6 +35,8 @@ #include "include/irq_service_interface.h" #include "dcn20/dcn20_resource.h" +#include "dml/dcn2x/dcn2x.h" + #include "dcn10/dcn10_hubp.h" #include "dcn10/dcn10_ipp.h" #include "dcn20_hubbub.h" @@ -1974,43 +1976,6 @@ void dcn20_split_stream_for_mpc( ASSERT(primary_pipe->plane_state); } -void dcn20_populate_dml_writeback_from_context( - struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes) -{ - int pipe_cnt, i; - - for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { - struct dc_writeback_info *wb_info = &res_ctx->pipe_ctx[i].stream->writeback_info[0]; - - if (!res_ctx->pipe_ctx[i].stream) - continue; - - /* Set writeback information */ - pipes[pipe_cnt].dout.wb_enable = (wb_info->wb_enabled == true) ? 1 : 0; - pipes[pipe_cnt].dout.num_active_wb++; - pipes[pipe_cnt].dout.wb.wb_src_height = wb_info->dwb_params.cnv_params.crop_height; - pipes[pipe_cnt].dout.wb.wb_src_width = wb_info->dwb_params.cnv_params.crop_width; - pipes[pipe_cnt].dout.wb.wb_dst_width = wb_info->dwb_params.dest_width; - pipes[pipe_cnt].dout.wb.wb_dst_height = wb_info->dwb_params.dest_height; - pipes[pipe_cnt].dout.wb.wb_htaps_luma = 1; - pipes[pipe_cnt].dout.wb.wb_vtaps_luma = 1; - pipes[pipe_cnt].dout.wb.wb_htaps_chroma = wb_info->dwb_params.scaler_taps.h_taps_c; - pipes[pipe_cnt].dout.wb.wb_vtaps_chroma = wb_info->dwb_params.scaler_taps.v_taps_c; - pipes[pipe_cnt].dout.wb.wb_hratio = 1.0; - pipes[pipe_cnt].dout.wb.wb_vratio = 1.0; - if (wb_info->dwb_params.out_format == dwb_scaler_mode_yuv420) { - if (wb_info->dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC) - pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_8; - else - pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_10; - } else - pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_444_32; - - pipe_cnt++; - } - -} - int dcn20_populate_dml_pipes_from_context( struct dc *dc, struct dc_state *context, @@ -2392,7 +2357,9 @@ int dcn20_populate_dml_pipes_from_context( } /* populate writeback information */ + DC_FP_START(); dc->res_pool->funcs->populate_dml_writeback_from_context(dc, res_ctx, pipes); + DC_FP_END(); return pipe_cnt; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h index c8f3127bbcdf..6ec8ff45f0f7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h @@ -58,8 +58,6 @@ struct pipe_ctx *dcn20_acquire_idle_pipe_for_layer( struct dc_state *state, const struct resource_pool *pool, struct dc_stream_state *stream); -void dcn20_populate_dml_writeback_from_context( - struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes); struct stream_encoder *dcn20_stream_encoder_create( enum engine_id eng_id, diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index f27fc2acac57..fbbdf9976183 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -35,6 +35,8 @@ #include "include/irq_service_interface.h" #include "dcn20/dcn20_resource.h" +#include "dml/dcn2x/dcn2x.h" + #include "clk_mgr.h" #include "dcn10/dcn10_hubp.h" #include "dcn10/dcn10_ipp.h" diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb_cm.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb_cm.c index 3fe9e41e4dbd..6a3d3a0ec0a3 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb_cm.c @@ -49,6 +49,11 @@ static void dwb3_get_reg_field_ogam(struct dcn30_dwbc *dwbc30, struct dcn3_xfer_func_reg *reg) { + reg->shifts.field_region_start_base = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION_START_BASE_B; + reg->masks.field_region_start_base = dwbc30->dwbc_mask->DWB_OGAM_RAMA_EXP_REGION_START_BASE_B; + reg->shifts.field_offset = dwbc30->dwbc_shift->DWB_OGAM_RAMA_OFFSET_B; + reg->masks.field_offset = dwbc30->dwbc_mask->DWB_OGAM_RAMA_OFFSET_B; + reg->shifts.exp_region0_lut_offset = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION0_LUT_OFFSET; reg->masks.exp_region0_lut_offset = dwbc30->dwbc_mask->DWB_OGAM_RAMA_EXP_REGION0_LUT_OFFSET; reg->shifts.exp_region0_num_segments = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS; @@ -66,8 +71,6 @@ static void dwb3_get_reg_field_ogam(struct dcn30_dwbc *dwbc30, reg->masks.field_region_end_base = dwbc30->dwbc_mask->DWB_OGAM_RAMA_EXP_REGION_END_BASE_B; reg->shifts.field_region_linear_slope = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_B; reg->masks.field_region_linear_slope = dwbc30->dwbc_mask->DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_B; - reg->masks.field_offset = dwbc30->dwbc_mask->DWB_OGAM_RAMA_OFFSET_B; - reg->shifts.field_offset = dwbc30->dwbc_shift->DWB_OGAM_RAMA_OFFSET_B; reg->shifts.exp_region_start = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION_START_B; reg->masks.exp_region_start = dwbc30->dwbc_mask->DWB_OGAM_RAMA_EXP_REGION_START_B; reg->shifts.exp_resion_start_segment = dwbc30->dwbc_shift->DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_B; @@ -147,18 +150,19 @@ static enum dc_lut_mode dwb3_get_ogam_current( uint32_t state_mode; uint32_t ram_select; - REG_GET(DWB_OGAM_CONTROL, - DWB_OGAM_MODE, &state_mode); - REG_GET(DWB_OGAM_CONTROL, - DWB_OGAM_SELECT, &ram_select); + REG_GET_2(DWB_OGAM_CONTROL, + DWB_OGAM_MODE_CURRENT, &state_mode, + DWB_OGAM_SELECT_CURRENT, &ram_select); if (state_mode == 0) { mode = LUT_BYPASS; } else if (state_mode == 2) { if (ram_select == 0) mode = LUT_RAM_A; - else + else if (ram_select == 1) mode = LUT_RAM_B; + else + mode = LUT_BYPASS; } else { // Reserved value mode = LUT_BYPASS; @@ -172,10 +176,10 @@ static void dwb3_configure_ogam_lut( struct dcn30_dwbc *dwbc30, bool is_ram_a) { - REG_UPDATE(DWB_OGAM_LUT_CONTROL, - DWB_OGAM_LUT_READ_COLOR_SEL, 7); - REG_UPDATE(DWB_OGAM_CONTROL, - DWB_OGAM_SELECT, is_ram_a == true ? 0 : 1); + REG_UPDATE_2(DWB_OGAM_LUT_CONTROL, + DWB_OGAM_LUT_WRITE_COLOR_MASK, 7, + DWB_OGAM_LUT_HOST_SEL, (is_ram_a == true) ? 0 : 1); + REG_SET(DWB_OGAM_LUT_INDEX, 0, DWB_OGAM_LUT_INDEX, 0); } @@ -185,17 +189,45 @@ static void dwb3_program_ogam_pwl(struct dcn30_dwbc *dwbc30, { uint32_t i; - // triple base implementation - for (i = 0; i < num/2; i++) { - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+0].red_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+0].green_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+0].blue_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+1].red_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+1].green_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+1].blue_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+2].red_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+2].green_reg); - REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[2*i+2].blue_reg); + uint32_t last_base_value_red = rgb[num-1].red_reg + rgb[num-1].delta_red_reg; + uint32_t last_base_value_green = rgb[num-1].green_reg + rgb[num-1].delta_green_reg; + uint32_t last_base_value_blue = rgb[num-1].blue_reg + rgb[num-1].delta_blue_reg; + + if (is_rgb_equal(rgb, num)) { + for (i = 0 ; i < num; i++) + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[i].red_reg); + + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, last_base_value_red); + + } else { + + REG_UPDATE(DWB_OGAM_LUT_CONTROL, + DWB_OGAM_LUT_WRITE_COLOR_MASK, 4); + + for (i = 0 ; i < num; i++) + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[i].red_reg); + + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, last_base_value_red); + + REG_SET(DWB_OGAM_LUT_INDEX, 0, DWB_OGAM_LUT_INDEX, 0); + + REG_UPDATE(DWB_OGAM_LUT_CONTROL, + DWB_OGAM_LUT_WRITE_COLOR_MASK, 2); + + for (i = 0 ; i < num; i++) + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[i].green_reg); + + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, last_base_value_green); + + REG_SET(DWB_OGAM_LUT_INDEX, 0, DWB_OGAM_LUT_INDEX, 0); + + REG_UPDATE(DWB_OGAM_LUT_CONTROL, + DWB_OGAM_LUT_WRITE_COLOR_MASK, 1); + + for (i = 0 ; i < num; i++) + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, rgb[i].blue_reg); + + REG_SET(DWB_OGAM_LUT_DATA, 0, DWB_OGAM_LUT_DATA, last_base_value_blue); } } @@ -211,6 +243,8 @@ static bool dwb3_program_ogam_lut( return false; } + REG_SET(DWB_OGAM_CONTROL, 0, DWB_OGAM_MODE, 2); + current_mode = dwb3_get_ogam_current(dwbc30); if (current_mode == LUT_BYPASS || current_mode == LUT_RAM_A) next_mode = LUT_RAM_B; @@ -227,8 +261,7 @@ static bool dwb3_program_ogam_lut( dwb3_program_ogam_pwl( dwbc30, params->rgb_resulted, params->hw_points_num); - REG_SET(DWB_OGAM_CONTROL, 0, DWB_OGAM_MODE, 2); - REG_SET(DWB_OGAM_CONTROL, 0, DWB_OGAM_SELECT, next_mode == LUT_RAM_A ? 0 : 1); + REG_UPDATE(DWB_OGAM_CONTROL, DWB_OGAM_SELECT, next_mode == LUT_RAM_A ? 0 : 1); return true; } @@ -271,14 +304,19 @@ static void dwb3_program_gamut_remap( struct color_matrices_reg gam_regs; - REG_UPDATE(DWB_GAMUT_REMAP_COEF_FORMAT, DWB_GAMUT_REMAP_COEF_FORMAT, coef_format); - if (regval == NULL || select == CM_GAMUT_REMAP_MODE_BYPASS) { REG_SET(DWB_GAMUT_REMAP_MODE, 0, DWB_GAMUT_REMAP_MODE, 0); return; } + REG_UPDATE(DWB_GAMUT_REMAP_COEF_FORMAT, DWB_GAMUT_REMAP_COEF_FORMAT, coef_format); + + gam_regs.shifts.csc_c11 = dwbc30->dwbc_shift->DWB_GAMUT_REMAPA_C11; + gam_regs.masks.csc_c11 = dwbc30->dwbc_mask->DWB_GAMUT_REMAPA_C11; + gam_regs.shifts.csc_c12 = dwbc30->dwbc_shift->DWB_GAMUT_REMAPA_C12; + gam_regs.masks.csc_c12 = dwbc30->dwbc_mask->DWB_GAMUT_REMAPA_C12; + switch (select) { case CM_GAMUT_REMAP_MODE_RAMA_COEFF: gam_regs.csc_c11_c12 = REG(DWB_GAMUT_REMAPA_C11_C12); diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 2e8ab9775fa3..fafed1e4a998 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -398,12 +398,22 @@ void dcn30_program_all_writeback_pipes_in_tree( for (i_pipe = 0; i_pipe < dc->res_pool->pipe_count; i_pipe++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i_pipe]; + if (!pipe_ctx->plane_state) + continue; + if (pipe_ctx->plane_state == wb_info.writeback_source_plane) { wb_info.mpcc_inst = pipe_ctx->plane_res.mpcc_inst; break; } } - ASSERT(wb_info.mpcc_inst != -1); + + if (wb_info.mpcc_inst == -1) { + /* Disable writeback pipe and disconnect from MPCC + * if source plane has been removed + */ + dc->hwss.disable_writeback(dc, wb_info.dwb_pipe_inst); + continue; + } ASSERT(wb_info.dwb_pipe_inst < dc->res_pool->res_cap->num_dwb); dwb = dc->res_pool->dwbc[wb_info.dwb_pipe_inst]; diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 596c97dce67e..a0de309475a9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -92,7 +92,7 @@ #define DC_LOGGER_INIT(logger) struct _vcs_dpi_ip_params_st dcn3_0_ip = { - .use_min_dcfclk = 1, + .use_min_dcfclk = 0, .clamp_min_dcfclk = 0, .odm_capable = 1, .gpuvm_enable = 0, @@ -1788,7 +1788,6 @@ static bool dcn30_split_stream_for_mpc_or_odm( } pri_pipe->next_odm_pipe = sec_pipe; sec_pipe->prev_odm_pipe = pri_pipe; - ASSERT(sec_pipe->top_pipe == NULL); if (!sec_pipe->top_pipe) sec_pipe->stream_res.opp = pool->opps[pipe_idx]; @@ -2399,16 +2398,37 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; if (bw_params->clk_table.entries[0].memclk_mhz) { + int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0; + + for (i = 0; i < MAX_NUM_DPM_LVL; i++) { + if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz) + max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz; + if (bw_params->clk_table.entries[i].dispclk_mhz > max_dispclk_mhz) + max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz; + if (bw_params->clk_table.entries[i].dppclk_mhz > max_dppclk_mhz) + max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz; + if (bw_params->clk_table.entries[i].phyclk_mhz > max_phyclk_mhz) + max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz; + } + + if (!max_dcfclk_mhz) + max_dcfclk_mhz = dcn3_0_soc.clock_limits[0].dcfclk_mhz; + if (!max_dispclk_mhz) + max_dispclk_mhz = dcn3_0_soc.clock_limits[0].dispclk_mhz; + if (!max_dppclk_mhz) + max_dppclk_mhz = dcn3_0_soc.clock_limits[0].dppclk_mhz; + if (!max_phyclk_mhz) + max_phyclk_mhz = dcn3_0_soc.clock_limits[0].phyclk_mhz; - if (bw_params->clk_table.entries[1].dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { // If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array - dcfclk_sta_targets[num_dcfclk_sta_targets] = bw_params->clk_table.entries[1].dcfclk_mhz; + dcfclk_sta_targets[num_dcfclk_sta_targets] = max_dcfclk_mhz; num_dcfclk_sta_targets++; - } else if (bw_params->clk_table.entries[1].dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + } else if (max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { // If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates for (i = 0; i < num_dcfclk_sta_targets; i++) { - if (dcfclk_sta_targets[i] > bw_params->clk_table.entries[1].dcfclk_mhz) { - dcfclk_sta_targets[i] = bw_params->clk_table.entries[1].dcfclk_mhz; + if (dcfclk_sta_targets[i] > max_dcfclk_mhz) { + dcfclk_sta_targets[i] = max_dcfclk_mhz; break; } } @@ -2448,7 +2468,7 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params dcfclk_mhz[num_states] = dcfclk_sta_targets[i]; dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++]; } else { - if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= bw_params->clk_table.entries[1].dcfclk_mhz) { + if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; } else { @@ -2463,11 +2483,12 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params } while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES && - optimal_dcfclk_for_uclk[j] <= bw_params->clk_table.entries[1].dcfclk_mhz) { + optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; } + dcn3_0_soc.num_states = num_states; for (i = 0; i < dcn3_0_soc.num_states; i++) { dcn3_0_soc.clock_limits[i].state = i; dcn3_0_soc.clock_limits[i].dcfclk_mhz = dcfclk_mhz[i]; @@ -2475,9 +2496,9 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params dcn3_0_soc.clock_limits[i].dram_speed_mts = dram_speed_mts[i]; /* Fill all states with max values of all other clocks */ - dcn3_0_soc.clock_limits[i].dispclk_mhz = bw_params->clk_table.entries[1].dispclk_mhz; - dcn3_0_soc.clock_limits[i].dppclk_mhz = bw_params->clk_table.entries[1].dppclk_mhz; - dcn3_0_soc.clock_limits[i].phyclk_mhz = bw_params->clk_table.entries[1].phyclk_mhz; + dcn3_0_soc.clock_limits[i].dispclk_mhz = max_dispclk_mhz; + dcn3_0_soc.clock_limits[i].dppclk_mhz = max_dppclk_mhz; + dcn3_0_soc.clock_limits[i].phyclk_mhz = max_phyclk_mhz; dcn3_0_soc.clock_limits[i].dtbclk_mhz = dcn3_0_soc.clock_limits[0].dtbclk_mhz; /* These clocks cannot come from bw_params, always fill from dcn3_0_soc[1] */ /* FCLK, PHYCLK_D18, SOCCLK, DSCCLK */ @@ -2490,11 +2511,6 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params if (dc->current_state) dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); } - - /* re-init DML with updated bb */ - dml_init_instance(&dc->dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); - if (dc->current_state) - dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); } static const struct resource_funcs dcn30_res_pool_funcs = { @@ -2617,6 +2633,26 @@ static bool dcn30_resource_construct( dc->caps.color.mpc.ogam_rom_caps.hlg = 0; dc->caps.color.mpc.ocsc = 1; + /* read VBIOS LTTPR caps */ + { + if (ctx->dc_bios->funcs->get_lttpr_caps) { + enum bp_result bp_query_result; + uint8_t is_vbios_lttpr_enable = 0; + + bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable); + dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable; + } + + if (ctx->dc_bios->funcs->get_lttpr_interop) { + enum bp_result bp_query_result; + uint8_t is_vbios_interop_enabled = 0; + + bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios, + &is_vbios_interop_enabled); + dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled; + } + } + if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV) dc->debug = debug_defaults_drv; else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c index 9776d1737818..912285fdce18 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c @@ -1622,106 +1622,12 @@ static void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *b dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30); } -static void calculate_wm_set_for_vlevel( - int vlevel, - struct wm_range_table_entry *table_entry, - struct dcn_watermarks *wm_set, - struct display_mode_lib *dml, - display_e2e_pipe_params_st *pipes, - int pipe_cnt) -{ - double dram_clock_change_latency_cached = dml->soc.dram_clock_change_latency_us; - - ASSERT(vlevel < dml->soc.num_states); - /* only pipe 0 is read for voltage and dcf/soc clocks */ - pipes[0].clks_cfg.voltage = vlevel; - pipes[0].clks_cfg.dcfclk_mhz = dml->soc.clock_limits[vlevel].dcfclk_mhz; - pipes[0].clks_cfg.socclk_mhz = dml->soc.clock_limits[vlevel].socclk_mhz; - - dml->soc.dram_clock_change_latency_us = table_entry->pstate_latency_us; - dml->soc.sr_exit_time_us = table_entry->sr_exit_time_us; - dml->soc.sr_enter_plus_exit_time_us = table_entry->sr_enter_plus_exit_time_us; - - wm_set->urgent_ns = get_wm_urgent(dml, pipes, pipe_cnt) * 1000; - wm_set->cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000; - wm_set->cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000; - wm_set->cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000; - wm_set->pte_meta_urgent_ns = get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000; - wm_set->frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000; - wm_set->frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000; - wm_set->urgent_latency_ns = get_urgent_latency(dml, pipes, pipe_cnt) * 1000; - dml->soc.dram_clock_change_latency_us = dram_clock_change_latency_cached; - -} - -static void dcn301_calculate_wm_and_dlg( - struct dc *dc, struct dc_state *context, - display_e2e_pipe_params_st *pipes, - int pipe_cnt, - int vlevel_req) -{ - int i, pipe_idx; - int vlevel, vlevel_max; - struct wm_range_table_entry *table_entry; - struct clk_bw_params *bw_params = dc->clk_mgr->bw_params; - - ASSERT(bw_params); - - vlevel_max = bw_params->clk_table.num_entries - 1; - - /* WM Set D */ - table_entry = &bw_params->wm_table.entries[WM_D]; - if (table_entry->wm_type == WM_TYPE_RETRAINING) - vlevel = 0; - else - vlevel = vlevel_max; - calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.d, - &context->bw_ctx.dml, pipes, pipe_cnt); - /* WM Set C */ - table_entry = &bw_params->wm_table.entries[WM_C]; - vlevel = min(max(vlevel_req, 2), vlevel_max); - calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.c, - &context->bw_ctx.dml, pipes, pipe_cnt); - /* WM Set B */ - table_entry = &bw_params->wm_table.entries[WM_B]; - vlevel = min(max(vlevel_req, 1), vlevel_max); - calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.b, - &context->bw_ctx.dml, pipes, pipe_cnt); - - /* WM Set A */ - table_entry = &bw_params->wm_table.entries[WM_A]; - vlevel = min(vlevel_req, vlevel_max); - calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.a, - &context->bw_ctx.dml, pipes, pipe_cnt); - - for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) { - if (!context->res_ctx.pipe_ctx[i].stream) - continue; - - pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt); - pipes[pipe_idx].clks_cfg.dppclk_mhz = get_dppclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx); - - if (dc->config.forced_clocks) { - pipes[pipe_idx].clks_cfg.dispclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dispclk_mhz; - pipes[pipe_idx].clks_cfg.dppclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dppclk_mhz; - } - if (dc->debug.min_disp_clk_khz > pipes[pipe_idx].clks_cfg.dispclk_mhz * 1000) - pipes[pipe_idx].clks_cfg.dispclk_mhz = dc->debug.min_disp_clk_khz / 1000.0; - if (dc->debug.min_dpp_clk_khz > pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000) - pipes[pipe_idx].clks_cfg.dppclk_mhz = dc->debug.min_dpp_clk_khz / 1000.0; - - pipe_idx++; - } - - dcn20_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel); -} - static struct resource_funcs dcn301_res_pool_funcs = { .destroy = dcn301_destroy_resource_pool, .link_enc_create = dcn301_link_encoder_create, .panel_cntl_create = dcn301_panel_cntl_create, .validate_bandwidth = dcn30_validate_bandwidth, - .calculate_wm_and_dlg = dcn301_calculate_wm_and_dlg, + .calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg, .update_soc_for_wm_a = dcn30_update_soc_for_wm_a, .populate_dml_pipes = dcn30_populate_dml_pipes_from_context, .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer, diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c index 833ab13fa834..dd38796ba30a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c @@ -146,8 +146,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_03_soc = { .min_dcfclk = 500.0, /* TODO: set this to actual min DCFCLK */ .num_states = 1, - .sr_exit_time_us = 26.5, - .sr_enter_plus_exit_time_us = 31, + .sr_exit_time_us = 35.5, + .sr_enter_plus_exit_time_us = 40, .urgent_latency_us = 4.0, .urgent_latency_pixel_data_only_us = 4.0, .urgent_latency_pixel_mixed_with_vm_data_us = 4.0, @@ -510,8 +510,12 @@ static struct stream_encoder *dcn303_stream_encoder_create(enum engine_id eng_id vpg = dcn303_vpg_create(ctx, vpg_inst); afmt = dcn303_afmt_create(ctx, afmt_inst); - if (!enc1 || !vpg || !afmt) + if (!enc1 || !vpg || !afmt) { + kfree(enc1); + kfree(vpg); + kfree(afmt); return NULL; + } dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id, vpg, afmt, &stream_enc_regs[eng_id], &se_shift, &se_mask); diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c index 83f7904630e6..3f2333ec67e2 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c @@ -407,6 +407,18 @@ void dcn31_update_info_frame(struct pipe_ctx *pipe_ctx) &pipe_ctx->stream_res.encoder_info_frame); } } +void dcn31_z10_save_init(struct dc *dc) +{ + union dmub_rb_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.dcn_restore.header.type = DMUB_CMD__IDLE_OPT; + cmd.dcn_restore.header.sub_type = DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT; + + dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); + dc_dmub_srv_wait_idle(dc->ctx->dmub_srv); +} void dcn31_z10_restore(struct dc *dc) { @@ -598,20 +610,3 @@ bool dcn31_is_abm_supported(struct dc *dc, } return false; } - -static void apply_riommu_invalidation_wa(struct dc *dc) -{ - struct dce_hwseq *hws = dc->hwseq; - - if (!hws->wa.early_riommu_invalidation) - return; - - REG_UPDATE(DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, 0); -} - -void dcn31_init_pipes(struct dc *dc, struct dc_state *context) -{ - dcn10_init_pipes(dc, context); - apply_riommu_invalidation_wa(dc); - -} diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.h index 40dfebe78fdd..140435e4f7ff 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.h @@ -44,6 +44,7 @@ void dcn31_enable_power_gating_plane( void dcn31_update_info_frame(struct pipe_ctx *pipe_ctx); void dcn31_z10_restore(struct dc *dc); +void dcn31_z10_save_init(struct dc *dc); void dcn31_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool power_on); int dcn31_init_sys_ctx(struct dce_hwseq *hws, struct dc *dc, struct dc_phy_addr_space_config *pa_config); diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c index aaf2dbd095fe..40011cd3c8ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c @@ -97,13 +97,14 @@ static const struct hw_sequencer_funcs dcn31_funcs = { .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, .z10_restore = dcn31_z10_restore, + .z10_save_init = dcn31_z10_save_init, .is_abm_supported = dcn31_is_abm_supported, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, .update_visual_confirm_color = dcn20_update_visual_confirm_color, }; static const struct hwseq_private_funcs dcn31_private_funcs = { - .init_pipes = dcn31_init_pipes, + .init_pipes = dcn10_init_pipes, .update_plane_addr = dcn20_update_plane_addr, .plane_atomic_disconnect = dcn10_plane_atomic_disconnect, .update_mpcc = dcn20_update_mpcc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c index 7db268da6976..3b3721386571 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c @@ -109,7 +109,7 @@ bool dcn31_is_panel_backlight_on(struct panel_cntl *panel_cntl) union dmub_rb_cmd cmd; if (!dcn31_query_backlight_info(panel_cntl, &cmd)) - return 0; + return false; return cmd.panel_cntl.data.is_backlight_on; } @@ -119,7 +119,7 @@ bool dcn31_is_panel_powered_on(struct panel_cntl *panel_cntl) union dmub_rb_cmd cmd; if (!dcn31_query_backlight_info(panel_cntl, &cmd)) - return 0; + return false; return cmd.panel_cntl.data.is_powered_on; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c index 38c010afade1..a7702d3c75cd 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c @@ -1302,7 +1302,6 @@ static struct dce_hwseq *dcn31_hwseq_create( hws->regs = &hwseq_reg; hws->shifts = &hwseq_shift; hws->masks = &hwseq_mask; - hws->wa.early_riommu_invalidation = true; } return hws; } @@ -1968,6 +1967,22 @@ static bool dcn31_resource_construct( dc->caps.color.mpc.ogam_rom_caps.hlg = 0; dc->caps.color.mpc.ocsc = 1; + /* read VBIOS LTTPR caps */ + { + if (ctx->dc_bios->funcs->get_lttpr_caps) { + enum bp_result bp_query_result; + uint8_t is_vbios_lttpr_enable = 0; + + bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable); + dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable; + } + + /* interop bit is implicit */ + { + dc->caps.vbios_lttpr_aware = true; + } + } + if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV) dc->debug = debug_defaults_drv; else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 45862167e6ce..56055df2e8d2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -58,6 +58,8 @@ CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_ccflags) ifdef CONFIG_DRM_AMD_DC_DCN CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_ccflags) +CFLAGS_$(AMDDALPATH)/dc/dml/dcn2x/dcn2x.o := $(dml_ccflags) +CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_ccflags) @@ -70,6 +72,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_mode_vba_31.o := $(dml_ccflags) $(fram CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_rq_dlg_calc_31.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_ccflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn2x/dcn2x.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_rcflags) @@ -91,6 +94,7 @@ DML = display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o \ ifdef CONFIG_DRM_AMD_DC_DCN DML += display_mode_vba.o dcn20/display_rq_dlg_calc_20.o dcn20/display_mode_vba_20.o +DML += dcn2x/dcn2x.o DML += dcn20/display_rq_dlg_calc_20v2.o dcn20/display_mode_vba_20v2.o DML += dcn21/display_rq_dlg_calc_21.o dcn21/display_mode_vba_21.o DML += dcn30/display_mode_vba_30.o dcn30/display_rq_dlg_calc_30.o diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index fbed5304692d..63bbdf8b8678 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -2641,7 +2641,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) { if (mode_lib->vba.PrefetchMode[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb] == 0) { if (mode_lib->vba.DRAMClockChangeWatermark > - dml_max(mode_lib->vba.StutterEnterPlusExitWatermark, mode_lib->vba.UrgentWatermark)) + dml_max(mode_lib->vba.StutterEnterPlusExitWatermark, mode_lib->vba.UrgentWatermark)) mode_lib->vba.MinTTUVBlank[k] += 25; } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c index 799bae229e67..2091dd8c252d 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c @@ -488,7 +488,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib, log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element // each 64b meta request for dcn is 8x8 meta elements and - // a meta element covers one 256b block of the the data surface. + // a meta element covers one 256b block of the data surface. log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256 log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element - log2_meta_req_height; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c index 6a6d5970d1d5..1a0c14e465fa 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c @@ -488,7 +488,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib, log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element // each 64b meta request for dcn is 8x8 meta elements and - // a meta element covers one 256b block of the the data surface. + // a meta element covers one 256b block of the data surface. log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256 log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element - log2_meta_req_height; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c index dc1c81a6e377..287e31052b30 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c @@ -482,7 +482,7 @@ static void get_meta_and_pte_attr( log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element // each 64b meta request for dcn is 8x8 meta elements and - // a meta element covers one 256b block of the the data surface. + // a meta element covers one 256b block of the data surface. log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256 log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element - log2_meta_req_height; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn2x/dcn2x.c b/drivers/gpu/drm/amd/display/dc/dml/dcn2x/dcn2x.c new file mode 100644 index 000000000000..c58522436291 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn2x/dcn2x.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "resource.h" + +#include "dcn2x.h" + +/** + * DOC: DCN2x FPU manipulation Overview + * + * The DCN architecture relies on FPU operations, which require special + * compilation flags and the use of kernel_fpu_begin/end functions; ideally, we + * want to avoid spreading FPU access across multiple files. With this idea in + * mind, this file aims to centralize all DCN20 and DCN2.1 (DCN2x) functions + * that require FPU access in a single place. Code in this file follows the + * following code pattern: + * + * 1. Functions that use FPU operations should be isolated in static functions. + * 2. The FPU functions should have the noinline attribute to ensure anything + * that deals with FP register is contained within this call. + * 3. All function that needs to be accessed outside this file requires a + * public interface that not uses any FPU reference. + * 4. Developers **must not** use DC_FP_START/END in this file, but they need + * to ensure that the caller invokes it before access any function available + * in this file. For this reason, public functions in this file must invoke + * dc_assert_fp_enabled(); + * + * Let's expand a little bit more the idea in the code pattern. To fully + * isolate FPU operations in a single place, we must avoid situations where + * compilers spill FP values to registers due to FP enable in a specific C + * file. Note that even if we isolate all FPU functions in a single file and + * call its interface from other files, the compiler might enable the use of + * FPU before we call DC_FP_START. Nevertheless, it is the programmer's + * responsibility to invoke DC_FP_START/END in the correct place. To highlight + * situations where developers forgot to use the FP protection before calling + * the DC FPU interface functions, we introduce a helper that checks if the + * function is invoked under FP protection. If not, it will trigger a kernel + * warning. + */ + +void dcn20_populate_dml_writeback_from_context(struct dc *dc, + struct resource_context *res_ctx, + display_e2e_pipe_params_st *pipes) +{ + int pipe_cnt, i; + + dc_assert_fp_enabled(); + + for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { + struct dc_writeback_info *wb_info = &res_ctx->pipe_ctx[i].stream->writeback_info[0]; + + if (!res_ctx->pipe_ctx[i].stream) + continue; + + /* Set writeback information */ + pipes[pipe_cnt].dout.wb_enable = (wb_info->wb_enabled == true) ? 1 : 0; + pipes[pipe_cnt].dout.num_active_wb++; + pipes[pipe_cnt].dout.wb.wb_src_height = wb_info->dwb_params.cnv_params.crop_height; + pipes[pipe_cnt].dout.wb.wb_src_width = wb_info->dwb_params.cnv_params.crop_width; + pipes[pipe_cnt].dout.wb.wb_dst_width = wb_info->dwb_params.dest_width; + pipes[pipe_cnt].dout.wb.wb_dst_height = wb_info->dwb_params.dest_height; + pipes[pipe_cnt].dout.wb.wb_htaps_luma = 1; + pipes[pipe_cnt].dout.wb.wb_vtaps_luma = 1; + pipes[pipe_cnt].dout.wb.wb_htaps_chroma = wb_info->dwb_params.scaler_taps.h_taps_c; + pipes[pipe_cnt].dout.wb.wb_vtaps_chroma = wb_info->dwb_params.scaler_taps.v_taps_c; + pipes[pipe_cnt].dout.wb.wb_hratio = 1.0; + pipes[pipe_cnt].dout.wb.wb_vratio = 1.0; + if (wb_info->dwb_params.out_format == dwb_scaler_mode_yuv420) { + if (wb_info->dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC) + pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_8; + else + pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_10; + } else { + pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_444_32; + } + + pipe_cnt++; + } +} diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.h b/drivers/gpu/drm/amd/display/dc/dml/dcn2x/dcn2x.h index ed422012c8c6..331547ba0713 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn2x/dcn2x.h @@ -1,5 +1,6 @@ +/* SPDX-License-Identifier: MIT */ /* - * Copyright 2014 Advanced Micro Devices, Inc. + * Copyright 2021 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -19,12 +20,15 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * + * Authors: AMD + * */ -#ifndef __DCE_VIRTUAL_H__ -#define __DCE_VIRTUAL_H__ - -extern const struct amdgpu_ip_block_version dce_virtual_ip_block; +#ifndef __DCN2X_H__ +#define __DCN2X_H__ -#endif +void dcn20_populate_dml_writeback_from_context(struct dc *dc, + struct resource_context *res_ctx, + display_e2e_pipe_params_st *pipes); +#endif /* __DCN2X_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index e1a961a62add..e3d9f1decdfc 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -3644,8 +3644,7 @@ static double TruncToValidBPP( void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib) { struct vba_vars_st *v = &mode_lib->vba; - int MinPrefetchMode = 0; - int MaxPrefetchMode = 2; + int MinPrefetchMode, MaxPrefetchMode; int i; unsigned int j, k, m; bool EnoughWritebackUnits = true; @@ -3657,6 +3656,10 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l /*MODE SUPPORT, VOLTAGE STATE AND SOC CONFIGURATION*/ + CalculateMinAndMaxPrefetchMode( + mode_lib->vba.AllowDRAMSelfRefreshOrDRAMClockChangeInVblank, + &MinPrefetchMode, &MaxPrefetchMode); + /*Scale Ratio, taps Support Check*/ v->ScaleRatioAndTapsSupport = true; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c index 04601a767a8f..0d934fae1c3a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c @@ -549,7 +549,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib, log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element // each 64b meta request for dcn is 8x8 meta elements and - // a meta element covers one 256b block of the the data surface. + // a meta element covers one 256b block of the data surface. log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256 log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element - log2_meta_req_height; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c index 3def093ef88e..c23905bc733a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c @@ -563,7 +563,7 @@ static void get_meta_and_pte_attr( log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element // each 64b meta request for dcn is 8x8 meta elements and - // a meta element covers one 256b block of the the data surface. + // a meta element covers one 256b block of the data surface. log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256 log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element - log2_meta_req_height; meta_req_width = 1 << log2_meta_req_width; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index 73f5be26abc4..0fad15020c74 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -244,6 +244,8 @@ static void fetch_socbb_params(struct display_mode_lib *mode_lib) mode_lib->vba.DRAMClockChangeSupportsVActive = !soc->disable_dram_clock_change_vactive_support || mode_lib->vba.DummyPStateCheck; mode_lib->vba.AllowDramClockChangeOneDisplayVactive = soc->allow_dram_clock_one_display_vactive; + mode_lib->vba.AllowDRAMSelfRefreshOrDRAMClockChangeInVblank = + soc->allow_dram_self_refresh_or_dram_clock_change_in_vblank; mode_lib->vba.Downspreading = soc->downspread_percent; mode_lib->vba.DRAMChannelWidth = soc->dram_channel_width_bytes; // new! @@ -733,8 +735,6 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) mode_lib->vba.OverrideHostVMPageTableLevels; } - mode_lib->vba.AllowDRAMSelfRefreshOrDRAMClockChangeInVblank = dm_try_to_allow_self_refresh_and_mclk_switch; - if (mode_lib->vba.OverrideGPUVMPageTableLevels) mode_lib->vba.GPUVMMaxPageTableLevels = mode_lib->vba.OverrideGPUVMPageTableLevels; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c index 414da64f5734..8f2b1684c231 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c @@ -456,7 +456,7 @@ static void dml1_rq_dlg_get_row_heights( log2_meta_req_bytes = 6; /* meta request is 64b and is 8x8byte meta element */ /* each 64b meta request for dcn is 8x8 meta elements and - * a meta element covers one 256b block of the the data surface. + * a meta element covers one 256b block of the data surface. */ log2_meta_req_height = log2_blk256_height + 3; /* meta req is 8x8 */ log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element @@ -718,7 +718,7 @@ static void get_surf_rq_param( log2_meta_req_bytes = 6; /* meta request is 64b and is 8x8byte meta element */ /* each 64b meta request for dcn is 8x8 meta elements and - * a meta element covers one 256b block of the the data surface. + * a meta element covers one 256b block of the data surface. */ log2_meta_req_height = log2_blk256_height + 3; /* meta req is 8x8 byte, each byte represent 1 blk256 */ log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 5ab008e62b82..ad5f2adcc40d 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -237,6 +237,7 @@ struct hw_sequencer_funcs { int width, int height, int offset); void (*z10_restore)(struct dc *dc); + void (*z10_save_init)(struct dc *dc); void (*update_visual_confirm_color)(struct dc *dc, struct pipe_ctx *pipe_ctx, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h index 082549f75978..f7f7e4fff0c2 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h @@ -41,7 +41,6 @@ struct dce_hwseq_wa { bool DEGVIDCN10_254; bool DEGVIDCN21; bool disallow_self_refresh_during_multi_plane_transition; - bool early_riommu_invalidation; }; struct hwseq_wa_state { diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h index 126c2f3a4dd3..f50cae252de4 100644 --- a/drivers/gpu/drm/amd/display/dc/os_types.h +++ b/drivers/gpu/drm/amd/display/dc/os_types.h @@ -51,38 +51,9 @@ #define dm_error(fmt, ...) DRM_ERROR(fmt, ##__VA_ARGS__) #if defined(CONFIG_DRM_AMD_DC_DCN) -#if defined(CONFIG_X86) -#include <asm/fpu/api.h> -#define DC_FP_START() kernel_fpu_begin() -#define DC_FP_END() kernel_fpu_end() -#elif defined(CONFIG_PPC64) -#include <asm/switch_to.h> -#include <asm/cputable.h> -#define DC_FP_START() { \ - if (cpu_has_feature(CPU_FTR_VSX_COMP)) { \ - preempt_disable(); \ - enable_kernel_vsx(); \ - } else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) { \ - preempt_disable(); \ - enable_kernel_altivec(); \ - } else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) { \ - preempt_disable(); \ - enable_kernel_fp(); \ - } \ -} -#define DC_FP_END() { \ - if (cpu_has_feature(CPU_FTR_VSX_COMP)) { \ - disable_kernel_vsx(); \ - preempt_enable(); \ - } else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) { \ - disable_kernel_altivec(); \ - preempt_enable(); \ - } else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) { \ - disable_kernel_fp(); \ - preempt_enable(); \ - } \ -} -#endif +#include "amdgpu_dm/dc_fpu.h" +#define DC_FP_START() dc_fpu_begin(__func__, __LINE__) +#define DC_FP_END() dc_fpu_end(__func__, __LINE__) #endif /* diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 02921ad22310..7efe9ba8706e 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -39,7 +39,6 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/delay.h> -#include <stdarg.h> #include "atomfirmware.h" @@ -47,10 +46,10 @@ /* Firmware versioning. */ #ifdef DMUB_EXPOSE_VERSION -#define DMUB_FW_VERSION_GIT_HASH 0xe599e0896 +#define DMUB_FW_VERSION_GIT_HASH 0x7383caadc #define DMUB_FW_VERSION_MAJOR 0 #define DMUB_FW_VERSION_MINOR 0 -#define DMUB_FW_VERSION_REVISION 76 +#define DMUB_FW_VERSION_REVISION 79 #define DMUB_FW_VERSION_TEST 0 #define DMUB_FW_VERSION_VBIOS 0 #define DMUB_FW_VERSION_HOTFIX 0 @@ -322,6 +321,10 @@ union dmub_fw_boot_status { uint32_t mailbox_rdy : 1; /**< 1 if mailbox ready */ uint32_t optimized_init_done : 1; /**< 1 if optimized init done */ uint32_t restore_required : 1; /**< 1 if driver should call restore */ + uint32_t defer_load : 1; /**< 1 if VBIOS data is deferred programmed */ + uint32_t reserved : 1; + uint32_t detection_required: 1; /**< if detection need to be triggered by driver */ + } bits; /**< status bits */ uint32_t all; /**< 32-bit access to status bits */ }; @@ -335,6 +338,7 @@ enum dmub_fw_boot_status_bit { DMUB_FW_BOOT_STATUS_BIT_OPTIMIZED_INIT_DONE = (1 << 2), /**< 1 if init done */ DMUB_FW_BOOT_STATUS_BIT_RESTORE_REQUIRED = (1 << 3), /**< 1 if driver should call restore */ DMUB_FW_BOOT_STATUS_BIT_DEFERRED_LOADED = (1 << 4), /**< 1 if VBIOS data is deferred programmed */ + DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED = (1 << 6), /**< 1 if detection need to be triggered by driver*/ }; /* Register bit definition for SCRATCH5 */ @@ -489,6 +493,11 @@ enum dmub_gpint_command { * RETURN: PSR residency in milli-percent. */ DMUB_GPINT__PSR_RESIDENCY = 9, + + /** + * DESC: Notifies DMCUB detection is done so detection required can be cleared. + */ + DMUB_GPINT__NOTIFY_DETECTION_DONE = 12, }; /** @@ -860,6 +869,11 @@ enum dmub_cmd_idle_opt_type { * DCN hardware restore. */ DMUB_CMD__IDLE_OPT_DCN_RESTORE = 0, + + /** + * DCN hardware save. + */ + DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT = 1 }; /** @@ -1438,7 +1452,7 @@ struct dmub_cmd_psr_set_level_data { * 16-bit value dicated by driver that will enable/disable different functionality. */ uint16_t psr_level; - /** + /** * PSR control version. */ uint8_t cmd_version; diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c index 61446170056e..fc667cb17eb0 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c @@ -83,7 +83,7 @@ static inline void dmub_dcn31_translate_addr(const union dmub_addr *addr_in, void dmub_dcn31_reset(struct dmub_srv *dmub) { union dmub_gpint_data_register cmd; - const uint32_t timeout = 30; + const uint32_t timeout = 100; uint32_t in_reset, scratch, i; REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &in_reset); @@ -98,26 +98,22 @@ void dmub_dcn31_reset(struct dmub_srv *dmub) /** * Timeout covers both the ACK and the wait * for remaining work to finish. - * - * This is mostly bound by the PHY disable sequence. - * Each register check will be greater than 1us, so - * don't bother using udelay. */ for (i = 0; i < timeout; ++i) { if (dmub->hw_funcs.is_gpint_acked(dmub, cmd)) break; + + udelay(1); } for (i = 0; i < timeout; ++i) { scratch = dmub->hw_funcs.get_gpint_response(dmub); if (scratch == DMUB_GPINT__STOP_FW_RESPONSE) break; - } - /* Clear the GPINT command manually so we don't reset again. */ - cmd.all = 0; - dmub->hw_funcs.set_gpint(dmub, cmd); + udelay(1); + } /* Force reset in case we timed out, DMCUB is likely hung. */ } @@ -130,6 +126,10 @@ void dmub_dcn31_reset(struct dmub_srv *dmub) REG_WRITE(DMCUB_OUTBOX1_RPTR, 0); REG_WRITE(DMCUB_OUTBOX1_WPTR, 0); REG_WRITE(DMCUB_SCRATCH0, 0); + + /* Clear the GPINT command manually so we don't send anything during boot. */ + cmd.all = 0; + dmub->hw_funcs.set_gpint(dmub, cmd); } void dmub_dcn31_reset_release(struct dmub_srv *dmub) @@ -270,11 +270,13 @@ void dmub_dcn31_set_outbox1_rptr(struct dmub_srv *dmub, uint32_t rptr_offset) bool dmub_dcn31_is_hw_init(struct dmub_srv *dmub) { - uint32_t is_hw_init; + union dmub_fw_boot_status status; + uint32_t is_enable; - REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_hw_init); + status.all = REG_READ(DMCUB_SCRATCH0); + REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_enable); - return is_hw_init != 0; + return is_enable != 0 && status.bits.dal_fw; } bool dmub_dcn31_is_supported(struct dmub_srv *dmub) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c index 06d60f031a06..3e81850a7ffe 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c @@ -145,6 +145,7 @@ static enum mod_hdcp_status transition(struct mod_hdcp *hdcp, } else { callback_in_ms(0, output); set_state_id(hdcp, output, HDCP_CP_NOT_DESIRED); + set_auth_complete(hdcp, output); } else if (is_hdmi_dvi_sl_hdcp(hdcp)) if (is_cp_desired_hdcp2(hdcp)) { @@ -156,10 +157,12 @@ static enum mod_hdcp_status transition(struct mod_hdcp *hdcp, } else { callback_in_ms(0, output); set_state_id(hdcp, output, HDCP_CP_NOT_DESIRED); + set_auth_complete(hdcp, output); } else { callback_in_ms(0, output); set_state_id(hdcp, output, HDCP_CP_NOT_DESIRED); + set_auth_complete(hdcp, output); } } else if (is_in_cp_not_desired_state(hdcp)) { increment_stay_counter(hdcp); @@ -520,7 +523,7 @@ enum mod_hdcp_status mod_hdcp_process_event(struct mod_hdcp *hdcp, /* reset authentication if needed */ if (trans_status == MOD_HDCP_STATUS_RESET_NEEDED) { - HDCP_FULL_DDC_TRACE(hdcp); + mod_hdcp_log_ddc_trace(hdcp); reset_status = reset_authentication(hdcp, output); if (reset_status != MOD_HDCP_STATUS_SUCCESS) push_error_status(hdcp, reset_status); diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h index 7123f0915706..399fbca8947b 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h @@ -324,6 +324,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_dp_transition(struct mod_hdcp *hdcp, /* log functions */ void mod_hdcp_dump_binary_message(uint8_t *msg, uint32_t msg_size, uint8_t *buf, uint32_t buf_size); +void mod_hdcp_log_ddc_trace(struct mod_hdcp *hdcp); /* TODO: add adjustment log */ /* psp functions */ @@ -494,6 +495,13 @@ static inline void set_watchdog_in_ms(struct mod_hdcp *hdcp, uint16_t time, output->watchdog_timer_delay = time; } +static inline void set_auth_complete(struct mod_hdcp *hdcp, + struct mod_hdcp_output *output) +{ + output->auth_complete = 1; + mod_hdcp_log_ddc_trace(hdcp); +} + /* connection topology helpers */ static inline uint8_t is_display_active(struct mod_hdcp_display *display) { diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c index 3dda8c1d83fc..7f011196ce98 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c @@ -89,7 +89,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_transition(struct mod_hdcp *hdcp, } else { callback_in_ms(0, output); set_state_id(hdcp, output, H1_A45_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); } break; case H1_A45_AUTHENTICATED: @@ -137,7 +137,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_transition(struct mod_hdcp *hdcp, } callback_in_ms(0, output); set_state_id(hdcp, output, H1_A45_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); break; default: status = MOD_HDCP_STATUS_INVALID_STATE; @@ -239,7 +239,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_dp_transition(struct mod_hdcp *hdcp, set_state_id(hdcp, output, D1_A6_WAIT_FOR_READY); } else { set_state_id(hdcp, output, D1_A4_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); } break; case D1_A4_AUTHENTICATED: @@ -311,7 +311,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_dp_transition(struct mod_hdcp *hdcp, break; } set_state_id(hdcp, output, D1_A4_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); break; default: fail_and_restart_in_ms(0, &status, output); diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_transition.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_transition.c index 70cb230d8f56..1f4095b26409 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_transition.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_transition.c @@ -242,7 +242,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_transition(struct mod_hdcp *hdcp, } callback_in_ms(0, output); set_state_id(hdcp, output, H2_A5_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); break; case H2_A5_AUTHENTICATED: if (input->rxstatus_read == FAIL || @@ -559,7 +559,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_dp_transition(struct mod_hdcp *hdcp, break; } set_state_id(hdcp, output, D2_A5_AUTHENTICATED); - HDCP_FULL_DDC_TRACE(hdcp); + set_auth_complete(hdcp, output); break; case D2_A5_AUTHENTICATED: if (input->rxstatus_read == FAIL || diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c index 1a0f7c3dc964..6b3b5f610907 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c @@ -51,6 +51,80 @@ void mod_hdcp_dump_binary_message(uint8_t *msg, uint32_t msg_size, } } +void mod_hdcp_log_ddc_trace(struct mod_hdcp *hdcp) +{ + if (is_hdcp1(hdcp)) { + HDCP_DDC_READ_TRACE(hdcp, "BKSV", hdcp->auth.msg.hdcp1.bksv, + sizeof(hdcp->auth.msg.hdcp1.bksv)); + HDCP_DDC_READ_TRACE(hdcp, "BCAPS", &hdcp->auth.msg.hdcp1.bcaps, + sizeof(hdcp->auth.msg.hdcp1.bcaps)); + HDCP_DDC_READ_TRACE(hdcp, "BSTATUS", + (uint8_t *)&hdcp->auth.msg.hdcp1.bstatus, + sizeof(hdcp->auth.msg.hdcp1.bstatus)); + HDCP_DDC_WRITE_TRACE(hdcp, "AN", hdcp->auth.msg.hdcp1.an, + sizeof(hdcp->auth.msg.hdcp1.an)); + HDCP_DDC_WRITE_TRACE(hdcp, "AKSV", hdcp->auth.msg.hdcp1.aksv, + sizeof(hdcp->auth.msg.hdcp1.aksv)); + HDCP_DDC_WRITE_TRACE(hdcp, "AINFO", &hdcp->auth.msg.hdcp1.ainfo, + sizeof(hdcp->auth.msg.hdcp1.ainfo)); + HDCP_DDC_READ_TRACE(hdcp, "RI' / R0'", + (uint8_t *)&hdcp->auth.msg.hdcp1.r0p, + sizeof(hdcp->auth.msg.hdcp1.r0p)); + HDCP_DDC_READ_TRACE(hdcp, "BINFO", + (uint8_t *)&hdcp->auth.msg.hdcp1.binfo_dp, + sizeof(hdcp->auth.msg.hdcp1.binfo_dp)); + HDCP_DDC_READ_TRACE(hdcp, "KSVLIST", hdcp->auth.msg.hdcp1.ksvlist, + hdcp->auth.msg.hdcp1.ksvlist_size); + HDCP_DDC_READ_TRACE(hdcp, "V'", hdcp->auth.msg.hdcp1.vp, + sizeof(hdcp->auth.msg.hdcp1.vp)); + } else if (is_hdcp2(hdcp)) { + HDCP_DDC_READ_TRACE(hdcp, "HDCP2Version", + &hdcp->auth.msg.hdcp2.hdcp2version_hdmi, + sizeof(hdcp->auth.msg.hdcp2.hdcp2version_hdmi)); + HDCP_DDC_READ_TRACE(hdcp, "Rx Caps", hdcp->auth.msg.hdcp2.rxcaps_dp, + sizeof(hdcp->auth.msg.hdcp2.rxcaps_dp)); + HDCP_DDC_WRITE_TRACE(hdcp, "AKE Init", hdcp->auth.msg.hdcp2.ake_init, + sizeof(hdcp->auth.msg.hdcp2.ake_init)); + HDCP_DDC_READ_TRACE(hdcp, "AKE Cert", hdcp->auth.msg.hdcp2.ake_cert, + sizeof(hdcp->auth.msg.hdcp2.ake_cert)); + HDCP_DDC_WRITE_TRACE(hdcp, "Stored KM", + hdcp->auth.msg.hdcp2.ake_stored_km, + sizeof(hdcp->auth.msg.hdcp2.ake_stored_km)); + HDCP_DDC_WRITE_TRACE(hdcp, "No Stored KM", + hdcp->auth.msg.hdcp2.ake_no_stored_km, + sizeof(hdcp->auth.msg.hdcp2.ake_no_stored_km)); + HDCP_DDC_READ_TRACE(hdcp, "H'", hdcp->auth.msg.hdcp2.ake_h_prime, + sizeof(hdcp->auth.msg.hdcp2.ake_h_prime)); + HDCP_DDC_READ_TRACE(hdcp, "Pairing Info", + hdcp->auth.msg.hdcp2.ake_pairing_info, + sizeof(hdcp->auth.msg.hdcp2.ake_pairing_info)); + HDCP_DDC_WRITE_TRACE(hdcp, "LC Init", hdcp->auth.msg.hdcp2.lc_init, + sizeof(hdcp->auth.msg.hdcp2.lc_init)); + HDCP_DDC_READ_TRACE(hdcp, "L'", hdcp->auth.msg.hdcp2.lc_l_prime, + sizeof(hdcp->auth.msg.hdcp2.lc_l_prime)); + HDCP_DDC_WRITE_TRACE(hdcp, "Exchange KS", hdcp->auth.msg.hdcp2.ske_eks, + sizeof(hdcp->auth.msg.hdcp2.ske_eks)); + HDCP_DDC_READ_TRACE(hdcp, "Rx Status", + (uint8_t *)&hdcp->auth.msg.hdcp2.rxstatus, + sizeof(hdcp->auth.msg.hdcp2.rxstatus)); + HDCP_DDC_READ_TRACE(hdcp, "Rx Id List", + hdcp->auth.msg.hdcp2.rx_id_list, + hdcp->auth.msg.hdcp2.rx_id_list_size); + HDCP_DDC_WRITE_TRACE(hdcp, "Rx Id List Ack", + hdcp->auth.msg.hdcp2.repeater_auth_ack, + sizeof(hdcp->auth.msg.hdcp2.repeater_auth_ack)); + HDCP_DDC_WRITE_TRACE(hdcp, "Content Stream Management", + hdcp->auth.msg.hdcp2.repeater_auth_stream_manage, + hdcp->auth.msg.hdcp2.stream_manage_size); + HDCP_DDC_READ_TRACE(hdcp, "Stream Ready", + hdcp->auth.msg.hdcp2.repeater_auth_stream_ready, + sizeof(hdcp->auth.msg.hdcp2.repeater_auth_stream_ready)); + HDCP_DDC_WRITE_TRACE(hdcp, "Content Stream Type", + hdcp->auth.msg.hdcp2.content_stream_type_dp, + sizeof(hdcp->auth.msg.hdcp2.content_stream_type_dp)); + } +} + char *mod_hdcp_status_to_str(int32_t status) { switch (status) { diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h index 47f8ee2832ff..eb6f9b9c504a 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h @@ -106,78 +106,6 @@ hdcp->config.index, msg_name,\ hdcp->buf); \ } while (0) -#define HDCP_FULL_DDC_TRACE(hdcp) do { \ - if (is_hdcp1(hdcp)) { \ - HDCP_DDC_READ_TRACE(hdcp, "BKSV", hdcp->auth.msg.hdcp1.bksv, \ - sizeof(hdcp->auth.msg.hdcp1.bksv)); \ - HDCP_DDC_READ_TRACE(hdcp, "BCAPS", &hdcp->auth.msg.hdcp1.bcaps, \ - sizeof(hdcp->auth.msg.hdcp1.bcaps)); \ - HDCP_DDC_READ_TRACE(hdcp, "BSTATUS", \ - (uint8_t *)&hdcp->auth.msg.hdcp1.bstatus, \ - sizeof(hdcp->auth.msg.hdcp1.bstatus)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "AN", hdcp->auth.msg.hdcp1.an, \ - sizeof(hdcp->auth.msg.hdcp1.an)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "AKSV", hdcp->auth.msg.hdcp1.aksv, \ - sizeof(hdcp->auth.msg.hdcp1.aksv)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "AINFO", &hdcp->auth.msg.hdcp1.ainfo, \ - sizeof(hdcp->auth.msg.hdcp1.ainfo)); \ - HDCP_DDC_READ_TRACE(hdcp, "RI' / R0'", \ - (uint8_t *)&hdcp->auth.msg.hdcp1.r0p, \ - sizeof(hdcp->auth.msg.hdcp1.r0p)); \ - HDCP_DDC_READ_TRACE(hdcp, "BINFO", \ - (uint8_t *)&hdcp->auth.msg.hdcp1.binfo_dp, \ - sizeof(hdcp->auth.msg.hdcp1.binfo_dp)); \ - HDCP_DDC_READ_TRACE(hdcp, "KSVLIST", hdcp->auth.msg.hdcp1.ksvlist, \ - hdcp->auth.msg.hdcp1.ksvlist_size); \ - HDCP_DDC_READ_TRACE(hdcp, "V'", hdcp->auth.msg.hdcp1.vp, \ - sizeof(hdcp->auth.msg.hdcp1.vp)); \ - } else { \ - HDCP_DDC_READ_TRACE(hdcp, "HDCP2Version", \ - &hdcp->auth.msg.hdcp2.hdcp2version_hdmi, \ - sizeof(hdcp->auth.msg.hdcp2.hdcp2version_hdmi)); \ - HDCP_DDC_READ_TRACE(hdcp, "Rx Caps", hdcp->auth.msg.hdcp2.rxcaps_dp, \ - sizeof(hdcp->auth.msg.hdcp2.rxcaps_dp)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "AKE Init", hdcp->auth.msg.hdcp2.ake_init, \ - sizeof(hdcp->auth.msg.hdcp2.ake_init)); \ - HDCP_DDC_READ_TRACE(hdcp, "AKE Cert", hdcp->auth.msg.hdcp2.ake_cert, \ - sizeof(hdcp->auth.msg.hdcp2.ake_cert)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "Stored KM", \ - hdcp->auth.msg.hdcp2.ake_stored_km, \ - sizeof(hdcp->auth.msg.hdcp2.ake_stored_km)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "No Stored KM", \ - hdcp->auth.msg.hdcp2.ake_no_stored_km, \ - sizeof(hdcp->auth.msg.hdcp2.ake_no_stored_km)); \ - HDCP_DDC_READ_TRACE(hdcp, "H'", hdcp->auth.msg.hdcp2.ake_h_prime, \ - sizeof(hdcp->auth.msg.hdcp2.ake_h_prime)); \ - HDCP_DDC_READ_TRACE(hdcp, "Pairing Info", \ - hdcp->auth.msg.hdcp2.ake_pairing_info, \ - sizeof(hdcp->auth.msg.hdcp2.ake_pairing_info)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "LC Init", hdcp->auth.msg.hdcp2.lc_init, \ - sizeof(hdcp->auth.msg.hdcp2.lc_init)); \ - HDCP_DDC_READ_TRACE(hdcp, "L'", hdcp->auth.msg.hdcp2.lc_l_prime, \ - sizeof(hdcp->auth.msg.hdcp2.lc_l_prime)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "Exchange KS", hdcp->auth.msg.hdcp2.ske_eks, \ - sizeof(hdcp->auth.msg.hdcp2.ske_eks)); \ - HDCP_DDC_READ_TRACE(hdcp, "Rx Status", \ - (uint8_t *)&hdcp->auth.msg.hdcp2.rxstatus, \ - sizeof(hdcp->auth.msg.hdcp2.rxstatus)); \ - HDCP_DDC_READ_TRACE(hdcp, "Rx Id List", \ - hdcp->auth.msg.hdcp2.rx_id_list, \ - hdcp->auth.msg.hdcp2.rx_id_list_size); \ - HDCP_DDC_WRITE_TRACE(hdcp, "Rx Id List Ack", \ - hdcp->auth.msg.hdcp2.repeater_auth_ack, \ - sizeof(hdcp->auth.msg.hdcp2.repeater_auth_ack)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "Content Stream Management", \ - hdcp->auth.msg.hdcp2.repeater_auth_stream_manage, \ - hdcp->auth.msg.hdcp2.stream_manage_size); \ - HDCP_DDC_READ_TRACE(hdcp, "Stream Ready", \ - hdcp->auth.msg.hdcp2.repeater_auth_stream_ready, \ - sizeof(hdcp->auth.msg.hdcp2.repeater_auth_stream_ready)); \ - HDCP_DDC_WRITE_TRACE(hdcp, "Content Stream Type", \ - hdcp->auth.msg.hdcp2.content_stream_type_dp, \ - sizeof(hdcp->auth.msg.hdcp2.content_stream_type_dp)); \ - } \ -} while (0) #define HDCP_TOP_ADD_DISPLAY_TRACE(hdcp, i) \ HDCP_LOG_TOP(hdcp, "[Link %d]\tadd display %d", \ hdcp->config.index, i) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index ade86a042398..e9bd84ec027d 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -54,7 +54,7 @@ static enum mod_hdcp_status remove_display_from_topology_v2( get_active_display_at_index(hdcp, index); enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; if (!display || !is_display_active(display)) return MOD_HDCP_STATUS_DISPLAY_NOT_FOUND; @@ -90,7 +90,7 @@ static enum mod_hdcp_status remove_display_from_topology_v3( get_active_display_at_index(hdcp, index); enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; if (!display || !is_display_active(display)) return MOD_HDCP_STATUS_DISPLAY_NOT_FOUND; @@ -128,13 +128,13 @@ static enum mod_hdcp_status add_display_to_topology_v2( struct mod_hdcp_link *link = &hdcp->connection.link; enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - if (!psp->dtm_context.dtm_initialized) { + if (!psp->dtm_context.context.initialized) { DRM_INFO("Failed to add display topology, DTM TA is not initialized."); display->state = MOD_HDCP_DISPLAY_INACTIVE; return MOD_HDCP_STATUS_FAILURE; } - dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; mutex_lock(&psp->dtm_context.mutex); memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); @@ -175,13 +175,13 @@ static enum mod_hdcp_status add_display_to_topology_v3( struct mod_hdcp_link *link = &hdcp->connection.link; enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - if (!psp->dtm_context.dtm_initialized) { + if (!psp->dtm_context.context.initialized) { DRM_INFO("Failed to add display topology, DTM TA is not initialized."); display->state = MOD_HDCP_DISPLAY_INACTIVE; return MOD_HDCP_STATUS_FAILURE; } - dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.context.mem_context.shared_buf; mutex_lock(&psp->dtm_context.mutex); memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); @@ -253,12 +253,12 @@ enum mod_hdcp_status mod_hdcp_hdcp1_create_session(struct mod_hdcp *hdcp) struct ta_hdcp_shared_memory *hdcp_cmd; enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - if (!psp->hdcp_context.hdcp_initialized) { + if (!psp->hdcp_context.context.initialized) { DRM_ERROR("Failed to create hdcp session. HDCP TA is not initialized."); return MOD_HDCP_STATUS_FAILURE; } - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; mutex_lock(&psp->hdcp_context.mutex); memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); @@ -293,7 +293,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_destroy_session(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp1_destroy_session.session_handle = hdcp->auth.id; @@ -325,7 +325,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_validate_rx(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp1_first_part_authentication.session_handle = hdcp->auth.id; @@ -367,7 +367,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_enable_encryption(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp1_enable_encryption.session_handle = hdcp->auth.id; @@ -393,7 +393,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_validate_ksvlist_vp(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp1_second_part_authentication.session_handle = hdcp->auth.id; @@ -436,7 +436,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_enable_dp_stream_encryption(struct mod_hdcp enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) { @@ -471,7 +471,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_link_maintenance(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); @@ -498,7 +498,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_create_session(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; - if (!psp->hdcp_context.hdcp_initialized) { + if (!psp->hdcp_context.context.initialized) { DRM_ERROR("Failed to create hdcp session, HDCP TA is not initialized"); return MOD_HDCP_STATUS_FAILURE; } @@ -508,7 +508,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_create_session(struct mod_hdcp *hdcp) mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp2_create_session_v2.display_handle = display->index; @@ -545,7 +545,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_destroy_session(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp2_destroy_session.session_handle = hdcp->auth.id; @@ -579,7 +579,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_prepare_ake_init(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -611,7 +611,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_ake_cert(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -671,7 +671,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_h_prime(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -717,7 +717,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_prepare_lc_init(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -750,7 +750,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_l_prime(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -785,7 +785,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_prepare_eks(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -833,7 +833,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_enable_encryption(struct mod_hdcp *hdcp) mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); hdcp_cmd->in_msg.hdcp2_set_encryption.session_handle = hdcp->auth.id; @@ -862,7 +862,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_rx_id_list(struct mod_hdcp *hdcp) mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -914,7 +914,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_enable_dp_stream_encryption(struct mod_hdcp enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -958,7 +958,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_prepare_stream_management(struct mod_hdcp *h enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; @@ -994,7 +994,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_stream_ready(struct mod_hdcp *hdcp) enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS; mutex_lock(&psp->hdcp_context.mutex); - hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; + hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.context.mem_context.shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); msg_in = &hdcp_cmd->in_msg.hdcp2_prepare_process_authentication_message_v2; diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h index a6eb86de8d5c..f37101f5a777 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h @@ -225,6 +225,7 @@ struct mod_hdcp_output { uint8_t watchdog_timer_stop; uint16_t callback_delay; uint16_t watchdog_timer_delay; + uint8_t auth_complete; }; /* used to represent per display info */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_offset.h index a9ad00e017a5..1a8a6a350789 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_offset.h @@ -508,6 +508,10 @@ #define regCP_ME2_PIPE2_INT_STATUS_BASE_IDX 0 #define regCP_ME2_PIPE3_INT_STATUS 0x1094 #define regCP_ME2_PIPE3_INT_STATUS_BASE_IDX 0 +#define regCP_ME1_INT_STAT_DEBUG 0x1095 +#define regCP_ME1_INT_STAT_DEBUG_BASE_IDX 0 +#define regCP_ME2_INT_STAT_DEBUG 0x1096 +#define regCP_ME2_INT_STAT_DEBUG_BASE_IDX 0 #define regCC_GC_EDC_CONFIG 0x1098 #define regCC_GC_EDC_CONFIG_BASE_IDX 0 #define regCP_ME1_PIPE_PRIORITY_CNTS 0x1099 diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_sh_mask.h index bc4d2997cb51..049221262e5c 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_2_sh_mask.h @@ -2989,6 +2989,60 @@ #define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000L #define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000L #define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000L +//CP_ME1_INT_STAT_DEBUG +#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT 0xc +#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd +#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe +#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT 0xf +#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT 0x10 +#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11 +#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17 +#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18 +#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a +#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b +#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d +#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e +#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f +#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK 0x00001000L +#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x00002000L +#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x00004000L +#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK 0x00008000L +#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK 0x00010000L +#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x00020000L +#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x00800000L +#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x01000000L +#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x04000000L +#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x08000000L +#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000L +#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000L +#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000L +//CP_ME2_INT_STAT_DEBUG +#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT 0xc +#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd +#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe +#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT 0xf +#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT 0x10 +#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11 +#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17 +#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18 +#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a +#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b +#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d +#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e +#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f +#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK 0x00001000L +#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x00002000L +#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x00004000L +#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK 0x00008000L +#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK 0x00010000L +#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x00020000L +#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x00800000L +#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x01000000L +#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x04000000L +#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x08000000L +#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000L +#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000L +#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000L //CP_ME2_PIPE0_INT_STATUS #define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT 0xc #define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd diff --git a/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_offset.h index a485526f3a51..8474f419caa5 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_offset.h @@ -38,6 +38,9 @@ #define mmCG_TACH_CTRL 0x006a #define mmCG_TACH_CTRL_BASE_IDX 0 +#define mmCG_TACH_STATUS 0x006b +#define mmCG_TACH_STATUS_BASE_IDX 0 + #define mmTHM_THERMAL_INT_ENA 0x000a #define mmTHM_THERMAL_INT_ENA_BASE_IDX 0 #define mmTHM_THERMAL_INT_CTRL 0x000b @@ -49,4 +52,7 @@ #define mmTHM_BACO_CNTL 0x0081 #define mmTHM_BACO_CNTL_BASE_IDX 0 +#define mmCG_THERMAL_STATUS 0x006C +#define mmCG_THERMAL_STATUS_BASE_IDX 0 + #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_sh_mask.h index d130d92aee19..f2f9eae9a68f 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_11_0_2_sh_mask.h @@ -92,5 +92,8 @@ #define THM_TCON_THERM_TRIP__RSVD3_MASK 0x7FFFC000L #define THM_TCON_THERM_TRIP__SW_THERM_TP_MASK 0x80000000L +#define CG_THERMAL_STATUS__FDO_PWM_DUTY__SHIFT 0x9 +#define CG_THERMAL_STATUS__FDO_PWM_DUTY_MASK 0x0001FE00L + #endif diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h index 3811e58dd857..44955458fe38 100644 --- a/drivers/gpu/drm/amd/include/atomfirmware.h +++ b/drivers/gpu/drm/amd/include/atomfirmware.h @@ -590,7 +590,7 @@ struct atom_firmware_info_v3_4 { uint8_t board_i2c_feature_id; // enum of atom_board_i2c_feature_id_def uint8_t board_i2c_feature_gpio_id; // i2c id find in gpio_lut data table gpio_id uint8_t board_i2c_feature_slave_addr; - uint8_t reserved3; + uint8_t ras_rom_i2c_slave_addr; uint16_t bootup_mvddq_mv; uint16_t bootup_mvpp_mv; uint32_t zfbstartaddrin16mb; diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 95c656d205ed..c84bd7b2cf59 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -44,6 +44,7 @@ struct kgd_mem; enum kfd_preempt_type { KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN = 0, KFD_PREEMPT_TYPE_WAVEFRONT_RESET, + KFD_PREEMPT_TYPE_WAVEFRONT_SAVE }; struct kfd_vm_fault_info { @@ -298,6 +299,8 @@ struct kfd2kgd_calls { void (*get_cu_occupancy)(struct kgd_dev *kgd, int pasid, int *wave_cnt, int *max_waves_per_cu); + void (*program_trap_handler_settings)(struct kgd_dev *kgd, + uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr); }; #endif /* KGD_KFD_INTERFACE_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index e38b191c7b7c..bac15c466733 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -306,8 +306,8 @@ struct amd_pm_funcs { /* export for sysfs */ void (*set_fan_control_mode)(void *handle, u32 mode); u32 (*get_fan_control_mode)(void *handle); - int (*set_fan_speed_percent)(void *handle, u32 speed); - int (*get_fan_speed_percent)(void *handle, u32 *speed); + int (*set_fan_speed_pwm)(void *handle, u32 speed); + int (*get_fan_speed_pwm)(void *handle, u32 *speed); int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask); int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf); int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level); diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 769f58d5ae1a..249cb0aeb5ae 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -2005,10 +2005,10 @@ static int ss_bias_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RW(power_dpm_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(power_dpm_force_performance_level, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), - AMDGPU_DEVICE_ATTR_RO(pp_num_states, ATTR_FLAG_BASIC), - AMDGPU_DEVICE_ATTR_RO(pp_cur_state, ATTR_FLAG_BASIC), - AMDGPU_DEVICE_ATTR_RW(pp_force_state, ATTR_FLAG_BASIC), - AMDGPU_DEVICE_ATTR_RW(pp_table, ATTR_FLAG_BASIC), + AMDGPU_DEVICE_ATTR_RO(pp_num_states, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RO(pp_cur_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_force_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_table, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_sclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), @@ -2094,14 +2094,19 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ *states = ATTR_STATE_UNSUPPORTED; } - if (asic_type == CHIP_ARCTURUS) { - /* Arcturus does not support standalone mclk/socclk/fclk level setting */ + switch (asic_type) { + case CHIP_ARCTURUS: + case CHIP_ALDEBARAN: + /* the Mi series card does not support standalone mclk/socclk/fclk level setting */ if (DEVICE_ATTR_IS(pp_dpm_mclk) || DEVICE_ATTR_IS(pp_dpm_socclk) || DEVICE_ATTR_IS(pp_dpm_fclk)) { dev_attr->attr.mode &= ~S_IWUGO; dev_attr->store = NULL; } + break; + default: + break; } if (DEVICE_ATTR_IS(pp_dpm_dcefclk)) { @@ -2379,7 +2384,7 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev, pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); - return sprintf(buf, "%u\n", pwm_mode); + return sysfs_emit(buf, "%u\n", pwm_mode); } static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, @@ -2424,14 +2429,14 @@ static ssize_t amdgpu_hwmon_get_pwm1_min(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%i\n", 0); + return sysfs_emit(buf, "%i\n", 0); } static ssize_t amdgpu_hwmon_get_pwm1_max(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%i\n", 255); + return sysfs_emit(buf, "%i\n", 255); } static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, @@ -2469,10 +2474,8 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, return err; } - value = (value * 100) / 255; - - if (adev->powerplay.pp_funcs->set_fan_speed_percent) - err = amdgpu_dpm_set_fan_speed_percent(adev, value); + if (adev->powerplay.pp_funcs->set_fan_speed_pwm) + err = amdgpu_dpm_set_fan_speed_pwm(adev, value); else err = -EINVAL; @@ -2504,8 +2507,8 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, return err; } - if (adev->powerplay.pp_funcs->get_fan_speed_percent) - err = amdgpu_dpm_get_fan_speed_percent(adev, &speed); + if (adev->powerplay.pp_funcs->get_fan_speed_pwm) + err = amdgpu_dpm_get_fan_speed_pwm(adev, &speed); else err = -EINVAL; @@ -2515,9 +2518,7 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, if (err) return err; - speed = (speed * 255) / 100; - - return sprintf(buf, "%i\n", speed); + return sysfs_emit(buf, "%i\n", speed); } static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, @@ -2550,7 +2551,7 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, if (err) return err; - return sprintf(buf, "%i\n", speed); + return sysfs_emit(buf, "%i\n", speed); } static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev, @@ -2647,7 +2648,7 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev, if (err) return err; - return sprintf(buf, "%i\n", rpm); + return sysfs_emit(buf, "%i\n", rpm); } static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, @@ -2729,7 +2730,7 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev, pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); - return sprintf(buf, "%i\n", pwm_mode == AMD_FAN_CTRL_AUTO ? 0 : 1); + return sysfs_emit(buf, "%i\n", pwm_mode == AMD_FAN_CTRL_AUTO ? 0 : 1); } static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, @@ -2899,7 +2900,7 @@ static ssize_t amdgpu_hwmon_show_power_cap_min(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%i\n", 0); + return sysfs_emit(buf, "%i\n", 0); } @@ -3174,6 +3175,9 @@ static ssize_t amdgpu_hwmon_show_mclk_label(struct device *dev, * * - fan[1-\*]_enable: Enable or disable the sensors.1: Enable 0: Disable * + * NOTE: DO NOT set the fan speed via "pwm1" and "fan[1-\*]_target" interfaces at the same time. + * That will get the former one overridden. + * * hwmon interfaces for GPU clocks: * * - freq1_input: the gfx/compute clock in hertz @@ -3349,13 +3353,13 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, if (!is_support_sw_smu(adev)) { /* mask fan attributes if we have no bindings for this asic to expose */ - if ((!adev->powerplay.pp_funcs->get_fan_speed_percent && + if ((!adev->powerplay.pp_funcs->get_fan_speed_pwm && attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */ (!adev->powerplay.pp_funcs->get_fan_control_mode && attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */ effective_mode &= ~S_IRUGO; - if ((!adev->powerplay.pp_funcs->set_fan_speed_percent && + if ((!adev->powerplay.pp_funcs->set_fan_speed_pwm && attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */ (!adev->powerplay.pp_funcs->set_fan_control_mode && attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */ @@ -3379,8 +3383,8 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, if (!is_support_sw_smu(adev)) { /* hide max/min values if we can't both query and manage the fan */ - if ((!adev->powerplay.pp_funcs->set_fan_speed_percent && - !adev->powerplay.pp_funcs->get_fan_speed_percent) && + if ((!adev->powerplay.pp_funcs->set_fan_speed_pwm && + !adev->powerplay.pp_funcs->get_fan_speed_pwm) && (!adev->powerplay.pp_funcs->set_fan_speed_rpm && !adev->powerplay.pp_funcs->get_fan_speed_rpm) && (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h index d03e6fa2bf1a..98f1b3d8c1d5 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h @@ -280,11 +280,11 @@ enum amdgpu_pcie_gen { #define amdgpu_dpm_get_fan_control_mode(adev) \ ((adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle)) -#define amdgpu_dpm_set_fan_speed_percent(adev, s) \ - ((adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s))) +#define amdgpu_dpm_set_fan_speed_pwm(adev, s) \ + ((adev)->powerplay.pp_funcs->set_fan_speed_pwm((adev)->powerplay.pp_handle, (s))) -#define amdgpu_dpm_get_fan_speed_percent(adev, s) \ - ((adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s))) +#define amdgpu_dpm_get_fan_speed_pwm(adev, s) \ + ((adev)->powerplay.pp_funcs->get_fan_speed_pwm((adev)->powerplay.pp_handle, (s))) #define amdgpu_dpm_get_fan_speed_rpm(adev, s) \ ((adev)->powerplay.pp_funcs->get_fan_speed_rpm)((adev)->powerplay.pp_handle, (s)) diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h index c2c201b8e3cf..8156729c370b 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h @@ -34,6 +34,8 @@ #define SMU_FW_NAME_LEN 0x24 #define SMU_DPM_USER_PROFILE_RESTORE (1 << 0) +#define SMU_CUSTOM_FAN_SPEED_RPM (1 << 1) +#define SMU_CUSTOM_FAN_SPEED_PWM (1 << 2) // Power Throttlers #define SMU_THROTTLER_PPT0_BIT 0 @@ -229,7 +231,8 @@ enum smu_memory_pool_size struct smu_user_dpm_profile { uint32_t fan_mode; uint32_t power_limit; - uint32_t fan_speed_percent; + uint32_t fan_speed_pwm; + uint32_t fan_speed_rpm; uint32_t flags; uint32_t user_od; @@ -540,7 +543,7 @@ struct smu_context struct work_struct interrupt_work; unsigned fan_max_rpm; - unsigned manual_fan_speed_percent; + unsigned manual_fan_speed_pwm; uint32_t gfx_default_hard_min_freq; uint32_t gfx_default_soft_max_freq; @@ -722,9 +725,14 @@ struct pptable_funcs { bool (*is_dpm_running)(struct smu_context *smu); /** - * @get_fan_speed_percent: Get the current fan speed in percent. + * @get_fan_speed_pwm: Get the current fan speed in PWM. */ - int (*get_fan_speed_percent)(struct smu_context *smu, uint32_t *speed); + int (*get_fan_speed_pwm)(struct smu_context *smu, uint32_t *speed); + + /** + * @get_fan_speed_rpm: Get the current fan speed in rpm. + */ + int (*get_fan_speed_rpm)(struct smu_context *smu, uint32_t *speed); /** * @set_watermarks_table: Configure and upload the watermarks tables to @@ -1043,9 +1051,14 @@ struct pptable_funcs { int (*set_fan_control_mode)(struct smu_context *smu, uint32_t mode); /** - * @set_fan_speed_percent: Set a static fan speed in percent. + * @set_fan_speed_pwm: Set a static fan speed in PWM. + */ + int (*set_fan_speed_pwm)(struct smu_context *smu, uint32_t speed); + + /** + * @set_fan_speed_rpm: Set a static fan speed in rpm. */ - int (*set_fan_speed_percent)(struct smu_context *smu, uint32_t speed); + int (*set_fan_speed_rpm)(struct smu_context *smu, uint32_t speed); /** * @set_xgmi_pstate: Set inter-chip global memory interconnect pstate. @@ -1322,6 +1335,30 @@ enum smu_cmn2asic_mapping_type { #define WORKLOAD_MAP(profile, workload) \ [profile] = {1, (workload)} +/** + * smu_memcpy_trailing - Copy the end of one structure into the middle of another + * + * @dst: Pointer to destination struct + * @first_dst_member: The member name in @dst where the overwrite begins + * @last_dst_member: The member name in @dst where the overwrite ends after + * @src: Pointer to the source struct + * @first_src_member: The member name in @src where the copy begins + * + */ +#define smu_memcpy_trailing(dst, first_dst_member, last_dst_member, \ + src, first_src_member) \ +({ \ + size_t __src_offset = offsetof(typeof(*(src)), first_src_member); \ + size_t __src_size = sizeof(*(src)) - __src_offset; \ + size_t __dst_offset = offsetof(typeof(*(dst)), first_dst_member); \ + size_t __dst_size = offsetofend(typeof(*(dst)), last_dst_member) - \ + __dst_offset; \ + BUILD_BUG_ON(__src_size != __dst_size); \ + __builtin_memcpy((u8 *)(dst) + __dst_offset, \ + (u8 *)(src) + __src_offset, \ + __dst_size); \ +}) + #if !defined(SWSMU_CODE_LAYER_L2) && !defined(SWSMU_CODE_LAYER_L3) && !defined(SWSMU_CODE_LAYER_L4) int smu_get_power_limit(void *handle, uint32_t *limit, diff --git a/drivers/gpu/drm/amd/pm/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/inc/hwmgr.h index 490371bd2520..8ed01071fe5a 100644 --- a/drivers/gpu/drm/amd/pm/inc/hwmgr.h +++ b/drivers/gpu/drm/amd/pm/inc/hwmgr.h @@ -278,9 +278,9 @@ struct pp_hwmgr_func { int (*get_fan_speed_info)(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info); void (*set_fan_control_mode)(struct pp_hwmgr *hwmgr, uint32_t mode); uint32_t (*get_fan_control_mode)(struct pp_hwmgr *hwmgr); - int (*set_fan_speed_percent)(struct pp_hwmgr *hwmgr, uint32_t percent); - int (*get_fan_speed_percent)(struct pp_hwmgr *hwmgr, uint32_t *speed); - int (*set_fan_speed_rpm)(struct pp_hwmgr *hwmgr, uint32_t percent); + int (*set_fan_speed_pwm)(struct pp_hwmgr *hwmgr, uint32_t speed); + int (*get_fan_speed_pwm)(struct pp_hwmgr *hwmgr, uint32_t *speed); + int (*set_fan_speed_rpm)(struct pp_hwmgr *hwmgr, uint32_t speed); int (*get_fan_speed_rpm)(struct pp_hwmgr *hwmgr, uint32_t *speed); int (*reset_fan_speed_to_default)(struct pp_hwmgr *hwmgr); int (*uninitialize_thermal_controller)(struct pp_hwmgr *hwmgr); diff --git a/drivers/gpu/drm/amd/pm/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 1d3765b873df..6f1b1b50d527 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h @@ -282,6 +282,7 @@ enum smu_clk_type { __SMU_DUMMY_MAP(TDC), \ __SMU_DUMMY_MAP(THERMAL), \ __SMU_DUMMY_MAP(GFX_PER_CU_CG), \ + __SMU_DUMMY_MAP(DATA_CALCULATIONS), \ __SMU_DUMMY_MAP(RM), \ __SMU_DUMMY_MAP(DS_DCEFCLK), \ __SMU_DUMMY_MAP(ACDC), \ @@ -297,7 +298,6 @@ enum smu_clk_type { __SMU_DUMMY_MAP(DS_FCLK), \ __SMU_DUMMY_MAP(DS_MP1CLK), \ __SMU_DUMMY_MAP(DS_MP0CLK), \ - __SMU_DUMMY_MAP(XGMI), \ __SMU_DUMMY_MAP(XGMI_PER_LINK_PWR_DWN), \ __SMU_DUMMY_MAP(DPM_GFX_PACE), \ __SMU_DUMMY_MAP(MEM_VDDCI_SCALING), \ diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h index 403bc1bf8a77..cbdae8a2c698 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h @@ -221,9 +221,18 @@ int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode); -int smu_v11_0_set_fan_speed_percent(struct smu_context *smu, +int smu_v11_0_set_fan_speed_pwm(struct smu_context *smu, uint32_t speed); +int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, + uint32_t speed); + +int smu_v11_0_get_fan_speed_pwm(struct smu_context *smu, + uint32_t *speed); + +int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed); + int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate); diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v13_0.h index 3fea2430dec0..dc91eb608791 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v13_0.h @@ -26,7 +26,7 @@ #include "amdgpu_smu.h" #define SMU13_DRIVER_IF_VERSION_INV 0xFFFFFFFF -#define SMU13_DRIVER_IF_VERSION_YELLOW_CARP 0x03 +#define SMU13_DRIVER_IF_VERSION_YELLOW_CARP 0x04 #define SMU13_DRIVER_IF_VERSION_ALDE 0x07 /* MP Apertures */ diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v13_0_1_pmfw.h b/drivers/gpu/drm/amd/pm/inc/smu_v13_0_1_pmfw.h index 5627de734246..c5e26d619bf0 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v13_0_1_pmfw.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v13_0_1_pmfw.h @@ -111,7 +111,9 @@ typedef struct { uint32_t InWhisperMode : 1; uint32_t spare0 : 1; uint32_t ZstateStatus : 4; - uint32_t spare1 :12; + uint32_t spare1 : 4; + uint32_t DstateFun : 4; + uint32_t DstateDev : 4; // MP1_EXT_SCRATCH2 uint32_t P2JobHandler :24; uint32_t RsmuPmiP2FinishedCnt : 8; diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index d2a38246a78a..321215003643 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -533,7 +533,7 @@ static uint32_t pp_dpm_get_fan_control_mode(void *handle) return mode; } -static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent) +static int pp_dpm_set_fan_speed_pwm(void *handle, uint32_t speed) { struct pp_hwmgr *hwmgr = handle; int ret = 0; @@ -541,17 +541,17 @@ static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent) if (!hwmgr || !hwmgr->pm_en) return -EINVAL; - if (hwmgr->hwmgr_func->set_fan_speed_percent == NULL) { + if (hwmgr->hwmgr_func->set_fan_speed_pwm == NULL) { pr_info_ratelimited("%s was not implemented.\n", __func__); return 0; } mutex_lock(&hwmgr->smu_lock); - ret = hwmgr->hwmgr_func->set_fan_speed_percent(hwmgr, percent); + ret = hwmgr->hwmgr_func->set_fan_speed_pwm(hwmgr, speed); mutex_unlock(&hwmgr->smu_lock); return ret; } -static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed) +static int pp_dpm_get_fan_speed_pwm(void *handle, uint32_t *speed) { struct pp_hwmgr *hwmgr = handle; int ret = 0; @@ -559,13 +559,13 @@ static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed) if (!hwmgr || !hwmgr->pm_en) return -EINVAL; - if (hwmgr->hwmgr_func->get_fan_speed_percent == NULL) { + if (hwmgr->hwmgr_func->get_fan_speed_pwm == NULL) { pr_info_ratelimited("%s was not implemented.\n", __func__); return 0; } mutex_lock(&hwmgr->smu_lock); - ret = hwmgr->hwmgr_func->get_fan_speed_percent(hwmgr, speed); + ret = hwmgr->hwmgr_func->get_fan_speed_pwm(hwmgr, speed); mutex_unlock(&hwmgr->smu_lock); return ret; } @@ -1691,8 +1691,8 @@ static const struct amd_pm_funcs pp_dpm_funcs = { .dispatch_tasks = pp_dpm_dispatch_tasks, .set_fan_control_mode = pp_dpm_set_fan_control_mode, .get_fan_control_mode = pp_dpm_get_fan_control_mode, - .set_fan_speed_percent = pp_dpm_set_fan_speed_percent, - .get_fan_speed_percent = pp_dpm_get_fan_speed_percent, + .set_fan_speed_pwm = pp_dpm_set_fan_speed_pwm, + .get_fan_speed_pwm = pp_dpm_get_fan_speed_pwm, .get_fan_speed_rpm = pp_dpm_get_fan_speed_rpm, .set_fan_speed_rpm = pp_dpm_set_fan_speed_rpm, .get_pp_num_states = pp_dpm_get_pp_num_states, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c index 43c3f6e755e7..1de3ae77e03e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c @@ -1036,13 +1036,13 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr, else i = 1; - size += sprintf(buf + size, "0: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", data->gfx_min_freq_limit/100, i == 0 ? "*" : ""); - size += sprintf(buf + size, "1: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", i == 1 ? now : SMU10_UMD_PSTATE_GFXCLK, i == 1 ? "*" : ""); - size += sprintf(buf + size, "2: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "2: %uMhz %s\n", data->gfx_max_freq_limit/100, i == 2 ? "*" : ""); break; @@ -1050,7 +1050,7 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr, smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetFclkFrequency, &now); for (i = 0; i < mclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, mclk_table->entries[i].clk / 100, ((mclk_table->entries[i].clk / 100) @@ -1065,10 +1065,10 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr, if (ret) return ret; - size = sprintf(buf, "%s:\n", "OD_SCLK"); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (data->gfx_actual_soft_min_freq > 0) ? data->gfx_actual_soft_min_freq : min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (data->gfx_actual_soft_max_freq > 0) ? data->gfx_actual_soft_max_freq : max_freq); } break; @@ -1081,8 +1081,8 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr, if (ret) return ret; - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n", min_freq, max_freq); } break; @@ -1456,11 +1456,11 @@ static int smu10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) if (!buf) return -EINVAL; - size += sprintf(buf + size, "%s %16s %s %s %s %s\n",title[0], + size += sysfs_emit_at(buf, size, "%s %16s %s %s %s %s\n",title[0], title[1], title[2], title[3], title[4], title[5]); for (i = 0; i <= PP_SMC_POWER_PROFILE_COMPUTE; i++) - size += sprintf(buf + size, "%3d %14s%s: %14d %3d %10d %14d\n", + size += sysfs_emit_at(buf, size, "%3d %14s%s: %14d %3d %10d %14d\n", i, profile_name[i], (i == hwmgr->power_profile_mode) ? "*" : " ", profile_mode_setting[i][0], profile_mode_setting[i][1], profile_mode_setting[i][2], profile_mode_setting[i][3]); @@ -1580,7 +1580,7 @@ static int smu10_set_fine_grain_clk_vol(struct pp_hwmgr *hwmgr, } if (smu10_data->gfx_actual_soft_min_freq > smu10_data->gfx_actual_soft_max_freq) { - pr_err("The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + pr_err("The setting minimum sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", smu10_data->gfx_actual_soft_min_freq, smu10_data->gfx_actual_soft_max_freq); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 0541bfc81c1b..e7803ce8f67a 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -27,6 +27,9 @@ #include <linux/pci.h> #include <linux/slab.h> #include <asm/div64.h> +#if IS_ENABLED(CONFIG_X86_64) +#include <asm/intel-family.h> +#endif #include <drm/amdgpu_drm.h> #include "ppatomctrl.h" #include "atombios.h" @@ -1733,6 +1736,17 @@ static int smu7_disable_dpm_tasks(struct pp_hwmgr *hwmgr) return result; } +static bool intel_core_rkl_chk(void) +{ +#if IS_ENABLED(CONFIG_X86_64) + struct cpuinfo_x86 *c = &cpu_data(0); + + return (c->x86 == 6 && c->x86_model == INTEL_FAM6_ROCKETLAKE); +#else + return false; +#endif +} + static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr) { struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend); @@ -1758,7 +1772,8 @@ static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr) data->mclk_dpm_key_disabled = hwmgr->feature_mask & PP_MCLK_DPM_MASK ? false : true; data->sclk_dpm_key_disabled = hwmgr->feature_mask & PP_SCLK_DPM_MASK ? false : true; - data->pcie_dpm_key_disabled = hwmgr->feature_mask & PP_PCIE_DPM_MASK ? false : true; + data->pcie_dpm_key_disabled = + intel_core_rkl_chk() || !(hwmgr->feature_mask & PP_PCIE_DPM_MASK); /* need to set voltage control types before EVV patching */ data->voltage_control = SMU7_VOLTAGE_CONTROL_NONE; data->vddci_control = SMU7_VOLTAGE_CONTROL_NONE; @@ -3212,7 +3227,7 @@ static int smu7_force_dpm_level(struct pp_hwmgr *hwmgr, if (!ret) { if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) - smu7_fan_ctrl_set_fan_speed_percent(hwmgr, 100); + smu7_fan_ctrl_set_fan_speed_pwm(hwmgr, 255); else if (level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) smu7_fan_ctrl_reset_fan_speed_to_default(hwmgr); } @@ -4896,8 +4911,8 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr, struct smu7_odn_dpm_table *odn_table = &(data->odn_dpm_table); struct phm_odn_clock_levels *odn_sclk_table = &(odn_table->odn_core_clock_dpm_levels); struct phm_odn_clock_levels *odn_mclk_table = &(odn_table->odn_memory_clock_dpm_levels); - int i, now, size = 0; - uint32_t clock, pcie_speed; + int size = 0; + uint32_t i, now, clock, pcie_speed; switch (type) { case PP_SCLK: @@ -4911,7 +4926,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr, now = i; for (i = 0; i < sclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, sclk_table->dpm_levels[i].value / 100, (i == now) ? "*" : ""); break; @@ -4926,7 +4941,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr, now = i; for (i = 0; i < mclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, mclk_table->dpm_levels[i].value / 100, (i == now) ? "*" : ""); break; @@ -4940,7 +4955,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr, now = i; for (i = 0; i < pcie_table->count; i++) - size += sprintf(buf + size, "%d: %s %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %s %s\n", i, (pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x8" : (pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" : (pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "", @@ -4948,32 +4963,32 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr, break; case OD_SCLK: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_SCLK"); + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); for (i = 0; i < odn_sclk_table->num_of_pl; i++) - size += sprintf(buf + size, "%d: %10uMHz %10umV\n", + size += sysfs_emit_at(buf, size, "%d: %10uMHz %10umV\n", i, odn_sclk_table->entries[i].clock/100, odn_sclk_table->entries[i].vddc); } break; case OD_MCLK: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_MCLK"); + size = sysfs_emit(buf, "%s:\n", "OD_MCLK"); for (i = 0; i < odn_mclk_table->num_of_pl; i++) - size += sprintf(buf + size, "%d: %10uMHz %10umV\n", + size += sysfs_emit_at(buf, size, "%d: %10uMHz %10umV\n", i, odn_mclk_table->entries[i].clock/100, odn_mclk_table->entries[i].vddc); } break; case OD_RANGE: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n", data->golden_dpm_table.sclk_table.dpm_levels[0].value/100, hwmgr->platform_descriptor.overdriveLimit.engineClock/100); - size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n", + size += sysfs_emit_at(buf, size, "MCLK: %7uMHz %10uMHz\n", data->golden_dpm_table.mclk_table.dpm_levels[0].value/100, hwmgr->platform_descriptor.overdriveLimit.memoryClock/100); - size += sprintf(buf + size, "VDDC: %7umV %11umV\n", + size += sysfs_emit_at(buf, size, "VDDC: %7umV %11umV\n", data->odn_dpm_table.min_vddc, data->odn_dpm_table.max_vddc); } @@ -4988,7 +5003,7 @@ static void smu7_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode) { switch (mode) { case AMD_FAN_CTRL_NONE: - smu7_fan_ctrl_set_fan_speed_percent(hwmgr, 100); + smu7_fan_ctrl_set_fan_speed_pwm(hwmgr, 255); break; case AMD_FAN_CTRL_MANUAL: if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, @@ -5503,7 +5518,7 @@ static int smu7_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) if (!buf) return -EINVAL; - size += sprintf(buf + size, "%s %16s %16s %16s %16s %16s %16s %16s\n", + size += sysfs_emit_at(buf, size, "%s %16s %16s %16s %16s %16s %16s %16s\n", title[0], title[1], title[2], title[3], title[4], title[5], title[6], title[7]); @@ -5511,7 +5526,7 @@ static int smu7_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) for (i = 0; i < len; i++) { if (i == hwmgr->power_profile_mode) { - size += sprintf(buf + size, "%3d %14s %s: %8d %16d %16d %16d %16d %16d\n", + size += sysfs_emit_at(buf, size, "%3d %14s %s: %8d %16d %16d %16d %16d %16d\n", i, profile_name[i], "*", data->current_profile_setting.sclk_up_hyst, data->current_profile_setting.sclk_down_hyst, @@ -5522,21 +5537,21 @@ static int smu7_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) continue; } if (smu7_profiling[i].bupdate_sclk) - size += sprintf(buf + size, "%3d %16s: %8d %16d %16d ", + size += sysfs_emit_at(buf, size, "%3d %16s: %8d %16d %16d ", i, profile_name[i], smu7_profiling[i].sclk_up_hyst, smu7_profiling[i].sclk_down_hyst, smu7_profiling[i].sclk_activity); else - size += sprintf(buf + size, "%3d %16s: %8s %16s %16s ", + size += sysfs_emit_at(buf, size, "%3d %16s: %8s %16s %16s ", i, profile_name[i], "-", "-", "-"); if (smu7_profiling[i].bupdate_mclk) - size += sprintf(buf + size, "%16d %16d %16d\n", + size += sysfs_emit_at(buf, size, "%16d %16d %16d\n", smu7_profiling[i].mclk_up_hyst, smu7_profiling[i].mclk_down_hyst, smu7_profiling[i].mclk_activity); else - size += sprintf(buf + size, "%16s %16s %16s\n", + size += sysfs_emit_at(buf, size, "%16s %16s %16s\n", "-", "-", "-"); } @@ -5692,8 +5707,8 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = { .set_max_fan_rpm_output = smu7_set_max_fan_rpm_output, .stop_thermal_controller = smu7_thermal_stop_thermal_controller, .get_fan_speed_info = smu7_fan_ctrl_get_fan_speed_info, - .get_fan_speed_percent = smu7_fan_ctrl_get_fan_speed_percent, - .set_fan_speed_percent = smu7_fan_ctrl_set_fan_speed_percent, + .get_fan_speed_pwm = smu7_fan_ctrl_get_fan_speed_pwm, + .set_fan_speed_pwm = smu7_fan_ctrl_set_fan_speed_pwm, .reset_fan_speed_to_default = smu7_fan_ctrl_reset_fan_speed_to_default, .get_fan_speed_rpm = smu7_fan_ctrl_get_fan_speed_rpm, .set_fan_speed_rpm = smu7_fan_ctrl_set_fan_speed_rpm, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c index 6cfe148ed45b..a6c3610db23e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.c @@ -51,7 +51,7 @@ int smu7_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, return 0; } -int smu7_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, +int smu7_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { uint32_t duty100; @@ -70,12 +70,9 @@ int smu7_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, return -EINVAL; - tmp64 = (uint64_t)duty * 100; + tmp64 = (uint64_t)duty * 255; do_div(tmp64, duty100); - *speed = (uint32_t)tmp64; - - if (*speed > 100) - *speed = 100; + *speed = MIN((uint32_t)tmp64, 255); return 0; } @@ -199,12 +196,11 @@ int smu7_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr) } /** - * smu7_fan_ctrl_set_fan_speed_percent - Set Fan Speed in percent. + * smu7_fan_ctrl_set_fan_speed_pwm - Set Fan Speed in PWM. * @hwmgr: the address of the powerplay hardware manager. - * @speed: is the percentage value (0% - 100%) to be set. - * Exception: Fails is the 100% setting appears to be 0. + * @speed: is the pwm value (0 - 255) to be set. */ -int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, +int smu7_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed) { uint32_t duty100; @@ -214,8 +210,7 @@ int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, if (hwmgr->thermal_controller.fanInfo.bNoFan) return 0; - if (speed > 100) - speed = 100; + speed = MIN(speed, 255); if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl)) smu7_fan_ctrl_stop_smc_fan_control(hwmgr); @@ -227,7 +222,7 @@ int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, return -EINVAL; tmp64 = (uint64_t)speed * duty100; - do_div(tmp64, 100); + do_div(tmp64, 255); duty = (uint32_t)tmp64; PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h index 42c1ba0fad78..a386a437e1f0 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_thermal.h @@ -41,10 +41,10 @@ extern int smu7_thermal_get_temperature(struct pp_hwmgr *hwmgr); extern int smu7_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr); extern int smu7_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info); -extern int smu7_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed); +extern int smu7_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed); extern int smu7_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr); extern int smu7_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode); -extern int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed); +extern int smu7_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed); extern int smu7_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr); extern int smu7_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr); extern int smu7_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c index d425b02b1418..b94a77e4e714 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c @@ -1547,7 +1547,8 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr, struct smu8_hwmgr *data = hwmgr->backend; struct phm_clock_voltage_dependency_table *sclk_table = hwmgr->dyn_state.vddc_dependency_on_sclk; - int i, now, size = 0; + uint32_t i, now; + int size = 0; switch (type) { case PP_SCLK: @@ -1558,7 +1559,7 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr, CURR_SCLK_INDEX); for (i = 0; i < sclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, sclk_table->entries[i].clk / 100, (i == now) ? "*" : ""); break; @@ -1570,7 +1571,7 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr, CURR_MCLK_INDEX); for (i = SMU8_NUM_NBPMEMORYCLOCK; i > 0; i--) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", SMU8_NUM_NBPMEMORYCLOCK-i, data->sys_info.nbp_memory_clock[i-1] / 100, (SMU8_NUM_NBPMEMORYCLOCK-i == now) ? "*" : ""); break; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 25979106fd25..c152a61ddd2c 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -4199,7 +4199,7 @@ static void vega10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode) switch (mode) { case AMD_FAN_CTRL_NONE: - vega10_fan_ctrl_set_fan_speed_percent(hwmgr, 100); + vega10_fan_ctrl_set_fan_speed_pwm(hwmgr, 255); break; case AMD_FAN_CTRL_MANUAL: if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl)) @@ -4553,13 +4553,13 @@ static int vega10_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf) "[EnableAllSmuFeatures] Failed to get enabled smc features!", return ret); - size += sprintf(buf + size, "Current ppfeatures: 0x%016llx\n", features_enabled); - size += sprintf(buf + size, "%-19s %-22s %s\n", + size += sysfs_emit_at(buf, size, "Current ppfeatures: 0x%016llx\n", features_enabled); + size += sysfs_emit_at(buf, size, "%-19s %-22s %s\n", output_title[0], output_title[1], output_title[2]); for (i = 0; i < GNLD_FEATURES_MAX; i++) { - size += sprintf(buf + size, "%-19s 0x%016llx %6s\n", + size += sysfs_emit_at(buf, size, "%-19s 0x%016llx %6s\n", ppfeature_name[i], 1ULL << i, (features_enabled & (1ULL << i)) ? "Y" : "N"); @@ -4650,7 +4650,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, else count = sclk_table->count; for (i = 0; i < count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, sclk_table->dpm_levels[i].value / 100, (i == now) ? "*" : ""); break; @@ -4661,7 +4661,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentUclkIndex, &now); for (i = 0; i < mclk_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, mclk_table->dpm_levels[i].value / 100, (i == now) ? "*" : ""); break; @@ -4672,7 +4672,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentSocclkIndex, &now); for (i = 0; i < soc_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, soc_table->dpm_levels[i].value / 100, (i == now) ? "*" : ""); break; @@ -4684,7 +4684,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, PPSMC_MSG_GetClockFreqMHz, CLK_DCEFCLK, &now); for (i = 0; i < dcef_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, dcef_table->dpm_levels[i].value / 100, (dcef_table->dpm_levels[i].value / 100 == now) ? "*" : ""); @@ -4698,7 +4698,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, gen_speed = pptable->PcieGenSpeed[i]; lane_width = pptable->PcieLaneCount[i]; - size += sprintf(buf + size, "%d: %s %s %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %s %s %s\n", i, (gen_speed == 0) ? "2.5GT/s," : (gen_speed == 1) ? "5.0GT/s," : (gen_speed == 2) ? "8.0GT/s," : @@ -4717,34 +4717,34 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr, case OD_SCLK: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_SCLK"); + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_sclk; for (i = 0; i < podn_vdd_dep->count; i++) - size += sprintf(buf + size, "%d: %10uMhz %10umV\n", + size += sysfs_emit_at(buf, size, "%d: %10uMhz %10umV\n", i, podn_vdd_dep->entries[i].clk / 100, podn_vdd_dep->entries[i].vddc); } break; case OD_MCLK: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_MCLK"); + size = sysfs_emit(buf, "%s:\n", "OD_MCLK"); podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_mclk; for (i = 0; i < podn_vdd_dep->count; i++) - size += sprintf(buf + size, "%d: %10uMhz %10umV\n", + size += sysfs_emit_at(buf, size, "%d: %10uMhz %10umV\n", i, podn_vdd_dep->entries[i].clk/100, podn_vdd_dep->entries[i].vddc); } break; case OD_RANGE: if (hwmgr->od_enabled) { - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n", data->golden_dpm_table.gfx_table.dpm_levels[0].value/100, hwmgr->platform_descriptor.overdriveLimit.engineClock/100); - size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n", + size += sysfs_emit_at(buf, size, "MCLK: %7uMHz %10uMHz\n", data->golden_dpm_table.mem_table.dpm_levels[0].value/100, hwmgr->platform_descriptor.overdriveLimit.memoryClock/100); - size += sprintf(buf + size, "VDDC: %7umV %11umV\n", + size += sysfs_emit_at(buf, size, "VDDC: %7umV %11umV\n", data->odn_dpm_table.min_vddc, data->odn_dpm_table.max_vddc); } @@ -5112,21 +5112,28 @@ static int vega10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) if (!buf) return -EINVAL; - size += sprintf(buf + size, "%s %16s %s %s %s %s\n",title[0], + size += sysfs_emit_at(buf, size, "%s %16s %s %s %s %s\n",title[0], title[1], title[2], title[3], title[4], title[5]); for (i = 0; i < PP_SMC_POWER_PROFILE_CUSTOM; i++) - size += sprintf(buf + size, "%3d %14s%s: %14d %3d %10d %14d\n", + size += sysfs_emit_at(buf, size, "%3d %14s%s: %14d %3d %10d %14d\n", i, profile_name[i], (i == hwmgr->power_profile_mode) ? "*" : " ", profile_mode_setting[i][0], profile_mode_setting[i][1], profile_mode_setting[i][2], profile_mode_setting[i][3]); - size += sprintf(buf + size, "%3d %14s%s: %14d %3d %10d %14d\n", i, + size += sysfs_emit_at(buf, size, "%3d %14s%s: %14d %3d %10d %14d\n", i, profile_name[i], (i == hwmgr->power_profile_mode) ? "*" : " ", data->custom_profile_mode[0], data->custom_profile_mode[1], data->custom_profile_mode[2], data->custom_profile_mode[3]); return size; } +static bool vega10_get_power_profile_mode_quirks(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + + return (adev->pdev->device == 0x6860); +} + static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, uint32_t size) { struct vega10_hwmgr *data = hwmgr->backend; @@ -5163,9 +5170,15 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui } out: - smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask, + if (vega10_get_power_profile_mode_quirks(hwmgr)) + smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask, + 1 << power_profile_mode, + NULL); + else + smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask, (!power_profile_mode) ? 0 : 1 << (power_profile_mode - 1), NULL); + hwmgr->power_profile_mode = power_profile_mode; return 0; @@ -5523,8 +5536,8 @@ static const struct pp_hwmgr_func vega10_hwmgr_funcs = { .force_dpm_level = vega10_dpm_force_dpm_level, .stop_thermal_controller = vega10_thermal_stop_thermal_controller, .get_fan_speed_info = vega10_fan_ctrl_get_fan_speed_info, - .get_fan_speed_percent = vega10_fan_ctrl_get_fan_speed_percent, - .set_fan_speed_percent = vega10_fan_ctrl_set_fan_speed_percent, + .get_fan_speed_pwm = vega10_fan_ctrl_get_fan_speed_pwm, + .set_fan_speed_pwm = vega10_fan_ctrl_set_fan_speed_pwm, .reset_fan_speed_to_default = vega10_fan_ctrl_reset_fan_speed_to_default, .get_fan_speed_rpm = vega10_fan_ctrl_get_fan_speed_rpm, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index 9b46b27bd30c..dad3e3741a4e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -64,7 +64,7 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, return 0; } -int vega10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, +int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { uint32_t current_rpm; @@ -78,11 +78,11 @@ int vega10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, if (hwmgr->thermal_controller. advanceFanControlParameters.usMaxFanRPM != 0) - percent = current_rpm * 100 / + percent = current_rpm * 255 / hwmgr->thermal_controller. advanceFanControlParameters.usMaxFanRPM; - *speed = percent > 100 ? 100 : percent; + *speed = MIN(percent, 255); return 0; } @@ -241,12 +241,11 @@ int vega10_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr) } /** - * vega10_fan_ctrl_set_fan_speed_percent - Set Fan Speed in percent. + * vega10_fan_ctrl_set_fan_speed_pwm - Set Fan Speed in PWM. * @hwmgr: the address of the powerplay hardware manager. - * @speed: is the percentage value (0% - 100%) to be set. - * Exception: Fails is the 100% setting appears to be 0. + * @speed: is the percentage value (0 - 255) to be set. */ -int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, +int vega10_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed) { struct amdgpu_device *adev = hwmgr->adev; @@ -257,8 +256,7 @@ int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, if (hwmgr->thermal_controller.fanInfo.bNoFan) return 0; - if (speed > 100) - speed = 100; + speed = MIN(speed, 255); if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl)) vega10_fan_ctrl_stop_smc_fan_control(hwmgr); @@ -270,7 +268,7 @@ int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, return -EINVAL; tmp64 = (uint64_t)speed * duty100; - do_div(tmp64, 100); + do_div(tmp64, 255); duty = (uint32_t)tmp64; WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h index 4a0ede7c1f07..6850a21a2991 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.h @@ -54,12 +54,12 @@ extern int vega10_thermal_get_temperature(struct pp_hwmgr *hwmgr); extern int vega10_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr); extern int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info); -extern int vega10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, +extern int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed); extern int vega10_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr); extern int vega10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode); -extern int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, +extern int vega10_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed); extern int vega10_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr); extern int vega10_thermal_ctrl_uninitialize_thermal_controller( diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c index 29e0d1d4035a..8558718e15a8 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c @@ -2146,13 +2146,13 @@ static int vega12_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf) "[EnableAllSmuFeatures] Failed to get enabled smc features!", return ret); - size += sprintf(buf + size, "Current ppfeatures: 0x%016llx\n", features_enabled); - size += sprintf(buf + size, "%-19s %-22s %s\n", + size += sysfs_emit_at(buf, size, "Current ppfeatures: 0x%016llx\n", features_enabled); + size += sysfs_emit_at(buf, size, "%-19s %-22s %s\n", output_title[0], output_title[1], output_title[2]); for (i = 0; i < GNLD_FEATURES_MAX; i++) { - size += sprintf(buf + size, "%-19s 0x%016llx %6s\n", + size += sysfs_emit_at(buf, size, "%-19s 0x%016llx %6s\n", ppfeature_name[i], 1ULL << i, (features_enabled & (1ULL << i)) ? "Y" : "N"); @@ -2256,7 +2256,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get gfx clk levels Failed!", return -1); for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : ""); break; @@ -2272,7 +2272,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get memory clk levels Failed!", return -1); for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : ""); break; @@ -2290,7 +2290,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get soc clk levels Failed!", return -1); for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : ""); break; @@ -2308,7 +2308,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get dcef clk levels Failed!", return -1); for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : ""); break; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c index 0791309586c5..0cf39c1244b1 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c @@ -2769,7 +2769,7 @@ static void vega20_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode) { switch (mode) { case AMD_FAN_CTRL_NONE: - vega20_fan_ctrl_set_fan_speed_percent(hwmgr, 100); + vega20_fan_ctrl_set_fan_speed_pwm(hwmgr, 255); break; case AMD_FAN_CTRL_MANUAL: if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl)) @@ -3243,13 +3243,13 @@ static int vega20_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf) "[EnableAllSmuFeatures] Failed to get enabled smc features!", return ret); - size += sprintf(buf + size, "Current ppfeatures: 0x%016llx\n", features_enabled); - size += sprintf(buf + size, "%-19s %-22s %s\n", + size += sysfs_emit_at(buf, size, "Current ppfeatures: 0x%016llx\n", features_enabled); + size += sysfs_emit_at(buf, size, "%-19s %-22s %s\n", output_title[0], output_title[1], output_title[2]); for (i = 0; i < GNLD_FEATURES_MAX; i++) { - size += sprintf(buf + size, "%-19s 0x%016llx %6s\n", + size += sysfs_emit_at(buf, size, "%-19s 0x%016llx %6s\n", ppfeature_name[i], 1ULL << i, (features_enabled & (1ULL << i)) ? "Y" : "N"); @@ -3372,13 +3372,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, return ret); if (vega20_get_sclks(hwmgr, &clocks)) { - size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n", now / 100); break; } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz == now * 10) ? "*" : ""); break; @@ -3390,13 +3390,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, return ret); if (vega20_get_memclocks(hwmgr, &clocks)) { - size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n", now / 100); break; } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz == now * 10) ? "*" : ""); break; @@ -3408,13 +3408,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, return ret); if (vega20_get_socclocks(hwmgr, &clocks)) { - size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n", now / 100); break; } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz == now * 10) ? "*" : ""); break; @@ -3426,7 +3426,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, return ret); for (i = 0; i < fclk_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, fclk_dpm_table->dpm_levels[i].value, fclk_dpm_table->dpm_levels[i].value == (now / 100) ? "*" : ""); break; @@ -3438,13 +3438,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, return ret); if (vega20_get_dcefclocks(hwmgr, &clocks)) { - size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n", now / 100); break; } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.data[i].clocks_in_khz == now * 10) ? "*" : ""); break; @@ -3458,7 +3458,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, gen_speed = pptable->PcieGenSpeed[i]; lane_width = pptable->PcieLaneCount[i]; - size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, (gen_speed == 0) ? "2.5GT/s," : (gen_speed == 1) ? "5.0GT/s," : (gen_speed == 2) ? "8.0GT/s," : @@ -3479,18 +3479,18 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, case OD_SCLK: if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id && od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) { - size = sprintf(buf, "%s:\n", "OD_SCLK"); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", od_table->GfxclkFmin); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", od_table->GfxclkFmax); } break; case OD_MCLK: if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) { - size = sprintf(buf, "%s:\n", "OD_MCLK"); - size += sprintf(buf + size, "1: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_MCLK"); + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", od_table->UclkFmax); } @@ -3503,14 +3503,14 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id && od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id && od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) { - size = sprintf(buf, "%s:\n", "OD_VDDC_CURVE"); - size += sprintf(buf + size, "0: %10uMhz %10dmV\n", + size = sysfs_emit(buf, "%s:\n", "OD_VDDC_CURVE"); + size += sysfs_emit_at(buf, size, "0: %10uMhz %10dmV\n", od_table->GfxclkFreq1, od_table->GfxclkVolt1 / VOLTAGE_SCALE); - size += sprintf(buf + size, "1: %10uMhz %10dmV\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz %10dmV\n", od_table->GfxclkFreq2, od_table->GfxclkVolt2 / VOLTAGE_SCALE); - size += sprintf(buf + size, "2: %10uMhz %10dmV\n", + size += sysfs_emit_at(buf, size, "2: %10uMhz %10dmV\n", od_table->GfxclkFreq3, od_table->GfxclkVolt3 / VOLTAGE_SCALE); } @@ -3518,17 +3518,17 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, break; case OD_RANGE: - size = sprintf(buf, "%s:\n", "OD_RANGE"); + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id && od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) { - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", od8_settings[OD8_SETTING_GFXCLK_FMIN].min_value, od8_settings[OD8_SETTING_GFXCLK_FMAX].max_value); } if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) { - size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", od8_settings[OD8_SETTING_UCLK_FMAX].min_value, od8_settings[OD8_SETTING_UCLK_FMAX].max_value); } @@ -3539,22 +3539,22 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id && od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id && od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) { - size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n", od8_settings[OD8_SETTING_GFXCLK_FREQ1].min_value, od8_settings[OD8_SETTING_GFXCLK_FREQ1].max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n", od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].min_value, od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].max_value); - size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n", od8_settings[OD8_SETTING_GFXCLK_FREQ2].min_value, od8_settings[OD8_SETTING_GFXCLK_FREQ2].max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n", od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].min_value, od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].max_value); - size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n", od8_settings[OD8_SETTING_GFXCLK_FREQ3].min_value, od8_settings[OD8_SETTING_GFXCLK_FREQ3].max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n", + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n", od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].min_value, od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].max_value); } @@ -4003,7 +4003,7 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) if (!buf) return -EINVAL; - size += sprintf(buf + size, "%16s %s %s %s %s %s %s %s %s %s %s\n", + size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s\n", title[0], title[1], title[2], title[3], title[4], title[5], title[6], title[7], title[8], title[9], title[10]); @@ -4016,10 +4016,10 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) "[GetPowerProfile] Failed to get activity monitor!", return result); - size += sprintf(buf + size, "%2d %14s%s:\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", i, profile_name[i], (i == hwmgr->power_profile_mode) ? "*" : " "); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 0, "GFXCLK", @@ -4033,7 +4033,7 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) activity_monitor.Gfx_PD_Data_error_coeff, activity_monitor.Gfx_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 1, "SOCCLK", @@ -4047,7 +4047,7 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) activity_monitor.Soc_PD_Data_error_coeff, activity_monitor.Soc_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 2, "UCLK", @@ -4061,7 +4061,7 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf) activity_monitor.Mem_PD_Data_error_coeff, activity_monitor.Mem_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 3, "FCLK", @@ -4409,8 +4409,8 @@ static const struct pp_hwmgr_func vega20_hwmgr_funcs = { .register_irq_handlers = smu9_register_irq_handlers, .disable_smc_firmware_ctf = vega20_thermal_disable_alert, /* fan control related */ - .get_fan_speed_percent = vega20_fan_ctrl_get_fan_speed_percent, - .set_fan_speed_percent = vega20_fan_ctrl_set_fan_speed_percent, + .get_fan_speed_pwm = vega20_fan_ctrl_get_fan_speed_pwm, + .set_fan_speed_pwm = vega20_fan_ctrl_set_fan_speed_pwm, .get_fan_speed_info = vega20_fan_ctrl_get_fan_speed_info, .get_fan_speed_rpm = vega20_fan_ctrl_get_fan_speed_rpm, .set_fan_speed_rpm = vega20_fan_ctrl_set_fan_speed_rpm, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c index 269dd7e95a44..f4f4efdbda79 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.c @@ -114,26 +114,29 @@ static int vega20_get_current_rpm(struct pp_hwmgr *hwmgr, uint32_t *current_rpm) return 0; } -int vega20_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, +int vega20_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { - struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); - PPTable_t *pp_table = &(data->smc_state_table.pp_table); - uint32_t current_rpm, percent = 0; - int ret = 0; + struct amdgpu_device *adev = hwmgr->adev; + uint32_t duty100, duty; + uint64_t tmp64; - ret = vega20_get_current_rpm(hwmgr, ¤t_rpm); - if (ret) - return ret; + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), + CG_THERMAL_STATUS, FDO_PWM_DUTY); - percent = current_rpm * 100 / pp_table->FanMaximumRpm; + if (!duty100) + return -EINVAL; - *speed = percent > 100 ? 100 : percent; + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); return 0; } -int vega20_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, +int vega20_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed) { struct amdgpu_device *adev = hwmgr->adev; @@ -141,8 +144,7 @@ int vega20_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t duty; uint64_t tmp64; - if (speed > 100) - speed = 100; + speed = MIN(speed, 255); if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl)) vega20_fan_ctrl_stop_smc_fan_control(hwmgr); @@ -154,7 +156,7 @@ int vega20_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, return -EINVAL; tmp64 = (uint64_t)speed * duty100; - do_div(tmp64, 100); + do_div(tmp64, 255); duty = (uint32_t)tmp64; WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h index 2d1769bbd24e..b18d09cf761e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_thermal.h @@ -56,9 +56,9 @@ extern int vega20_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed); extern int vega20_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed); -extern int vega20_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, +extern int vega20_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed); -extern int vega20_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, +extern int vega20_fan_ctrl_set_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t speed); extern int vega20_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr); extern int vega20_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr); diff --git a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c index 15c0b8af376f..bdbbeb959c68 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c @@ -6539,7 +6539,7 @@ static int si_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev) } } -static int si_dpm_get_fan_speed_percent(void *handle, +static int si_dpm_get_fan_speed_pwm(void *handle, u32 *speed) { u32 duty, duty100; @@ -6555,17 +6555,14 @@ static int si_dpm_get_fan_speed_percent(void *handle, if (duty100 == 0) return -EINVAL; - tmp64 = (u64)duty * 100; + tmp64 = (u64)duty * 255; do_div(tmp64, duty100); - *speed = (u32)tmp64; - - if (*speed > 100) - *speed = 100; + *speed = MIN((u32)tmp64, 255); return 0; } -static int si_dpm_set_fan_speed_percent(void *handle, +static int si_dpm_set_fan_speed_pwm(void *handle, u32 speed) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -6580,7 +6577,7 @@ static int si_dpm_set_fan_speed_percent(void *handle, if (si_pi->fan_is_controlled_by_smc) return -EINVAL; - if (speed > 100) + if (speed > 255) return -EINVAL; duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; @@ -6589,7 +6586,7 @@ static int si_dpm_set_fan_speed_percent(void *handle, return -EINVAL; tmp64 = (u64)speed * duty100; - do_div(tmp64, 100); + do_div(tmp64, 255); duty = (u32)tmp64; tmp = RREG32(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK; @@ -8059,8 +8056,8 @@ static const struct amd_pm_funcs si_dpm_funcs = { .vblank_too_short = &si_dpm_vblank_too_short, .set_fan_control_mode = &si_dpm_set_fan_control_mode, .get_fan_control_mode = &si_dpm_get_fan_control_mode, - .set_fan_speed_percent = &si_dpm_set_fan_speed_percent, - .get_fan_speed_percent = &si_dpm_get_fan_speed_percent, + .set_fan_speed_pwm = &si_dpm_set_fan_speed_pwm, + .get_fan_speed_pwm = &si_dpm_get_fan_speed_pwm, .check_state_equal = &si_check_state_equal, .get_vce_clock_state = amdgpu_get_vce_clock_state, .read_sensor = &si_dpm_read_sensor, diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 71afc2d20b12..3ab1ce4d3419 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -58,7 +58,7 @@ static int smu_handle_task(struct smu_context *smu, enum amd_pp_task task_id, bool lock_needed); static int smu_reset(struct smu_context *smu); -static int smu_set_fan_speed_percent(void *handle, u32 speed); +static int smu_set_fan_speed_pwm(void *handle, u32 speed); static int smu_set_fan_control_mode(struct smu_context *smu, int value); static int smu_set_power_limit(void *handle, uint32_t limit); static int smu_set_fan_speed_rpm(void *handle, uint32_t speed); @@ -403,17 +403,26 @@ static void smu_restore_dpm_user_profile(struct smu_context *smu) } /* set the user dpm fan configurations */ - if (smu->user_dpm_profile.fan_mode == AMD_FAN_CTRL_MANUAL) { + if (smu->user_dpm_profile.fan_mode == AMD_FAN_CTRL_MANUAL || + smu->user_dpm_profile.fan_mode == AMD_FAN_CTRL_NONE) { ret = smu_set_fan_control_mode(smu, smu->user_dpm_profile.fan_mode); if (ret) { + smu->user_dpm_profile.fan_speed_pwm = 0; + smu->user_dpm_profile.fan_speed_rpm = 0; + smu->user_dpm_profile.fan_mode = AMD_FAN_CTRL_AUTO; dev_err(smu->adev->dev, "Failed to set manual fan control mode\n"); - return; } - if (!ret && smu->user_dpm_profile.fan_speed_percent) { - ret = smu_set_fan_speed_percent(smu, smu->user_dpm_profile.fan_speed_percent); + if (smu->user_dpm_profile.fan_speed_pwm) { + ret = smu_set_fan_speed_pwm(smu, smu->user_dpm_profile.fan_speed_pwm); if (ret) - dev_err(smu->adev->dev, "Failed to set manual fan speed\n"); + dev_err(smu->adev->dev, "Failed to set manual fan speed in pwm\n"); + } + + if (smu->user_dpm_profile.fan_speed_rpm) { + ret = smu_set_fan_speed_rpm(smu, smu->user_dpm_profile.fan_speed_rpm); + if (ret) + dev_err(smu->adev->dev, "Failed to set manual fan speed in rpm\n"); } } @@ -620,6 +629,7 @@ static int smu_early_init(void *handle) mutex_init(&smu->smu_baco.mutex); smu->smu_baco.state = SMU_BACO_STATE_EXIT; smu->smu_baco.platform_support = false; + smu->user_dpm_profile.fan_mode = -1; adev->powerplay.pp_handle = smu; adev->powerplay.pp_funcs = &swsmu_pm_funcs; @@ -2179,7 +2189,6 @@ static int smu_set_gfx_cgpg(struct smu_context *smu, bool enabled) static int smu_set_fan_speed_rpm(void *handle, uint32_t speed) { struct smu_context *smu = handle; - u32 percent; int ret = 0; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) @@ -2187,11 +2196,16 @@ static int smu_set_fan_speed_rpm(void *handle, uint32_t speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_percent) { - percent = speed * 100 / smu->fan_max_rpm; - ret = smu->ppt_funcs->set_fan_speed_percent(smu, percent); - if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) - smu->user_dpm_profile.fan_speed_percent = percent; + if (smu->ppt_funcs->set_fan_speed_rpm) { + ret = smu->ppt_funcs->set_fan_speed_rpm(smu, speed); + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) { + smu->user_dpm_profile.flags |= SMU_CUSTOM_FAN_SPEED_RPM; + smu->user_dpm_profile.fan_speed_rpm = speed; + + /* Override custom PWM setting as they cannot co-exist */ + smu->user_dpm_profile.flags &= ~SMU_CUSTOM_FAN_SPEED_PWM; + smu->user_dpm_profile.fan_speed_pwm = 0; + } } mutex_unlock(&smu->mutex); @@ -2551,8 +2565,11 @@ static int smu_set_fan_control_mode(struct smu_context *smu, int value) /* reset user dpm fan speed */ if (!ret && value != AMD_FAN_CTRL_MANUAL && - !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) - smu->user_dpm_profile.fan_speed_percent = 0; + !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) { + smu->user_dpm_profile.fan_speed_pwm = 0; + smu->user_dpm_profile.fan_speed_rpm = 0; + smu->user_dpm_profile.flags &= ~(SMU_CUSTOM_FAN_SPEED_RPM | SMU_CUSTOM_FAN_SPEED_PWM); + } return ret; } @@ -2565,31 +2582,25 @@ static void smu_pp_set_fan_control_mode(void *handle, u32 value) } -static int smu_get_fan_speed_percent(void *handle, u32 *speed) +static int smu_get_fan_speed_pwm(void *handle, u32 *speed) { struct smu_context *smu = handle; int ret = 0; - uint32_t percent; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_percent) { - ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); - if (!ret) { - *speed = percent > 100 ? 100 : percent; - } - } + if (smu->ppt_funcs->get_fan_speed_pwm) + ret = smu->ppt_funcs->get_fan_speed_pwm(smu, speed); mutex_unlock(&smu->mutex); - return ret; } -static int smu_set_fan_speed_percent(void *handle, u32 speed) +static int smu_set_fan_speed_pwm(void *handle, u32 speed) { struct smu_context *smu = handle; int ret = 0; @@ -2599,12 +2610,16 @@ static int smu_set_fan_speed_percent(void *handle, u32 speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_percent) { - if (speed > 100) - speed = 100; - ret = smu->ppt_funcs->set_fan_speed_percent(smu, speed); - if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) - smu->user_dpm_profile.fan_speed_percent = speed; + if (smu->ppt_funcs->set_fan_speed_pwm) { + ret = smu->ppt_funcs->set_fan_speed_pwm(smu, speed); + if (!ret && !(smu->user_dpm_profile.flags & SMU_DPM_USER_PROFILE_RESTORE)) { + smu->user_dpm_profile.flags |= SMU_CUSTOM_FAN_SPEED_PWM; + smu->user_dpm_profile.fan_speed_pwm = speed; + + /* Override custom RPM setting as they cannot co-exist */ + smu->user_dpm_profile.flags &= ~SMU_CUSTOM_FAN_SPEED_RPM; + smu->user_dpm_profile.fan_speed_rpm = 0; + } } mutex_unlock(&smu->mutex); @@ -2616,17 +2631,14 @@ static int smu_get_fan_speed_rpm(void *handle, uint32_t *speed) { struct smu_context *smu = handle; int ret = 0; - u32 percent; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_percent) { - ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); - *speed = percent * smu->fan_max_rpm / 100; - } + if (smu->ppt_funcs->get_fan_speed_rpm) + ret = smu->ppt_funcs->get_fan_speed_rpm(smu, speed); mutex_unlock(&smu->mutex); @@ -3043,8 +3055,8 @@ static const struct amd_pm_funcs swsmu_pm_funcs = { /* export for sysfs */ .set_fan_control_mode = smu_pp_set_fan_control_mode, .get_fan_control_mode = smu_get_fan_control_mode, - .set_fan_speed_percent = smu_set_fan_speed_percent, - .get_fan_speed_percent = smu_get_fan_speed_percent, + .set_fan_speed_pwm = smu_set_fan_speed_pwm, + .get_fan_speed_pwm = smu_get_fan_speed_pwm, .force_clock_level = smu_force_ppclk_levels, .print_clock_levels = smu_print_ppclk_levels, .force_performance_level = smu_force_performance_level, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 6ec8492f71f5..e343cc218990 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -81,6 +81,24 @@ #define smnPCIE_ESM_CTRL 0x111003D0 +#define mmCG_FDO_CTRL0_ARCT 0x8B +#define mmCG_FDO_CTRL0_ARCT_BASE_IDX 0 + +#define mmCG_FDO_CTRL1_ARCT 0x8C +#define mmCG_FDO_CTRL1_ARCT_BASE_IDX 0 + +#define mmCG_FDO_CTRL2_ARCT 0x8D +#define mmCG_FDO_CTRL2_ARCT_BASE_IDX 0 + +#define mmCG_TACH_CTRL_ARCT 0x8E +#define mmCG_TACH_CTRL_ARCT_BASE_IDX 0 + +#define mmCG_TACH_STATUS_ARCT 0x8F +#define mmCG_TACH_STATUS_ARCT_BASE_IDX 0 + +#define mmCG_THERMAL_STATUS_ARCT 0x90 +#define mmCG_THERMAL_STATUS_ARCT_BASE_IDX 0 + static const struct cmn2asic_msg_mapping arcturus_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 0), MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), @@ -163,14 +181,14 @@ static const struct cmn2asic_mapping arcturus_feature_mask_map[SMU_FEATURE_COUNT FEA_MAP(DPM_SOCCLK), FEA_MAP(DPM_FCLK), FEA_MAP(DPM_MP0CLK), - ARCTURUS_FEA_MAP(SMU_FEATURE_XGMI_BIT, FEATURE_DPM_XGMI_BIT), + FEA_MAP(DPM_XGMI), FEA_MAP(DS_GFXCLK), FEA_MAP(DS_SOCCLK), FEA_MAP(DS_LCLK), FEA_MAP(DS_FCLK), FEA_MAP(DS_UCLK), FEA_MAP(GFX_ULV), - ARCTURUS_FEA_MAP(SMU_FEATURE_VCN_PG_BIT, FEATURE_DPM_VCN_BIT), + ARCTURUS_FEA_MAP(SMU_FEATURE_VCN_DPM_BIT, FEATURE_DPM_VCN_BIT), FEA_MAP(RSMU_SMN_CG), FEA_MAP(WAFL_CG), FEA_MAP(PPT), @@ -465,10 +483,8 @@ static int arcturus_append_powerplay_table(struct smu_context *smu) if ((smc_dpm_table->table_header.format_revision == 4) && (smc_dpm_table->table_header.content_revision == 6)) - memcpy(&smc_pptable->MaxVoltageStepGfx, - &smc_dpm_table->maxvoltagestepgfx, - sizeof(*smc_dpm_table) - offsetof(struct atom_smc_dpm_info_v4_6, maxvoltagestepgfx)); - + smu_memcpy_trailing(smc_pptable, MaxVoltageStepGfx, BoardReserved, + smc_dpm_table, maxvoltagestepgfx); return 0; } @@ -721,13 +737,13 @@ static int arcturus_get_current_clk_freq_by_table(struct smu_context *smu, member_type = METRICS_AVERAGE_SOCCLK; break; case PPCLK_VCLK: - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) + if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_DPM_BIT)) member_type = METRICS_CURR_VCLK; else member_type = METRICS_AVERAGE_VCLK; break; case PPCLK_DCLK: - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) + if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_DPM_BIT)) member_type = METRICS_CURR_DCLK; else member_type = METRICS_AVERAGE_DCLK; @@ -756,7 +772,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, uint32_t gen_speed, lane_width; if (amdgpu_ras_intr_triggered()) - return snprintf(buf, PAGE_SIZE, "unavailable\n"); + return sysfs_emit(buf, "unavailable\n"); dpm_context = smu_dpm->dpm_context; @@ -780,7 +796,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, * And it's safe to assume that is always the current clock. */ for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -803,7 +819,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -826,7 +842,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -849,7 +865,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -872,7 +888,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -895,7 +911,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (arcturus_freqs_in_same_level( @@ -906,7 +922,7 @@ static int arcturus_print_clk_levels(struct smu_context *smu, case SMU_PCIE: gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu); lane_width = smu_v11_0_get_current_pcie_link_width_level(smu); - size += sprintf(buf + size, "0: %s %s %dMhz *\n", + size += sysfs_emit_at(buf, size, "0: %s %s %dMhz *\n", (gen_speed == 0) ? "2.5GT/s," : (gen_speed == 1) ? "5.0GT/s," : (gen_speed == 2) ? "8.0GT/s," : @@ -1162,11 +1178,29 @@ static int arcturus_read_sensor(struct smu_context *smu, return ret; } -static int arcturus_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int arcturus_set_fan_static_mode(struct smu_context *smu, + uint32_t mode) { - int ret; - u32 rpm; + struct amdgpu_device *adev = smu->adev; + + WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2_ARCT, + REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2_ARCT), + CG_FDO_CTRL2, TMIN, 0)); + WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2_ARCT, + REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2_ARCT), + CG_FDO_CTRL2, FDO_PWM_MODE, mode)); + + return 0; +} + +static int arcturus_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t crystal_clock_freq = 2500; + uint32_t tach_status; + uint64_t tmp64; + int ret = 0; if (!speed) return -EINVAL; @@ -1175,14 +1209,112 @@ static int arcturus_get_fan_speed_percent(struct smu_context *smu, case AMD_FAN_CTRL_AUTO: ret = arcturus_get_smu_metrics_data(smu, METRICS_CURR_FANSPEED, - &rpm); - if (!ret && smu->fan_max_rpm) - *speed = rpm * 100 / smu->fan_max_rpm; - return ret; + speed); + break; default: - *speed = smu->user_dpm_profile.fan_speed_percent; + /* + * For pre Sienna Cichlid ASICs, the 0 RPM may be not correctly + * detected via register retrieving. To workaround this, we will + * report the fan speed as 0 RPM if user just requested such. + */ + if ((smu->user_dpm_profile.flags & SMU_CUSTOM_FAN_SPEED_RPM) + && !smu->user_dpm_profile.fan_speed_rpm) { + *speed = 0; + return 0; + } + + tmp64 = (uint64_t)crystal_clock_freq * 60 * 10000; + tach_status = RREG32_SOC15(THM, 0, mmCG_TACH_STATUS_ARCT); + if (tach_status) { + do_div(tmp64, tach_status); + *speed = (uint32_t)tmp64; + } else { + *speed = 0; + } + + break; + } + + return ret; +} + +static int arcturus_set_fan_speed_pwm(struct smu_context *smu, + uint32_t speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t duty100, duty; + uint64_t tmp64; + + speed = MIN(speed, 255); + + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1_ARCT), + CG_FDO_CTRL1, FMAX_DUTY100); + if (!duty100) + return -EINVAL; + + tmp64 = (uint64_t)speed * duty100; + do_div(tmp64, 255); + duty = (uint32_t)tmp64; + + WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0_ARCT, + REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL0_ARCT), + CG_FDO_CTRL0, FDO_STATIC_DUTY, duty)); + + return arcturus_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC); +} + +static int arcturus_set_fan_speed_rpm(struct smu_context *smu, + uint32_t speed) +{ + struct amdgpu_device *adev = smu->adev; + /* + * crystal_clock_freq used for fan speed rpm calculation is + * always 25Mhz. So, hardcode it as 2500(in 10K unit). + */ + uint32_t crystal_clock_freq = 2500; + uint32_t tach_period; + + tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed); + WREG32_SOC15(THM, 0, mmCG_TACH_CTRL_ARCT, + REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL_ARCT), + CG_TACH_CTRL, TARGET_PERIOD, + tach_period)); + + return arcturus_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC_RPM); +} + +static int arcturus_get_fan_speed_pwm(struct smu_context *smu, + uint32_t *speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t duty100, duty; + uint64_t tmp64; + + /* + * For pre Sienna Cichlid ASICs, the 0 RPM may be not correctly + * detected via register retrieving. To workaround this, we will + * report the fan speed as 0 PWM if user just requested such. + */ + if ((smu->user_dpm_profile.flags & SMU_CUSTOM_FAN_SPEED_PWM) + && !smu->user_dpm_profile.fan_speed_pwm) { + *speed = 0; return 0; } + + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1_ARCT), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS_ARCT), + CG_THERMAL_STATUS, FDO_PWM_DUTY); + + if (duty100) { + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); + } else { + *speed = 0; + } + + return 0; } static int arcturus_get_fan_parameters(struct smu_context *smu) @@ -1272,11 +1404,11 @@ static int arcturus_get_power_profile_mode(struct smu_context *smu, return result; if (smu_version >= 0x360d00) - size += sprintf(buf + size, "%16s %s %s %s %s %s %s %s %s %s %s\n", + size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s\n", title[0], title[1], title[2], title[3], title[4], title[5], title[6], title[7], title[8], title[9], title[10]); else - size += sprintf(buf + size, "%16s\n", + size += sysfs_emit_at(buf, size, "%16s\n", title[0]); for (i = 0; i <= PP_SMC_POWER_PROFILE_CUSTOM; i++) { @@ -1302,11 +1434,11 @@ static int arcturus_get_power_profile_mode(struct smu_context *smu, } } - size += sprintf(buf + size, "%2d %14s%s\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); if (smu_version >= 0x360d00) { - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 0, "GFXCLK", @@ -1320,7 +1452,7 @@ static int arcturus_get_power_profile_mode(struct smu_context *smu, activity_monitor.Gfx_PD_Data_error_coeff, activity_monitor.Gfx_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 1, "UCLK", @@ -1916,16 +2048,16 @@ static int arcturus_dpm_set_vcn_enable(struct smu_context *smu, bool enable) int ret = 0; if (enable) { - if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_VCN_PG_BIT, 1); + if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_DPM_BIT)) { + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_VCN_DPM_BIT, 1); if (ret) { dev_err(smu->adev->dev, "[EnableVCNDPM] failed!\n"); return ret; } } } else { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_VCN_PG_BIT, 0); + if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_DPM_BIT)) { + ret = smu_cmn_feature_set_enabled(smu, SMU_FEATURE_VCN_DPM_BIT, 0); if (ret) { dev_err(smu->adev->dev, "[DisableVCNDPM] failed!\n"); return ret; @@ -2270,7 +2402,8 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .print_clk_levels = arcturus_print_clk_levels, .force_clk_levels = arcturus_force_clk_levels, .read_sensor = arcturus_read_sensor, - .get_fan_speed_percent = arcturus_get_fan_speed_percent, + .get_fan_speed_pwm = arcturus_get_fan_speed_pwm, + .get_fan_speed_rpm = arcturus_get_fan_speed_rpm, .get_power_profile_mode = arcturus_get_power_profile_mode, .set_power_profile_mode = arcturus_set_power_profile_mode, .set_performance_level = arcturus_set_performance_level, @@ -2315,7 +2448,8 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, + .set_fan_speed_pwm = arcturus_set_fan_speed_pwm, + .set_fan_speed_rpm = arcturus_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index d7722c229ddd..a5fc5d7cb6c7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -431,16 +431,16 @@ static int navi10_append_powerplay_table(struct smu_context *smu) switch (smc_dpm_table->table_header.content_revision) { case 5: /* nv10 and nv14 */ - memcpy(smc_pptable->I2cControllers, smc_dpm_table->I2cControllers, - sizeof(*smc_dpm_table) - sizeof(smc_dpm_table->table_header)); + smu_memcpy_trailing(smc_pptable, I2cControllers, BoardReserved, + smc_dpm_table, I2cControllers); break; case 7: /* nv12 */ ret = amdgpu_atombios_get_data_table(adev, index, NULL, NULL, NULL, (uint8_t **)&smc_dpm_table_v4_7); if (ret) return ret; - memcpy(smc_pptable->I2cControllers, smc_dpm_table_v4_7->I2cControllers, - sizeof(*smc_dpm_table_v4_7) - sizeof(smc_dpm_table_v4_7->table_header)); + smu_memcpy_trailing(smc_pptable, I2cControllers, BoardReserved, + smc_dpm_table_v4_7, I2cControllers); break; default: dev_err(smu->adev->dev, "smc_dpm_info with unsupported content revision %d!\n", @@ -1303,7 +1303,7 @@ static int navi10_print_clk_levels(struct smu_context *smu, if (ret) return size; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); } } else { @@ -1321,7 +1321,7 @@ static int navi10_print_clk_levels(struct smu_context *smu, freq_values[1] = (freq_values[0] + freq_values[2]) / 2; for (i = 0; i < 3; i++) { - size += sprintf(buf + size, "%d: %uMhz %s\n", i, freq_values[i], + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i], i == mark_index ? "*" : ""); } @@ -1331,7 +1331,7 @@ static int navi10_print_clk_levels(struct smu_context *smu, gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu); lane_width = smu_v11_0_get_current_pcie_link_width_level(smu); for (i = 0; i < NUM_LINK_LEVELS; i++) - size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," : (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 1) ? "5.0GT/s," : (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 2) ? "8.0GT/s," : @@ -1352,23 +1352,24 @@ static int navi10_print_clk_levels(struct smu_context *smu, break; if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_LIMITS)) break; - size += sprintf(buf + size, "OD_SCLK:\n"); - size += sprintf(buf + size, "0: %uMhz\n1: %uMhz\n", od_table->GfxclkFmin, od_table->GfxclkFmax); + size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); + size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", + od_table->GfxclkFmin, od_table->GfxclkFmax); break; case SMU_OD_MCLK: if (!smu->od_enabled || !od_table || !od_settings) break; if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_UCLK_MAX)) break; - size += sprintf(buf + size, "OD_MCLK:\n"); - size += sprintf(buf + size, "1: %uMHz\n", od_table->UclkFmax); + size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); + size += sysfs_emit_at(buf, size, "1: %uMHz\n", od_table->UclkFmax); break; case SMU_OD_VDDC_CURVE: if (!smu->od_enabled || !od_table || !od_settings) break; if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_CURVE)) break; - size += sprintf(buf + size, "OD_VDDC_CURVE:\n"); + size += sysfs_emit_at(buf, size, "OD_VDDC_CURVE:\n"); for (i = 0; i < 3; i++) { switch (i) { case 0: @@ -1383,55 +1384,57 @@ static int navi10_print_clk_levels(struct smu_context *smu, default: break; } - size += sprintf(buf + size, "%d: %uMHz %umV\n", i, curve_settings[0], curve_settings[1] / NAVI10_VOLTAGE_SCALE); + size += sysfs_emit_at(buf, size, "%d: %uMHz %umV\n", + i, curve_settings[0], + curve_settings[1] / NAVI10_VOLTAGE_SCALE); } break; case SMU_OD_RANGE: if (!smu->od_enabled || !od_table || !od_settings) break; - size = sprintf(buf, "%s:\n", "OD_RANGE"); + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_LIMITS)) { navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMIN, &min_value, NULL); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMAX, NULL, &max_value); - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", min_value, max_value); } if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_UCLK_MAX)) { navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_UCLKFMAX, &min_value, &max_value); - size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", min_value, max_value); } if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_CURVE)) { navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P1, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n", + min_value, max_value); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P1, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n", + min_value, max_value); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P2, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n", + min_value, max_value); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P2, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n", + min_value, max_value); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P3, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n", + min_value, max_value); navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P3, &min_value, &max_value); - size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n", - min_value, max_value); + size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n", + min_value, max_value); } break; @@ -1668,27 +1671,27 @@ static bool navi10_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int navi10_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int navi10_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed) { - int ret; - u32 rpm; + int ret = 0; if (!speed) return -EINVAL; switch (smu_v11_0_get_fan_control_mode(smu)) { case AMD_FAN_CTRL_AUTO: - ret = navi1x_get_smu_metrics_data(smu, + ret = navi10_get_smu_metrics_data(smu, METRICS_CURR_FANSPEED, - &rpm); - if (!ret && smu->fan_max_rpm) - *speed = rpm * 100 / smu->fan_max_rpm; - return ret; + speed); + break; default: - *speed = smu->user_dpm_profile.fan_speed_percent; - return 0; + ret = smu_v11_0_get_fan_speed_rpm(smu, + speed); + break; } + + return ret; } static int navi10_get_fan_parameters(struct smu_context *smu) @@ -1730,7 +1733,7 @@ static int navi10_get_power_profile_mode(struct smu_context *smu, char *buf) if (!buf) return -EINVAL; - size += sprintf(buf + size, "%16s %s %s %s %s %s %s %s %s %s %s\n", + size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s\n", title[0], title[1], title[2], title[3], title[4], title[5], title[6], title[7], title[8], title[9], title[10]); @@ -1750,10 +1753,10 @@ static int navi10_get_power_profile_mode(struct smu_context *smu, char *buf) return result; } - size += sprintf(buf + size, "%2d %14s%s:\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 0, "GFXCLK", @@ -1767,7 +1770,7 @@ static int navi10_get_power_profile_mode(struct smu_context *smu, char *buf) activity_monitor.Gfx_PD_Data_error_coeff, activity_monitor.Gfx_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 1, "SOCCLK", @@ -1781,7 +1784,7 @@ static int navi10_get_power_profile_mode(struct smu_context *smu, char *buf) activity_monitor.Soc_PD_Data_error_coeff, activity_monitor.Soc_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 2, "MEMLK", @@ -3224,7 +3227,8 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_config_changed = navi10_display_config_changed, .notify_smc_display_config = navi10_notify_smc_display_config, .is_dpm_running = navi10_is_dpm_running, - .get_fan_speed_percent = navi10_get_fan_speed_percent, + .get_fan_speed_pwm = smu_v11_0_get_fan_speed_pwm, + .get_fan_speed_rpm = navi10_get_fan_speed_rpm, .get_power_profile_mode = navi10_get_power_profile_mode, .set_power_profile_mode = navi10_set_power_profile_mode, .set_watermarks_table = navi10_set_watermarks_table, @@ -3267,7 +3271,8 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, + .set_fan_speed_pwm = smu_v11_0_set_fan_speed_pwm, + .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 90e40aacb8f6..5e292c3f5050 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -353,8 +353,7 @@ static void sienna_cichlid_check_bxco_support(struct smu_context *smu) struct amdgpu_device *adev = smu->adev; uint32_t val; - if (powerplay_table->platform_caps & SMU_11_0_7_PP_PLATFORM_CAP_BACO || - powerplay_table->platform_caps & SMU_11_0_7_PP_PLATFORM_CAP_MACO) { + if (powerplay_table->platform_caps & SMU_11_0_7_PP_PLATFORM_CAP_BACO) { val = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP0); smu_baco->platform_support = (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) ? true : @@ -1089,7 +1088,7 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, if (ret) goto print_clk_out; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); } } else { @@ -1111,7 +1110,7 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, } for (i = 0; i < count; i++) { - size += sprintf(buf + size, "%d: %uMhz %s\n", i, freq_values[i], + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i], cur_value == freq_values[i] ? "*" : ""); } @@ -1122,7 +1121,7 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, lane_width = smu_v11_0_get_current_pcie_link_width_level(smu); GET_PPTABLE_MEMBER(LclkFreq, &table_member); for (i = 0; i < NUM_LINK_LEVELS; i++) - size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i, (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," : (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 1) ? "5.0GT/s," : (dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 2) ? "8.0GT/s," : @@ -1145,8 +1144,8 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) break; - size += sprintf(buf + size, "OD_SCLK:\n"); - size += sprintf(buf + size, "0: %uMhz\n1: %uMhz\n", od_table->GfxclkFmin, od_table->GfxclkFmax); + size += sysfs_emit_at(buf, size, "OD_SCLK:\n"); + size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", od_table->GfxclkFmin, od_table->GfxclkFmax); break; case SMU_OD_MCLK: @@ -1156,8 +1155,8 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) break; - size += sprintf(buf + size, "OD_MCLK:\n"); - size += sprintf(buf + size, "0: %uMhz\n1: %uMHz\n", od_table->UclkFmin, od_table->UclkFmax); + size += sysfs_emit_at(buf, size, "OD_MCLK:\n"); + size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMHz\n", od_table->UclkFmin, od_table->UclkFmax); break; case SMU_OD_VDDGFX_OFFSET: @@ -1173,22 +1172,22 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, (smu_version < 0x003a2900)) break; - size += sprintf(buf + size, "OD_VDDGFX_OFFSET:\n"); - size += sprintf(buf + size, "%dmV\n", od_table->VddGfxOffset); + size += sysfs_emit_at(buf, size, "OD_VDDGFX_OFFSET:\n"); + size += sysfs_emit_at(buf, size, "%dmV\n", od_table->VddGfxOffset); break; case SMU_OD_RANGE: if (!smu->od_enabled || !od_table || !od_settings) break; - size = sprintf(buf, "%s:\n", "OD_RANGE"); + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) { sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMIN, &min_value, NULL); sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMAX, NULL, &max_value); - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", min_value, max_value); } @@ -1197,7 +1196,7 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, &min_value, NULL); sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_UCLKFMAX, NULL, &max_value); - size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n", min_value, max_value); } break; @@ -1355,27 +1354,20 @@ static bool sienna_cichlid_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int sienna_cichlid_get_fan_speed_percent(struct smu_context *smu, - uint32_t *speed) +static int sienna_cichlid_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed) { - int ret; - u32 rpm; - if (!speed) return -EINVAL; - switch (smu_v11_0_get_fan_control_mode(smu)) { - case AMD_FAN_CTRL_AUTO: - ret = sienna_cichlid_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - &rpm); - if (!ret && smu->fan_max_rpm) - *speed = rpm * 100 / smu->fan_max_rpm; - return ret; - default: - *speed = smu->user_dpm_profile.fan_speed_percent; - return 0; - } + /* + * For Sienna_Cichlid and later, the fan speed(rpm) reported + * by pmfw is always trustable(even when the fan control feature + * disabled or 0 RPM kicked in). + */ + return sienna_cichlid_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + speed); } static int sienna_cichlid_get_fan_parameters(struct smu_context *smu) @@ -1420,7 +1412,7 @@ static int sienna_cichlid_get_power_profile_mode(struct smu_context *smu, char * if (!buf) return -EINVAL; - size += sprintf(buf + size, "%16s %s %s %s %s %s %s %s %s %s %s\n", + size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s\n", title[0], title[1], title[2], title[3], title[4], title[5], title[6], title[7], title[8], title[9], title[10]); @@ -1440,10 +1432,10 @@ static int sienna_cichlid_get_power_profile_mode(struct smu_context *smu, char * return result; } - size += sprintf(buf + size, "%2d %14s%s:\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s:\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 0, "GFXCLK", @@ -1457,7 +1449,7 @@ static int sienna_cichlid_get_power_profile_mode(struct smu_context *smu, char * activity_monitor->Gfx_PD_Data_error_coeff, activity_monitor->Gfx_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 1, "SOCCLK", @@ -1471,7 +1463,7 @@ static int sienna_cichlid_get_power_profile_mode(struct smu_context *smu, char * activity_monitor->Fclk_PD_Data_error_coeff, activity_monitor->Fclk_PD_Data_error_rate_coeff); - size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", + size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n", " ", 2, "MEMLK", @@ -3860,7 +3852,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_config_changed = sienna_cichlid_display_config_changed, .notify_smc_display_config = sienna_cichlid_notify_smc_display_config, .is_dpm_running = sienna_cichlid_is_dpm_running, - .get_fan_speed_percent = sienna_cichlid_get_fan_speed_percent, + .get_fan_speed_pwm = smu_v11_0_get_fan_speed_pwm, + .get_fan_speed_rpm = sienna_cichlid_get_fan_speed_rpm, .get_power_profile_mode = sienna_cichlid_get_power_profile_mode, .set_power_profile_mode = sienna_cichlid_set_power_profile_mode, .set_watermarks_table = sienna_cichlid_set_watermarks_table, @@ -3903,7 +3896,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_clock_voltage_request = smu_v11_0_display_clock_voltage_request, .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, - .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, + .set_fan_speed_pwm = smu_v11_0_set_fan_speed_pwm, + .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index b5419e8eba89..87b055466a33 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -1200,17 +1200,13 @@ smu_v11_0_set_fan_static_mode(struct smu_context *smu, uint32_t mode) } int -smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) +smu_v11_0_set_fan_speed_pwm(struct smu_context *smu, uint32_t speed) { struct amdgpu_device *adev = smu->adev; uint32_t duty100, duty; uint64_t tmp64; - if (speed > 100) - speed = 100; - - if (smu_v11_0_auto_fan_control(smu, 0)) - return -EINVAL; + speed = MIN(speed, 255); duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), CG_FDO_CTRL1, FMAX_DUTY100); @@ -1218,7 +1214,7 @@ smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) return -EINVAL; tmp64 = (uint64_t)speed * duty100; - do_div(tmp64, 100); + do_div(tmp64, 255); duty = (uint32_t)tmp64; WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0, @@ -1228,6 +1224,99 @@ smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) return smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC); } +int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, + uint32_t speed) +{ + struct amdgpu_device *adev = smu->adev; + /* + * crystal_clock_freq used for fan speed rpm calculation is + * always 25Mhz. So, hardcode it as 2500(in 10K unit). + */ + uint32_t crystal_clock_freq = 2500; + uint32_t tach_period; + + /* + * To prevent from possible overheat, some ASICs may have requirement + * for minimum fan speed: + * - For some NV10 SKU, the fan speed cannot be set lower than + * 700 RPM. + * - For some Sienna Cichlid SKU, the fan speed cannot be set + * lower than 500 RPM. + */ + tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed); + WREG32_SOC15(THM, 0, mmCG_TACH_CTRL, + REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), + CG_TACH_CTRL, TARGET_PERIOD, + tach_period)); + + return smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC_RPM); +} + +int smu_v11_0_get_fan_speed_pwm(struct smu_context *smu, + uint32_t *speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t duty100, duty; + uint64_t tmp64; + + /* + * For pre Sienna Cichlid ASICs, the 0 RPM may be not correctly + * detected via register retrieving. To workaround this, we will + * report the fan speed as 0 PWM if user just requested such. + */ + if ((smu->user_dpm_profile.flags & SMU_CUSTOM_FAN_SPEED_PWM) + && !smu->user_dpm_profile.fan_speed_pwm) { + *speed = 0; + return 0; + } + + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), + CG_THERMAL_STATUS, FDO_PWM_DUTY); + if (!duty100) + return -EINVAL; + + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); + + return 0; +} + +int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, + uint32_t *speed) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t crystal_clock_freq = 2500; + uint32_t tach_status; + uint64_t tmp64; + + /* + * For pre Sienna Cichlid ASICs, the 0 RPM may be not correctly + * detected via register retrieving. To workaround this, we will + * report the fan speed as 0 RPM if user just requested such. + */ + if ((smu->user_dpm_profile.flags & SMU_CUSTOM_FAN_SPEED_RPM) + && !smu->user_dpm_profile.fan_speed_rpm) { + *speed = 0; + return 0; + } + + tmp64 = (uint64_t)crystal_clock_freq * 60 * 10000; + + tach_status = RREG32_SOC15(THM, 0, mmCG_TACH_STATUS); + if (tach_status) { + do_div(tmp64, tach_status); + *speed = (uint32_t)tmp64; + } else { + dev_warn_once(adev->dev, "Got zero output on CG_TACH_STATUS reading!\n"); + *speed = 0; + } + + return 0; +} + int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode) @@ -1236,7 +1325,9 @@ smu_v11_0_set_fan_control_mode(struct smu_context *smu, switch (mode) { case AMD_FAN_CTRL_NONE: - ret = smu_v11_0_set_fan_speed_percent(smu, 100); + ret = smu_v11_0_auto_fan_control(smu, 0); + if (!ret) + ret = smu_v11_0_set_fan_speed_pwm(smu, 255); break; case AMD_FAN_CTRL_MANUAL: ret = smu_v11_0_auto_fan_control(smu, 0); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 18681dc458da..3a3421452e57 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -256,7 +256,7 @@ static int vangogh_tables_init(struct smu_context *smu) return 0; err3_out: - kfree(smu_table->clocks_table); + kfree(smu_table->watermarks_table); err2_out: kfree(smu_table->gpu_metrics_table); err1_out: @@ -592,28 +592,28 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu, switch (clk_type) { case SMU_OD_SCLK: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "%s:\n", "OD_SCLK"); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); } break; case SMU_OD_CCLK: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); } break; case SMU_OD_RANGE: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); - size += sprintf(buf + size, "CCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); } break; @@ -656,14 +656,14 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu, return ret; if (!value) continue; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); if (cur_value == value) cur_value_match_level = true; } if (!cur_value_match_level) - size += sprintf(buf + size, " %uMhz *\n", cur_value); + size += sysfs_emit_at(buf, size, " %uMhz *\n", cur_value); break; default: break; @@ -691,28 +691,28 @@ static int vangogh_print_clk_levels(struct smu_context *smu, switch (clk_type) { case SMU_OD_SCLK: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "%s:\n", "OD_SCLK"); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); } break; case SMU_OD_CCLK: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); } break; case SMU_OD_RANGE: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); - size += sprintf(buf + size, "CCLK: %7uMhz %10uMhz\n", + size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); } break; @@ -755,14 +755,14 @@ static int vangogh_print_clk_levels(struct smu_context *smu, return ret; if (!value) continue; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); if (cur_value == value) cur_value_match_level = true; } if (!cur_value_match_level) - size += sprintf(buf + size, " %uMhz *\n", cur_value); + size += sysfs_emit_at(buf, size, " %uMhz *\n", cur_value); break; default: break; @@ -1035,7 +1035,7 @@ static int vangogh_get_power_profile_mode(struct smu_context *smu, if (workload_type < 0) continue; - size += sprintf(buf + size, "%2d %14s%s\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); } @@ -1869,7 +1869,7 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB } else { if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { dev_err(smu->adev->dev, - "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + "The setting minimum sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", smu->gfx_actual_hard_min_freq, smu->gfx_actual_soft_max_freq); return -EINVAL; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 9a9c24a6ec35..5aa175e12a78 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -426,7 +426,7 @@ static int renoir_od_edit_dpm_table(struct smu_context *smu, } else { if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { dev_err(smu->adev->dev, - "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + "The setting minimum sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", smu->gfx_actual_hard_min_freq, smu->gfx_actual_soft_max_freq); return -EINVAL; @@ -510,16 +510,16 @@ static int renoir_print_clk_levels(struct smu_context *smu, 0, &max); if (ret) return ret; - size += sprintf(buf + size, "OD_RANGE\nSCLK: %10uMhz %10uMhz\n", min, max); + size += sysfs_emit_at(buf, size, "OD_RANGE\nSCLK: %10uMhz %10uMhz\n", min, max); } break; case SMU_OD_SCLK: if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq; max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq; - size += sprintf(buf + size, "OD_SCLK\n"); - size += sprintf(buf + size, "0:%10uMhz\n", min); - size += sprintf(buf + size, "1:%10uMhz\n", max); + size += sysfs_emit_at(buf, size, "OD_SCLK\n"); + size += sysfs_emit_at(buf, size, "0:%10uMhz\n", min); + size += sysfs_emit_at(buf, size, "1:%10uMhz\n", max); } break; case SMU_GFXCLK: @@ -536,12 +536,12 @@ static int renoir_print_clk_levels(struct smu_context *smu, else i = 1; - size += sprintf(buf + size, "0: %uMhz %s\n", min, + size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", min, i == 0 ? "*" : ""); - size += sprintf(buf + size, "1: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "1: %uMhz %s\n", i == 1 ? cur_value : RENOIR_UMD_PSTATE_GFXCLK, i == 1 ? "*" : ""); - size += sprintf(buf + size, "2: %uMhz %s\n", max, + size += sysfs_emit_at(buf, size, "2: %uMhz %s\n", max, i == 2 ? "*" : ""); } return size; @@ -588,14 +588,14 @@ static int renoir_print_clk_levels(struct smu_context *smu, return ret; if (!value) continue; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); if (cur_value == value) cur_value_match_level = true; } if (!cur_value_match_level) - size += sprintf(buf + size, " %uMhz *\n", cur_value); + size += sysfs_emit_at(buf, size, " %uMhz *\n", cur_value); break; default: @@ -1118,7 +1118,7 @@ static int renoir_get_power_profile_mode(struct smu_context *smu, if (workload_type < 0) continue; - size += sprintf(buf + size, "%2d %14s%s\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c index 856eeaf293b8..ab652028e003 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c @@ -90,8 +90,8 @@ static const struct cmn2asic_msg_mapping aldebaran_message_map[SMU_MSG_MAX_COUNT MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 1), MSG_MAP(EnableAllSmuFeatures, PPSMC_MSG_EnableAllSmuFeatures, 0), MSG_MAP(DisableAllSmuFeatures, PPSMC_MSG_DisableAllSmuFeatures, 0), - MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetEnabledSmuFeaturesLow, 0), - MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetEnabledSmuFeaturesHigh, 0), + MSG_MAP(GetEnabledSmuFeaturesLow, PPSMC_MSG_GetEnabledSmuFeaturesLow, 1), + MSG_MAP(GetEnabledSmuFeaturesHigh, PPSMC_MSG_GetEnabledSmuFeaturesHigh, 1), MSG_MAP(SetDriverDramAddrHigh, PPSMC_MSG_SetDriverDramAddrHigh, 1), MSG_MAP(SetDriverDramAddrLow, PPSMC_MSG_SetDriverDramAddrLow, 1), MSG_MAP(SetToolsDramAddrHigh, PPSMC_MSG_SetToolsDramAddrHigh, 0), @@ -150,20 +150,20 @@ static const struct cmn2asic_mapping aldebaran_clk_map[SMU_CLK_COUNT] = { }; static const struct cmn2asic_mapping aldebaran_feature_mask_map[SMU_FEATURE_COUNT] = { - ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_PREFETCHER_BIT, FEATURE_DATA_CALCULATIONS), + ALDEBARAN_FEA_MAP(SMU_FEATURE_DATA_CALCULATIONS_BIT, FEATURE_DATA_CALCULATIONS), ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_GFXCLK_BIT, FEATURE_DPM_GFXCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_UCLK_BIT, FEATURE_DPM_UCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_SOCCLK_BIT, FEATURE_DPM_SOCCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_FCLK_BIT, FEATURE_DPM_FCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_LCLK_BIT, FEATURE_DPM_LCLK_BIT), - ALDEBARAN_FEA_MAP(SMU_FEATURE_XGMI_BIT, FEATURE_DPM_XGMI_BIT), + ALDEBARAN_FEA_MAP(SMU_FEATURE_DPM_XGMI_BIT, FEATURE_DPM_XGMI_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DS_GFXCLK_BIT, FEATURE_DS_GFXCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DS_SOCCLK_BIT, FEATURE_DS_SOCCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DS_LCLK_BIT, FEATURE_DS_LCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DS_FCLK_BIT, FEATURE_DS_FCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_DS_UCLK_BIT, FEATURE_DS_UCLK_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_GFX_SS_BIT, FEATURE_GFX_SS_BIT), - ALDEBARAN_FEA_MAP(SMU_FEATURE_VCN_PG_BIT, FEATURE_DPM_VCN_BIT), + ALDEBARAN_FEA_MAP(SMU_FEATURE_VCN_DPM_BIT, FEATURE_DPM_VCN_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_RSMU_SMN_CG_BIT, FEATURE_RSMU_SMN_CG_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_WAFL_CG_BIT, FEATURE_WAFL_CG_BIT), ALDEBARAN_FEA_MAP(SMU_FEATURE_PPT_BIT, FEATURE_PPT_BIT), @@ -409,9 +409,8 @@ static int aldebaran_append_powerplay_table(struct smu_context *smu) if ((smc_dpm_table->table_header.format_revision == 4) && (smc_dpm_table->table_header.content_revision == 10)) - memcpy(&smc_pptable->GfxMaxCurrent, - &smc_dpm_table->GfxMaxCurrent, - sizeof(*smc_dpm_table) - offsetof(struct atom_smc_dpm_info_v4_10, GfxMaxCurrent)); + smu_memcpy_trailing(smc_pptable, GfxMaxCurrent, reserved, + smc_dpm_table, GfxMaxCurrent); return 0; } @@ -735,14 +734,14 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, uint32_t min_clk, max_clk; if (amdgpu_ras_intr_triggered()) - return snprintf(buf, PAGE_SIZE, "unavailable\n"); + return sysfs_emit(buf, "unavailable\n"); dpm_context = smu_dpm->dpm_context; switch (type) { case SMU_OD_SCLK: - size = sprintf(buf, "%s:\n", "GFXCLK"); + size = sysfs_emit(buf, "%s:\n", "GFXCLK"); fallthrough; case SMU_SCLK: ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_GFXCLK, &now); @@ -779,8 +778,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, */ if (display_levels == clocks.num_levels) { for (i = 0; i < clocks.num_levels; i++) - size += sprintf( - buf + size, "%d: %uMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i], (clocks.num_levels == 1) ? "*" : @@ -790,14 +788,14 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, "")); } else { for (i = 0; i < display_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", i, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i], i == 1 ? "*" : ""); } break; case SMU_OD_MCLK: - size = sprintf(buf, "%s:\n", "MCLK"); + size = sysfs_emit(buf, "%s:\n", "MCLK"); fallthrough; case SMU_MCLK: ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_UCLK, &now); @@ -814,7 +812,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.num_levels == 1) ? "*" : (aldebaran_freqs_in_same_level( @@ -837,7 +835,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, } for (i = 0; i < clocks.num_levels; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, clocks.data[i].clocks_in_khz / 1000, (clocks.num_levels == 1) ? "*" : (aldebaran_freqs_in_same_level( @@ -860,7 +858,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (aldebaran_freqs_in_same_level( @@ -883,7 +881,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (aldebaran_freqs_in_same_level( @@ -906,7 +904,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu, } for (i = 0; i < single_dpm_table->count; i++) - size += sprintf(buf + size, "%d: %uMhz %s\n", + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, single_dpm_table->dpm_levels[i].value, (clocks.num_levels == 1) ? "*" : (aldebaran_freqs_in_same_level( @@ -1194,8 +1192,19 @@ static int aldebaran_get_power_limit(struct smu_context *smu, uint32_t power_limit = 0; int ret; - if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_PPT_BIT)) - return -EINVAL; + if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_PPT_BIT)) { + if (current_power_limit) + *current_power_limit = 0; + if (default_power_limit) + *default_power_limit = 0; + if (max_power_limit) + *max_power_limit = 0; + + dev_warn(smu->adev->dev, + "PPT feature is not enabled, power values can't be fetched."); + + return 0; + } /* Valid power data is available only from primary die. * For secondary die show the value as 0. diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index a421ba85bd6d..a0e50f23b1dd 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -85,6 +85,10 @@ int smu_v13_0_init_microcode(struct smu_context *smu) const struct common_firmware_header *header; struct amdgpu_firmware_info *ucode = NULL; + /* doesn't need to load smu firmware in IOV mode */ + if (amdgpu_sriov_vf(adev)) + return 0; + switch (adev->asic_type) { case CHIP_ALDEBARAN: chip_name = "aldebaran"; @@ -268,52 +272,86 @@ static int smu_v13_0_set_pptable_v2_1(struct smu_context *smu, void **table, return 0; } -int smu_v13_0_setup_pptable(struct smu_context *smu) +static int smu_v13_0_get_pptable_from_vbios(struct smu_context *smu, void **table, uint32_t *size) { struct amdgpu_device *adev = smu->adev; - const struct smc_firmware_header_v1_0 *hdr; - int ret, index; - uint32_t size = 0; uint16_t atom_table_size; uint8_t frev, crev; - void *table; - uint16_t version_major, version_minor; + int ret, index; + dev_info(adev->dev, "use vbios provided pptable\n"); + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + powerplayinfo); - if (amdgpu_smu_pptable_id >= 0) { - smu->smu_table.boot_values.pp_table_id = amdgpu_smu_pptable_id; - dev_info(adev->dev, "override pptable id %d\n", amdgpu_smu_pptable_id); - } + ret = amdgpu_atombios_get_data_table(adev, index, &atom_table_size, &frev, &crev, + (uint8_t **)table); + if (ret) + return ret; + + if (size) + *size = atom_table_size; + + return 0; +} + +static int smu_v13_0_get_pptable_from_firmware(struct smu_context *smu, void **table, uint32_t *size, + uint32_t pptable_id) +{ + const struct smc_firmware_header_v1_0 *hdr; + struct amdgpu_device *adev = smu->adev; + uint16_t version_major, version_minor; + int ret; hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data; + if (!hdr) + return -EINVAL; + + dev_info(adev->dev, "use driver provided pptable %d\n", pptable_id); + version_major = le16_to_cpu(hdr->header.header_version_major); version_minor = le16_to_cpu(hdr->header.header_version_minor); - if (version_major == 2 && smu->smu_table.boot_values.pp_table_id > 0) { - dev_info(adev->dev, "use driver provided pptable %d\n", smu->smu_table.boot_values.pp_table_id); - switch (version_minor) { - case 1: - ret = smu_v13_0_set_pptable_v2_1(smu, &table, &size, - smu->smu_table.boot_values.pp_table_id); - break; - default: - ret = -EINVAL; - break; - } - if (ret) - return ret; + if (version_major != 2) { + dev_err(adev->dev, "Unsupported smu firmware version %d.%d\n", + version_major, version_minor); + return -EINVAL; + } - } else { - dev_info(adev->dev, "use vbios provided pptable\n"); - index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, - powerplayinfo); + switch (version_minor) { + case 1: + ret = smu_v13_0_set_pptable_v2_1(smu, table, size, pptable_id); + break; + default: + ret = -EINVAL; + break; + } - ret = amdgpu_atombios_get_data_table(adev, index, &atom_table_size, &frev, &crev, - (uint8_t **)&table); - if (ret) - return ret; - size = atom_table_size; + return ret; +} + +int smu_v13_0_setup_pptable(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t size = 0, pptable_id = 0; + void *table; + int ret = 0; + + /* override pptable_id from driver parameter */ + if (amdgpu_smu_pptable_id >= 0) { + pptable_id = amdgpu_smu_pptable_id; + dev_info(adev->dev, "override pptable id %d\n", pptable_id); + } else { + pptable_id = smu->smu_table.boot_values.pp_table_id; } + /* force using vbios pptable in sriov mode */ + if (amdgpu_sriov_vf(adev) || !pptable_id) + ret = smu_v13_0_get_pptable_from_vbios(smu, &table, &size); + else + ret = smu_v13_0_get_pptable_from_firmware(smu, &table, &size, pptable_id); + + if (ret) + return ret; + if (!smu->smu_table.power_play_table) smu->smu_table.power_play_table = table; if (!smu->smu_table.power_play_table_size) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c index 0cfeb9fc7c03..627ba2eec7fd 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c @@ -572,7 +572,7 @@ static int yellow_carp_get_power_profile_mode(struct smu_context *smu, if (workload_type < 0) continue; - size += sprintf(buf + size, "%2d %14s%s\n", + size += sysfs_emit_at(buf, size, "%2d %14s%s\n", i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); } @@ -731,7 +731,7 @@ static int yellow_carp_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM } else { if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { dev_err(smu->adev->dev, - "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + "The setting minimum sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", smu->gfx_actual_hard_min_freq, smu->gfx_actual_soft_max_freq); return -EINVAL; @@ -1054,15 +1054,15 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu, switch (clk_type) { case SMU_OD_SCLK: - size = sprintf(buf, "%s:\n", "OD_SCLK"); - size += sprintf(buf + size, "0: %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); + size += sysfs_emit_at(buf, size, "0: %10uMhz\n", (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); - size += sprintf(buf + size, "1: %10uMhz\n", + size += sysfs_emit_at(buf, size, "1: %10uMhz\n", (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); break; case SMU_OD_RANGE: - size = sprintf(buf, "%s:\n", "OD_RANGE"); - size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); + size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); break; case SMU_SOCCLK: @@ -1083,7 +1083,7 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu, if (ret) goto print_clk_out; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value, cur_value == value ? "*" : ""); } break; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 415be74df28c..66711ab24c15 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -710,7 +710,7 @@ size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu, return 0; } - size = sprintf(buf + size, "features high: 0x%08x low: 0x%08x\n", + size = sysfs_emit_at(buf, size, "features high: 0x%08x low: 0x%08x\n", feature_mask[1], feature_mask[0]); memset(sort_feature, -1, sizeof(sort_feature)); @@ -725,14 +725,14 @@ size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu, sort_feature[feature_index] = i; } - size += sprintf(buf + size, "%-2s. %-20s %-3s : %-s\n", + size += sysfs_emit_at(buf, size, "%-2s. %-20s %-3s : %-s\n", "No", "Feature", "Bit", "State"); for (i = 0; i < SMU_FEATURE_COUNT; i++) { if (sort_feature[i] < 0) continue; - size += sprintf(buf + size, "%02d. %-20s (%2d) : %s\n", + size += sysfs_emit_at(buf, size, "%02d. %-20s (%2d) : %s\n", count++, smu_get_feature_name(smu, sort_feature[i]), i, diff --git a/drivers/gpu/drm/drm_dp_aux_bus.c b/drivers/gpu/drm/drm_dp_aux_bus.c index e49a70f3691b..298ea7a49591 100644 --- a/drivers/gpu/drm/drm_dp_aux_bus.c +++ b/drivers/gpu/drm/drm_dp_aux_bus.c @@ -67,9 +67,8 @@ static int dp_aux_ep_probe(struct device *dev) * * Calls through to the endpoint driver remove. * - * Return: 0 if no error or negative error code. */ -static int dp_aux_ep_remove(struct device *dev) +static void dp_aux_ep_remove(struct device *dev) { struct dp_aux_ep_driver *aux_ep_drv = to_dp_aux_ep_drv(dev->driver); struct dp_aux_ep_device *aux_ep = to_dp_aux_ep_dev(dev); @@ -77,8 +76,6 @@ static int dp_aux_ep_remove(struct device *dev) if (aux_ep_drv->remove) aux_ep_drv->remove(aux_ep); dev_pm_domain_detach(dev, true); - - return 0; } /** diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c index d29907955ff7..5d82891c3222 100644 --- a/drivers/gpu/drm/drm_ioc32.c +++ b/drivers/gpu/drm/drm_ioc32.c @@ -855,8 +855,6 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd, req.request.sequence = req32.request.sequence; req.request.signal = req32.request.signal; err = drm_ioctl_kernel(file, drm_wait_vblank_ioctl, &req, DRM_UNLOCKED); - if (err) - return err; req32.reply.type = req.reply.type; req32.reply.sequence = req.reply.sequence; @@ -865,7 +863,7 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd, if (copy_to_user(argp, &req32, sizeof(req32))) return -EFAULT; - return 0; + return err; } #if defined(CONFIG_X86) diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 1d009494af8b..deb23dbec8b5 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -807,8 +807,8 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, struct page **pages, unsigned int nr_pages) { struct sg_table *sg; - struct scatterlist *sge; size_t max_segment = 0; + int err; sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (!sg) @@ -818,13 +818,12 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, max_segment = dma_max_mapping_size(dev->dev); if (max_segment == 0) max_segment = UINT_MAX; - sge = __sg_alloc_table_from_pages(sg, pages, nr_pages, 0, - nr_pages << PAGE_SHIFT, - max_segment, - NULL, 0, GFP_KERNEL); - if (IS_ERR(sge)) { + err = sg_alloc_table_from_pages_segment(sg, pages, nr_pages, 0, + nr_pages << PAGE_SHIFT, + max_segment, GFP_KERNEL); + if (err) { kfree(sg); - sg = ERR_CAST(sge); + sg = ERR_PTR(err); } return sg; } diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c index 111b932cf2a9..f783d4963d4b 100644 --- a/drivers/gpu/drm/drm_print.c +++ b/drivers/gpu/drm/drm_print.c @@ -25,7 +25,7 @@ #define DEBUG /* for pr_debug() */ -#include <stdarg.h> +#include <linux/stdarg.h> #include <linux/io.h> #include <linux/moduleparam.h> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index f0a07278ad04..7dcc6392792d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -468,17 +468,7 @@ static const struct drm_ioctl_desc etnaviv_ioctls[] = { ETNA_IOCTL(PM_QUERY_SIG, pm_query_sig, DRM_RENDER_ALLOW), }; -static const struct file_operations fops = { - .owner = THIS_MODULE, - .open = drm_open, - .release = drm_release, - .unlocked_ioctl = drm_ioctl, - .compat_ioctl = drm_compat_ioctl, - .poll = drm_poll, - .read = drm_read, - .llseek = no_llseek, - .mmap = etnaviv_gem_mmap, -}; +DEFINE_DRM_GEM_FOPS(fops); static const struct drm_driver etnaviv_drm_driver = { .driver_features = DRIVER_GEM | DRIVER_RENDER, @@ -487,7 +477,7 @@ static const struct drm_driver etnaviv_drm_driver = { .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table, - .gem_prime_mmap = etnaviv_gem_prime_mmap, + .gem_prime_mmap = drm_gem_prime_mmap, #ifdef CONFIG_DEBUG_FS .debugfs_init = etnaviv_debugfs_init, #endif diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h index 003288ebd896..049ae87de9be 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h @@ -47,12 +47,9 @@ struct etnaviv_drm_private { int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file); -int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma); int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset); struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj); int etnaviv_gem_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map); -int etnaviv_gem_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma); struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sg); int etnaviv_gem_prime_pin(struct drm_gem_object *obj); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index b8fa6ed3dd73..8f1b5af47dd6 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -130,8 +130,7 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, { pgprot_t vm_page_prot; - vma->vm_flags &= ~VM_PFNMAP; - vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP; vm_page_prot = vm_get_page_prot(vma->vm_flags); @@ -154,19 +153,11 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, return 0; } -int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma) +static int etnaviv_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { - struct etnaviv_gem_object *obj; - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) { - DBG("mmap failed: %d", ret); - return ret; - } + struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - obj = to_etnaviv_bo(vma->vm_private_data); - return obj->ops->mmap(obj, vma); + return etnaviv_obj->ops->mmap(etnaviv_obj, vma); } static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) @@ -567,6 +558,7 @@ static const struct drm_gem_object_funcs etnaviv_gem_object_funcs = { .unpin = etnaviv_gem_prime_unpin, .get_sg_table = etnaviv_gem_prime_get_sg_table, .vmap = etnaviv_gem_prime_vmap, + .mmap = etnaviv_gem_mmap, .vm_ops = &vm_ops, }; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index d741b1d735f7..6d8bed9c739d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -34,19 +34,6 @@ int etnaviv_gem_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) return 0; } -int etnaviv_gem_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma) -{ - struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - int ret; - - ret = drm_gem_mmap_obj(obj, obj->size, vma); - if (ret < 0) - return ret; - - return etnaviv_obj->ops->mmap(etnaviv_obj, vma); -} - int etnaviv_gem_prime_pin(struct drm_gem_object *obj) { if (!obj->import_attach) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 4102bcea3341..c297fffe06eb 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -613,6 +613,12 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) etnaviv_is_model_rev(gpu, GC2000, 0x5108)) pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX; + /* Disable SE, RA and TX clock gating on affected core revisions. */ + if (etnaviv_is_model_rev(gpu, GC7000, 0x6202)) + pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_SE | + VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA | + VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX; + pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ; pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c index dfc0f536b3b9..f2fc645c7956 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c @@ -39,6 +39,37 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = { }, { .model = 0x7000, + .revision = 0x6202, + .product_id = 0x70003, + .customer_id = 0, + .eco_id = 0, + .stream_count = 8, + .register_max = 64, + .thread_count = 512, + .shader_core_count = 2, + .vertex_cache_size = 16, + .vertex_output_buffer_size = 1024, + .pixel_pipes = 1, + .instruction_count = 512, + .num_constants = 320, + .buffer_size = 0, + .varyings_count = 16, + .features = 0xe0287cad, + .minor_features0 = 0xc1489eff, + .minor_features1 = 0xfefbfad9, + .minor_features2 = 0xeb9d4fbf, + .minor_features3 = 0xedfffced, + .minor_features4 = 0xdb0dafc7, + .minor_features5 = 0x3b5ac333, + .minor_features6 = 0xfccee201, + .minor_features7 = 0x03fffa6f, + .minor_features8 = 0x00e10ef0, + .minor_features9 = 0x0088003c, + .minor_features10 = 0x00004040, + .minor_features11 = 0x00000024, + }, + { + .model = 0x7000, .revision = 0x6204, .product_id = ~0U, .customer_id = ~0U, diff --git a/drivers/gpu/drm/exynos/exynos_drm_dma.c b/drivers/gpu/drm/exynos/exynos_drm_dma.c index 0644936afee2..bf33c3084cb4 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dma.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dma.c @@ -115,6 +115,8 @@ int exynos_drm_register_dma(struct drm_device *drm, struct device *dev, EXYNOS_DEV_ADDR_START, EXYNOS_DEV_ADDR_SIZE); else if (IS_ENABLED(CONFIG_IOMMU_DMA)) mapping = iommu_get_domain_for_dev(priv->dma_dev); + else + mapping = ERR_PTR(-ENODEV); if (IS_ERR(mapping)) return PTR_ERR(mapping); diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index cab4d2c370a7..b00230626c6a 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -4,6 +4,7 @@ * Authors: Joonyoung Shim <jy0922.shim@samsung.com> */ +#include <linux/refcount.h> #include <linux/clk.h> #include <linux/component.h> #include <linux/delay.h> @@ -208,7 +209,7 @@ struct g2d_cmdlist_userptr { struct page **pages; unsigned int npages; struct sg_table *sgt; - atomic_t refcount; + refcount_t refcount; bool in_pool; bool out_of_list; }; @@ -386,9 +387,9 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, if (force) goto out; - atomic_dec(&g2d_userptr->refcount); + refcount_dec(&g2d_userptr->refcount); - if (atomic_read(&g2d_userptr->refcount) > 0) + if (refcount_read(&g2d_userptr->refcount) > 0) return; if (g2d_userptr->in_pool) @@ -436,7 +437,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, * and different size. */ if (g2d_userptr->size == size) { - atomic_inc(&g2d_userptr->refcount); + refcount_inc(&g2d_userptr->refcount); *obj = g2d_userptr; return &g2d_userptr->dma_addr; @@ -461,7 +462,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, if (!g2d_userptr) return ERR_PTR(-ENOMEM); - atomic_set(&g2d_userptr->refcount, 1); + refcount_set(&g2d_userptr->refcount, 1); g2d_userptr->size = size; start = userptr & PAGE_MASK; @@ -897,13 +898,14 @@ static void g2d_runqueue_worker(struct work_struct *work) ret = pm_runtime_resume_and_get(g2d->dev); if (ret < 0) { dev_err(g2d->dev, "failed to enable G2D device.\n"); - return; + goto out; } g2d_dma_start(g2d, g2d->runqueue_node); } } +out: mutex_unlock(&g2d->runqueue_mutex); } diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index 72a38f28393f..47e845353ffa 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -207,6 +207,8 @@ config DRM_I915_LOW_LEVEL_TRACEPOINTS This provides the ability to precisely monitor engine utilisation and also analyze the request dependency resolving timeline. + Recommended for driver developers only. + If in doubt, say "N". config DRM_I915_DEBUG_VBLANK_EVADE @@ -220,6 +222,8 @@ config DRM_I915_DEBUG_VBLANK_EVADE is exceeded, even if there isn't an actual risk of missing the vblank. + Recommended for driver developers only. + If in doubt, say "N". config DRM_I915_DEBUG_RUNTIME_PM @@ -232,4 +236,6 @@ config DRM_I915_DEBUG_RUNTIME_PM runtime PM functionality. This may introduce overhead during driver loading, suspend and resume operations. + Recommended for driver developers only. + If in doubt, say "N" diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index f57dfc74d6ce..642a5b5a1b81 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -38,6 +38,7 @@ i915-y += i915_drv.o \ i915_irq.o \ i915_getparam.o \ i915_mitigations.o \ + i915_module.o \ i915_params.o \ i915_pci.o \ i915_scatterlist.o \ @@ -89,7 +90,6 @@ gt-y += \ gt/gen8_ppgtt.o \ gt/intel_breadcrumbs.o \ gt/intel_context.o \ - gt/intel_context_param.o \ gt/intel_context_sseu.o \ gt/intel_engine_cs.o \ gt/intel_engine_heartbeat.o \ @@ -108,6 +108,7 @@ gt-y += \ gt/intel_gtt.o \ gt/intel_llc.o \ gt/intel_lrc.o \ + gt/intel_migrate.o \ gt/intel_mocs.o \ gt/intel_ppgtt.o \ gt/intel_rc6.o \ @@ -135,7 +136,6 @@ i915-y += $(gt-y) gem-y += \ gem/i915_gem_busy.o \ gem/i915_gem_clflush.o \ - gem/i915_gem_client_blt.o \ gem/i915_gem_context.o \ gem/i915_gem_create.o \ gem/i915_gem_dmabuf.o \ @@ -143,7 +143,6 @@ gem-y += \ gem/i915_gem_execbuffer.o \ gem/i915_gem_internal.o \ gem/i915_gem_object.o \ - gem/i915_gem_object_blt.o \ gem/i915_gem_lmem.o \ gem/i915_gem_mman.o \ gem/i915_gem_pages.o \ @@ -162,15 +161,17 @@ gem-y += \ i915-y += \ $(gem-y) \ i915_active.o \ + i915_buddy.o \ i915_cmd_parser.o \ i915_gem_evict.o \ i915_gem_gtt.o \ + i915_gem_ww.o \ i915_gem.o \ - i915_globals.o \ i915_query.o \ i915_request.o \ i915_scheduler.o \ i915_trace_points.o \ + i915_ttm_buddy_manager.o \ i915_vma.o \ intel_wopcm.o @@ -185,6 +186,8 @@ i915-y += gt/uc/intel_uc.o \ gt/uc/intel_guc_fw.o \ gt/uc/intel_guc_log.o \ gt/uc/intel_guc_log_debugfs.o \ + gt/uc/intel_guc_rc.o \ + gt/uc/intel_guc_slpc.o \ gt/uc/intel_guc_submission.o \ gt/uc/intel_huc.o \ gt/uc/intel_huc_debugfs.o \ @@ -266,6 +269,7 @@ i915-y += \ display/intel_pps.o \ display/intel_qp_tables.o \ display/intel_sdvo.o \ + display/intel_snps_phy.o \ display/intel_tv.o \ display/intel_vdsc.o \ display/intel_vrr.o \ @@ -277,7 +281,9 @@ i915-y += i915_perf.o # Post-mortem debug and GPU hang state capture i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o i915-$(CONFIG_DRM_I915_SELFTEST) += \ + gem/selftests/i915_gem_client_blt.o \ gem/selftests/igt_gem_utils.o \ + selftests/intel_scheduler_helpers.o \ selftests/i915_random.o \ selftests/i915_selftest.o \ selftests/igt_atomic.o \ diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c index 9643c45a2209..b1439ba78f67 100644 --- a/drivers/gpu/drm/i915/display/i9xx_plane.c +++ b/drivers/gpu/drm/i915/display/i9xx_plane.c @@ -912,7 +912,7 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) zpos = 0; drm_plane_create_zpos_immutable_property(&plane->base, zpos); - drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); + intel_plane_helper_add(plane); return plane; diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 833d0c1be4f1..7cfe91fc05f2 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -19,6 +19,12 @@ static const guid_t intel_dsm_guid = GUID_INIT(0x7ed873d3, 0xc2d0, 0x4e4f, 0xa8, 0x54, 0x0f, 0x13, 0x17, 0xb0, 0x1c, 0x2c); +#define INTEL_DSM_FN_GET_BIOS_DATA_FUNCS_SUPPORTED 0 /* No args */ + +static const guid_t intel_dsm_guid2 = + GUID_INIT(0x3e5b41c6, 0xeb1d, 0x4260, + 0x9d, 0x15, 0xc7, 0x1f, 0xba, 0xda, 0xe4, 0x14); + static char *intel_dsm_port_name(u8 id) { switch (id) { @@ -176,6 +182,19 @@ void intel_unregister_dsm_handler(void) { } +void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915) +{ + struct pci_dev *pdev = to_pci_dev(i915->drm.dev); + acpi_handle dhandle; + + dhandle = ACPI_HANDLE(&pdev->dev); + if (!dhandle) + return; + + acpi_evaluate_dsm(dhandle, &intel_dsm_guid2, INTEL_DSM_REVISION_ID, + INTEL_DSM_FN_GET_BIOS_DATA_FUNCS_SUPPORTED, NULL); +} + /* * ACPI Specification, Revision 5.0, Appendix B.3.2 _DOD (Enumerate All Devices * Attached to the Display Adapter). diff --git a/drivers/gpu/drm/i915/display/intel_acpi.h b/drivers/gpu/drm/i915/display/intel_acpi.h index e8b068661d22..9f197401c313 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.h +++ b/drivers/gpu/drm/i915/display/intel_acpi.h @@ -11,11 +11,14 @@ struct drm_i915_private; #ifdef CONFIG_ACPI void intel_register_dsm_handler(void); void intel_unregister_dsm_handler(void); +void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915); void intel_acpi_device_id_update(struct drm_i915_private *i915); #else static inline void intel_register_dsm_handler(void) { return; } static inline void intel_unregister_dsm_handler(void) { return; } static inline +void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915) { return; } +static inline void intel_acpi_device_id_update(struct drm_i915_private *i915) { return; } #endif /* CONFIG_ACPI */ diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index 36f52a1d7552..47234d898549 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -601,7 +601,12 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, return 0; } -const struct drm_plane_helper_funcs intel_plane_helper_funcs = { +static const struct drm_plane_helper_funcs intel_plane_helper_funcs = { .prepare_fb = intel_prepare_plane_fb, .cleanup_fb = intel_cleanup_plane_fb, }; + +void intel_plane_helper_add(struct intel_plane *plane) +{ + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); +} diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.h b/drivers/gpu/drm/i915/display/intel_atomic_plane.h index dc4d05e75e1c..62e5a2a77fd4 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.h +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.h @@ -17,8 +17,6 @@ struct intel_crtc_state; struct intel_plane; struct intel_plane_state; -extern const struct drm_plane_helper_funcs intel_plane_helper_funcs; - unsigned int intel_adjusted_rate(const struct drm_rect *src, const struct drm_rect *dst, unsigned int rate); @@ -65,5 +63,6 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, bool can_position); void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state); +void intel_plane_helper_add(struct intel_plane *plane); #endif /* __INTEL_ATOMIC_PLANE_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c index 5f4f316b3ab5..532237588511 100644 --- a/drivers/gpu/drm/i915/display/intel_audio.c +++ b/drivers/gpu/drm/i915/display/intel_audio.c @@ -1001,7 +1001,7 @@ static unsigned long i915_audio_component_get_power(struct device *kdev) /* Catch potential impedance mismatches before they occur! */ BUILD_BUG_ON(sizeof(intel_wakeref_t) > sizeof(unsigned long)); - ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); + ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO_PLAYBACK); if (dev_priv->audio_power_refcount++ == 0) { if (DISPLAY_VER(dev_priv) >= 9) { @@ -1034,7 +1034,7 @@ static void i915_audio_component_put_power(struct device *kdev, if (IS_GEMINILAKE(dev_priv)) glk_force_audio_cdclk(dev_priv, false); - intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO, cookie); + intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO_PLAYBACK, cookie); } static void i915_audio_component_codec_wake_override(struct device *kdev, diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 5b6922e28ef2..e86e6ed2d3bf 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -1871,12 +1871,12 @@ intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata) static bool is_port_valid(struct drm_i915_private *i915, enum port port) { /* - * On some ICL/CNL SKUs port F is not present, but broken VBTs mark + * On some ICL SKUs port F is not present, but broken VBTs mark * the port as present. Only try to initialize port F for the * SKUs that may actually have it. */ - if (port == PORT_F && (IS_ICELAKE(i915) || IS_CANNONLAKE(i915))) - return IS_ICL_WITH_PORT_F(i915) || IS_CNL_WITH_PORT_F(i915); + if (port == PORT_F && IS_ICELAKE(i915)) + return IS_ICL_WITH_PORT_F(i915); return true; } @@ -1998,7 +1998,7 @@ static void parse_ddi_port(struct drm_i915_private *i915, "Port %c VBT HDMI boost level: %d\n", port_name(port), hdmi_boost_level); - /* DP max link rate for CNL+ */ + /* DP max link rate for GLK+ */ if (i915->vbt.version >= 216) { if (i915->vbt.version >= 230) info->dp_max_link_rate = parse_bdb_230_dp_max_link_rate(child->dp_max_link_rate); @@ -2166,7 +2166,8 @@ static void init_vbt_missing_defaults(struct drm_i915_private *i915) { enum port port; - int ports = PORT_A | PORT_B | PORT_C | PORT_D | PORT_E | PORT_F; + int ports = BIT(PORT_A) | BIT(PORT_B) | BIT(PORT_C) | + BIT(PORT_D) | BIT(PORT_E) | BIT(PORT_F); if (!HAS_DDI(i915) && !IS_CHERRYVIEW(i915)) return; diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index bfb398f0432e..e91e0e0191fb 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -17,12 +17,53 @@ struct intel_qgv_point { u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd; }; +struct intel_psf_gv_point { + u8 clk; /* clock in multiples of 16.6666 MHz */ +}; + struct intel_qgv_info { struct intel_qgv_point points[I915_NUM_QGV_POINTS]; + struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS]; u8 num_points; + u8 num_psf_points; u8 t_bl; }; +static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv, + struct intel_qgv_point *sp, + int point) +{ + u32 dclk_ratio, dclk_reference; + u32 val; + + val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC); + dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val); + if (val & DG1_QCLK_REFERENCE) + dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */ + else + dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */ + sp->dclk = dclk_ratio * dclk_reference; + + val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); + if (val & DG1_GEAR_TYPE) + sp->dclk *= 2; + + if (sp->dclk == 0) + return -EINVAL; + + val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR); + sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val); + sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val); + + val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH); + sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val); + sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val); + + sp->t_rc = sp->t_rp + sp->t_ras; + + return 0; +} + static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, struct intel_qgv_point *sp, int point) @@ -49,6 +90,28 @@ static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, return 0; } +static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv, + struct intel_psf_gv_point *points) +{ + u32 val = 0; + int ret; + int i; + + ret = sandybridge_pcode_read(dev_priv, + ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, + &val, NULL); + if (ret) + return ret; + + for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) { + points[i].clk = val & 0xff; + val >>= 8; + } + + return 0; +} + int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, u32 points_mask) { @@ -62,7 +125,7 @@ int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, 1); if (ret < 0) { - drm_err(&dev_priv->drm, "Failed to disable qgv points (%d)\n", ret); + drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask); return ret; } @@ -76,6 +139,7 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, int i, ret; qi->num_points = dram_info->num_qgv_points; + qi->num_psf_points = dram_info->num_psf_gv_points; if (DISPLAY_VER(dev_priv) == 12) switch (dram_info->type) { @@ -99,7 +163,11 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, for (i = 0; i < qi->num_points; i++) { struct intel_qgv_point *sp = &qi->points[i]; - ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i); + if (IS_DG1(dev_priv)) + ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i); + else + ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i); + if (ret) return ret; @@ -109,6 +177,19 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, sp->t_rcd, sp->t_rc); } + if (qi->num_psf_points > 0) { + ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points); + if (ret) { + drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n"); + qi->num_psf_points = 0; + } + + for (i = 0; i < qi->num_psf_points; i++) + drm_dbg_kms(&dev_priv->drm, + "PSF GV %d: CLK=%d \n", + i, qi->psf_points[i].clk); + } + return 0; } @@ -118,6 +199,16 @@ static int icl_calc_bw(int dclk, int num, int den) return DIV_ROUND_CLOSEST(num * dclk * 100, den * 6); } +static int adl_calc_psf_bw(int clk) +{ + /* + * clk is multiples of 16.666MHz (100/6) + * According to BSpec PSF GV bandwidth is + * calculated as BW = 64 * clk * 16.666Mhz + */ + return DIV_ROUND_CLOSEST(64 * clk * 100, 6); +} + static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) { u16 dclk = 0; @@ -194,6 +285,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; bi->num_qgv_points = qi.num_points; + bi->num_psf_gv_points = qi.num_psf_points; for (j = 0; j < qi.num_points; j++) { const struct intel_qgv_point *sp = &qi.points[j]; @@ -217,6 +309,16 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel i, j, bi->num_planes, bi->deratedbw[j]); } + for (j = 0; j < qi.num_psf_points; j++) { + const struct intel_psf_gv_point *sp = &qi.psf_points[j]; + + bi->psf_bw[j] = adl_calc_psf_bw(sp->clk); + + drm_dbg_kms(&dev_priv->drm, + "BW%d / PSF GV %d: num_planes=%d bw=%u\n", + i, j, bi->num_planes, bi->psf_bw[j]); + } + if (bi->num_planes == 1) break; } @@ -234,6 +336,26 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel return 0; } +static void dg2_get_bw_info(struct drm_i915_private *i915) +{ + struct intel_bw_info *bi = &i915->max_bw[0]; + + /* + * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth + * that doesn't depend on the number of planes enabled. Create a + * single dummy QGV point to reflect that. DG2-G10 platforms have a + * constant 50 GB/s bandwidth, whereas DG2-G11 platforms have 38 GB/s. + */ + bi->num_planes = 1; + bi->num_qgv_points = 1; + if (IS_DG2_G11(i915)) + bi->deratedbw[0] = 38000; + else + bi->deratedbw[0] = 50000; + + i915->sagv_status = I915_SAGV_NOT_CONTROLLED; +} + static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, int num_planes, int qgv_point) { @@ -262,12 +384,23 @@ static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, return 0; } +static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv, + int psf_gv_point) +{ + const struct intel_bw_info *bi = + &dev_priv->max_bw[0]; + + return bi->psf_bw[psf_gv_point]; +} + void intel_bw_init_hw(struct drm_i915_private *dev_priv) { if (!HAS_DISPLAY(dev_priv)) return; - if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) + if (IS_DG2(dev_priv)) + dg2_get_bw_info(dev_priv); + else if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) icl_get_bw_info(dev_priv, &adls_sa_info); else if (IS_ROCKETLAKE(dev_priv)) icl_get_bw_info(dev_priv, &rkl_sa_info); @@ -534,12 +667,24 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) u32 allowed_points = 0; unsigned int max_bw_point = 0, max_bw = 0; unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; - u32 mask = (1 << num_qgv_points) - 1; + unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points; + u32 mask = 0; /* FIXME earlier gens need some checks too */ if (DISPLAY_VER(dev_priv) < 11) return 0; + /* + * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects + * it with failure if we try masking any unadvertised points. + * So need to operate only with those returned from PCode. + */ + if (num_qgv_points > 0) + mask |= REG_GENMASK(num_qgv_points - 1, 0); + + if (num_psf_gv_points > 0) + mask |= REG_GENMASK(num_psf_gv_points - 1, 0) << ADLS_PSF_PT_SHIFT; + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { unsigned int old_data_rate = @@ -602,23 +747,44 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) max_bw = max_data_rate; } if (max_data_rate >= data_rate) - allowed_points |= BIT(i); + allowed_points |= REG_FIELD_PREP(ADLS_QGV_PT_MASK, BIT(i)); + drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n", i, max_data_rate, data_rate); } + for (i = 0; i < num_psf_gv_points; i++) { + unsigned int max_data_rate = adl_psf_bw(dev_priv, i); + + if (max_data_rate >= data_rate) + allowed_points |= REG_FIELD_PREP(ADLS_PSF_PT_MASK, BIT(i)); + + drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d" + " required %d\n", + i, max_data_rate, data_rate); + } + /* * BSpec states that we always should have at least one allowed point * left, so if we couldn't - simply reject the configuration for obvious * reasons. */ - if (allowed_points == 0) { + if ((allowed_points & ADLS_QGV_PT_MASK) == 0) { drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory" " bandwidth %d for display configuration(%d active planes).\n", data_rate, num_active_planes); return -EINVAL; } + if (num_psf_gv_points > 0) { + if ((allowed_points & ADLS_PSF_PT_MASK) == 0) { + drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory" + " bandwidth %d for display configuration(%d active planes).\n", + data_rate, num_active_planes); + return -EINVAL; + } + } + /* * Leave only single point with highest bandwidth, if * we can't enable SAGV due to the increased memory latency it may diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 613ffcc68eba..34fa4130d5c4 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -1195,17 +1195,6 @@ static const struct intel_cdclk_vals glk_cdclk_table[] = { {} }; -static const struct intel_cdclk_vals cnl_cdclk_table[] = { - { .refclk = 19200, .cdclk = 168000, .divider = 4, .ratio = 35 }, - { .refclk = 19200, .cdclk = 336000, .divider = 2, .ratio = 35 }, - { .refclk = 19200, .cdclk = 528000, .divider = 2, .ratio = 55 }, - - { .refclk = 24000, .cdclk = 168000, .divider = 4, .ratio = 28 }, - { .refclk = 24000, .cdclk = 336000, .divider = 2, .ratio = 28 }, - { .refclk = 24000, .cdclk = 528000, .divider = 2, .ratio = 44 }, - {} -}; - static const struct intel_cdclk_vals icl_cdclk_table[] = { { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 }, { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 }, @@ -1290,6 +1279,16 @@ static const struct intel_cdclk_vals adlp_cdclk_table[] = { {} }; +static const struct intel_cdclk_vals dg2_cdclk_table[] = { + { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio = 9 }, + { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 }, + { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 }, + { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 }, + { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 }, + { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 }, + {} +}; + static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk) { const struct intel_cdclk_vals *table = dev_priv->cdclk.table; @@ -1329,16 +1328,6 @@ static u8 bxt_calc_voltage_level(int cdclk) return DIV_ROUND_UP(cdclk, 25000); } -static u8 cnl_calc_voltage_level(int cdclk) -{ - if (cdclk > 336000) - return 2; - else if (cdclk > 168000) - return 1; - else - return 0; -} - static u8 icl_calc_voltage_level(int cdclk) { if (cdclk > 556800) @@ -1373,15 +1362,6 @@ static u8 tgl_calc_voltage_level(int cdclk) return 0; } -static void cnl_readout_refclk(struct drm_i915_private *dev_priv, - struct intel_cdclk_config *cdclk_config) -{ - if (intel_de_read(dev_priv, SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz) - cdclk_config->ref = 24000; - else - cdclk_config->ref = 19200; -} - static void icl_readout_refclk(struct drm_i915_private *dev_priv, struct intel_cdclk_config *cdclk_config) { @@ -1408,10 +1388,10 @@ static void bxt_de_pll_readout(struct drm_i915_private *dev_priv, { u32 val, ratio; - if (DISPLAY_VER(dev_priv) >= 11) + if (IS_DG2(dev_priv)) + cdclk_config->ref = 38400; + else if (DISPLAY_VER(dev_priv) >= 11) icl_readout_refclk(dev_priv, cdclk_config); - else if (IS_CANNONLAKE(dev_priv)) - cnl_readout_refclk(dev_priv, cdclk_config); else cdclk_config->ref = 19200; @@ -1427,11 +1407,11 @@ static void bxt_de_pll_readout(struct drm_i915_private *dev_priv, } /* - * CNL+ have the ratio directly in the PLL enable register, gen9lp had - * it in a separate PLL control register. + * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register, + * gen9lp had it in a separate PLL control register. */ - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) - ratio = val & CNL_CDCLK_PLL_RATIO_MASK; + if (DISPLAY_VER(dev_priv) >= 11) + ratio = val & ICL_CDCLK_PLL_RATIO_MASK; else ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK; @@ -1518,7 +1498,7 @@ static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco) dev_priv->cdclk.hw.vco = vco; } -static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv) +static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv) { intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE, 0); @@ -1530,12 +1510,12 @@ static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv) dev_priv->cdclk.hw.vco = 0; } -static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) +static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) { int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); u32 val; - val = CNL_CDCLK_PLL_RATIO(ratio); + val = ICL_CDCLK_PLL_RATIO(ratio); intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); val |= BXT_DE_PLL_PLL_ENABLE; @@ -1548,18 +1528,13 @@ static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) dev_priv->cdclk.hw.vco = vco; } -static bool has_cdclk_crawl(struct drm_i915_private *i915) -{ - return INTEL_INFO(i915)->has_cdclk_crawl; -} - static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco) { int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); u32 val; /* Write PLL ratio without disabling */ - val = CNL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE; + val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE; intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); /* Submit freq change request */ @@ -1628,7 +1603,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int ret; /* Inform power controller of upcoming frequency change. */ - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, SKL_CDCLK_PREPARE_FOR_CHANGE, SKL_CDCLK_READY_FOR_CHANGE, @@ -1649,16 +1624,16 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, return; } - if (has_cdclk_crawl(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) { + if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) { if (dev_priv->cdclk.hw.vco != vco) adlp_cdclk_pll_crawl(dev_priv, vco); - } else if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) { + } else if (DISPLAY_VER(dev_priv) >= 11) { if (dev_priv->cdclk.hw.vco != 0 && dev_priv->cdclk.hw.vco != vco) - cnl_cdclk_pll_disable(dev_priv); + icl_cdclk_pll_disable(dev_priv); if (dev_priv->cdclk.hw.vco != vco) - cnl_cdclk_pll_enable(dev_priv, vco); + icl_cdclk_pll_enable(dev_priv, vco); } else { if (dev_priv->cdclk.hw.vco != 0 && dev_priv->cdclk.hw.vco != vco) @@ -1684,7 +1659,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, if (pipe != INVALID_PIPE) intel_wait_for_vblank(dev_priv, pipe); - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) { + if (DISPLAY_VER(dev_priv) >= 11) { ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, cdclk_config->voltage_level); } else { @@ -1709,7 +1684,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, intel_update_cdclk(dev_priv); - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) /* * Can't read out the voltage level :( * Let's just assume everything is as expected. @@ -1857,7 +1832,7 @@ static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv, { int a_div, b_div; - if (!has_cdclk_crawl(dev_priv)) + if (!HAS_CDCLK_CRAWL(dev_priv)) return false; /* @@ -2118,7 +2093,7 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) crtc_state->port_clock >= 540000 && crtc_state->lane_count == 4) { if (DISPLAY_VER(dev_priv) == 10) { - /* Display WA #1145: glk,cnl */ + /* Display WA #1145: glk */ min_cdclk = max(316800, min_cdclk); } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) { /* Display WA #1144: skl,bxt */ @@ -2239,7 +2214,7 @@ static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state) /* * Account for port clock min voltage level requirements. - * This only really does something on CNL+ but can be + * This only really does something on DISPLA_VER >= 11 but can be * called on earlier platforms as well. * * Note that this functions assumes that 0 is @@ -2653,8 +2628,6 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) dev_priv->max_cdclk_freq = 648000; else dev_priv->max_cdclk_freq = 652800; - } else if (IS_CANNONLAKE(dev_priv)) { - dev_priv->max_cdclk_freq = 528000; } else if (IS_GEMINILAKE(dev_priv)) { dev_priv->max_cdclk_freq = 316800; } else if (IS_BROXTON(dev_priv)) { @@ -2878,13 +2851,19 @@ u32 intel_read_rawclk(struct drm_i915_private *dev_priv) */ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) { - if (IS_ALDERLAKE_P(dev_priv)) { + if (IS_DG2(dev_priv)) { + dev_priv->display.set_cdclk = bxt_set_cdclk; + dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; + dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; + dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; + dev_priv->cdclk.table = dg2_cdclk_table; + } else if (IS_ALDERLAKE_P(dev_priv)) { dev_priv->display.set_cdclk = bxt_set_cdclk; dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; - /* Wa_22011320316:adlp[a0] */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + /* Wa_22011320316:adl-p[a0] */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) dev_priv->cdclk.table = adlp_a_step_cdclk_table; else dev_priv->cdclk.table = adlp_cdclk_table; @@ -2912,12 +2891,6 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; dev_priv->display.calc_voltage_level = icl_calc_voltage_level; dev_priv->cdclk.table = icl_cdclk_table; - } else if (IS_CANNONLAKE(dev_priv)) { - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = cnl_calc_voltage_level; - dev_priv->cdclk.table = cnl_cdclk_table; } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; dev_priv->display.set_cdclk = bxt_set_cdclk; diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index dab892d2251b..afcb4bf3826c 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -305,13 +305,12 @@ static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) ilk_csc_postoff_limited_range); } else if (crtc_state->csc_enable) { /* - * On GLK+ both pipe CSC and degamma LUT are controlled + * On GLK both pipe CSC and degamma LUT are controlled * by csc_enable. Hence for the cases where the degama * LUT is needed but CSC is not we need to load an * identity matrix. */ - drm_WARN_ON(&dev_priv->drm, !IS_CANNONLAKE(dev_priv) && - !IS_GEMINILAKE(dev_priv)); + drm_WARN_ON(&dev_priv->drm, !IS_GEMINILAKE(dev_priv)); ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ilk_csc_coeff_identity, diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c index 487c54cd5982..bacdf8a16bcb 100644 --- a/drivers/gpu/drm/i915/display/intel_combo_phy.c +++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c @@ -23,9 +23,9 @@ enum { PROCMON_1_05V_DOT_1, }; -static const struct cnl_procmon { +static const struct icl_procmon { u32 dw1, dw9, dw10; -} cnl_procmon_values[] = { +} icl_procmon_values[] = { [PROCMON_0_85V_DOT_0] = { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, }, [PROCMON_0_95V_DOT_0] = @@ -38,15 +38,10 @@ static const struct cnl_procmon { { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, }, }; -/* - * CNL has just one set of registers, while gen11 has a set for each combo PHY. - * The CNL registers are equivalent to the gen11 PHY A registers, that's why we - * call the ICL macros even though the function has CNL on its name. - */ -static const struct cnl_procmon * -cnl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) +static const struct icl_procmon * +icl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) { - const struct cnl_procmon *procmon; + const struct icl_procmon *procmon; u32 val; val = intel_de_read(dev_priv, ICL_PORT_COMP_DW3(phy)); @@ -55,32 +50,32 @@ cnl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) MISSING_CASE(val); fallthrough; case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0]; + procmon = &icl_procmon_values[PROCMON_0_85V_DOT_0]; break; case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0]; + procmon = &icl_procmon_values[PROCMON_0_95V_DOT_0]; break; case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1: - procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1]; + procmon = &icl_procmon_values[PROCMON_0_95V_DOT_1]; break; case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0]; + procmon = &icl_procmon_values[PROCMON_1_05V_DOT_0]; break; case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1: - procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1]; + procmon = &icl_procmon_values[PROCMON_1_05V_DOT_1]; break; } return procmon; } -static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv, +static void icl_set_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) { - const struct cnl_procmon *procmon; + const struct icl_procmon *procmon; u32 val; - procmon = cnl_get_procmon_ref_values(dev_priv, phy); + procmon = icl_get_procmon_ref_values(dev_priv, phy); val = intel_de_read(dev_priv, ICL_PORT_COMP_DW1(phy)); val &= ~((0xff << 16) | 0xff); @@ -109,13 +104,13 @@ static bool check_phy_reg(struct drm_i915_private *dev_priv, return true; } -static bool cnl_verify_procmon_ref_values(struct drm_i915_private *dev_priv, +static bool icl_verify_procmon_ref_values(struct drm_i915_private *dev_priv, enum phy phy) { - const struct cnl_procmon *procmon; + const struct icl_procmon *procmon; bool ret; - procmon = cnl_get_procmon_ref_values(dev_priv, phy); + procmon = icl_get_procmon_ref_values(dev_priv, phy); ret = check_phy_reg(dev_priv, phy, ICL_PORT_COMP_DW1(phy), (0xff << 16) | 0xff, procmon->dw1); @@ -127,61 +122,6 @@ static bool cnl_verify_procmon_ref_values(struct drm_i915_private *dev_priv, return ret; } -static bool cnl_combo_phy_enabled(struct drm_i915_private *dev_priv) -{ - return !(intel_de_read(dev_priv, CHICKEN_MISC_2) & CNL_COMP_PWR_DOWN) && - (intel_de_read(dev_priv, CNL_PORT_COMP_DW0) & COMP_INIT); -} - -static bool cnl_combo_phy_verify_state(struct drm_i915_private *dev_priv) -{ - enum phy phy = PHY_A; - bool ret; - - if (!cnl_combo_phy_enabled(dev_priv)) - return false; - - ret = cnl_verify_procmon_ref_values(dev_priv, phy); - - ret &= check_phy_reg(dev_priv, phy, CNL_PORT_CL1CM_DW5, - CL_POWER_DOWN_ENABLE, CL_POWER_DOWN_ENABLE); - - return ret; -} - -static void cnl_combo_phys_init(struct drm_i915_private *dev_priv) -{ - u32 val; - - val = intel_de_read(dev_priv, CHICKEN_MISC_2); - val &= ~CNL_COMP_PWR_DOWN; - intel_de_write(dev_priv, CHICKEN_MISC_2, val); - - /* Dummy PORT_A to get the correct CNL register from the ICL macro */ - cnl_set_procmon_ref_values(dev_priv, PHY_A); - - val = intel_de_read(dev_priv, CNL_PORT_COMP_DW0); - val |= COMP_INIT; - intel_de_write(dev_priv, CNL_PORT_COMP_DW0, val); - - val = intel_de_read(dev_priv, CNL_PORT_CL1CM_DW5); - val |= CL_POWER_DOWN_ENABLE; - intel_de_write(dev_priv, CNL_PORT_CL1CM_DW5, val); -} - -static void cnl_combo_phys_uninit(struct drm_i915_private *dev_priv) -{ - u32 val; - - if (!cnl_combo_phy_verify_state(dev_priv)) - drm_warn(&dev_priv->drm, - "Combo PHY HW state changed unexpectedly.\n"); - - val = intel_de_read(dev_priv, CHICKEN_MISC_2); - val |= CNL_COMP_PWR_DOWN; - intel_de_write(dev_priv, CHICKEN_MISC_2, val); -} - static bool has_phy_misc(struct drm_i915_private *i915, enum phy phy) { /* @@ -291,7 +231,7 @@ static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, DCC_MODE_SELECT_CONTINUOSLY); } - ret &= cnl_verify_procmon_ref_values(dev_priv, phy); + ret &= icl_verify_procmon_ref_values(dev_priv, phy); if (phy_is_master(dev_priv, phy)) { ret &= check_phy_reg(dev_priv, phy, ICL_PORT_COMP_DW8(phy), @@ -415,7 +355,7 @@ skip_phy_misc: intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), val); } - cnl_set_procmon_ref_values(dev_priv, phy); + icl_set_procmon_ref_values(dev_priv, phy); if (phy_is_master(dev_priv, phy)) { val = intel_de_read(dev_priv, ICL_PORT_COMP_DW8(phy)); @@ -474,16 +414,10 @@ skip_phy_misc: void intel_combo_phy_init(struct drm_i915_private *i915) { - if (DISPLAY_VER(i915) >= 11) - icl_combo_phys_init(i915); - else if (IS_CANNONLAKE(i915)) - cnl_combo_phys_init(i915); + icl_combo_phys_init(i915); } void intel_combo_phy_uninit(struct drm_i915_private *i915) { - if (DISPLAY_VER(i915) >= 11) - icl_combo_phys_uninit(i915); - else if (IS_CANNONLAKE(i915)) - cnl_combo_phys_uninit(i915); + icl_combo_phys_uninit(i915); } diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c index 448c4d99ac35..254e67141a77 100644 --- a/drivers/gpu/drm/i915/display/intel_crtc.c +++ b/drivers/gpu/drm/i915/display/intel_crtc.c @@ -335,7 +335,7 @@ int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc; } - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) drm_crtc_create_scaling_filter_property(&crtc->base, BIT(DRM_SCALING_FILTER_DEFAULT) | BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 966e020331fb..c7618fef0143 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -383,7 +383,7 @@ static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state, if (plane_state->hw.rotation & DRM_MODE_ROTATE_180) cntl |= MCURSOR_ROTATE_180; - /* Wa_22012358565:adlp */ + /* Wa_22012358565:adl-p */ if (DISPLAY_VER(dev_priv) == 13) cntl |= MCURSOR_ARB_SLOTS(1); @@ -629,12 +629,16 @@ intel_legacy_cursor_update(struct drm_plane *_plane, /* * When crtc is inactive or there is a modeset pending, - * wait for it to complete in the slowpath + * wait for it to complete in the slowpath. + * PSR2 selective fetch also requires the slow path as + * PSR2 plane and transcoder registers can only be updated during + * vblank. * * FIXME bigjoiner fastpath would be good */ if (!crtc_state->hw.active || intel_crtc_needs_modeset(crtc_state) || - crtc_state->update_pipe || crtc_state->bigjoiner) + crtc_state->update_pipe || crtc_state->bigjoiner || + crtc_state->enable_psr2_sel_fetch) goto slow; /* @@ -801,7 +805,7 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, if (DISPLAY_VER(dev_priv) >= 12) drm_plane_enable_fb_damage_clips(&cursor->base); - drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs); + intel_plane_helper_add(cursor); return cursor; diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index 26a3aa73fcc4..9903a78df896 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -51,6 +51,7 @@ #include "intel_panel.h" #include "intel_pps.h" #include "intel_psr.h" +#include "intel_snps_phy.h" #include "intel_sprite.h" #include "intel_tc.h" #include "intel_vdsc.h" @@ -171,14 +172,18 @@ void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, static void intel_wait_ddi_buf_active(struct drm_i915_private *dev_priv, enum port port) { + int ret; + /* Wait > 518 usecs for DDI_BUF_CTL to be non idle */ if (DISPLAY_VER(dev_priv) < 10) { usleep_range(518, 1000); return; } - if (wait_for_us(!(intel_de_read(dev_priv, DDI_BUF_CTL(port)) & - DDI_BUF_IS_IDLE), 500)) + ret = _wait_for(!(intel_de_read(dev_priv, DDI_BUF_CTL(port)) & + DDI_BUF_IS_IDLE), IS_DG2(dev_priv) ? 1200 : 500, 10, 10); + + if (ret) drm_err(&dev_priv->drm, "Timeout waiting for DDI BUF %c to get active\n", port_name(port)); } @@ -822,7 +827,7 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, static enum intel_display_power_domain intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port) { - /* CNL+ HW requires corresponding AUX IOs to be powered up for PSR with + /* ICL+ HW requires corresponding AUX IOs to be powered up for PSR with * DC states enabled at the same time, while for driver initiated AUX * transfers we need the same AUX IOs to be powered but with DC states * disabled. Accordingly use the AUX power domain here which leaves DC @@ -1017,126 +1022,6 @@ static u8 intel_ddi_dp_preemph_max(struct intel_dp *intel_dp) return DP_TRAIN_PRE_EMPH_LEVEL_3; } -static void cnl_ddi_vswing_program(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_ddi_buf_trans *ddi_translations; - enum port port = encoder->port; - int n_entries, ln; - u32 val; - - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) - return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; - - /* Set PORT_TX_DW5 Scaling Mode Sel to 010b. */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW5_LN0(port)); - val &= ~SCALING_MODE_SEL_MASK; - val |= SCALING_MODE_SEL(2); - intel_de_write(dev_priv, CNL_PORT_TX_DW5_GRP(port), val); - - /* Program PORT_TX_DW2 */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW2_LN0(port)); - val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | - RCOMP_SCALAR_MASK); - val |= SWING_SEL_UPPER(ddi_translations->entries[level].cnl.dw2_swing_sel); - val |= SWING_SEL_LOWER(ddi_translations->entries[level].cnl.dw2_swing_sel); - /* Rcomp scalar is fixed as 0x98 for every table entry */ - val |= RCOMP_SCALAR(0x98); - intel_de_write(dev_priv, CNL_PORT_TX_DW2_GRP(port), val); - - /* Program PORT_TX_DW4 */ - /* We cannot write to GRP. It would overrite individual loadgen */ - for (ln = 0; ln < 4; ln++) { - val = intel_de_read(dev_priv, CNL_PORT_TX_DW4_LN(ln, port)); - val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | - CURSOR_COEFF_MASK); - val |= POST_CURSOR_1(ddi_translations->entries[level].cnl.dw4_post_cursor_1); - val |= POST_CURSOR_2(ddi_translations->entries[level].cnl.dw4_post_cursor_2); - val |= CURSOR_COEFF(ddi_translations->entries[level].cnl.dw4_cursor_coeff); - intel_de_write(dev_priv, CNL_PORT_TX_DW4_LN(ln, port), val); - } - - /* Program PORT_TX_DW5 */ - /* All DW5 values are fixed for every table entry */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW5_LN0(port)); - val &= ~RTERM_SELECT_MASK; - val |= RTERM_SELECT(6); - val |= TAP3_DISABLE; - intel_de_write(dev_priv, CNL_PORT_TX_DW5_GRP(port), val); - - /* Program PORT_TX_DW7 */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW7_LN0(port)); - val &= ~N_SCALAR_MASK; - val |= N_SCALAR(ddi_translations->entries[level].cnl.dw7_n_scalar); - intel_de_write(dev_priv, CNL_PORT_TX_DW7_GRP(port), val); -} - -static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum port port = encoder->port; - int width, rate, ln; - u32 val; - - width = crtc_state->lane_count; - rate = crtc_state->port_clock; - - /* - * 1. If port type is eDP or DP, - * set PORT_PCS_DW1 cmnkeeper_enable to 1b, - * else clear to 0b. - */ - val = intel_de_read(dev_priv, CNL_PORT_PCS_DW1_LN0(port)); - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - val &= ~COMMON_KEEPER_EN; - else - val |= COMMON_KEEPER_EN; - intel_de_write(dev_priv, CNL_PORT_PCS_DW1_GRP(port), val); - - /* 2. Program loadgen select */ - /* - * Program PORT_TX_DW4_LN depending on Bit rate and used lanes - * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1) - * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0) - * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0) - */ - for (ln = 0; ln <= 3; ln++) { - val = intel_de_read(dev_priv, CNL_PORT_TX_DW4_LN(ln, port)); - val &= ~LOADGEN_SELECT; - - if ((rate <= 600000 && width == 4 && ln >= 1) || - (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) { - val |= LOADGEN_SELECT; - } - intel_de_write(dev_priv, CNL_PORT_TX_DW4_LN(ln, port), val); - } - - /* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */ - val = intel_de_read(dev_priv, CNL_PORT_CL1CM_DW5); - val |= SUS_CLOCK_CONFIG; - intel_de_write(dev_priv, CNL_PORT_CL1CM_DW5, val); - - /* 4. Clear training enable to change swing values */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW5_LN0(port)); - val &= ~TX_TRAINING_EN; - intel_de_write(dev_priv, CNL_PORT_TX_DW5_GRP(port), val); - - /* 5. Program swing and de-emphasis */ - cnl_ddi_vswing_program(encoder, crtc_state, level); - - /* 6. Set training enable to trigger update */ - val = intel_de_read(dev_priv, CNL_PORT_TX_DW5_LN0(port)); - val |= TX_TRAINING_EN; - intel_de_write(dev_priv, CNL_PORT_TX_DW5_GRP(port), val); -} - static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int level) @@ -1175,8 +1060,8 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, val = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN0(phy)); val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | RCOMP_SCALAR_MASK); - val |= SWING_SEL_UPPER(ddi_translations->entries[level].cnl.dw2_swing_sel); - val |= SWING_SEL_LOWER(ddi_translations->entries[level].cnl.dw2_swing_sel); + val |= SWING_SEL_UPPER(ddi_translations->entries[level].icl.dw2_swing_sel); + val |= SWING_SEL_LOWER(ddi_translations->entries[level].icl.dw2_swing_sel); /* Program Rcomp scalar for every table entry */ val |= RCOMP_SCALAR(0x98); intel_de_write(dev_priv, ICL_PORT_TX_DW2_GRP(phy), val); @@ -1187,16 +1072,16 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, val = intel_de_read(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy)); val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | CURSOR_COEFF_MASK); - val |= POST_CURSOR_1(ddi_translations->entries[level].cnl.dw4_post_cursor_1); - val |= POST_CURSOR_2(ddi_translations->entries[level].cnl.dw4_post_cursor_2); - val |= CURSOR_COEFF(ddi_translations->entries[level].cnl.dw4_cursor_coeff); + val |= POST_CURSOR_1(ddi_translations->entries[level].icl.dw4_post_cursor_1); + val |= POST_CURSOR_2(ddi_translations->entries[level].icl.dw4_post_cursor_2); + val |= CURSOR_COEFF(ddi_translations->entries[level].icl.dw4_cursor_coeff); intel_de_write(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy), val); } /* Program PORT_TX_DW7 */ val = intel_de_read(dev_priv, ICL_PORT_TX_DW7_LN0(phy)); val &= ~N_SCALAR_MASK; - val |= N_SCALAR(ddi_translations->entries[level].cnl.dw7_n_scalar); + val |= N_SCALAR(ddi_translations->entries[level].icl.dw7_n_scalar); intel_de_write(dev_priv, ICL_PORT_TX_DW7_GRP(phy), val); } @@ -1496,33 +1381,33 @@ static int intel_ddi_dp_level(struct intel_dp *intel_dp) } static void -tgl_set_signal_levels(struct intel_dp *intel_dp, +dg2_set_signal_levels(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; int level = intel_ddi_dp_level(intel_dp); - tgl_ddi_vswing_sequence(encoder, crtc_state, level); + intel_snps_phy_ddi_vswing_sequence(encoder, level); } static void -icl_set_signal_levels(struct intel_dp *intel_dp, +tgl_set_signal_levels(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; int level = intel_ddi_dp_level(intel_dp); - icl_ddi_vswing_sequence(encoder, crtc_state, level); + tgl_ddi_vswing_sequence(encoder, crtc_state, level); } static void -cnl_set_signal_levels(struct intel_dp *intel_dp, +icl_set_signal_levels(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; int level = intel_ddi_dp_level(intel_dp); - cnl_ddi_vswing_sequence(encoder, crtc_state, level); + icl_ddi_vswing_sequence(encoder, crtc_state, level); } static void @@ -1560,7 +1445,7 @@ hsw_set_signal_levels(struct intel_dp *intel_dp, intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); } -static void _cnl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg, +static void _icl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_sel_mask, u32 clk_sel, u32 clk_off) { mutex_lock(&i915->dpll.lock); @@ -1576,7 +1461,7 @@ static void _cnl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg, mutex_unlock(&i915->dpll.lock); } -static void _cnl_ddi_disable_clock(struct drm_i915_private *i915, i915_reg_t reg, +static void _icl_ddi_disable_clock(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_off) { mutex_lock(&i915->dpll.lock); @@ -1586,14 +1471,14 @@ static void _cnl_ddi_disable_clock(struct drm_i915_private *i915, i915_reg_t reg mutex_unlock(&i915->dpll.lock); } -static bool _cnl_ddi_is_clock_enabled(struct drm_i915_private *i915, i915_reg_t reg, +static bool _icl_ddi_is_clock_enabled(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_off) { return !(intel_de_read(i915, reg) & clk_off); } static struct intel_shared_dpll * -_cnl_ddi_get_pll(struct drm_i915_private *i915, i915_reg_t reg, +_icl_ddi_get_pll(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_sel_mask, u32 clk_sel_shift) { enum intel_dpll_id id; @@ -1613,7 +1498,7 @@ static void adls_ddi_enable_clock(struct intel_encoder *encoder, if (drm_WARN_ON(&i915->drm, !pll)) return; - _cnl_ddi_enable_clock(i915, ADLS_DPCLKA_CFGCR(phy), + _icl_ddi_enable_clock(i915, ADLS_DPCLKA_CFGCR(phy), ADLS_DPCLKA_CFGCR_DDI_CLK_SEL_MASK(phy), pll->info->id << ADLS_DPCLKA_CFGCR_DDI_SHIFT(phy), ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); @@ -1624,7 +1509,7 @@ static void adls_ddi_disable_clock(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - _cnl_ddi_disable_clock(i915, ADLS_DPCLKA_CFGCR(phy), + _icl_ddi_disable_clock(i915, ADLS_DPCLKA_CFGCR(phy), ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1633,7 +1518,7 @@ static bool adls_ddi_is_clock_enabled(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_is_clock_enabled(i915, ADLS_DPCLKA_CFGCR(phy), + return _icl_ddi_is_clock_enabled(i915, ADLS_DPCLKA_CFGCR(phy), ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1642,7 +1527,7 @@ static struct intel_shared_dpll *adls_ddi_get_pll(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_get_pll(i915, ADLS_DPCLKA_CFGCR(phy), + return _icl_ddi_get_pll(i915, ADLS_DPCLKA_CFGCR(phy), ADLS_DPCLKA_CFGCR_DDI_CLK_SEL_MASK(phy), ADLS_DPCLKA_CFGCR_DDI_SHIFT(phy)); } @@ -1657,7 +1542,7 @@ static void rkl_ddi_enable_clock(struct intel_encoder *encoder, if (drm_WARN_ON(&i915->drm, !pll)) return; - _cnl_ddi_enable_clock(i915, ICL_DPCLKA_CFGCR0, + _icl_ddi_enable_clock(i915, ICL_DPCLKA_CFGCR0, RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy), RKL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy), RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); @@ -1668,7 +1553,7 @@ static void rkl_ddi_disable_clock(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - _cnl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0, + _icl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0, RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1677,7 +1562,7 @@ static bool rkl_ddi_is_clock_enabled(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0, + return _icl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0, RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1686,7 +1571,7 @@ static struct intel_shared_dpll *rkl_ddi_get_pll(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0, + return _icl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0, RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy), RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)); } @@ -1710,7 +1595,7 @@ static void dg1_ddi_enable_clock(struct intel_encoder *encoder, (pll->info->id >= DPLL_ID_DG1_DPLL2 && phy < PHY_C))) return; - _cnl_ddi_enable_clock(i915, DG1_DPCLKA_CFGCR0(phy), + _icl_ddi_enable_clock(i915, DG1_DPCLKA_CFGCR0(phy), DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy), DG1_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy), DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); @@ -1721,7 +1606,7 @@ static void dg1_ddi_disable_clock(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - _cnl_ddi_disable_clock(i915, DG1_DPCLKA_CFGCR0(phy), + _icl_ddi_disable_clock(i915, DG1_DPCLKA_CFGCR0(phy), DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1730,7 +1615,7 @@ static bool dg1_ddi_is_clock_enabled(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_is_clock_enabled(i915, DG1_DPCLKA_CFGCR0(phy), + return _icl_ddi_is_clock_enabled(i915, DG1_DPCLKA_CFGCR0(phy), DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1767,7 +1652,7 @@ static void icl_ddi_combo_enable_clock(struct intel_encoder *encoder, if (drm_WARN_ON(&i915->drm, !pll)) return; - _cnl_ddi_enable_clock(i915, ICL_DPCLKA_CFGCR0, + _icl_ddi_enable_clock(i915, ICL_DPCLKA_CFGCR0, ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy), ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy), ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); @@ -1778,7 +1663,7 @@ static void icl_ddi_combo_disable_clock(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - _cnl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0, + _icl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0, ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1787,7 +1672,7 @@ static bool icl_ddi_combo_is_clock_enabled(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0, + return _icl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0, ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); } @@ -1796,7 +1681,7 @@ struct intel_shared_dpll *icl_ddi_combo_get_pll(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - return _cnl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0, + return _icl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0, ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy), ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)); } @@ -1929,50 +1814,6 @@ static struct intel_shared_dpll *icl_ddi_tc_get_pll(struct intel_encoder *encode return intel_get_shared_dpll_by_id(i915, id); } -static void cnl_ddi_enable_clock(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct intel_shared_dpll *pll = crtc_state->shared_dpll; - enum port port = encoder->port; - - if (drm_WARN_ON(&i915->drm, !pll)) - return; - - _cnl_ddi_enable_clock(i915, DPCLKA_CFGCR0, - DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port), - DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port), - DPCLKA_CFGCR0_DDI_CLK_OFF(port)); -} - -static void cnl_ddi_disable_clock(struct intel_encoder *encoder) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - enum port port = encoder->port; - - _cnl_ddi_disable_clock(i915, DPCLKA_CFGCR0, - DPCLKA_CFGCR0_DDI_CLK_OFF(port)); -} - -static bool cnl_ddi_is_clock_enabled(struct intel_encoder *encoder) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - enum port port = encoder->port; - - return _cnl_ddi_is_clock_enabled(i915, DPCLKA_CFGCR0, - DPCLKA_CFGCR0_DDI_CLK_OFF(port)); -} - -static struct intel_shared_dpll *cnl_ddi_get_pll(struct intel_encoder *encoder) -{ - struct drm_i915_private *i915 = to_i915(encoder->base.dev); - enum port port = encoder->port; - - return _cnl_ddi_get_pll(i915, DPCLKA_CFGCR0, - DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port), - DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)); -} - static struct intel_shared_dpll *bxt_ddi_get_pll(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); @@ -2196,7 +2037,7 @@ void intel_ddi_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) ddi_clk_needed = false; } - if (ddi_clk_needed || !encoder->disable_clock || + if (ddi_clk_needed || !encoder->is_clock_enabled || !encoder->is_clock_enabled(encoder)) return; @@ -2410,6 +2251,15 @@ static void intel_ddi_power_up_lanes(struct intel_encoder *encoder, } } +/* Splitter enable for eDP MSO is limited to certain pipes. */ +static u8 intel_ddi_splitter_pipe_mask(struct drm_i915_private *i915) +{ + if (IS_ALDERLAKE_P(i915)) + return BIT(PIPE_A) | BIT(PIPE_B); + else + return BIT(PIPE_A); +} + static void intel_ddi_mso_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { @@ -2427,8 +2277,7 @@ static void intel_ddi_mso_get_config(struct intel_encoder *encoder, if (!pipe_config->splitter.enable) return; - /* Splitter enable is supported for pipe A only. */ - if (drm_WARN_ON(&i915->drm, pipe != PIPE_A)) { + if (drm_WARN_ON(&i915->drm, !(intel_ddi_splitter_pipe_mask(i915) & BIT(pipe)))) { pipe_config->splitter.enable = false; return; } @@ -2460,10 +2309,6 @@ static void intel_ddi_mso_configure(const struct intel_crtc_state *crtc_state) return; if (crtc_state->splitter.enable) { - /* Splitter enable is supported for pipe A only. */ - if (drm_WARN_ON(&i915->drm, pipe != PIPE_A)) - return; - dss1 |= SPLITTER_ENABLE; dss1 |= OVERLAP_PIXELS(crtc_state->splitter.pixel_overlap); if (crtc_state->splitter.link_count == 2) @@ -2477,6 +2322,116 @@ static void intel_ddi_mso_configure(const struct intel_crtc_state *crtc_state) OVERLAP_PIXELS_MASK, dss1); } +static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + struct intel_digital_port *dig_port = enc_to_dig_port(encoder); + bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); + int level = intel_ddi_dp_level(intel_dp); + + intel_dp_set_link_params(intel_dp, crtc_state->port_clock, + crtc_state->lane_count); + + /* + * 1. Enable Power Wells + * + * This was handled at the beginning of intel_atomic_commit_tail(), + * before we called down into this function. + */ + + /* 2. Enable Panel Power if PPS is required */ + intel_pps_on(intel_dp); + + /* + * 3. Enable the port PLL. + */ + intel_ddi_enable_clock(encoder, crtc_state); + + /* 4. Enable IO power */ + if (!intel_phy_is_tc(dev_priv, phy) || + dig_port->tc_mode != TC_PORT_TBT_ALT) + dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, + dig_port->ddi_io_power_domain); + + /* + * 5. The rest of the below are substeps under the bspec's "Enable and + * Train Display Port" step. Note that steps that are specific to + * MST will be handled by intel_mst_pre_enable_dp() before/after it + * calls into this function. Also intel_mst_pre_enable_dp() only calls + * us when active_mst_links==0, so any steps designated for "single + * stream or multi-stream master transcoder" can just be performed + * unconditionally here. + */ + + /* + * 5.a Configure Transcoder Clock Select to direct the Port clock to the + * Transcoder. + */ + intel_ddi_enable_pipe_clock(encoder, crtc_state); + + /* 5.b Not relevant to i915 for now */ + + /* + * 5.c Configure TRANS_DDI_FUNC_CTL DDI Select, DDI Mode Select & MST + * Transport Select + */ + intel_ddi_config_transcoder_func(encoder, crtc_state); + + /* + * 5.d Configure & enable DP_TP_CTL with link training pattern 1 + * selected + * + * This will be handled by the intel_dp_start_link_train() farther + * down this function. + */ + + /* 5.e Configure voltage swing and related IO settings */ + intel_snps_phy_ddi_vswing_sequence(encoder, level); + + /* + * 5.f Configure and enable DDI_BUF_CTL + * 5.g Wait for DDI_BUF_CTL DDI Idle Status = 0b (Not Idle), timeout + * after 1200 us. + * + * We only configure what the register value will be here. Actual + * enabling happens during link training farther down. + */ + intel_ddi_init_dp_buf_reg(encoder, crtc_state); + + if (!is_mst) + intel_dp_set_power(intel_dp, DP_SET_POWER_D0); + + intel_dp_sink_set_decompression_state(intel_dp, crtc_state, true); + /* + * DDI FEC: "anticipates enabling FEC encoding sets the FEC_READY bit + * in the FEC_CONFIGURATION register to 1 before initiating link + * training + */ + intel_dp_sink_set_fec_ready(intel_dp, crtc_state); + + /* + * 5.h Follow DisplayPort specification training sequence (see notes for + * failure handling) + * 5.i If DisplayPort multi-stream - Set DP_TP_CTL link training to Idle + * Pattern, wait for 5 idle patterns (DP_TP_STATUS Min_Idles_Sent) + * (timeout after 800 us) + */ + intel_dp_start_link_train(intel_dp, crtc_state); + + /* 5.j Set DP_TP_CTL link training to Normal */ + if (!is_trans_port_sync_mode(crtc_state)) + intel_dp_stop_link_train(intel_dp, crtc_state); + + /* 5.k Configure and enable FEC if needed */ + intel_ddi_enable_fec(encoder, crtc_state); + intel_dsc_enable(encoder, crtc_state); +} + static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, @@ -2657,8 +2612,6 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, if (DISPLAY_VER(dev_priv) >= 11) icl_ddi_vswing_sequence(encoder, crtc_state, level); - else if (IS_CANNONLAKE(dev_priv)) - cnl_ddi_vswing_sequence(encoder, crtc_state, level); else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) bxt_ddi_vswing_sequence(encoder, crtc_state, level); else @@ -2694,7 +2647,9 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - if (DISPLAY_VER(dev_priv) >= 12) + if (IS_DG2(dev_priv)) + dg2_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); + else if (DISPLAY_VER(dev_priv) >= 12) tgl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); else hsw_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); @@ -3101,12 +3056,12 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, "[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n", connector->base.id, connector->name); - if (DISPLAY_VER(dev_priv) >= 12) + if (IS_DG2(dev_priv)) + intel_snps_phy_ddi_vswing_sequence(encoder, U32_MAX); + else if (DISPLAY_VER(dev_priv) >= 12) tgl_ddi_vswing_sequence(encoder, crtc_state, level); else if (DISPLAY_VER(dev_priv) == 11) icl_ddi_vswing_sequence(encoder, crtc_state, level); - else if (IS_CANNONLAKE(dev_priv)) - cnl_ddi_vswing_sequence(encoder, crtc_state, level); else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) bxt_ddi_vswing_sequence(encoder, crtc_state, level); else @@ -3204,12 +3159,6 @@ static void intel_disable_ddi_dp(struct intel_atomic_state *state, intel_dp->link_trained = false; - if (old_crtc_state->has_audio) - intel_audio_codec_disable(encoder, - old_crtc_state, old_conn_state); - - intel_edp_drrs_disable(intel_dp, old_crtc_state); - intel_psr_disable(intel_dp, old_crtc_state); intel_edp_backlight_off(old_conn_state); /* Disable the decompression in DP Sink */ intel_dp_sink_set_decompression_state(intel_dp, old_crtc_state, @@ -3227,10 +3176,6 @@ static void intel_disable_ddi_hdmi(struct intel_atomic_state *state, struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct drm_connector *connector = old_conn_state->connector; - if (old_crtc_state->has_audio) - intel_audio_codec_disable(encoder, - old_crtc_state, old_conn_state); - if (!intel_hdmi_handle_sink_scrambling(encoder, connector, false, false)) drm_dbg_kms(&i915->drm, @@ -3238,6 +3183,25 @@ static void intel_disable_ddi_hdmi(struct intel_atomic_state *state, connector->base.id, connector->name); } +static void intel_pre_disable_ddi(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *old_crtc_state, + const struct drm_connector_state *old_conn_state) +{ + struct intel_dp *intel_dp; + + if (old_crtc_state->has_audio) + intel_audio_codec_disable(encoder, old_crtc_state, + old_conn_state); + + if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI)) + return; + + intel_dp = enc_to_intel_dp(encoder); + intel_edp_drrs_disable(intel_dp, old_crtc_state); + intel_psr_disable(intel_dp, old_crtc_state); +} + static void intel_disable_ddi(struct intel_atomic_state *state, struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, @@ -3454,7 +3418,7 @@ static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv, if (cpu_transcoder == TRANSCODER_EDP) return false; - if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO)) + if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO_MMIO)) return false; return intel_de_read(dev_priv, HSW_AUD_PIN_ELD_CP_VLD) & @@ -3470,8 +3434,6 @@ void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, crtc_state->min_voltage_level = 3; else if (DISPLAY_VER(dev_priv) >= 11 && crtc_state->port_clock > 594000) crtc_state->min_voltage_level = 1; - else if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000) - crtc_state->min_voltage_level = 2; } static enum transcoder bdw_transcoder_master_readout(struct drm_i915_private *dev_priv, @@ -3745,6 +3707,15 @@ void intel_ddi_get_clock(struct intel_encoder *encoder, &crtc_state->dpll_hw_state); } +static void dg2_ddi_get_config(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state) +{ + intel_mpllb_readout_hw_state(encoder, &crtc_state->mpllb_state); + crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder, &crtc_state->mpllb_state); + + intel_ddi_get_config(encoder, crtc_state); +} + static void adls_ddi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) { @@ -3812,13 +3783,6 @@ static void icl_ddi_tc_get_config(struct intel_encoder *encoder, intel_ddi_get_config(encoder, crtc_state); } -static void cnl_ddi_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) -{ - intel_ddi_get_clock(encoder, crtc_state, cnl_ddi_get_pll(encoder)); - intel_ddi_get_config(encoder, crtc_state); -} - static void bxt_ddi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) { @@ -4065,12 +4029,12 @@ intel_ddi_init_dp_connector(struct intel_digital_port *dig_port) dig_port->dp.set_link_train = intel_ddi_set_link_train; dig_port->dp.set_idle_link_train = intel_ddi_set_idle_link_train; - if (DISPLAY_VER(dev_priv) >= 12) + if (IS_DG2(dev_priv)) + dig_port->dp.set_signal_levels = dg2_set_signal_levels; + else if (DISPLAY_VER(dev_priv) >= 12) dig_port->dp.set_signal_levels = tgl_set_signal_levels; else if (DISPLAY_VER(dev_priv) >= 11) dig_port->dp.set_signal_levels = icl_set_signal_levels; - else if (IS_CANNONLAKE(dev_priv)) - dig_port->dp.set_signal_levels = cnl_set_signal_levels; else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) dig_port->dp.set_signal_levels = bxt_set_signal_levels; else @@ -4317,15 +4281,6 @@ static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) return true; - /* Cannonlake: Most of SKUs don't support DDI_E, and the only - * one who does also have a full A/E split called - * DDI_F what makes DDI_E useless. However for this - * case let's trust VBT info. - */ - if (IS_CANNONLAKE(dev_priv) && - !intel_bios_is_port_present(dev_priv, PORT_E)) - return true; - return false; } @@ -4430,15 +4385,6 @@ static enum hpd_pin ehl_hpd_pin(struct drm_i915_private *dev_priv, return HPD_PORT_A + port - PORT_A; } -static enum hpd_pin cnl_hpd_pin(struct drm_i915_private *dev_priv, - enum port port) -{ - if (port == PORT_F) - return HPD_PORT_E; - - return HPD_PORT_A + port - PORT_A; -} - static enum hpd_pin skl_hpd_pin(struct drm_i915_private *dev_priv, enum port port) { if (HAS_PCH_TGP(dev_priv)) @@ -4590,6 +4536,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->enable = intel_enable_ddi; encoder->pre_pll_enable = intel_ddi_pre_pll_enable; encoder->pre_enable = intel_ddi_pre_enable; + encoder->pre_disable = intel_pre_disable_ddi; encoder->disable = intel_disable_ddi; encoder->post_disable = intel_ddi_post_disable; encoder->update_pipe = intel_ddi_update_pipe; @@ -4606,7 +4553,11 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->cloneable = 0; encoder->pipe_mask = ~0; - if (IS_ALDERLAKE_S(dev_priv)) { + if (IS_DG2(dev_priv)) { + encoder->enable_clock = intel_mpllb_enable; + encoder->disable_clock = intel_mpllb_disable; + encoder->get_config = dg2_ddi_get_config; + } else if (IS_ALDERLAKE_S(dev_priv)) { encoder->enable_clock = adls_ddi_enable_clock; encoder->disable_clock = adls_ddi_disable_clock; encoder->is_clock_enabled = adls_ddi_is_clock_enabled; @@ -4645,11 +4596,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->is_clock_enabled = icl_ddi_combo_is_clock_enabled; encoder->get_config = icl_ddi_combo_get_config; } - } else if (IS_CANNONLAKE(dev_priv)) { - encoder->enable_clock = cnl_ddi_enable_clock; - encoder->disable_clock = cnl_ddi_disable_clock; - encoder->is_clock_enabled = cnl_ddi_is_clock_enabled; - encoder->get_config = cnl_ddi_get_config; } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { /* BXT/GLK have fixed PLL->port mapping */ encoder->get_config = bxt_ddi_get_config; @@ -4679,8 +4625,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->hpd_pin = ehl_hpd_pin(dev_priv, port); else if (DISPLAY_VER(dev_priv) == 11) encoder->hpd_pin = icl_hpd_pin(dev_priv, port); - else if (IS_CANNONLAKE(dev_priv)) - encoder->hpd_pin = cnl_hpd_pin(dev_priv, port); else if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv)) encoder->hpd_pin = skl_hpd_pin(dev_priv, port); else @@ -4723,12 +4667,8 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) dig_port->hpd_pulse = intel_dp_hpd_pulse; - /* Splitter enable for eDP MSO is limited to certain pipes. */ - if (dig_port->dp.mso_link_count) { - encoder->pipe_mask = BIT(PIPE_A); - if (IS_ALDERLAKE_P(dev_priv)) - encoder->pipe_mask |= BIT(PIPE_B); - } + if (dig_port->dp.mso_link_count) + encoder->pipe_mask = intel_ddi_splitter_pipe_mask(dev_priv); } /* In theory we don't need the encoder->type check, but leave it just in diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c index 63b1ae830d9a..ba2c08f1a797 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c @@ -417,199 +417,19 @@ static const struct intel_ddi_buf_trans bxt_ddi_translations_hdmi = { .hdmi_default_entry = ARRAY_SIZE(_bxt_ddi_translations_hdmi) - 1, }; -/* Voltage Swing Programming for VccIO 0.85V for DP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_dp_0_85V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x5D, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x6A, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */ - { .cnl = { 0xB, 0x7A, 0x32, 0x00, 0x0D } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7C, 0x2D, 0x00, 0x12 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x69, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xB, 0x7A, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7C, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ - { .cnl = { 0xB, 0x7D, 0x3C, 0x00, 0x03 } }, /* 650 725 0.9 */ - { .cnl = { 0x6, 0x7C, 0x34, 0x00, 0x0B } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7B, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_dp_0_85V = { - .entries = _cnl_ddi_translations_dp_0_85V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_dp_0_85V), -}; - -/* Voltage Swing Programming for VccIO 0.85V for HDMI */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_hdmi_0_85V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x60, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ - { .cnl = { 0xB, 0x73, 0x36, 0x00, 0x09 } }, /* 450 650 3.2 */ - { .cnl = { 0x6, 0x7F, 0x31, 0x00, 0x0E } }, /* 450 850 5.5 */ - { .cnl = { 0xB, 0x73, 0x3F, 0x00, 0x00 } }, /* 650 650 0.0 */ - { .cnl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 650 850 2.3 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 850 850 0.0 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_hdmi_0_85V = { - .entries = _cnl_ddi_translations_hdmi_0_85V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_hdmi_0_85V), - .hdmi_default_entry = ARRAY_SIZE(_cnl_ddi_translations_hdmi_0_85V) - 1, -}; - -/* Voltage Swing Programming for VccIO 0.85V for eDP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_edp_0_85V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x66, 0x3A, 0x00, 0x05 } }, /* 384 500 2.3 */ - { .cnl = { 0x0, 0x7F, 0x38, 0x00, 0x07 } }, /* 153 200 2.3 */ - { .cnl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 192 250 2.3 */ - { .cnl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 230 300 2.3 */ - { .cnl = { 0x9, 0x7F, 0x38, 0x00, 0x07 } }, /* 269 350 2.3 */ - { .cnl = { 0xA, 0x66, 0x3C, 0x00, 0x03 } }, /* 446 500 1.0 */ - { .cnl = { 0xB, 0x70, 0x3C, 0x00, 0x03 } }, /* 460 600 2.3 */ - { .cnl = { 0xC, 0x75, 0x3C, 0x00, 0x03 } }, /* 537 700 2.3 */ - { .cnl = { 0x2, 0x7F, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_edp_0_85V = { - .entries = _cnl_ddi_translations_edp_0_85V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_edp_0_85V), -}; - -/* Voltage Swing Programming for VccIO 0.95V for DP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_dp_0_95V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x5D, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x6A, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */ - { .cnl = { 0xB, 0x7A, 0x32, 0x00, 0x0D } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7C, 0x2D, 0x00, 0x12 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x69, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xB, 0x7A, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7C, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ - { .cnl = { 0xB, 0x7D, 0x3C, 0x00, 0x03 } }, /* 650 725 0.9 */ - { .cnl = { 0x6, 0x7C, 0x34, 0x00, 0x0B } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7B, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_dp_0_95V = { - .entries = _cnl_ddi_translations_dp_0_95V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_dp_0_95V), -}; - -/* Voltage Swing Programming for VccIO 0.95V for HDMI */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_hdmi_0_95V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x5C, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ - { .cnl = { 0xB, 0x69, 0x37, 0x00, 0x08 } }, /* 400 600 3.5 */ - { .cnl = { 0x5, 0x76, 0x31, 0x00, 0x0E } }, /* 400 800 6.0 */ - { .cnl = { 0xA, 0x5E, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ - { .cnl = { 0xB, 0x69, 0x3F, 0x00, 0x00 } }, /* 600 600 0.0 */ - { .cnl = { 0xB, 0x79, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ - { .cnl = { 0x6, 0x7D, 0x32, 0x00, 0x0D } }, /* 600 1000 4.4 */ - { .cnl = { 0x5, 0x76, 0x3F, 0x00, 0x00 } }, /* 800 800 0.0 */ - { .cnl = { 0x6, 0x7D, 0x39, 0x00, 0x06 } }, /* 800 1000 1.9 */ - { .cnl = { 0x6, 0x7F, 0x39, 0x00, 0x06 } }, /* 850 1050 1.8 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1050 1050 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_hdmi_0_95V = { - .entries = _cnl_ddi_translations_hdmi_0_95V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_hdmi_0_95V), - .hdmi_default_entry = ARRAY_SIZE(_cnl_ddi_translations_hdmi_0_95V) - 1, -}; - -/* Voltage Swing Programming for VccIO 0.95V for eDP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_edp_0_95V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x61, 0x3A, 0x00, 0x05 } }, /* 384 500 2.3 */ - { .cnl = { 0x0, 0x7F, 0x38, 0x00, 0x07 } }, /* 153 200 2.3 */ - { .cnl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 192 250 2.3 */ - { .cnl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 230 300 2.3 */ - { .cnl = { 0x9, 0x7F, 0x38, 0x00, 0x07 } }, /* 269 350 2.3 */ - { .cnl = { 0xA, 0x61, 0x3C, 0x00, 0x03 } }, /* 446 500 1.0 */ - { .cnl = { 0xB, 0x68, 0x39, 0x00, 0x06 } }, /* 460 600 2.3 */ - { .cnl = { 0xC, 0x6E, 0x39, 0x00, 0x06 } }, /* 537 700 2.3 */ - { .cnl = { 0x4, 0x7F, 0x3A, 0x00, 0x05 } }, /* 460 600 2.3 */ - { .cnl = { 0x2, 0x7F, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_edp_0_95V = { - .entries = _cnl_ddi_translations_edp_0_95V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_edp_0_95V), -}; - -/* Voltage Swing Programming for VccIO 1.05V for DP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_dp_1_05V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x58, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ - { .cnl = { 0xB, 0x64, 0x37, 0x00, 0x08 } }, /* 400 600 3.5 */ - { .cnl = { 0x5, 0x70, 0x31, 0x00, 0x0E } }, /* 400 800 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 400 1050 8.4 */ - { .cnl = { 0xB, 0x64, 0x3F, 0x00, 0x00 } }, /* 600 600 0.0 */ - { .cnl = { 0x5, 0x73, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ - { .cnl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 550 1050 5.6 */ - { .cnl = { 0x5, 0x76, 0x3E, 0x00, 0x01 } }, /* 850 900 0.5 */ - { .cnl = { 0x6, 0x7F, 0x36, 0x00, 0x09 } }, /* 750 1050 2.9 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1050 1050 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_dp_1_05V = { - .entries = _cnl_ddi_translations_dp_1_05V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_dp_1_05V), -}; - -/* Voltage Swing Programming for VccIO 1.05V for HDMI */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_hdmi_1_05V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x58, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ - { .cnl = { 0xB, 0x64, 0x37, 0x00, 0x08 } }, /* 400 600 3.5 */ - { .cnl = { 0x5, 0x70, 0x31, 0x00, 0x0E } }, /* 400 800 6.0 */ - { .cnl = { 0xA, 0x5B, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ - { .cnl = { 0xB, 0x64, 0x3F, 0x00, 0x00 } }, /* 600 600 0.0 */ - { .cnl = { 0x5, 0x73, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ - { .cnl = { 0x6, 0x7C, 0x32, 0x00, 0x0D } }, /* 600 1000 4.4 */ - { .cnl = { 0x5, 0x70, 0x3F, 0x00, 0x00 } }, /* 800 800 0.0 */ - { .cnl = { 0x6, 0x7C, 0x39, 0x00, 0x06 } }, /* 800 1000 1.9 */ - { .cnl = { 0x6, 0x7F, 0x39, 0x00, 0x06 } }, /* 850 1050 1.8 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1050 1050 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_hdmi_1_05V = { - .entries = _cnl_ddi_translations_hdmi_1_05V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_hdmi_1_05V), - .hdmi_default_entry = ARRAY_SIZE(_cnl_ddi_translations_hdmi_1_05V) - 1, -}; - -/* Voltage Swing Programming for VccIO 1.05V for eDP */ -static const union intel_ddi_buf_trans_entry _cnl_ddi_translations_edp_1_05V[] = { - /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x5E, 0x3A, 0x00, 0x05 } }, /* 384 500 2.3 */ - { .cnl = { 0x0, 0x7F, 0x38, 0x00, 0x07 } }, /* 153 200 2.3 */ - { .cnl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 192 250 2.3 */ - { .cnl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 230 300 2.3 */ - { .cnl = { 0x9, 0x7F, 0x38, 0x00, 0x07 } }, /* 269 350 2.3 */ - { .cnl = { 0xA, 0x5E, 0x3C, 0x00, 0x03 } }, /* 446 500 1.0 */ - { .cnl = { 0xB, 0x64, 0x39, 0x00, 0x06 } }, /* 460 600 2.3 */ - { .cnl = { 0xE, 0x6A, 0x39, 0x00, 0x06 } }, /* 537 700 2.3 */ - { .cnl = { 0x2, 0x7F, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ -}; - -static const struct intel_ddi_buf_trans cnl_ddi_translations_edp_1_05V = { - .entries = _cnl_ddi_translations_edp_1_05V, - .num_entries = ARRAY_SIZE(_cnl_ddi_translations_edp_1_05V), -}; - /* icl_combo_phy_ddi_translations */ static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x71, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x6C, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x71, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x6C, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3 = { @@ -619,16 +439,16 @@ static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hbr2_e static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_edp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0x0, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ - { .cnl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ - { .cnl = { 0x1, 0x7F, 0x33, 0x00, 0x0C } }, /* 200 300 3.5 */ - { .cnl = { 0x9, 0x7F, 0x31, 0x00, 0x0E } }, /* 200 350 4.9 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ - { .cnl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ - { .cnl = { 0x9, 0x7F, 0x35, 0x00, 0x0A } }, /* 250 350 2.9 */ - { .cnl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ - { .cnl = { 0x9, 0x7F, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ - { .cnl = { 0x9, 0x7F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0x0, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ + { .icl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ + { .icl = { 0x1, 0x7F, 0x33, 0x00, 0x0C } }, /* 200 300 3.5 */ + { .icl = { 0x9, 0x7F, 0x31, 0x00, 0x0E } }, /* 200 350 4.9 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ + { .icl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ + { .icl = { 0x9, 0x7F, 0x35, 0x00, 0x0A } }, /* 250 350 2.9 */ + { .icl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ + { .icl = { 0x9, 0x7F, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ + { .icl = { 0x9, 0x7F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr2 = { @@ -638,13 +458,13 @@ static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr2 static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_hdmi[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x60, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ - { .cnl = { 0xB, 0x73, 0x36, 0x00, 0x09 } }, /* 450 650 3.2 */ - { .cnl = { 0x6, 0x7F, 0x31, 0x00, 0x0E } }, /* 450 850 5.5 */ - { .cnl = { 0xB, 0x73, 0x3F, 0x00, 0x00 } }, /* 650 650 0.0 ALS */ - { .cnl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 650 850 2.3 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 850 850 0.0 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ + { .icl = { 0xA, 0x60, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ + { .icl = { 0xB, 0x73, 0x36, 0x00, 0x09 } }, /* 450 650 3.2 */ + { .icl = { 0x6, 0x7F, 0x31, 0x00, 0x0E } }, /* 450 850 5.5 */ + { .icl = { 0xB, 0x73, 0x3F, 0x00, 0x00 } }, /* 650 650 0.0 ALS */ + { .icl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 650 850 2.3 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 850 850 0.0 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ }; static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi = { @@ -655,16 +475,16 @@ static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi = { static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_dp[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x33, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x47, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x64, 0x34, 0x00, 0x0B } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x46, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x64, 0x38, 0x00, 0x07 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x61, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x38, 0x00, 0x07 } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x33, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x47, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x64, 0x34, 0x00, 0x0B } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x46, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x64, 0x38, 0x00, 0x07 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x61, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x38, 0x00, 0x07 } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_dp = { @@ -674,16 +494,16 @@ static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_dp = { static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_edp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ - { .cnl = { 0x1, 0x7F, 0x3D, 0x00, 0x02 } }, /* 200 300 3.5 */ - { .cnl = { 0xA, 0x35, 0x39, 0x00, 0x06 } }, /* 200 350 4.9 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ - { .cnl = { 0x1, 0x7F, 0x3C, 0x00, 0x03 } }, /* 250 300 1.6 */ - { .cnl = { 0xA, 0x35, 0x39, 0x00, 0x06 } }, /* 250 350 2.9 */ - { .cnl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ - { .cnl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ + { .icl = { 0x1, 0x7F, 0x3D, 0x00, 0x02 } }, /* 200 300 3.5 */ + { .icl = { 0xA, 0x35, 0x39, 0x00, 0x06 } }, /* 200 350 4.9 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ + { .icl = { 0x1, 0x7F, 0x3C, 0x00, 0x03 } }, /* 250 300 1.6 */ + { .icl = { 0xA, 0x35, 0x39, 0x00, 0x06 } }, /* 250 350 2.9 */ + { .icl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ + { .icl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_edp_hbr2 = { @@ -693,16 +513,16 @@ static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_edp_hbr2 static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp_hbr[] = { /* NT mV Trans mV db */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ - { .cnl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ - { .cnl = { 0x1, 0x7F, 0x33, 0x00, 0x0C } }, /* 200 300 3.5 */ - { .cnl = { 0xA, 0x35, 0x36, 0x00, 0x09 } }, /* 200 350 4.9 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ - { .cnl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ - { .cnl = { 0xA, 0x35, 0x35, 0x00, 0x0A } }, /* 250 350 2.9 */ - { .cnl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ - { .cnl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ + { .icl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ + { .icl = { 0x1, 0x7F, 0x33, 0x00, 0x0C } }, /* 200 300 3.5 */ + { .icl = { 0xA, 0x35, 0x36, 0x00, 0x09 } }, /* 200 350 4.9 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ + { .icl = { 0x1, 0x7F, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ + { .icl = { 0xA, 0x35, 0x35, 0x00, 0x0A } }, /* 250 350 2.9 */ + { .icl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ + { .icl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr = { @@ -712,16 +532,16 @@ static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr = static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ - { .cnl = { 0x1, 0x7F, 0x3D, 0x00, 0x02 } }, /* 200 300 3.5 */ - { .cnl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 200 350 4.9 */ - { .cnl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ - { .cnl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 300 1.6 */ - { .cnl = { 0xA, 0x35, 0x3A, 0x00, 0x05 } }, /* 250 350 2.9 */ - { .cnl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ - { .cnl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ + { .icl = { 0x1, 0x7F, 0x3D, 0x00, 0x02 } }, /* 200 300 3.5 */ + { .icl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 200 350 4.9 */ + { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 250 0.0 */ + { .icl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 250 300 1.6 */ + { .icl = { 0xA, 0x35, 0x3A, 0x00, 0x05 } }, /* 250 350 2.9 */ + { .icl = { 0x1, 0x7F, 0x3F, 0x00, 0x00 } }, /* 300 300 0.0 */ + { .icl = { 0xA, 0x35, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr2 = { @@ -731,16 +551,16 @@ static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr2 static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_rbr_hbr[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x43, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x60, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x60, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x43, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x60, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x60, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans dg1_combo_phy_ddi_translations_dp_rbr_hbr = { @@ -750,16 +570,16 @@ static const struct intel_ddi_buf_trans dg1_combo_phy_ddi_translations_dp_rbr_hb static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x43, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x60, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x58, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x43, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x60, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x58, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans dg1_combo_phy_ddi_translations_dp_hbr2_hbr3 = { @@ -885,16 +705,16 @@ static const struct intel_ddi_buf_trans tgl_dkl_phy_ddi_translations_hdmi = { static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_hbr[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x71, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7D, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x6C, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x71, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7D, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x6C, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr = { @@ -904,16 +724,16 @@ static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr = static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x63, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x61, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7B, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x63, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x61, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7B, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr2 = { @@ -923,16 +743,16 @@ static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr2 = static const union intel_ddi_buf_trans_entry _tgl_uy_combo_phy_ddi_translations_dp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x60, 0x32, 0x00, 0x0D } }, /* 350 700 6.0 */ - { .cnl = { 0xC, 0x7F, 0x2D, 0x00, 0x12 } }, /* 350 900 8.2 */ - { .cnl = { 0xC, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x6F, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7D, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ - { .cnl = { 0x6, 0x60, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x34, 0x00, 0x0B } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x60, 0x32, 0x00, 0x0D } }, /* 350 700 6.0 */ + { .icl = { 0xC, 0x7F, 0x2D, 0x00, 0x12 } }, /* 350 900 8.2 */ + { .icl = { 0xC, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x6F, 0x36, 0x00, 0x09 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7D, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ + { .icl = { 0x6, 0x60, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x34, 0x00, 0x0B } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans tgl_uy_combo_phy_ddi_translations_dp_hbr2 = { @@ -946,15 +766,15 @@ static const struct intel_ddi_buf_trans tgl_uy_combo_phy_ddi_translations_dp_hbr */ static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_edp_hbr2_hobl[] = { /* VS pre-emp */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 0 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 1 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 2 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 3 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 0 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 1 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 2 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 2 0 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 2 1 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 0 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 1 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 2 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 3 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 0 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 1 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1 2 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 2 0 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 2 1 */ }; static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_edp_hbr2_hobl = { @@ -964,16 +784,16 @@ static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_edp_hbr2_ static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_hbr[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x2F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7D, 0x2A, 0x00, 0x15 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x6E, 0x3E, 0x00, 0x01 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x2F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7D, 0x2A, 0x00, 0x15 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x6E, 0x3E, 0x00, 0x01 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr = { @@ -983,16 +803,16 @@ static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr = static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x50, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x61, 0x33, 0x00, 0x0C } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2E, 0x00, 0x11 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x5F, 0x38, 0x00, 0x07 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x5F, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7E, 0x36, 0x00, 0x09 } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x50, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x61, 0x33, 0x00, 0x0C } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2E, 0x00, 0x11 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x5F, 0x38, 0x00, 0x07 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x5F, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7E, 0x36, 0x00, 0x09 } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr2_hbr3 = { @@ -1002,16 +822,16 @@ static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr2_h static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x63, 0x30, 0x00, 0x0F } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x63, 0x37, 0x00, 0x08 } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x31, 0x00, 0x0E } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x61, 0x3C, 0x00, 0x03 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7B, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x31, 0x00, 0x0E } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x63, 0x37, 0x00, 0x08 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x73, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x58, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_dp_hbr2_hbr3 = { @@ -1021,16 +841,16 @@ static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_dp_hbr2_ static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_edp_hbr2[] = { /* NT mV Trans mV db */ - { .cnl = { 0x9, 0x70, 0x3C, 0x00, 0x03 } }, /* 200 200 0.0 */ - { .cnl = { 0x9, 0x6D, 0x3A, 0x00, 0x05 } }, /* 200 250 1.9 */ - { .cnl = { 0x9, 0x7F, 0x36, 0x00, 0x09 } }, /* 200 300 3.5 */ - { .cnl = { 0x4, 0x59, 0x32, 0x00, 0x0D } }, /* 200 350 4.9 */ - { .cnl = { 0x2, 0x77, 0x3A, 0x00, 0x05 } }, /* 250 250 0.0 */ - { .cnl = { 0x2, 0x7F, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ - { .cnl = { 0x4, 0x5A, 0x36, 0x00, 0x09 } }, /* 250 350 2.9 */ - { .cnl = { 0x4, 0x5E, 0x3D, 0x00, 0x04 } }, /* 300 300 0.0 */ - { .cnl = { 0x4, 0x65, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ - { .cnl = { 0x4, 0x6F, 0x3A, 0x00, 0x05 } }, /* 350 350 0.0 */ + { .icl = { 0x9, 0x73, 0x3D, 0x00, 0x02 } }, /* 200 200 0.0 */ + { .icl = { 0x9, 0x7A, 0x3C, 0x00, 0x03 } }, /* 200 250 1.9 */ + { .icl = { 0x9, 0x7F, 0x3B, 0x00, 0x04 } }, /* 200 300 3.5 */ + { .icl = { 0x4, 0x6C, 0x33, 0x00, 0x0C } }, /* 200 350 4.9 */ + { .icl = { 0x2, 0x73, 0x3A, 0x00, 0x05 } }, /* 250 250 0.0 */ + { .icl = { 0x2, 0x7C, 0x38, 0x00, 0x07 } }, /* 250 300 1.6 */ + { .icl = { 0x4, 0x5A, 0x36, 0x00, 0x09 } }, /* 250 350 2.9 */ + { .icl = { 0x4, 0x57, 0x3D, 0x00, 0x02 } }, /* 300 300 0.0 */ + { .icl = { 0x4, 0x65, 0x38, 0x00, 0x07 } }, /* 300 350 1.3 */ + { .icl = { 0x4, 0x6C, 0x3A, 0x00, 0x05 } }, /* 350 350 0.0 */ }; static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr2 = { @@ -1040,16 +860,16 @@ static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr2 static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_edp_hbr3[] = { /* NT mV Trans mV db */ - { .cnl = { 0xA, 0x5E, 0x34, 0x00, 0x0B } }, /* 350 350 0.0 */ - { .cnl = { 0xA, 0x69, 0x32, 0x00, 0x0D } }, /* 350 500 3.1 */ - { .cnl = { 0xC, 0x74, 0x31, 0x00, 0x0E } }, /* 350 700 6.0 */ - { .cnl = { 0x6, 0x7F, 0x2E, 0x00, 0x11 } }, /* 350 900 8.2 */ - { .cnl = { 0xA, 0x5C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ - { .cnl = { 0xC, 0x7F, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ - { .cnl = { 0x6, 0x7F, 0x33, 0x00, 0x0C } }, /* 500 900 5.1 */ - { .cnl = { 0xC, 0x7F, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ - { .cnl = { 0x6, 0x7F, 0x3C, 0x00, 0x03 } }, /* 600 900 3.5 */ - { .cnl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x63, 0x31, 0x00, 0x0E } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x47, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x63, 0x37, 0x00, 0x08 } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x73, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x58, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr3 = { @@ -1057,6 +877,74 @@ static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr3 .num_entries = ARRAY_SIZE(_adls_combo_phy_ddi_translations_edp_hbr3), }; +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_hdmi[] = { + /* NT mV Trans mV db */ + { .icl = { 0x6, 0x60, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ + { .icl = { 0x6, 0x68, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xA, 0x73, 0x3F, 0x00, 0x00 } }, /* 650 650 0.0 ALS */ + { .icl = { 0xA, 0x78, 0x3F, 0x00, 0x00 } }, /* 800 800 0.0 */ + { .icl = { 0xB, 0x7F, 0x3F, 0x00, 0x00 } }, /* 1000 1000 0.0 Re-timer */ + { .icl = { 0xB, 0x7F, 0x3B, 0x00, 0x04 } }, /* Full Red -1.5 */ + { .icl = { 0xB, 0x7F, 0x39, 0x00, 0x06 } }, /* Full Red -1.8 */ + { .icl = { 0xB, 0x7F, 0x37, 0x00, 0x08 } }, /* Full Red -2.0 CRLS */ + { .icl = { 0xB, 0x7F, 0x35, 0x00, 0x0A } }, /* Full Red -2.5 */ + { .icl = { 0xB, 0x7F, 0x33, 0x00, 0x0C } }, /* Full Red -3.0 */ +}; + +static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_hdmi = { + .entries = _adlp_combo_phy_ddi_translations_hdmi, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_hdmi) - 1, +}; + +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp_hbr[] = { + /* NT mV Trans mV db */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x71, 0x31, 0x00, 0x0E } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2C, 0x00, 0x13 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x73, 0x3E, 0x00, 0x01 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ +}; + +static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_dp_hbr = { + .entries = _adlp_combo_phy_ddi_translations_dp_hbr, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_dp_hbr), +}; + +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { + /* NT mV Trans mV db */ + { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ + { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ + { .icl = { 0xC, 0x71, 0x2F, 0x00, 0x10 } }, /* 350 700 6.0 */ + { .icl = { 0x6, 0x7F, 0x2B, 0x00, 0x14 } }, /* 350 900 8.2 */ + { .icl = { 0xA, 0x4C, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ + { .icl = { 0xC, 0x73, 0x34, 0x00, 0x0B } }, /* 500 700 2.9 */ + { .icl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 500 900 5.1 */ + { .icl = { 0xC, 0x63, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */ + { .icl = { 0x6, 0x7F, 0x38, 0x00, 0x07 } }, /* 600 900 3.5 */ + { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ +}; + +static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_dp_hbr2_hbr3 = { + .entries = _adlp_combo_phy_ddi_translations_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_dp_hbr2_hbr3), +}; + +static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_edp_hbr3 = { + .entries = _icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3), +}; + +static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_edp_up_to_hbr2 = { + .entries = _icl_combo_phy_ddi_translations_edp_hbr2, + .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_edp_hbr2), +}; + static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_ddi_translations_dp_hbr[] = { /* VS pre-emp Non-trans mV Pre-emph dB */ { .dkl = { 0x7, 0x0, 0x01 } }, /* 0 0 400mV 0 dB */ @@ -1270,88 +1158,6 @@ bxt_get_buf_trans(struct intel_encoder *encoder, } static const struct intel_ddi_buf_trans * -cnl_get_buf_trans_hdmi(struct intel_encoder *encoder, int *n_entries) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 voltage = intel_de_read(dev_priv, CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; - - if (voltage == VOLTAGE_INFO_0_85V) { - return intel_get_buf_trans(&cnl_ddi_translations_hdmi_0_85V, - n_entries); - } else if (voltage == VOLTAGE_INFO_0_95V) { - return intel_get_buf_trans(&cnl_ddi_translations_hdmi_0_95V, - n_entries); - } else if (voltage == VOLTAGE_INFO_1_05V) { - return intel_get_buf_trans(&cnl_ddi_translations_hdmi_1_05V, - n_entries); - } else { - *n_entries = 1; /* shut up gcc */ - MISSING_CASE(voltage); - } - return NULL; -} - -static const struct intel_ddi_buf_trans * -cnl_get_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 voltage = intel_de_read(dev_priv, CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; - - if (voltage == VOLTAGE_INFO_0_85V) { - return intel_get_buf_trans(&cnl_ddi_translations_dp_0_85V, - n_entries); - } else if (voltage == VOLTAGE_INFO_0_95V) { - return intel_get_buf_trans(&cnl_ddi_translations_dp_0_95V, - n_entries); - } else if (voltage == VOLTAGE_INFO_1_05V) { - return intel_get_buf_trans(&cnl_ddi_translations_dp_1_05V, - n_entries); - } else { - *n_entries = 1; /* shut up gcc */ - MISSING_CASE(voltage); - } - return NULL; -} - -static const struct intel_ddi_buf_trans * -cnl_get_buf_trans_edp(struct intel_encoder *encoder, int *n_entries) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 voltage = intel_de_read(dev_priv, CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; - - if (dev_priv->vbt.edp.low_vswing) { - if (voltage == VOLTAGE_INFO_0_85V) { - return intel_get_buf_trans(&cnl_ddi_translations_edp_0_85V, - n_entries); - } else if (voltage == VOLTAGE_INFO_0_95V) { - return intel_get_buf_trans(&cnl_ddi_translations_edp_0_95V, - n_entries); - } else if (voltage == VOLTAGE_INFO_1_05V) { - return intel_get_buf_trans(&cnl_ddi_translations_edp_1_05V, - n_entries); - } else { - *n_entries = 1; /* shut up gcc */ - MISSING_CASE(voltage); - } - return NULL; - } else { - return cnl_get_buf_trans_dp(encoder, n_entries); - } -} - -static const struct intel_ddi_buf_trans * -cnl_get_buf_trans(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int *n_entries) -{ - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return cnl_get_buf_trans_hdmi(encoder, n_entries); - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) - return cnl_get_buf_trans_edp(encoder, n_entries); - return cnl_get_buf_trans_dp(encoder, n_entries); -} - -static const struct intel_ddi_buf_trans * icl_get_combo_buf_trans_dp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries) @@ -1662,6 +1468,52 @@ adls_get_combo_buf_trans(struct intel_encoder *encoder, } static const struct intel_ddi_buf_trans * +adlp_get_combo_buf_trans_dp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + if (crtc_state->port_clock > 270000) + return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_dp_hbr2_hbr3, n_entries); + else + return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_dp_hbr, n_entries); +} + +static const struct intel_ddi_buf_trans * +adlp_get_combo_buf_trans_edp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + if (crtc_state->port_clock > 540000) { + return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_edp_hbr3, + n_entries); + } else if (dev_priv->vbt.edp.hobl && !intel_dp->hobl_failed) { + return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, + n_entries); + } else if (dev_priv->vbt.edp.low_vswing) { + return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_edp_up_to_hbr2, + n_entries); + } + + return adlp_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); +} + +static const struct intel_ddi_buf_trans * +adlp_get_combo_buf_trans(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_hdmi, n_entries); + else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) + return adlp_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); + else + return adlp_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); +} + +static const struct intel_ddi_buf_trans * tgl_get_dkl_buf_trans_dp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries) @@ -1738,7 +1590,7 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder) if (IS_ALDERLAKE_P(i915)) { if (intel_phy_is_combo(i915, phy)) - encoder->get_buf_trans = tgl_get_combo_buf_trans; + encoder->get_buf_trans = adlp_get_combo_buf_trans; else encoder->get_buf_trans = adlp_get_dkl_buf_trans; } else if (IS_ALDERLAKE_S(i915)) { @@ -1761,8 +1613,6 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder) encoder->get_buf_trans = icl_get_combo_buf_trans; else encoder->get_buf_trans = icl_get_mg_buf_trans; - } else if (IS_CANNONLAKE(i915)) { - encoder->get_buf_trans = cnl_get_buf_trans; } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) { encoder->get_buf_trans = bxt_get_buf_trans; } else if (IS_CML_ULX(i915) || IS_CFL_ULX(i915) || IS_KBL_ULX(i915)) { diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h index 05226eb46cd6..2acd720f9d4f 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h @@ -25,7 +25,7 @@ struct bxt_ddi_buf_trans { u8 deemphasis; }; -struct cnl_ddi_buf_trans { +struct icl_ddi_buf_trans { u8 dw2_swing_sel; u8 dw7_n_scalar; u8 dw4_cursor_coeff; @@ -48,7 +48,7 @@ struct tgl_dkl_phy_ddi_buf_trans { union intel_ddi_buf_trans_entry { struct hsw_ddi_buf_trans hsw; struct bxt_ddi_buf_trans bxt; - struct cnl_ddi_buf_trans cnl; + struct icl_ddi_buf_trans icl; struct icl_mg_phy_ddi_buf_trans mg; struct tgl_dkl_phy_ddi_buf_trans dkl; }; diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index eec6c9e9cda7..134a6acbd8fb 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -59,6 +59,7 @@ #include "display/intel_hdmi.h" #include "display/intel_lvds.h" #include "display/intel_sdvo.h" +#include "display/intel_snps_phy.h" #include "display/intel_tv.h" #include "display/intel_vdsc.h" #include "display/intel_vrr.h" @@ -975,7 +976,7 @@ void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) /* FIXME: assert CPU port conditions for SNB+ */ } - /* Wa_22012358565:adlp */ + /* Wa_22012358565:adl-p */ if (DISPLAY_VER(dev_priv) == 13) intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe), 0, PIPE_ARB_USE_PROG_SLOTS); @@ -1035,6 +1036,10 @@ void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state) if (!IS_I830(dev_priv)) val &= ~PIPECONF_ENABLE; + if (DISPLAY_VER(dev_priv) >= 12) + intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), + FECSTALL_DIS_DPTSTREAM_DPTTG, 0); + intel_de_write(dev_priv, reg, val); if ((val & PIPECONF_ENABLE) == 0) intel_wait_for_pipe_off(old_crtc_state); @@ -1331,6 +1336,9 @@ retry: ret = i915_gem_object_lock(obj, &ww); if (!ret && phys_cursor) ret = i915_gem_object_attach_phys(obj, alignment); + else if (!ret && HAS_LMEM(dev_priv)) + ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM); + /* TODO: Do we need to sync when migration becomes async? */ if (!ret) ret = i915_gem_object_pin_pages(obj); if (ret) @@ -2204,8 +2212,29 @@ unlock: clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags); } -static void icl_set_pipe_chicken(struct intel_crtc *crtc) +static bool underrun_recovery_supported(const struct intel_crtc_state *crtc_state) +{ + if (crtc_state->pch_pfit.enabled && + (crtc_state->pipe_src_w > drm_rect_width(&crtc_state->pch_pfit.dst) || + crtc_state->pipe_src_h > drm_rect_height(&crtc_state->pch_pfit.dst) || + crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)) + return false; + + if (crtc_state->dsc.compression_enable) + return false; + + if (crtc_state->has_psr2) + return false; + + if (crtc_state->splitter.enable) + return false; + + return true; +} + +static void icl_set_pipe_chicken(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; u32 tmp; @@ -2226,19 +2255,19 @@ static void icl_set_pipe_chicken(struct intel_crtc *crtc) */ tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU; - /* - * "The underrun recovery mechanism should be disabled - * when the following is enabled for this pipe: - * WiDi - * Downscaling (this includes YUV420 fullblend) - * COG - * DSC - * PSR2" - * - * FIXME: enable whenever possible... - */ - if (IS_ALDERLAKE_P(dev_priv)) - tmp |= UNDERRUN_RECOVERY_DISABLE; + if (IS_DG2(dev_priv)) { + /* + * Underrun recovery must always be disabled on DG2. However + * the chicken bit meaning is inverted compared to other + * platforms. + */ + tmp &= ~UNDERRUN_RECOVERY_ENABLE_DG2; + } else if (DISPLAY_VER(dev_priv) >= 13) { + if (underrun_recovery_supported(crtc_state)) + tmp &= ~UNDERRUN_RECOVERY_DISABLE_ADLP; + else + tmp |= UNDERRUN_RECOVERY_DISABLE_ADLP; + } intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp); } @@ -3188,6 +3217,28 @@ static void intel_encoders_enable(struct intel_atomic_state *state, } } +static void intel_encoders_pre_disable(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + const struct drm_connector_state *old_conn_state; + struct drm_connector *conn; + int i; + + for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) { + struct intel_encoder *encoder = + to_intel_encoder(old_conn_state->best_encoder); + + if (old_conn_state->crtc != &crtc->base) + continue; + + if (encoder->pre_disable) + encoder->pre_disable(state, encoder, old_crtc_state, + old_conn_state); + } +} + static void intel_encoders_disable(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -3397,13 +3448,17 @@ static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv, intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val); } -static void icl_pipe_mbus_enable(struct intel_crtc *crtc) +static void icl_pipe_mbus_enable(struct intel_crtc *crtc, bool joined_mbus) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; u32 val; - val = MBUS_DBOX_A_CREDIT(2); + /* Wa_22010947358:adl-p */ + if (IS_ALDERLAKE_P(dev_priv)) + val = joined_mbus ? MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4); + else + val = MBUS_DBOX_A_CREDIT(2); if (DISPLAY_VER(dev_priv) >= 12) { val |= MBUS_DBOX_BW_CREDIT(2); @@ -3471,7 +3526,8 @@ static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state, * Enable sequence steps 1-7 on bigjoiner master */ intel_encoders_pre_pll_enable(state, master); - intel_enable_shared_dpll(master_crtc_state); + if (master_crtc_state->shared_dpll) + intel_enable_shared_dpll(master_crtc_state); intel_encoders_pre_enable(state, master); /* and DSC on slave */ @@ -3529,7 +3585,7 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, crtc->active = true; - /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */ + /* Display WA #1180: WaDisableScalarClockGating: glk */ psl_clkgate_wa = DISPLAY_VER(dev_priv) == 10 && new_crtc_state->pch_pfit.enabled; if (psl_clkgate_wa) @@ -3553,13 +3609,17 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, hsw_set_linetime_wm(new_crtc_state); if (DISPLAY_VER(dev_priv) >= 11) - icl_set_pipe_chicken(crtc); + icl_set_pipe_chicken(new_crtc_state); if (dev_priv->display.initial_watermarks) dev_priv->display.initial_watermarks(state, crtc); - if (DISPLAY_VER(dev_priv) >= 11) - icl_pipe_mbus_enable(crtc); + if (DISPLAY_VER(dev_priv) >= 11) { + const struct intel_dbuf_state *dbuf_state = + intel_atomic_get_new_dbuf_state(state); + + icl_pipe_mbus_enable(crtc, dbuf_state->joined_mbus); + } if (new_crtc_state->bigjoiner_slave) intel_crtc_vblank_on(new_crtc_state); @@ -3693,6 +3753,13 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) { if (phy == PHY_NONE) return false; + else if (IS_DG2(dev_priv)) + /* + * DG2 outputs labelled as "combo PHY" in the bspec use + * SNPS PHYs with completely different programming, + * hence we always return false here. + */ + return false; else if (IS_ALDERLAKE_S(dev_priv)) return phy <= PHY_E; else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) @@ -3707,7 +3774,10 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy) { - if (IS_ALDERLAKE_P(dev_priv)) + if (IS_DG2(dev_priv)) + /* DG2's "TC1" output uses a SNPS PHY */ + return false; + else if (IS_ALDERLAKE_P(dev_priv)) return phy >= PHY_F && phy <= PHY_I; else if (IS_TIGERLAKE(dev_priv)) return phy >= PHY_D && phy <= PHY_I; @@ -3717,6 +3787,20 @@ bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy) return false; } +bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy) +{ + if (phy == PHY_NONE) + return false; + else if (IS_DG2(dev_priv)) + /* + * All four "combo" ports and the TC1 port (PHY E) use + * Synopsis PHYs. + */ + return phy <= PHY_E; + + return false; +} + enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port) { if (DISPLAY_VER(i915) >= 13 && port >= PORT_D_XELPD) @@ -3861,7 +3945,7 @@ static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state) } if (HAS_DDI(dev_priv) && crtc_state->has_audio) - mask |= BIT_ULL(POWER_DOMAIN_AUDIO); + mask |= BIT_ULL(POWER_DOMAIN_AUDIO_MMIO); if (crtc_state->shared_dpll) mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE); @@ -5757,16 +5841,18 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) switch (crtc_state->pipe_bpp) { case 18: - val |= PIPEMISC_DITHER_6_BPC; + val |= PIPEMISC_6_BPC; break; case 24: - val |= PIPEMISC_DITHER_8_BPC; + val |= PIPEMISC_8_BPC; break; case 30: - val |= PIPEMISC_DITHER_10_BPC; + val |= PIPEMISC_10_BPC; break; case 36: - val |= PIPEMISC_DITHER_12_BPC; + /* Port output 12BPC defined for ADLP+ */ + if (DISPLAY_VER(dev_priv) > 12) + val |= PIPEMISC_12_BPC_ADLP; break; default: MISSING_CASE(crtc_state->pipe_bpp); @@ -5819,15 +5905,27 @@ int bdw_get_pipemisc_bpp(struct intel_crtc *crtc) tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe)); - switch (tmp & PIPEMISC_DITHER_BPC_MASK) { - case PIPEMISC_DITHER_6_BPC: + switch (tmp & PIPEMISC_BPC_MASK) { + case PIPEMISC_6_BPC: return 18; - case PIPEMISC_DITHER_8_BPC: + case PIPEMISC_8_BPC: return 24; - case PIPEMISC_DITHER_10_BPC: + case PIPEMISC_10_BPC: return 30; - case PIPEMISC_DITHER_12_BPC: - return 36; + /* + * PORT OUTPUT 12 BPC defined for ADLP+. + * + * TODO: + * For previous platforms with DSI interface, bits 5:7 + * are used for storing pipe_bpp irrespective of dithering. + * Since the value of 12 BPC is not defined for these bits + * on older platforms, need to find a workaround for 12 BPC + * MIPI DSI HW readout. + */ + case PIPEMISC_12_BPC_ADLP: + if (DISPLAY_VER(dev_priv) > 12) + return 36; + fallthrough; default: MISSING_CASE(tmp); return 0; @@ -8630,10 +8728,11 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_BOOL(double_wide); - PIPE_CONF_CHECK_P(shared_dpll); + if (dev_priv->dpll.mgr) + PIPE_CONF_CHECK_P(shared_dpll); /* FIXME do the readout properly and get rid of this quirk */ - if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) { + if (dev_priv->dpll.mgr && !PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) { PIPE_CONF_CHECK_X(dpll_hw_state.dpll); PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md); PIPE_CONF_CHECK_X(dpll_hw_state.fp0); @@ -8665,7 +8764,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc); PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias); PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias); + } + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) { PIPE_CONF_CHECK_X(dsi_pll.ctrl); PIPE_CONF_CHECK_X(dsi_pll.div); @@ -9003,6 +9104,10 @@ verify_crtc_state(struct intel_crtc *crtc, if (!new_crtc_state->hw.active) return; + if (new_crtc_state->bigjoiner_slave) + /* No PLLs set for slave */ + pipe_config->shared_dpll = NULL; + intel_pipe_config_sanity_check(dev_priv, pipe_config); if (!intel_pipe_config_compare(new_crtc_state, @@ -9106,6 +9211,55 @@ verify_shared_dpll_state(struct intel_crtc *crtc, } static void +verify_mpllb_state(struct intel_atomic_state *state, + struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_mpllb_state mpllb_hw_state = { 0 }; + struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->mpllb_state; + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct intel_encoder *encoder; + + if (!IS_DG2(i915)) + return; + + if (!new_crtc_state->hw.active) + return; + + if (new_crtc_state->bigjoiner_slave) + return; + + encoder = intel_get_crtc_new_encoder(state, new_crtc_state); + intel_mpllb_readout_hw_state(encoder, &mpllb_hw_state); + +#define MPLLB_CHECK(name) do { \ + if (mpllb_sw_state->name != mpllb_hw_state.name) { \ + pipe_config_mismatch(false, crtc, "MPLLB:" __stringify(name), \ + "(expected 0x%08x, found 0x%08x)", \ + mpllb_sw_state->name, \ + mpllb_hw_state.name); \ + } \ +} while (0) + + MPLLB_CHECK(mpllb_cp); + MPLLB_CHECK(mpllb_div); + MPLLB_CHECK(mpllb_div2); + MPLLB_CHECK(mpllb_fracn1); + MPLLB_CHECK(mpllb_fracn2); + MPLLB_CHECK(mpllb_sscen); + MPLLB_CHECK(mpllb_sscstep); + + /* + * ref_control is handled by the hardware/firemware and never + * programmed by the software, but the proper values are supplied + * in the bspec for verification purposes. + */ + MPLLB_CHECK(ref_control); + +#undef MPLLB_CHECK +} + +static void intel_modeset_verify_crtc(struct intel_crtc *crtc, struct intel_atomic_state *state, struct intel_crtc_state *old_crtc_state, @@ -9118,6 +9272,7 @@ intel_modeset_verify_crtc(struct intel_crtc *crtc, verify_connector_state(state, crtc); verify_crtc_state(crtc, old_crtc_state, new_crtc_state); verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state); + verify_mpllb_state(state, new_crtc_state); } static void @@ -9743,7 +9898,7 @@ static int intel_atomic_check_async(struct intel_atomic_state *state) /* * FIXME: This check is kept generic for all platforms. - * Need to verify this for all gen9 and gen10 platforms to enable + * Need to verify this for all gen9 platforms to enable * this selectively if required. */ switch (new_plane_state->hw.fb->modifier) { @@ -10154,7 +10309,7 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state, hsw_set_linetime_wm(new_crtc_state); if (DISPLAY_VER(dev_priv) >= 11) - icl_set_pipe_chicken(crtc); + icl_set_pipe_chicken(new_crtc_state); } static void commit_pipe_pre_planes(struct intel_atomic_state *state, @@ -10288,6 +10443,8 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state, drm_WARN_ON(&dev_priv->drm, old_crtc_state->bigjoiner_slave); + intel_encoders_pre_disable(state, crtc); + intel_crtc_disable_planes(state, crtc); /* @@ -11322,7 +11479,12 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - if (IS_ALDERLAKE_P(dev_priv)) { + if (IS_DG2(dev_priv)) { + intel_ddi_init(dev_priv, PORT_A); + intel_ddi_init(dev_priv, PORT_B); + intel_ddi_init(dev_priv, PORT_C); + intel_ddi_init(dev_priv, PORT_D_XELPD); + } else if (IS_ALDERLAKE_P(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_B); intel_ddi_init(dev_priv, PORT_TC1); @@ -11375,7 +11537,6 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) intel_ddi_init(dev_priv, PORT_C); intel_ddi_init(dev_priv, PORT_D); intel_ddi_init(dev_priv, PORT_E); - intel_ddi_init(dev_priv, PORT_F); } else if (HAS_DDI(dev_priv)) { u32 found; @@ -11778,7 +11939,7 @@ intel_user_framebuffer_create(struct drm_device *dev, /* object is backed with LMEM for discrete */ i915 = to_i915(obj->base.dev); - if (HAS_LMEM(i915) && !i915_gem_object_validates_to_lmem(obj)) { + if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM)) { /* object is "remote", not in local memory */ i915_gem_object_put(obj); return ERR_PTR(-EREMOTE); @@ -13124,7 +13285,7 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv) static void intel_early_display_was(struct drm_i915_private *dev_priv) { /* - * Display WA #1185 WaDisableDARBFClkGating:cnl,glk,icl,ehl,tgl + * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl * Also known as Wa_14010480278. */ if (IS_DISPLAY_VER(dev_priv, 10, 12)) diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index c9dbaf074d77..284936f0ddab 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -561,6 +561,7 @@ struct drm_display_mode * intel_encoder_current_mode(struct intel_encoder *encoder); bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy); bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy); +bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy); enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port); int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index d5af5708c9da..8fdacb252bb1 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -2256,6 +2256,11 @@ static int i915_lpsp_capability_show(struct seq_file *m, void *data) if (connector->status != connector_status_connected) return -ENODEV; + if (DISPLAY_VER(i915) >= 13) { + LPSP_CAPABLE(encoder->port <= PORT_B); + return 0; + } + switch (DISPLAY_VER(i915)) { case 12: /* @@ -2390,6 +2395,73 @@ static const struct file_operations i915_dsc_fec_support_fops = { .write = i915_dsc_fec_support_write }; +static int i915_dsc_bpp_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct drm_device *dev = connector->dev; + struct drm_crtc *crtc; + struct intel_crtc_state *crtc_state; + struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector)); + int ret; + + if (!encoder) + return -ENODEV; + + ret = drm_modeset_lock_single_interruptible(&dev->mode_config.connection_mutex); + if (ret) + return ret; + + crtc = connector->state->crtc; + if (connector->status != connector_status_connected || !crtc) { + ret = -ENODEV; + goto out; + } + + crtc_state = to_intel_crtc_state(crtc->state); + seq_printf(m, "Compressed_BPP: %d\n", crtc_state->dsc.compressed_bpp); + +out: drm_modeset_unlock(&dev->mode_config.connection_mutex); + + return ret; +} + +static ssize_t i915_dsc_bpp_write(struct file *file, + const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct drm_connector *connector = + ((struct seq_file *)file->private_data)->private; + struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector)); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + int dsc_bpp = 0; + int ret; + + ret = kstrtoint_from_user(ubuf, len, 0, &dsc_bpp); + if (ret < 0) + return ret; + + intel_dp->force_dsc_bpp = dsc_bpp; + *offp += len; + + return len; +} + +static int i915_dsc_bpp_open(struct inode *inode, + struct file *file) +{ + return single_open(file, i915_dsc_bpp_show, + inode->i_private); +} + +static const struct file_operations i915_dsc_bpp_fops = { + .owner = THIS_MODULE, + .open = i915_dsc_bpp_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = i915_dsc_bpp_write +}; + /** * intel_connector_debugfs_add - add i915 specific connector debugfs files * @connector: pointer to a registered drm_connector @@ -2428,10 +2500,17 @@ int intel_connector_debugfs_add(struct drm_connector *connector) connector, &i915_hdcp_sink_capability_fops); } - if ((DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) && ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort && !to_intel_connector(connector)->mst_port) || connector->connector_type == DRM_MODE_CONNECTOR_eDP)) - debugfs_create_file("i915_dsc_fec_support", S_IRUGO, root, + if (DISPLAY_VER(dev_priv) >= 11 && + ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort && + !to_intel_connector(connector)->mst_port) || + connector->connector_type == DRM_MODE_CONNECTOR_eDP)) { + debugfs_create_file("i915_dsc_fec_support", 0644, root, connector, &i915_dsc_fec_support_fops); + debugfs_create_file("i915_dsc_bpp", 0644, root, + connector, &i915_dsc_bpp_fops); + } + /* Legacy panels doesn't lpsp on any platform */ if ((DISPLAY_VER(dev_priv) >= 9 || IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) && diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 285380079aab..cce1a926fcc1 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -18,6 +18,7 @@ #include "intel_pm.h" #include "intel_pps.h" #include "intel_sideband.h" +#include "intel_snps_phy.h" #include "intel_tc.h" #include "intel_vga.h" @@ -106,8 +107,10 @@ intel_display_power_domain_str(enum intel_display_power_domain domain) return "PORT_OTHER"; case POWER_DOMAIN_VGA: return "VGA"; - case POWER_DOMAIN_AUDIO: - return "AUDIO"; + case POWER_DOMAIN_AUDIO_MMIO: + return "AUDIO_MMIO"; + case POWER_DOMAIN_AUDIO_PLAYBACK: + return "AUDIO_PLAYBACK"; case POWER_DOMAIN_AUX_A: return "AUX_A"; case POWER_DOMAIN_AUX_B: @@ -341,6 +344,17 @@ static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv, { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; int pw_idx = power_well->desc->hsw.idx; + int enable_delay = power_well->desc->hsw.fixed_enable_delay; + + /* + * For some power wells we're not supposed to watch the status bit for + * an ack, but rather just wait a fixed amount of time and then + * proceed. This is only used on DG2. + */ + if (IS_DG2(dev_priv) && enable_delay) { + usleep_range(enable_delay, 2 * enable_delay); + return; + } /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */ if (intel_de_wait_for_set(dev_priv, regs->driver, @@ -436,17 +450,6 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv, hsw_wait_for_power_well_enable(dev_priv, power_well, false); - /* Display WA #1178: cnl */ - if (IS_CANNONLAKE(dev_priv) && - pw_idx >= GLK_PW_CTL_IDX_AUX_B && - pw_idx <= CNL_PW_CTL_IDX_AUX_F) { - u32 val; - - val = intel_de_read(dev_priv, CNL_AUX_ANAOVRD1(pw_idx)); - val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS; - intel_de_write(dev_priv, CNL_AUX_ANAOVRD1(pw_idx), val); - } - if (power_well->desc->hsw.has_fuses) { enum skl_power_gate pg; @@ -2508,7 +2511,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DSI) | \ BIT_ULL(POWER_DOMAIN_PORT_CRT) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ BIT_ULL(POWER_DOMAIN_AUX_C) | \ BIT_ULL(POWER_DOMAIN_GMBUS) | \ @@ -2558,7 +2562,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT_ULL(POWER_DOMAIN_PORT_DSI) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ BIT_ULL(POWER_DOMAIN_AUX_C) | \ BIT_ULL(POWER_DOMAIN_AUX_D) | \ @@ -2591,7 +2596,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */ \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define BDW_DISPLAY_POWER_DOMAINS ( \ @@ -2607,7 +2613,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */ \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \ @@ -2625,7 +2632,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_AUX_B) | \ BIT_ULL(POWER_DOMAIN_AUX_C) | \ BIT_ULL(POWER_DOMAIN_AUX_D) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS ( \ @@ -2660,7 +2668,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ BIT_ULL(POWER_DOMAIN_AUX_C) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS ( \ @@ -2693,7 +2702,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ BIT_ULL(POWER_DOMAIN_AUX_C) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS ( \ @@ -2732,63 +2742,6 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_GMBUS) | \ BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \ - BIT_ULL(POWER_DOMAIN_PIPE_B) | \ - BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \ - BIT_ULL(POWER_DOMAIN_PIPE_C) | \ - BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \ - BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \ - BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) | \ - BIT_ULL(POWER_DOMAIN_AUX_B) | \ - BIT_ULL(POWER_DOMAIN_AUX_C) | \ - BIT_ULL(POWER_DOMAIN_AUX_D) | \ - BIT_ULL(POWER_DOMAIN_AUX_F) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ - BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_AUX_A_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_AUX_A) | \ - BIT_ULL(POWER_DOMAIN_AUX_IO_A) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_AUX_B_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_AUX_B) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_AUX_C_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_AUX_C) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_AUX_D_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_AUX_D) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_AUX_F_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_AUX_F) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS ( \ - BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) | \ - BIT_ULL(POWER_DOMAIN_INIT)) -#define CNL_DISPLAY_DC_OFF_POWER_DOMAINS ( \ - CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ - BIT_ULL(POWER_DOMAIN_GT_IRQ) | \ - BIT_ULL(POWER_DOMAIN_MODESET) | \ - BIT_ULL(POWER_DOMAIN_AUX_A) | \ - BIT_ULL(POWER_DOMAIN_INIT)) - /* * ICL PW_0/PG_0 domains (HW/DMC control): * - PCI @@ -2830,7 +2783,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_AUX_E_TBT) | \ BIT_ULL(POWER_DOMAIN_AUX_F_TBT) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_INIT)) /* * - transcoder WD @@ -2922,7 +2876,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_AUX_TBT5) | \ BIT_ULL(POWER_DOMAIN_AUX_TBT6) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_INIT)) #define TGL_PW_2_POWER_DOMAINS ( \ @@ -2992,7 +2947,8 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, RKL_PW_4_POWER_DOMAINS | \ BIT_ULL(POWER_DOMAIN_PIPE_B) | \ BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) | \ @@ -3030,6 +2986,35 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_INIT)) /* + * DG1 onwards Audio MMIO/VERBS lies in PG0 power well. + */ +#define DG1_PW_3_POWER_DOMAINS ( \ + TGL_PW_4_POWER_DOMAINS | \ + BIT_ULL(POWER_DOMAIN_PIPE_B) | \ + BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \ + BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \ + BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) | \ + BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) | \ + BIT_ULL(POWER_DOMAIN_AUX_USBC1) | \ + BIT_ULL(POWER_DOMAIN_AUX_USBC2) | \ + BIT_ULL(POWER_DOMAIN_VGA) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ + BIT_ULL(POWER_DOMAIN_INIT)) + +#define DG1_PW_2_POWER_DOMAINS ( \ + DG1_PW_3_POWER_DOMAINS | \ + BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) | \ + BIT_ULL(POWER_DOMAIN_INIT)) + +#define DG1_DISPLAY_DC_OFF_POWER_DOMAINS ( \ + DG1_PW_3_POWER_DOMAINS | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ + BIT_ULL(POWER_DOMAIN_MODESET) | \ + BIT_ULL(POWER_DOMAIN_AUX_A) | \ + BIT_ULL(POWER_DOMAIN_AUX_B) | \ + BIT_ULL(POWER_DOMAIN_INIT)) + +/* * XE_LPD Power Domains * * Previous platforms required that PG(n-1) be enabled before PG(n). That @@ -3074,7 +3059,7 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, XELPD_PW_B_POWER_DOMAINS | \ XELPD_PW_C_POWER_DOMAINS | \ XELPD_PW_D_POWER_DOMAINS | \ - BIT_ULL(POWER_DOMAIN_AUDIO) | \ + BIT_ULL(POWER_DOMAIN_AUDIO_PLAYBACK) | \ BIT_ULL(POWER_DOMAIN_VGA) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_D_XELPD) | \ @@ -3115,6 +3100,7 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, #define XELPD_DISPLAY_DC_OFF_POWER_DOMAINS ( \ XELPD_PW_2_POWER_DOMAINS | \ + BIT_ULL(POWER_DOMAIN_AUDIO_MMIO) | \ BIT_ULL(POWER_DOMAIN_MODESET) | \ BIT_ULL(POWER_DOMAIN_AUX_A) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ @@ -3695,148 +3681,6 @@ static const struct i915_power_well_desc glk_power_wells[] = { }, }; -static const struct i915_power_well_desc cnl_power_wells[] = { - { - .name = "always-on", - .always_on = true, - .domains = POWER_DOMAIN_MASK, - .ops = &i9xx_always_on_power_well_ops, - .id = DISP_PW_ID_NONE, - }, - { - .name = "power well 1", - /* Handled by the DMC firmware */ - .always_on = true, - .domains = 0, - .ops = &hsw_power_well_ops, - .id = SKL_DISP_PW_1, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = SKL_PW_CTL_IDX_PW_1, - .hsw.has_fuses = true, - }, - }, - { - .name = "AUX A", - .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = GLK_PW_CTL_IDX_AUX_A, - }, - }, - { - .name = "AUX B", - .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = GLK_PW_CTL_IDX_AUX_B, - }, - }, - { - .name = "AUX C", - .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = GLK_PW_CTL_IDX_AUX_C, - }, - }, - { - .name = "AUX D", - .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = CNL_PW_CTL_IDX_AUX_D, - }, - }, - { - .name = "DC off", - .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS, - .ops = &gen9_dc_off_power_well_ops, - .id = SKL_DISP_DC_OFF, - }, - { - .name = "power well 2", - .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = SKL_DISP_PW_2, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = SKL_PW_CTL_IDX_PW_2, - .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C), - .hsw.has_vga = true, - .hsw.has_fuses = true, - }, - }, - { - .name = "DDI A IO power well", - .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = GLK_PW_CTL_IDX_DDI_A, - }, - }, - { - .name = "DDI B IO power well", - .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = SKL_PW_CTL_IDX_DDI_B, - }, - }, - { - .name = "DDI C IO power well", - .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = SKL_PW_CTL_IDX_DDI_C, - }, - }, - { - .name = "DDI D IO power well", - .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = SKL_PW_CTL_IDX_DDI_D, - }, - }, - { - .name = "DDI F IO power well", - .domains = CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = CNL_DISP_PW_DDI_F_IO, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = CNL_PW_CTL_IDX_DDI_F, - }, - }, - { - .name = "AUX F", - .domains = CNL_DISPLAY_AUX_F_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = CNL_DISP_PW_DDI_F_AUX, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = CNL_PW_CTL_IDX_AUX_F, - }, - }, -}; - static const struct i915_power_well_ops icl_aux_power_well_ops = { .sync_hw = hsw_power_well_sync_hw, .enable = icl_aux_power_well_enable, @@ -4643,6 +4487,165 @@ static const struct i915_power_well_desc rkl_power_wells[] = { }, }; +static const struct i915_power_well_desc dg1_power_wells[] = { + { + .name = "always-on", + .always_on = true, + .domains = POWER_DOMAIN_MASK, + .ops = &i9xx_always_on_power_well_ops, + .id = DISP_PW_ID_NONE, + }, + { + .name = "power well 1", + /* Handled by the DMC firmware */ + .always_on = true, + .domains = 0, + .ops = &hsw_power_well_ops, + .id = SKL_DISP_PW_1, + { + .hsw.regs = &hsw_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_PW_1, + .hsw.has_fuses = true, + }, + }, + { + .name = "DC off", + .domains = DG1_DISPLAY_DC_OFF_POWER_DOMAINS, + .ops = &gen9_dc_off_power_well_ops, + .id = SKL_DISP_DC_OFF, + }, + { + .name = "power well 2", + .domains = DG1_PW_2_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = SKL_DISP_PW_2, + { + .hsw.regs = &hsw_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_PW_2, + .hsw.has_fuses = true, + }, + }, + { + .name = "power well 3", + .domains = DG1_PW_3_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = ICL_DISP_PW_3, + { + .hsw.regs = &hsw_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_PW_3, + .hsw.irq_pipe_mask = BIT(PIPE_B), + .hsw.has_vga = true, + .hsw.has_fuses = true, + }, + }, + { + .name = "DDI A IO", + .domains = ICL_DDI_IO_A_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_ddi_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_DDI_A, + } + }, + { + .name = "DDI B IO", + .domains = ICL_DDI_IO_B_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_ddi_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_DDI_B, + } + }, + { + .name = "DDI IO TC1", + .domains = TGL_DDI_IO_TC1_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_ddi_power_well_regs, + .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1, + }, + }, + { + .name = "DDI IO TC2", + .domains = TGL_DDI_IO_TC2_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_ddi_power_well_regs, + .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2, + }, + }, + { + .name = "AUX A", + .domains = TGL_AUX_A_IO_POWER_DOMAINS, + .ops = &icl_aux_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_aux_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_AUX_A, + }, + }, + { + .name = "AUX B", + .domains = TGL_AUX_B_IO_POWER_DOMAINS, + .ops = &icl_aux_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_aux_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_AUX_B, + }, + }, + { + .name = "AUX USBC1", + .domains = TGL_AUX_IO_USBC1_POWER_DOMAINS, + .ops = &icl_aux_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_aux_power_well_regs, + .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1, + .hsw.is_tc_tbt = false, + }, + }, + { + .name = "AUX USBC2", + .domains = TGL_AUX_IO_USBC2_POWER_DOMAINS, + .ops = &icl_aux_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &icl_aux_power_well_regs, + .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2, + .hsw.is_tc_tbt = false, + }, + }, + { + .name = "power well 4", + .domains = TGL_PW_4_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &hsw_power_well_regs, + .hsw.idx = ICL_PW_CTL_IDX_PW_4, + .hsw.has_fuses = true, + .hsw.irq_pipe_mask = BIT(PIPE_C), + } + }, + { + .name = "power well 5", + .domains = TGL_PW_5_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + .id = DISP_PW_ID_NONE, + { + .hsw.regs = &hsw_power_well_regs, + .hsw.idx = TGL_PW_CTL_IDX_PW_5, + .hsw.has_fuses = true, + .hsw.irq_pipe_mask = BIT(PIPE_D), + }, + }, +}; + static const struct i915_power_well_desc xelpd_power_wells[] = { { .name = "always-on", @@ -4828,6 +4831,7 @@ static const struct i915_power_well_desc xelpd_power_wells[] = { { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_A, + .hsw.fixed_enable_delay = 600, }, }, { @@ -4838,6 +4842,7 @@ static const struct i915_power_well_desc xelpd_power_wells[] = { { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_B, + .hsw.fixed_enable_delay = 600, }, }, { @@ -4848,6 +4853,7 @@ static const struct i915_power_well_desc xelpd_power_wells[] = { { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_C, + .hsw.fixed_enable_delay = 600, }, }, { @@ -4858,6 +4864,7 @@ static const struct i915_power_well_desc xelpd_power_wells[] = { { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = XELPD_PW_CTL_IDX_AUX_D, + .hsw.fixed_enable_delay = 600, }, }, { @@ -4878,6 +4885,7 @@ static const struct i915_power_well_desc xelpd_power_wells[] = { { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1, + .hsw.fixed_enable_delay = 600, }, }, { @@ -5122,7 +5130,9 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) err = 0; } else if (DISPLAY_VER(dev_priv) >= 13) { err = set_power_wells(power_domains, xelpd_power_wells); - } else if (IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv)) { + } else if (IS_DG1(dev_priv)) { + err = set_power_wells(power_domains, dg1_power_wells); + } else if (IS_ALDERLAKE_S(dev_priv)) { err = set_power_wells_mask(power_domains, tgl_power_wells, BIT_ULL(TGL_DISP_PW_TC_COLD_OFF)); } else if (IS_ROCKETLAKE(dev_priv)) { @@ -5131,12 +5141,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) err = set_power_wells(power_domains, tgl_power_wells); } else if (DISPLAY_VER(dev_priv) == 11) { err = set_power_wells(power_domains, icl_power_wells); - } else if (IS_CNL_WITH_PORT_F(dev_priv)) { - err = set_power_wells(power_domains, cnl_power_wells); - } else if (IS_CANNONLAKE(dev_priv)) { - err = set_power_wells_mask(power_domains, cnl_power_wells, - BIT_ULL(CNL_DISP_PW_DDI_F_IO) | - BIT_ULL(CNL_DISP_PW_DDI_F_AUX)); } else if (IS_GEMINILAKE(dev_priv)) { err = set_power_wells(power_domains, glk_power_wells); } else if (IS_BROXTON(dev_priv)) { @@ -5691,75 +5695,6 @@ static void bxt_display_core_uninit(struct drm_i915_private *dev_priv) usleep_range(10, 30); /* 10 us delay per Bspec */ } -static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *well; - - gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); - - /* 1. Enable PCH Reset Handshake */ - intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv)); - - if (!HAS_DISPLAY(dev_priv)) - return; - - /* 2-3. */ - intel_combo_phy_init(dev_priv); - - /* - * 4. Enable Power Well 1 (PG1). - * The AUX IO power wells will be enabled on demand. - */ - mutex_lock(&power_domains->lock); - well = lookup_power_well(dev_priv, SKL_DISP_PW_1); - intel_power_well_enable(dev_priv, well); - mutex_unlock(&power_domains->lock); - - /* 5. Enable CD clock */ - intel_cdclk_init_hw(dev_priv); - - /* 6. Enable DBUF */ - gen9_dbuf_enable(dev_priv); - - if (resume && intel_dmc_has_payload(dev_priv)) - intel_dmc_load_program(dev_priv); -} - -static void cnl_display_core_uninit(struct drm_i915_private *dev_priv) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *well; - - if (!HAS_DISPLAY(dev_priv)) - return; - - gen9_disable_dc_states(dev_priv); - - /* 1. Disable all display engine functions -> aready done */ - - /* 2. Disable DBUF */ - gen9_dbuf_disable(dev_priv); - - /* 3. Disable CD clock */ - intel_cdclk_uninit_hw(dev_priv); - - /* - * 4. Disable Power Well 1 (PG1). - * The AUX IO power wells are toggled on demand, so they are already - * disabled at this point. - */ - mutex_lock(&power_domains->lock); - well = lookup_power_well(dev_priv, SKL_DISP_PW_1); - intel_power_well_disable(dev_priv, well); - mutex_unlock(&power_domains->lock); - - usleep_range(10, 30); /* 10 us delay per Bspec */ - - /* 5. */ - intel_combo_phy_uninit(dev_priv); -} - struct buddy_page_mask { u32 page_mask; u8 type; @@ -5798,9 +5733,14 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) unsigned long abox_mask = INTEL_INFO(dev_priv)->abox_mask; int config, i; + /* BW_BUDDY registers are not used on dgpu's beyond DG1 */ + if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv)) + return; + if (IS_ALDERLAKE_S(dev_priv) || - IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0) || - IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) + IS_DG1_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) || + IS_RKL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) || + IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) /* Wa_1409767108:tgl,dg1,adl-s */ table = wa_1409767108_buddy_page_masks; else @@ -5822,10 +5762,11 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i), table[config].page_mask); - /* Wa_22010178259:tgl,rkl */ - intel_de_rmw(dev_priv, BW_BUDDY_CTL(i), - BW_BUDDY_TLB_REQ_TIMER_MASK, - BW_BUDDY_TLB_REQ_TIMER(0x8)); + /* Wa_22010178259:tgl,dg1,rkl,adl-s */ + if (DISPLAY_VER(dev_priv) == 12) + intel_de_rmw(dev_priv, BW_BUDDY_CTL(i), + BW_BUDDY_TLB_REQ_TIMER_MASK, + BW_BUDDY_TLB_REQ_TIMER(0x8)); } } } @@ -5879,11 +5820,15 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv, if (DISPLAY_VER(dev_priv) >= 12) tgl_bw_buddy_init(dev_priv); + /* 8. Ensure PHYs have completed calibration and adaptation */ + if (IS_DG2(dev_priv)) + intel_snps_phy_wait_for_calibration(dev_priv); + if (resume && intel_dmc_has_payload(dev_priv)) intel_dmc_load_program(dev_priv); - /* Wa_14011508470 */ - if (DISPLAY_VER(dev_priv) == 12) { + /* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p */ + if (DISPLAY_VER(dev_priv) >= 12) { val = DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM | DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR; intel_uncore_rmw(&dev_priv->uncore, GEN11_CHICKEN_DCPR_2, 0, val); @@ -6098,8 +6043,6 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume) if (DISPLAY_VER(i915) >= 11) { icl_display_core_init(i915, resume); - } else if (IS_CANNONLAKE(i915)) { - cnl_display_core_init(i915, resume); } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) { bxt_display_core_init(i915, resume); } else if (DISPLAY_VER(i915) == 9) { @@ -6259,8 +6202,6 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, if (DISPLAY_VER(i915) >= 11) icl_display_core_uninit(i915); - else if (IS_CANNONLAKE(i915)) - cnl_display_core_uninit(i915); else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) bxt_display_core_uninit(i915); else if (DISPLAY_VER(i915) == 9) @@ -6388,13 +6329,13 @@ void intel_display_power_suspend_late(struct drm_i915_private *i915) if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) || IS_BROXTON(i915)) { bxt_enable_dc9(i915); - /* Tweaked Wa_14010685332:icp,jsp,mcc */ - if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC) - intel_de_rmw(i915, SOUTH_CHICKEN1, - SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS); } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_enable_pc8(i915); } + + /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */ + if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1) + intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS); } void intel_display_power_resume_early(struct drm_i915_private *i915) @@ -6403,13 +6344,13 @@ void intel_display_power_resume_early(struct drm_i915_private *i915) IS_BROXTON(i915)) { gen9_sanitize_dc_state(i915); bxt_disable_dc9(i915); - /* Tweaked Wa_14010685332:icp,jsp,mcc */ - if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC) - intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0); - } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_disable_pc8(i915); } + + /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */ + if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1) + intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0); } void intel_display_power_suspend(struct drm_i915_private *i915) diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h index 4f0917df4375..978531841fa3 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.h +++ b/drivers/gpu/drm/i915/display/intel_display_power.h @@ -76,7 +76,8 @@ enum intel_display_power_domain { POWER_DOMAIN_PORT_CRT, POWER_DOMAIN_PORT_OTHER, POWER_DOMAIN_VGA, - POWER_DOMAIN_AUDIO, + POWER_DOMAIN_AUDIO_MMIO, + POWER_DOMAIN_AUDIO_PLAYBACK, POWER_DOMAIN_AUX_A, POWER_DOMAIN_AUX_B, POWER_DOMAIN_AUX_C, @@ -142,8 +143,6 @@ enum i915_power_well_id { SKL_DISP_PW_MISC_IO, SKL_DISP_PW_1, SKL_DISP_PW_2, - CNL_DISP_PW_DDI_F_IO, - CNL_DISP_PW_DDI_F_AUX, ICL_DISP_PW_3, SKL_DISP_DC_OFF, TGL_DISP_PW_TC_COLD_OFF, @@ -223,6 +222,12 @@ struct i915_power_well_desc { u8 idx; /* Mask of pipes whose IRQ logic is backed by the pw */ u8 irq_pipe_mask; + /* + * Instead of waiting for the status bit to ack enables, + * just wait a specific amount of time and then consider + * the well enabled. + */ + u16 fixed_enable_delay; /* The pw is backing the VGA functionality */ bool has_vga:1; bool has_fuses:1; @@ -386,6 +391,10 @@ intel_display_power_put_all_in_set(struct drm_i915_private *i915, intel_display_power_put_mask_in_set(i915, power_domain_set, power_domain_set->mask); } +/* + * FIXME: We should probably switch this to a 0-based scheme to be consistent + * with how we now name/number DBUF_CTL instances. + */ enum dbuf_slice { DBUF_S1, DBUF_S2, diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index d94f361b548b..6beeeeba1bed 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -196,6 +196,10 @@ struct intel_encoder { void (*update_complete)(struct intel_atomic_state *, struct intel_encoder *, struct intel_crtc *); + void (*pre_disable)(struct intel_atomic_state *, + struct intel_encoder *, + const struct intel_crtc_state *, + const struct drm_connector_state *); void (*disable)(struct intel_atomic_state *, struct intel_encoder *, const struct intel_crtc_state *, @@ -884,6 +888,18 @@ enum intel_output_format { INTEL_OUTPUT_FORMAT_YCBCR444, }; +struct intel_mpllb_state { + u32 clock; /* in KHz */ + u32 ref_control; + u32 mpllb_cp; + u32 mpllb_div; + u32 mpllb_div2; + u32 mpllb_fracn1; + u32 mpllb_fracn2; + u32 mpllb_sscen; + u32 mpllb_sscstep; +}; + struct intel_crtc_state { /* * uapi (drm) state. This is the software state shown to userspace. @@ -1018,7 +1034,10 @@ struct intel_crtc_state { struct intel_shared_dpll *shared_dpll; /* Actual register state of the dpll, for shared dpll cross-checking. */ - struct intel_dpll_hw_state dpll_hw_state; + union { + struct intel_dpll_hw_state dpll_hw_state; + struct intel_mpllb_state mpllb_state; + }; /* * ICL reserved DPLLs for the CRTC/port. The active PLL is selected by @@ -1612,6 +1631,7 @@ struct intel_dp { /* Display stream compression testing */ bool force_dsc_en; + int force_dsc_bpp; bool hobl_failed; bool hobl_active; diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index f8789d4543bf..3c3c6cb5c0df 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -57,12 +57,12 @@ MODULE_FIRMWARE(ADLS_DMC_PATH); #define DG1_DMC_VERSION_REQUIRED DMC_VERSION(2, 2) MODULE_FIRMWARE(DG1_DMC_PATH); -#define RKL_DMC_PATH DMC_PATH(rkl, 2, 02) -#define RKL_DMC_VERSION_REQUIRED DMC_VERSION(2, 2) +#define RKL_DMC_PATH DMC_PATH(rkl, 2, 03) +#define RKL_DMC_VERSION_REQUIRED DMC_VERSION(2, 3) MODULE_FIRMWARE(RKL_DMC_PATH); -#define TGL_DMC_PATH DMC_PATH(tgl, 2, 08) -#define TGL_DMC_VERSION_REQUIRED DMC_VERSION(2, 8) +#define TGL_DMC_PATH DMC_PATH(tgl, 2, 12) +#define TGL_DMC_VERSION_REQUIRED DMC_VERSION(2, 12) MODULE_FIRMWARE(TGL_DMC_PATH); #define ICL_DMC_PATH DMC_PATH(icl, 1, 09) @@ -70,11 +70,6 @@ MODULE_FIRMWARE(TGL_DMC_PATH); #define ICL_DMC_MAX_FW_SIZE 0x6000 MODULE_FIRMWARE(ICL_DMC_PATH); -#define CNL_DMC_PATH DMC_PATH(cnl, 1, 07) -#define CNL_DMC_VERSION_REQUIRED DMC_VERSION(1, 7) -#define CNL_DMC_MAX_FW_SIZE GLK_DMC_MAX_FW_SIZE -MODULE_FIRMWARE(CNL_DMC_PATH); - #define GLK_DMC_PATH DMC_PATH(glk, 1, 04) #define GLK_DMC_VERSION_REQUIRED DMC_VERSION(1, 4) #define GLK_DMC_MAX_FW_SIZE 0x4000 @@ -247,50 +242,15 @@ bool intel_dmc_has_payload(struct drm_i915_private *i915) return i915->dmc.dmc_info[DMC_FW_MAIN].payload; } -static const struct stepping_info skl_stepping_info[] = { - {'A', '0'}, {'B', '0'}, {'C', '0'}, - {'D', '0'}, {'E', '0'}, {'F', '0'}, - {'G', '0'}, {'H', '0'}, {'I', '0'}, - {'J', '0'}, {'K', '0'} -}; - -static const struct stepping_info bxt_stepping_info[] = { - {'A', '0'}, {'A', '1'}, {'A', '2'}, - {'B', '0'}, {'B', '1'}, {'B', '2'} -}; - -static const struct stepping_info icl_stepping_info[] = { - {'A', '0'}, {'A', '1'}, {'A', '2'}, - {'B', '0'}, {'B', '2'}, - {'C', '0'} -}; - -static const struct stepping_info no_stepping_info = { '*', '*' }; - static const struct stepping_info * -intel_get_stepping_info(struct drm_i915_private *dev_priv) +intel_get_stepping_info(struct drm_i915_private *i915, + struct stepping_info *si) { - const struct stepping_info *si; - unsigned int size; - - if (IS_ICELAKE(dev_priv)) { - size = ARRAY_SIZE(icl_stepping_info); - si = icl_stepping_info; - } else if (IS_SKYLAKE(dev_priv)) { - size = ARRAY_SIZE(skl_stepping_info); - si = skl_stepping_info; - } else if (IS_BROXTON(dev_priv)) { - size = ARRAY_SIZE(bxt_stepping_info); - si = bxt_stepping_info; - } else { - size = 0; - si = NULL; - } - - if (INTEL_REVID(dev_priv) < size) - return si + INTEL_REVID(dev_priv); + const char *step_name = intel_step_name(RUNTIME_INFO(i915)->step.display_step); - return &no_stepping_info; + si->stepping = step_name[0]; + si->substepping = step_name[1]; + return si; } static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) @@ -616,7 +576,8 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv, struct intel_package_header *package_header; struct intel_dmc_header_base *dmc_header; struct intel_dmc *dmc = &dev_priv->dmc; - const struct stepping_info *si = intel_get_stepping_info(dev_priv); + struct stepping_info display_info = { '*', '*'}; + const struct stepping_info *si = intel_get_stepping_info(dev_priv, &display_info); u32 readcount = 0; u32 r, offset; int id; @@ -752,10 +713,6 @@ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv) dmc->fw_path = ICL_DMC_PATH; dmc->required_version = ICL_DMC_VERSION_REQUIRED; dmc->max_fw_size = ICL_DMC_MAX_FW_SIZE; - } else if (IS_CANNONLAKE(dev_priv)) { - dmc->fw_path = CNL_DMC_PATH; - dmc->required_version = CNL_DMC_VERSION_REQUIRED; - dmc->max_fw_size = CNL_DMC_MAX_FW_SIZE; } else if (IS_GEMINILAKE(dev_priv)) { dmc->fw_path = GLK_DMC_PATH; dmc->required_version = GLK_DMC_VERSION_REQUIRED; diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 5b52beaddada..04175f359fd6 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -222,29 +222,6 @@ bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp) encoder->port != PORT_A); } -static int cnl_max_source_rate(struct intel_dp *intel_dp) -{ - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum port port = dig_port->base.port; - - u32 voltage = intel_de_read(dev_priv, CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; - - /* Low voltage SKUs are limited to max of 5.4G */ - if (voltage == VOLTAGE_INFO_0_85V) - return 540000; - - /* For this SKU 8.1G is supported in all ports */ - if (IS_CNL_WITH_PORT_F(dev_priv)) - return 810000; - - /* For other SKUs, max rate on ports A and D is 5.4G */ - if (port == PORT_A || port == PORT_D) - return 540000; - - return 810000; -} - static int icl_max_source_rate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); @@ -270,7 +247,7 @@ static void intel_dp_set_source_rates(struct intel_dp *intel_dp) { /* The values must be in increasing order */ - static const int cnl_rates[] = { + static const int icl_rates[] = { 162000, 216000, 270000, 324000, 432000, 540000, 648000, 810000 }; static const int bxt_rates[] = { @@ -295,12 +272,10 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) drm_WARN_ON(&dev_priv->drm, intel_dp->source_rates || intel_dp->num_source_rates); - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) { - source_rates = cnl_rates; - size = ARRAY_SIZE(cnl_rates); - if (DISPLAY_VER(dev_priv) == 10) - max_rate = cnl_max_source_rate(intel_dp); - else if (IS_JSL_EHL(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) { + source_rates = icl_rates; + size = ARRAY_SIZE(icl_rates); + if (IS_JSL_EHL(dev_priv)) max_rate = ehl_max_source_rate(intel_dp); else max_rate = icl_max_source_rate(intel_dp); @@ -1274,6 +1249,23 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, pipe_config->pipe_bpp); pipe_config->dsc.slice_count = dsc_dp_slice_count; } + + /* As of today we support DSC for only RGB */ + if (intel_dp->force_dsc_bpp) { + if (intel_dp->force_dsc_bpp >= 8 && + intel_dp->force_dsc_bpp < pipe_bpp) { + drm_dbg_kms(&dev_priv->drm, + "DSC BPP forced to %d", + intel_dp->force_dsc_bpp); + pipe_config->dsc.compressed_bpp = + intel_dp->force_dsc_bpp; + } else { + drm_dbg_kms(&dev_priv->drm, + "Invalid DSC BPP %d", + intel_dp->force_dsc_bpp); + } + } + /* * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate * is greater than the maximum Cdclock and if slice count is even @@ -3339,6 +3331,9 @@ static void intel_dp_process_phy_request(struct intel_dp *intel_dp, intel_dp_autotest_phy_ddi_enable(intel_dp, crtc_state); + drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET, + intel_dp->train_set, crtc_state->lane_count); + drm_dp_set_phy_test_pattern(&intel_dp->aux, data, link_status[DP_DPCD_REV]); } @@ -3847,23 +3842,18 @@ static void intel_dp_check_device_service_irq(struct intel_dp *intel_dp) static void intel_dp_check_link_service_irq(struct intel_dp *intel_dp) { - struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 val; if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) return; if (drm_dp_dpcd_readb(&intel_dp->aux, - DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &val) != 1 || !val) { - drm_dbg_kms(&i915->drm, "Error in reading link service irq vector\n"); + DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &val) != 1 || !val) return; - } if (drm_dp_dpcd_writeb(&intel_dp->aux, - DP_LINK_SERVICE_IRQ_VECTOR_ESI0, val) != 1) { - drm_dbg_kms(&i915->drm, "Error in writing link service irq vector\n"); + DP_LINK_SERVICE_IRQ_VECTOR_ESI0, val) != 1) return; - } if (val & HDMI_LINK_STATUS_CHANGED) intel_dp_handle_hdmi_link_status_change(intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c index 7c048d2ecf43..f483f479dd0b 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c @@ -158,7 +158,6 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, /* * Max timeout values: * SKL-GLK: 1.6ms - * CNL: 3.2ms * ICL+: 4ms */ ret = DP_AUX_CH_CTL_SEND_BUSY | diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 08bceae40aa8..053a3c2f7267 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -206,7 +206,6 @@ int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) return lttpr_count; } -EXPORT_SYMBOL(intel_dp_init_lttpr_and_dprx_caps); static u8 dp_voltage_max(u8 preemph) { diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 3661cd19ce48..8d13d7b26a25 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -348,6 +348,16 @@ static void wait_for_act_sent(struct intel_encoder *encoder, drm_dp_check_act_status(&intel_dp->mst_mgr); } +static void intel_mst_pre_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *old_crtc_state, + const struct drm_connector_state *old_conn_state) +{ + if (old_crtc_state->has_audio) + intel_audio_codec_disable(encoder, old_crtc_state, + old_conn_state); +} + static void intel_mst_disable_dp(struct intel_atomic_state *state, struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, @@ -372,9 +382,6 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state, if (ret) { drm_dbg_kms(&i915->drm, "failed to update payload %d\n", ret); } - if (old_crtc_state->has_audio) - intel_audio_codec_disable(encoder, - old_crtc_state, old_conn_state); } static void intel_mst_post_disable_dp(struct intel_atomic_state *state, @@ -542,7 +549,7 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, struct intel_digital_port *dig_port = intel_mst->primary; struct intel_dp *intel_dp = &dig_port->dp; struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 val; + enum transcoder trans = pipe_config->cpu_transcoder; drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder); @@ -550,12 +557,8 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_transcoder_func(encoder, pipe_config); - val = intel_de_read(dev_priv, - TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder)); - val |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC; - intel_de_write(dev_priv, - TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder), - val); + intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(trans), 0, + TRANS_DDI_DP_VC_PAYLOAD_ALLOC); drm_dbg_kms(&dev_priv->drm, "active links %d\n", intel_dp->active_mst_links); @@ -564,6 +567,10 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, drm_dp_update_payload_part2(&intel_dp->mst_mgr); + if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable) + intel_de_rmw(dev_priv, CHICKEN_TRANS(trans), 0, + FECSTALL_DIS_DPTSTREAM_DPTTG); + intel_enable_pipe(pipe_config); intel_crtc_vblank_on(pipe_config); @@ -903,6 +910,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *dig_port, enum pipe intel_encoder->compute_config = intel_dp_mst_compute_config; intel_encoder->compute_config_late = intel_dp_mst_compute_config_late; + intel_encoder->pre_disable = intel_mst_pre_disable_dp; intel_encoder->disable = intel_mst_disable_dp; intel_encoder->post_disable = intel_mst_post_disable_dp; intel_encoder->update_pipe = intel_ddi_update_pipe; diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index 89635da9f6f6..14515e62c05e 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -11,6 +11,7 @@ #include "intel_lvds.h" #include "intel_panel.h" #include "intel_sideband.h" +#include "display/intel_snps_phy.h" struct intel_limit { struct { @@ -923,12 +924,13 @@ static int hsw_crtc_compute_clock(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state); + struct intel_encoder *encoder = + intel_get_crtc_new_encoder(state, crtc_state); - if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) || - DISPLAY_VER(dev_priv) >= 11) { - struct intel_encoder *encoder = - intel_get_crtc_new_encoder(state, crtc_state); - + if (IS_DG2(dev_priv)) { + return intel_mpllb_calc_state(crtc_state, encoder); + } else if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) || + DISPLAY_VER(dev_priv) >= 11) { if (!intel_reserve_shared_dplls(state, crtc, encoder)) { drm_dbg_kms(&dev_priv->drm, "failed to find PLL for pipe %c\n", diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index 71ac57670043..5c91d125a337 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -168,7 +168,7 @@ intel_combo_pll_enable_reg(struct drm_i915_private *i915, else if (IS_JSL_EHL(i915) && (pll->info->id == DPLL_ID_EHL_DPLL4)) return MG_PLL_ENABLE(0); - return CNL_DPLL_ENABLE(pll->info->id); + return ICL_DPLL_ENABLE(pll->info->id); } static i915_reg_t @@ -2346,160 +2346,7 @@ static const struct intel_dpll_mgr bxt_pll_mgr = { .dump_hw_state = bxt_dump_hw_state, }; -static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll) -{ - const enum intel_dpll_id id = pll->info->id; - u32 val; - - /* 1. Enable DPLL power in DPLL_ENABLE. */ - val = intel_de_read(dev_priv, CNL_DPLL_ENABLE(id)); - val |= PLL_POWER_ENABLE; - intel_de_write(dev_priv, CNL_DPLL_ENABLE(id), val); - - /* 2. Wait for DPLL power state enabled in DPLL_ENABLE. */ - if (intel_de_wait_for_set(dev_priv, CNL_DPLL_ENABLE(id), - PLL_POWER_STATE, 5)) - drm_err(&dev_priv->drm, "PLL %d Power not enabled\n", id); - - /* - * 3. Configure DPLL_CFGCR0 to set SSC enable/disable, - * select DP mode, and set DP link rate. - */ - val = pll->state.hw_state.cfgcr0; - intel_de_write(dev_priv, CNL_DPLL_CFGCR0(id), val); - - /* 4. Reab back to ensure writes completed */ - intel_de_posting_read(dev_priv, CNL_DPLL_CFGCR0(id)); - - /* 3. Configure DPLL_CFGCR0 */ - /* Avoid touch CFGCR1 if HDMI mode is not enabled */ - if (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_HDMI_MODE) { - val = pll->state.hw_state.cfgcr1; - intel_de_write(dev_priv, CNL_DPLL_CFGCR1(id), val); - /* 4. Reab back to ensure writes completed */ - intel_de_posting_read(dev_priv, CNL_DPLL_CFGCR1(id)); - } - - /* - * 5. If the frequency will result in a change to the voltage - * requirement, follow the Display Voltage Frequency Switching - * Sequence Before Frequency Change - * - * Note: DVFS is actually handled via the cdclk code paths, - * hence we do nothing here. - */ - - /* 6. Enable DPLL in DPLL_ENABLE. */ - val = intel_de_read(dev_priv, CNL_DPLL_ENABLE(id)); - val |= PLL_ENABLE; - intel_de_write(dev_priv, CNL_DPLL_ENABLE(id), val); - - /* 7. Wait for PLL lock status in DPLL_ENABLE. */ - if (intel_de_wait_for_set(dev_priv, CNL_DPLL_ENABLE(id), PLL_LOCK, 5)) - drm_err(&dev_priv->drm, "PLL %d not locked\n", id); - - /* - * 8. If the frequency will result in a change to the voltage - * requirement, follow the Display Voltage Frequency Switching - * Sequence After Frequency Change - * - * Note: DVFS is actually handled via the cdclk code paths, - * hence we do nothing here. - */ - - /* - * 9. turn on the clock for the DDI and map the DPLL to the DDI - * Done at intel_ddi_clk_select - */ -} - -static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll) -{ - const enum intel_dpll_id id = pll->info->id; - u32 val; - - /* - * 1. Configure DPCLKA_CFGCR0 to turn off the clock for the DDI. - * Done at intel_ddi_post_disable - */ - - /* - * 2. If the frequency will result in a change to the voltage - * requirement, follow the Display Voltage Frequency Switching - * Sequence Before Frequency Change - * - * Note: DVFS is actually handled via the cdclk code paths, - * hence we do nothing here. - */ - - /* 3. Disable DPLL through DPLL_ENABLE. */ - val = intel_de_read(dev_priv, CNL_DPLL_ENABLE(id)); - val &= ~PLL_ENABLE; - intel_de_write(dev_priv, CNL_DPLL_ENABLE(id), val); - - /* 4. Wait for PLL not locked status in DPLL_ENABLE. */ - if (intel_de_wait_for_clear(dev_priv, CNL_DPLL_ENABLE(id), PLL_LOCK, 5)) - drm_err(&dev_priv->drm, "PLL %d locked\n", id); - - /* - * 5. If the frequency will result in a change to the voltage - * requirement, follow the Display Voltage Frequency Switching - * Sequence After Frequency Change - * - * Note: DVFS is actually handled via the cdclk code paths, - * hence we do nothing here. - */ - - /* 6. Disable DPLL power in DPLL_ENABLE. */ - val = intel_de_read(dev_priv, CNL_DPLL_ENABLE(id)); - val &= ~PLL_POWER_ENABLE; - intel_de_write(dev_priv, CNL_DPLL_ENABLE(id), val); - - /* 7. Wait for DPLL power state disabled in DPLL_ENABLE. */ - if (intel_de_wait_for_clear(dev_priv, CNL_DPLL_ENABLE(id), - PLL_POWER_STATE, 5)) - drm_err(&dev_priv->drm, "PLL %d Power not disabled\n", id); -} - -static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll, - struct intel_dpll_hw_state *hw_state) -{ - const enum intel_dpll_id id = pll->info->id; - intel_wakeref_t wakeref; - u32 val; - bool ret; - - wakeref = intel_display_power_get_if_enabled(dev_priv, - POWER_DOMAIN_DISPLAY_CORE); - if (!wakeref) - return false; - - ret = false; - - val = intel_de_read(dev_priv, CNL_DPLL_ENABLE(id)); - if (!(val & PLL_ENABLE)) - goto out; - - val = intel_de_read(dev_priv, CNL_DPLL_CFGCR0(id)); - hw_state->cfgcr0 = val; - - /* avoid reading back stale values if HDMI mode is not enabled */ - if (val & DPLL_CFGCR0_HDMI_MODE) { - hw_state->cfgcr1 = intel_de_read(dev_priv, - CNL_DPLL_CFGCR1(id)); - } - ret = true; - -out: - intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref); - - return ret; -} - -static void cnl_wrpll_get_multipliers(int bestdiv, int *pdiv, +static void icl_wrpll_get_multipliers(int bestdiv, int *pdiv, int *qdiv, int *kdiv) { /* even dividers */ @@ -2538,7 +2385,7 @@ static void cnl_wrpll_get_multipliers(int bestdiv, int *pdiv, } } -static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, +static void icl_wrpll_params_populate(struct skl_wrpll_params *params, u32 dco_freq, u32 ref_freq, int pdiv, int qdiv, int kdiv) { @@ -2586,349 +2433,19 @@ static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, params->dco_fraction = dco & 0x7fff; } -static bool -__cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, - struct skl_wrpll_params *wrpll_params, - int ref_clock) -{ - u32 afe_clock = crtc_state->port_clock * 5; - u32 dco_min = 7998000; - u32 dco_max = 10000000; - u32 dco_mid = (dco_min + dco_max) / 2; - static const int dividers[] = { 2, 4, 6, 8, 10, 12, 14, 16, - 18, 20, 24, 28, 30, 32, 36, 40, - 42, 44, 48, 50, 52, 54, 56, 60, - 64, 66, 68, 70, 72, 76, 78, 80, - 84, 88, 90, 92, 96, 98, 100, 102, - 3, 5, 7, 9, 15, 21 }; - u32 dco, best_dco = 0, dco_centrality = 0; - u32 best_dco_centrality = U32_MAX; /* Spec meaning of 999999 MHz */ - int d, best_div = 0, pdiv = 0, qdiv = 0, kdiv = 0; - - for (d = 0; d < ARRAY_SIZE(dividers); d++) { - dco = afe_clock * dividers[d]; - - if ((dco <= dco_max) && (dco >= dco_min)) { - dco_centrality = abs(dco - dco_mid); - - if (dco_centrality < best_dco_centrality) { - best_dco_centrality = dco_centrality; - best_div = dividers[d]; - best_dco = dco; - } - } - } - - if (best_div == 0) - return false; - - cnl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv); - cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, - pdiv, qdiv, kdiv); - - return true; -} - -static bool -cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, - struct skl_wrpll_params *wrpll_params) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - - return __cnl_ddi_calculate_wrpll(crtc_state, wrpll_params, - i915->dpll.ref_clks.nssc); -} - -static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) -{ - u32 cfgcr0, cfgcr1; - struct skl_wrpll_params wrpll_params = { 0, }; - - cfgcr0 = DPLL_CFGCR0_HDMI_MODE; - - if (!cnl_ddi_calculate_wrpll(crtc_state, &wrpll_params)) - return false; - - cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) | - wrpll_params.dco_integer; - - cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(wrpll_params.qdiv_ratio) | - DPLL_CFGCR1_QDIV_MODE(wrpll_params.qdiv_mode) | - DPLL_CFGCR1_KDIV(wrpll_params.kdiv) | - DPLL_CFGCR1_PDIV(wrpll_params.pdiv) | - DPLL_CFGCR1_CENTRAL_FREQ; - - memset(&crtc_state->dpll_hw_state, 0, - sizeof(crtc_state->dpll_hw_state)); - - crtc_state->dpll_hw_state.cfgcr0 = cfgcr0; - crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; - return true; -} - /* - * Display WA #22010492432: ehl, tgl + * Display WA #22010492432: ehl, tgl, adl-p * Program half of the nominal DCO divider fraction value. */ static bool ehl_combo_pll_div_frac_wa_needed(struct drm_i915_private *i915) { return ((IS_PLATFORM(i915, INTEL_ELKHARTLAKE) && - IS_JSL_EHL_REVID(i915, EHL_REVID_B0, REVID_FOREVER)) || - IS_TIGERLAKE(i915)) && + IS_JSL_EHL_DISPLAY_STEP(i915, STEP_B0, STEP_FOREVER)) || + IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) && i915->dpll.ref_clks.nssc == 38400; } -static int __cnl_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state, - int ref_clock) -{ - u32 dco_fraction; - u32 p0, p1, p2, dco_freq; - - p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK; - p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK; - - if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) - p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >> - DPLL_CFGCR1_QDIV_RATIO_SHIFT; - else - p1 = 1; - - - switch (p0) { - case DPLL_CFGCR1_PDIV_2: - p0 = 2; - break; - case DPLL_CFGCR1_PDIV_3: - p0 = 3; - break; - case DPLL_CFGCR1_PDIV_5: - p0 = 5; - break; - case DPLL_CFGCR1_PDIV_7: - p0 = 7; - break; - } - - switch (p2) { - case DPLL_CFGCR1_KDIV_1: - p2 = 1; - break; - case DPLL_CFGCR1_KDIV_2: - p2 = 2; - break; - case DPLL_CFGCR1_KDIV_3: - p2 = 3; - break; - } - - dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * - ref_clock; - - dco_fraction = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> - DPLL_CFGCR0_DCO_FRACTION_SHIFT; - - if (ehl_combo_pll_div_frac_wa_needed(dev_priv)) - dco_fraction *= 2; - - dco_freq += (dco_fraction * ref_clock) / 0x8000; - - if (drm_WARN_ON(&dev_priv->drm, p0 == 0 || p1 == 0 || p2 == 0)) - return 0; - - return dco_freq / (p0 * p1 * p2 * 5); -} - -static int cnl_ddi_wrpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state) -{ - return __cnl_ddi_wrpll_get_freq(i915, pll, pll_state, - i915->dpll.ref_clks.nssc); -} - -static bool -cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) -{ - u32 cfgcr0; - - cfgcr0 = DPLL_CFGCR0_SSC_ENABLE; - - switch (crtc_state->port_clock / 2) { - case 81000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_810; - break; - case 135000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1350; - break; - case 270000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2700; - break; - /* eDP 1.4 rates */ - case 162000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1620; - break; - case 108000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1080; - break; - case 216000: - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2160; - break; - case 324000: - /* Some SKUs may require elevated I/O voltage to support this */ - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_3240; - break; - case 405000: - /* Some SKUs may require elevated I/O voltage to support this */ - cfgcr0 |= DPLL_CFGCR0_LINK_RATE_4050; - break; - } - - memset(&crtc_state->dpll_hw_state, 0, - sizeof(crtc_state->dpll_hw_state)); - - crtc_state->dpll_hw_state.cfgcr0 = cfgcr0; - - return true; -} - -static int cnl_ddi_lcpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state) -{ - int link_clock = 0; - - switch (pll_state->cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK) { - case DPLL_CFGCR0_LINK_RATE_810: - link_clock = 81000; - break; - case DPLL_CFGCR0_LINK_RATE_1080: - link_clock = 108000; - break; - case DPLL_CFGCR0_LINK_RATE_1350: - link_clock = 135000; - break; - case DPLL_CFGCR0_LINK_RATE_1620: - link_clock = 162000; - break; - case DPLL_CFGCR0_LINK_RATE_2160: - link_clock = 216000; - break; - case DPLL_CFGCR0_LINK_RATE_2700: - link_clock = 270000; - break; - case DPLL_CFGCR0_LINK_RATE_3240: - link_clock = 324000; - break; - case DPLL_CFGCR0_LINK_RATE_4050: - link_clock = 405000; - break; - default: - drm_WARN(&i915->drm, 1, "Unsupported link rate\n"); - break; - } - - return link_clock * 2; -} - -static bool cnl_get_dpll(struct intel_atomic_state *state, - struct intel_crtc *crtc, - struct intel_encoder *encoder) -{ - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - struct intel_shared_dpll *pll; - bool bret; - - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { - bret = cnl_ddi_hdmi_pll_dividers(crtc_state); - if (!bret) { - drm_dbg_kms(&i915->drm, - "Could not get HDMI pll dividers.\n"); - return false; - } - } else if (intel_crtc_has_dp_encoder(crtc_state)) { - bret = cnl_ddi_dp_set_dpll_hw_state(crtc_state); - if (!bret) { - drm_dbg_kms(&i915->drm, - "Could not set DP dpll HW state.\n"); - return false; - } - } else { - drm_dbg_kms(&i915->drm, - "Skip DPLL setup for output_types 0x%x\n", - crtc_state->output_types); - return false; - } - - pll = intel_find_shared_dpll(state, crtc, - &crtc_state->dpll_hw_state, - BIT(DPLL_ID_SKL_DPLL2) | - BIT(DPLL_ID_SKL_DPLL1) | - BIT(DPLL_ID_SKL_DPLL0)); - if (!pll) { - drm_dbg_kms(&i915->drm, "No PLL selected\n"); - return false; - } - - intel_reference_shared_dpll(state, crtc, - pll, &crtc_state->dpll_hw_state); - - crtc_state->shared_dpll = pll; - - return true; -} - -static int cnl_ddi_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state) -{ - if (pll_state->cfgcr0 & DPLL_CFGCR0_HDMI_MODE) - return cnl_ddi_wrpll_get_freq(i915, pll, pll_state); - else - return cnl_ddi_lcpll_get_freq(i915, pll, pll_state); -} - -static void cnl_update_dpll_ref_clks(struct drm_i915_private *i915) -{ - /* No SSC reference */ - i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; -} - -static void cnl_dump_hw_state(struct drm_i915_private *dev_priv, - const struct intel_dpll_hw_state *hw_state) -{ - drm_dbg_kms(&dev_priv->drm, "dpll_hw_state: " - "cfgcr0: 0x%x, cfgcr1: 0x%x\n", - hw_state->cfgcr0, - hw_state->cfgcr1); -} - -static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = { - .enable = cnl_ddi_pll_enable, - .disable = cnl_ddi_pll_disable, - .get_hw_state = cnl_ddi_pll_get_hw_state, - .get_freq = cnl_ddi_pll_get_freq, -}; - -static const struct dpll_info cnl_plls[] = { - { "DPLL 0", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 }, - { "DPLL 1", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, - { "DPLL 2", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, - { }, -}; - -static const struct intel_dpll_mgr cnl_pll_mgr = { - .dpll_info = cnl_plls, - .get_dplls = cnl_get_dpll, - .put_dplls = intel_put_dpll, - .update_ref_clks = cnl_update_dpll_ref_clks, - .dump_hw_state = cnl_dump_hw_state, -}; - struct icl_combo_pll_params { int clock; struct skl_wrpll_params wrpll; @@ -3105,17 +2622,104 @@ icl_calc_wrpll(struct intel_crtc_state *crtc_state, struct skl_wrpll_params *wrpll_params) { struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + int ref_clock = icl_wrpll_ref_clock(i915); + u32 afe_clock = crtc_state->port_clock * 5; + u32 dco_min = 7998000; + u32 dco_max = 10000000; + u32 dco_mid = (dco_min + dco_max) / 2; + static const int dividers[] = { 2, 4, 6, 8, 10, 12, 14, 16, + 18, 20, 24, 28, 30, 32, 36, 40, + 42, 44, 48, 50, 52, 54, 56, 60, + 64, 66, 68, 70, 72, 76, 78, 80, + 84, 88, 90, 92, 96, 98, 100, 102, + 3, 5, 7, 9, 15, 21 }; + u32 dco, best_dco = 0, dco_centrality = 0; + u32 best_dco_centrality = U32_MAX; /* Spec meaning of 999999 MHz */ + int d, best_div = 0, pdiv = 0, qdiv = 0, kdiv = 0; - return __cnl_ddi_calculate_wrpll(crtc_state, wrpll_params, - icl_wrpll_ref_clock(i915)); + for (d = 0; d < ARRAY_SIZE(dividers); d++) { + dco = afe_clock * dividers[d]; + + if (dco <= dco_max && dco >= dco_min) { + dco_centrality = abs(dco - dco_mid); + + if (dco_centrality < best_dco_centrality) { + best_dco_centrality = dco_centrality; + best_div = dividers[d]; + best_dco = dco; + } + } + } + + if (best_div == 0) + return false; + + icl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv); + icl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, + pdiv, qdiv, kdiv); + + return true; } static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915, const struct intel_shared_dpll *pll, const struct intel_dpll_hw_state *pll_state) { - return __cnl_ddi_wrpll_get_freq(i915, pll, pll_state, - icl_wrpll_ref_clock(i915)); + int ref_clock = icl_wrpll_ref_clock(i915); + u32 dco_fraction; + u32 p0, p1, p2, dco_freq; + + p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK; + p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK; + + if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) + p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >> + DPLL_CFGCR1_QDIV_RATIO_SHIFT; + else + p1 = 1; + + switch (p0) { + case DPLL_CFGCR1_PDIV_2: + p0 = 2; + break; + case DPLL_CFGCR1_PDIV_3: + p0 = 3; + break; + case DPLL_CFGCR1_PDIV_5: + p0 = 5; + break; + case DPLL_CFGCR1_PDIV_7: + p0 = 7; + break; + } + + switch (p2) { + case DPLL_CFGCR1_KDIV_1: + p2 = 1; + break; + case DPLL_CFGCR1_KDIV_2: + p2 = 2; + break; + case DPLL_CFGCR1_KDIV_3: + p2 = 3; + break; + } + + dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * + ref_clock; + + dco_fraction = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> + DPLL_CFGCR0_DCO_FRACTION_SHIFT; + + if (ehl_combo_pll_div_frac_wa_needed(i915)) + dco_fraction *= 2; + + dco_freq += (dco_fraction * ref_clock) / 0x8000; + + if (drm_WARN_ON(&i915->drm, p0 == 0 || p1 == 0 || p2 == 0)) + return 0; + + return dco_freq / (p0 * p1 * p2 * 5); } static void icl_calc_dpll_state(struct drm_i915_private *i915, @@ -4131,6 +3735,31 @@ static void icl_pll_enable(struct drm_i915_private *dev_priv, drm_err(&dev_priv->drm, "PLL %d not locked\n", pll->info->id); } +static void adlp_cmtg_clock_gating_wa(struct drm_i915_private *i915, struct intel_shared_dpll *pll) +{ + u32 val; + + if (!IS_ADLP_DISPLAY_STEP(i915, STEP_A0, STEP_B0) || + pll->info->id != DPLL_ID_ICL_DPLL0) + return; + /* + * Wa_16011069516:adl-p[a0] + * + * All CMTG regs are unreliable until CMTG clock gating is disabled, + * so we can only assume the default TRANS_CMTG_CHICKEN reg value and + * sanity check this assumption with a double read, which presumably + * returns the correct value even with clock gating on. + * + * Instead of the usual place for workarounds we apply this one here, + * since TRANS_CMTG_CHICKEN is only accessible while DPLL0 is enabled. + */ + val = intel_de_read(i915, TRANS_CMTG_CHICKEN); + val = intel_de_read(i915, TRANS_CMTG_CHICKEN); + intel_de_write(i915, TRANS_CMTG_CHICKEN, DISABLE_DPT_CLK_GATING); + if (drm_WARN_ON(&i915->drm, val & ~DISABLE_DPT_CLK_GATING)) + drm_dbg_kms(&i915->drm, "Unexpected flags in TRANS_CMTG_CHICKEN: %08x\n", val); +} + static void combo_pll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { @@ -4160,6 +3789,8 @@ static void combo_pll_enable(struct drm_i915_private *dev_priv, icl_pll_enable(dev_priv, pll, enable_reg); + adlp_cmtg_clock_gating_wa(dev_priv, pll); + /* DVFS post sequence would be here. See the comment above. */ } @@ -4462,7 +4093,10 @@ void intel_shared_dpll_init(struct drm_device *dev) const struct dpll_info *dpll_info; int i; - if (IS_ALDERLAKE_P(dev_priv)) + if (IS_DG2(dev_priv)) + /* No shared DPLLs on DG2; port PLLs are part of the PHY */ + dpll_mgr = NULL; + else if (IS_ALDERLAKE_P(dev_priv)) dpll_mgr = &adlp_pll_mgr; else if (IS_ALDERLAKE_S(dev_priv)) dpll_mgr = &adls_pll_mgr; @@ -4476,8 +4110,6 @@ void intel_shared_dpll_init(struct drm_device *dev) dpll_mgr = &ehl_pll_mgr; else if (DISPLAY_VER(dev_priv) >= 11) dpll_mgr = &icl_pll_mgr; - else if (IS_CANNONLAKE(dev_priv)) - dpll_mgr = &cnl_pll_mgr; else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) dpll_mgr = &bxt_pll_mgr; else if (DISPLAY_VER(dev_priv) == 9) @@ -4668,7 +4300,12 @@ void intel_dpll_readout_hw_state(struct drm_i915_private *i915) static void sanitize_dpll_state(struct drm_i915_private *i915, struct intel_shared_dpll *pll) { - if (!pll->on || pll->active_mask) + if (!pll->on) + return; + + adlp_cmtg_clock_gating_wa(i915, pll); + + if (pll->active_mask) return; drm_dbg_kms(&i915->drm, diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h index 7fd031a70cfd..30e0aa5ca109 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h @@ -204,9 +204,8 @@ struct intel_dpll_hw_state { /* HDMI only, 0 when used for DP */ u32 cfgcr1, cfgcr2; - /* cnl */ + /* icl */ u32 cfgcr0; - /* CNL also uses cfgcr1 */ /* bxt */ u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12; diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 82effb64a3b9..ddfc17e21668 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -912,11 +912,11 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) } /* - * Tigerlake is not supporting FBC with PSR2. + * Display 12+ is not supporting FBC with PSR2. * Recommendation is to keep this combination disabled * Bspec: 50422 HSD: 14010260002 */ - if (fbc->state_cache.psr2_active && IS_TIGERLAKE(dev_priv)) { + if (fbc->state_cache.psr2_active && DISPLAY_VER(dev_priv) >= 12) { fbc->no_fbc_reason = "not supported with PSR2"; return false; } diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index fcf47f98ea36..ceb1bf8a8c3c 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -600,7 +600,7 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num, int i = 0, inc, try = 0; int ret = 0; - /* Display WA #0868: skl,bxt,kbl,cfl,glk,cnl */ + /* Display WA #0868: skl,bxt,kbl,cfl,glk */ if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) bxt_gmbus_clock_gating(dev_priv, false); else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv)) @@ -713,7 +713,7 @@ timeout: ret = -EAGAIN; out: - /* Display WA #0868: skl,bxt,kbl,cfl,glk,cnl */ + /* Display WA #0868: skl,bxt,kbl,cfl,glk */ if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) bxt_gmbus_clock_gating(dev_priv, true); else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv)) diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 852af2b23540..b04685bb6439 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -51,6 +51,7 @@ #include "intel_hdmi.h" #include "intel_lspcon.h" #include "intel_panel.h" +#include "intel_snps_phy.h" static struct drm_device *intel_hdmi_to_dev(struct intel_hdmi *intel_hdmi) { @@ -1850,6 +1851,16 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi, if (IS_CHERRYVIEW(dev_priv) && clock > 216000 && clock < 240000) return MODE_CLOCK_RANGE; + /* + * SNPS PHYs' MPLLB table-based programming can only handle a fixed + * set of link rates. + * + * FIXME: We will hopefully get an algorithmic way of programming + * the MPLLB for HDMI in the future. + */ + if (IS_DG2(dev_priv)) + return intel_snps_phy_check_hdmi_link_rate(clock); + return MODE_OK; } diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c index dfd724e506b5..3855fba70980 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.c +++ b/drivers/gpu/drm/i915/display/intel_opregion.c @@ -1078,6 +1078,9 @@ void intel_opregion_resume(struct drm_i915_private *i915) opregion->asle->ardy = ASLE_ARDY_READY; } + /* Some platforms abuse the _DSM to enable MUX */ + intel_dsm_get_bios_data_funcs_supported(i915); + intel_opregion_notify_adapter(i915, PCI_D0); } diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 9643624fe160..1b0daf649e82 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -32,6 +32,7 @@ #include "intel_dp_aux.h" #include "intel_hdmi.h" #include "intel_psr.h" +#include "intel_snps_phy.h" #include "intel_sprite.h" #include "skl_universal_plane.h" @@ -534,19 +535,21 @@ static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp) static void hsw_activate_psr2(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 val; + u32 val = EDP_PSR2_ENABLE; + + val |= psr_compute_idle_frames(intel_dp) << EDP_PSR2_IDLE_FRAME_SHIFT; - val = psr_compute_idle_frames(intel_dp) << EDP_PSR2_IDLE_FRAME_SHIFT; + if (!IS_ALDERLAKE_P(dev_priv)) + val |= EDP_SU_TRACK_ENABLE; - val |= EDP_PSR2_ENABLE | EDP_SU_TRACK_ENABLE; if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) <= 12) val |= EDP_Y_COORDINATE_ENABLE; val |= EDP_PSR2_FRAME_BEFORE_SU(intel_dp->psr.sink_sync_latency + 1); val |= intel_psr2_get_tp_time(intel_dp); - /* Wa_22012278275:adlp */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_D1)) { + /* Wa_22012278275:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_E0)) { static const u8 map[] = { 2, /* 5 lines */ 1, /* 6 lines */ @@ -592,9 +595,8 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) val |= EDP_PSR2_SU_SDP_SCANLINE; if (intel_dp->psr.psr2_sel_fetch_enabled) { - /* WA 1408330847 */ - if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0) || - IS_RKL_REVID(dev_priv, RKL_REVID_A0, RKL_REVID_A0)) + /* Wa_1408330847 */ + if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) intel_de_rmw(dev_priv, CHICKEN_PAR1_1, DIS_RAM_BYPASS_PSR2_MAN_TRACK, DIS_RAM_BYPASS_PSR2_MAN_TRACK); @@ -733,8 +735,8 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state)) return; - /* Wa_16011303918:adlp */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + /* Wa_16011303918:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) return; /* @@ -781,7 +783,7 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, } /* Wa_14010254185 Wa_14010103792 */ - if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1)) { + if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) { drm_dbg_kms(&dev_priv->drm, "PSR2 sel fetch not enabled, missing the implementation of WAs\n"); return false; @@ -793,6 +795,7 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, static bool psr2_granularity_check(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state) { + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay; const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay; u16 y_granularity = 0; @@ -809,10 +812,13 @@ static bool psr2_granularity_check(struct intel_dp *intel_dp, return intel_dp->psr.su_y_granularity == 4; /* - * For SW tracking we can adjust the y to match sink requirement if - * multiple of 4 + * adl_p has 1 line granularity. For other platforms with SW tracking we + * can adjust the y coordinates to match sink requirement if multiple of + * 4. */ - if (intel_dp->psr.su_y_granularity <= 2) + if (IS_ALDERLAKE_P(dev_priv)) + y_granularity = intel_dp->psr.su_y_granularity; + else if (intel_dp->psr.su_y_granularity <= 2) y_granularity = 4; else if ((intel_dp->psr.su_y_granularity % 4) == 0) y_granularity = intel_dp->psr.su_y_granularity; @@ -865,7 +871,8 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, } /* Wa_16011181250 */ - if (IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv)) { + if (IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv) || + IS_DG2(dev_priv)) { drm_dbg_kms(&dev_priv->drm, "PSR2 is defeatured for this platform\n"); return false; } @@ -940,7 +947,7 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, /* Wa_2209313811 */ if (!crtc_state->enable_psr2_sel_fetch && - IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1)) { + IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) { drm_dbg_kms(&dev_priv->drm, "PSR2 HW tracking is not supported this Display stepping\n"); return false; } @@ -965,9 +972,9 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, return false; } - /* Wa_16011303918:adlp */ + /* Wa_16011303918:adl-p */ if (crtc_state->vrr.enable && - IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) { + IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) { drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, not compatible with HW stepping + VRR\n"); return false; @@ -1160,8 +1167,8 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp) intel_dp->psr.psr2_sel_fetch_enabled ? IGNORE_PSR2_HW_TRACKING : 0); - /* Wa_16011168373:adlp */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0) && + /* Wa_16011168373:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) && intel_dp->psr.psr2_enabled) intel_de_rmw(dev_priv, TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), @@ -1206,6 +1213,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); struct intel_encoder *encoder = &dig_port->base; u32 val; @@ -1231,6 +1239,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state, &intel_dp->psr.vsc); intel_write_dp_vsc_sdp(encoder, crtc_state, &intel_dp->psr.vsc); + intel_snps_phy_update_psr_power_state(dev_priv, phy, true); intel_psr_enable_sink(intel_dp); intel_psr_enable_source(intel_dp); intel_dp->psr.enabled = true; @@ -1327,6 +1336,8 @@ static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp) static void intel_psr_disable_locked(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + enum phy phy = intel_port_to_phy(dev_priv, + dp_to_dig_port(intel_dp)->base.port); lockdep_assert_held(&intel_dp->psr.lock); @@ -1339,20 +1350,21 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) intel_psr_exit(intel_dp); intel_psr_wait_exit_locked(intel_dp); - /* WA 1408330847 */ + /* Wa_1408330847 */ if (intel_dp->psr.psr2_sel_fetch_enabled && - (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0) || - IS_RKL_REVID(dev_priv, RKL_REVID_A0, RKL_REVID_A0))) + IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) intel_de_rmw(dev_priv, CHICKEN_PAR1_1, DIS_RAM_BYPASS_PSR2_MAN_TRACK, 0); - /* Wa_16011168373:adlp */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0) && + /* Wa_16011168373:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) && intel_dp->psr.psr2_enabled) intel_de_rmw(dev_priv, TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), TRANS_SET_CONTEXT_LATENCY_MASK, 0); + intel_snps_phy_update_psr_power_state(dev_priv, phy, false); + /* Disable PSR on Sink */ drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0); @@ -1525,21 +1537,32 @@ void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_st static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state, struct drm_rect *clip, bool full_update) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 val = PSR2_MAN_TRK_CTL_ENABLE; if (full_update) { - val |= PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; + if (IS_ALDERLAKE_P(dev_priv)) + val |= ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; + else + val |= PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; + goto exit; } if (clip->y1 == -1) goto exit; - drm_WARN_ON(crtc_state->uapi.crtc->dev, clip->y1 % 4 || clip->y2 % 4); + if (IS_ALDERLAKE_P(dev_priv)) { + val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1); + val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2); + } else { + drm_WARN_ON(crtc_state->uapi.crtc->dev, clip->y1 % 4 || clip->y2 % 4); - val |= PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE; - val |= PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1 / 4 + 1); - val |= PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 / 4 + 1); + val |= PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE; + val |= PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1 / 4 + 1); + val |= PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 / 4 + 1); + } exit: crtc_state->psr2_man_track_ctl = val; } @@ -1563,11 +1586,15 @@ static void clip_area_update(struct drm_rect *overlap_damage_area, static void intel_psr2_sel_fetch_pipe_alignment(const struct intel_crtc_state *crtc_state, struct drm_rect *pipe_clip) { + struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); const u16 y_alignment = crtc_state->su_y_granularity; pipe_clip->y1 -= pipe_clip->y1 % y_alignment; if (pipe_clip->y2 % y_alignment) pipe_clip->y2 = ((pipe_clip->y2 / y_alignment) + 1) * y_alignment; + + if (IS_ALDERLAKE_P(dev_priv) && crtc_state->dsc.compression_enable) + drm_warn(&dev_priv->drm, "Missing PSR2 sel fetch alignment with DSC\n"); } int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, @@ -1699,6 +1726,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, sel_fetch_area = &new_plane_state->psr2_sel_fetch_area; sel_fetch_area->y1 = inter.y1 - new_plane_state->uapi.dst.y1; sel_fetch_area->y2 = inter.y2 - new_plane_state->uapi.dst.y1; + crtc_state->update_planes |= BIT(plane->id); } skip_sel_fetch_set_loop: diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c new file mode 100644 index 000000000000..18b52b64af95 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include <linux/util_macros.h> + +#include "intel_de.h" +#include "intel_display_types.h" +#include "intel_snps_phy.h" + +/** + * DOC: Synopsis PHY support + * + * Synopsis PHYs are primarily programmed by looking up magic register values + * in tables rather than calculating the necessary values at runtime. + * + * Of special note is that the SNPS PHYs include a dedicated port PLL, known as + * an "MPLLB." The MPLLB replaces the shared DPLL functionality used on other + * platforms and must be programming directly during the modeset sequence + * since it is not handled by the shared DPLL framework as on other platforms. + */ + +void intel_snps_phy_wait_for_calibration(struct drm_i915_private *dev_priv) +{ + enum phy phy; + + for_each_phy_masked(phy, ~0) { + if (!intel_phy_is_snps(dev_priv, phy)) + continue; + + if (intel_de_wait_for_clear(dev_priv, ICL_PHY_MISC(phy), + DG2_PHY_DP_TX_ACK_MASK, 25)) + DRM_ERROR("SNPS PHY %c failed to calibrate after 25ms.\n", + phy); + } +} + +void intel_snps_phy_update_psr_power_state(struct drm_i915_private *dev_priv, + enum phy phy, bool enable) +{ + u32 val; + + if (!intel_phy_is_snps(dev_priv, phy)) + return; + + val = REG_FIELD_PREP(SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR, + enable ? 2 : 3); + intel_uncore_rmw(&dev_priv->uncore, SNPS_PHY_TX_REQ(phy), + SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR, val); +} + +static const u32 dg2_ddi_translations[] = { + /* VS 0, pre-emph 0 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 26), + + /* VS 0, pre-emph 1 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 33) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 6), + + /* VS 0, pre-emph 2 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 38) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 12), + + /* VS 0, pre-emph 3 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 43) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 19), + + /* VS 1, pre-emph 0 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 39), + + /* VS 1, pre-emph 1 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 44) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 8), + + /* VS 1, pre-emph 2 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 47) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 15), + + /* VS 2, pre-emph 0 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 52), + + /* VS 2, pre-emph 1 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 51) | + REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 10), + + /* VS 3, pre-emph 0 */ + REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 62), +}; + +void intel_snps_phy_ddi_vswing_sequence(struct intel_encoder *encoder, + u32 level) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + int n_entries, ln; + + n_entries = ARRAY_SIZE(dg2_ddi_translations); + if (level >= n_entries) + level = n_entries - 1; + + for (ln = 0; ln < 4; ln++) + intel_de_write(dev_priv, SNPS_PHY_TX_EQ(ln, phy), + dg2_ddi_translations[level]); +} + +/* + * Basic DP link rates with 100 MHz reference clock. + */ + +static const struct intel_mpllb_state dg2_dp_rbr_100 = { + .clock = 162000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 20) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 226), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 3), +}; + +static const struct intel_mpllb_state dg2_dp_hbr1_100 = { + .clock = 270000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 20) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 184), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), +}; + +static const struct intel_mpllb_state dg2_dp_hbr2_100 = { + .clock = 540000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 20) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 184), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), +}; + +static const struct intel_mpllb_state dg2_dp_hbr3_100 = { + .clock = 810000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 19) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 292), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), +}; + +static const struct intel_mpllb_state *dg2_dp_100_tables[] = { + &dg2_dp_rbr_100, + &dg2_dp_hbr1_100, + &dg2_dp_hbr2_100, + &dg2_dp_hbr3_100, + NULL, +}; + +/* + * Basic DP link rates with 38.4 MHz reference clock. + */ + +static const struct intel_mpllb_state dg2_dp_rbr_38_4 = { + .clock = 162000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 304), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 49152), +}; + +static const struct intel_mpllb_state dg2_dp_hbr1_38_4 = { + .clock = 270000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 248), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40960), +}; + +static const struct intel_mpllb_state dg2_dp_hbr2_38_4 = { + .clock = 540000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 248), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40960), +}; + +static const struct intel_mpllb_state dg2_dp_hbr3_38_4 = { + .clock = 810000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 26) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 388), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 61440), +}; + +static const struct intel_mpllb_state *dg2_dp_38_4_tables[] = { + &dg2_dp_rbr_38_4, + &dg2_dp_hbr1_38_4, + &dg2_dp_hbr2_38_4, + &dg2_dp_hbr3_38_4, + NULL, +}; + +/* + * eDP link rates with 100 MHz reference clock. + */ + +static const struct intel_mpllb_state dg2_edp_r216 = { + .clock = 216000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 19) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 312), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 4), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 50961), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 65752), +}; + +static const struct intel_mpllb_state dg2_edp_r243 = { + .clock = 243000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 20) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 356), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 57331), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 73971), +}; + +static const struct intel_mpllb_state dg2_edp_r324 = { + .clock = 324000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 20) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 226), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 3), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 38221), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 49314), +}; + +static const struct intel_mpllb_state dg2_edp_r432 = { + .clock = 432000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 19) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 312), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 4), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 50961), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 65752), +}; + +static const struct intel_mpllb_state *dg2_edp_tables[] = { + &dg2_dp_rbr_100, + &dg2_edp_r216, + &dg2_edp_r243, + &dg2_dp_hbr1_100, + &dg2_edp_r324, + &dg2_edp_r432, + &dg2_dp_hbr2_100, + &dg2_dp_hbr3_100, + NULL, +}; + +/* + * HDMI link rates with 100 MHz reference clock. + */ + +static const struct intel_mpllb_state dg2_hdmi_25_175 = { + .clock = 25175, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 128) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 143), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36663) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 71), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_27_0 = { + .clock = 27000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 140) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_74_25 = { + .clock = 74250, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_148_5 = { + .clock = 148500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_594 = { + .clock = 594000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 5), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state *dg2_hdmi_tables[] = { + &dg2_hdmi_25_175, + &dg2_hdmi_27_0, + &dg2_hdmi_74_25, + &dg2_hdmi_148_5, + &dg2_hdmi_594, + NULL, +}; + +static const struct intel_mpllb_state ** +intel_mpllb_tables_get(struct intel_crtc_state *crtc_state, + struct intel_encoder *encoder) +{ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) { + return dg2_edp_tables; + } else if (intel_crtc_has_dp_encoder(crtc_state)) { + /* + * FIXME: Initially we're just enabling the "combo" outputs on + * port A-D. The MPLLB for those ports takes an input from the + * "Display Filter PLL" which always has an output frequency + * of 100 MHz, hence the use of the _100 tables below. + * + * Once we enable port TC1 it will either use the same 100 MHz + * "Display Filter PLL" (when strapped to support a native + * display connection) or different 38.4 MHz "Filter PLL" when + * strapped to support a USB connection, so we'll need to check + * that to determine which table to use. + */ + if (0) + return dg2_dp_38_4_tables; + else + return dg2_dp_100_tables; + } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { + return dg2_hdmi_tables; + } + + MISSING_CASE(encoder->type); + return NULL; +} + +int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state, + struct intel_encoder *encoder) +{ + const struct intel_mpllb_state **tables; + int i; + + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { + if (intel_snps_phy_check_hdmi_link_rate(crtc_state->port_clock) + != MODE_OK) { + /* + * FIXME: Can only support fixed HDMI frequencies + * until we have a proper algorithm under a valid + * license. + */ + DRM_DEBUG_KMS("Can't support HDMI link rate %d\n", + crtc_state->port_clock); + return -EINVAL; + } + } + + tables = intel_mpllb_tables_get(crtc_state, encoder); + if (!tables) + return -EINVAL; + + for (i = 0; tables[i]; i++) { + if (crtc_state->port_clock <= tables[i]->clock) { + crtc_state->mpllb_state = *tables[i]; + return 0; + } + } + + return -EINVAL; +} + +void intel_mpllb_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + const struct intel_mpllb_state *pll_state = &crtc_state->mpllb_state; + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + i915_reg_t enable_reg = (phy <= PHY_D ? + DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0)); + + /* + * 3. Software programs the following PLL registers for the desired + * frequency. + */ + intel_de_write(dev_priv, SNPS_PHY_MPLLB_CP(phy), pll_state->mpllb_cp); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_DIV(phy), pll_state->mpllb_div); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_DIV2(phy), pll_state->mpllb_div2); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_SSCEN(phy), pll_state->mpllb_sscen); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_SSCSTEP(phy), pll_state->mpllb_sscstep); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_FRACN1(phy), pll_state->mpllb_fracn1); + intel_de_write(dev_priv, SNPS_PHY_MPLLB_FRACN2(phy), pll_state->mpllb_fracn2); + + /* + * 4. If the frequency will result in a change to the voltage + * requirement, follow the Display Voltage Frequency Switching - + * Sequence Before Frequency Change. + * + * We handle this step in bxt_set_cdclk(). + */ + + /* 5. Software sets DPLL_ENABLE [PLL Enable] to "1". */ + intel_uncore_rmw(&dev_priv->uncore, enable_reg, 0, PLL_ENABLE); + + /* + * 9. Software sets SNPS_PHY_MPLLB_DIV dp_mpllb_force_en to "1". This + * will keep the PLL running during the DDI lane programming and any + * typeC DP cable disconnect. Do not set the force before enabling the + * PLL because that will start the PLL before it has sampled the + * divider values. + */ + intel_de_write(dev_priv, SNPS_PHY_MPLLB_DIV(phy), + pll_state->mpllb_div | SNPS_PHY_MPLLB_FORCE_EN); + + /* + * 10. Software polls on register DPLL_ENABLE [PLL Lock] to confirm PLL + * is locked at new settings. This register bit is sampling PHY + * dp_mpllb_state interface signal. + */ + if (intel_de_wait_for_set(dev_priv, enable_reg, PLL_LOCK, 5)) + DRM_ERROR("Port %c PLL not locked\n", phy_name(phy)); + + /* + * 11. If the frequency will result in a change to the voltage + * requirement, follow the Display Voltage Frequency Switching - + * Sequence After Frequency Change. + * + * We handle this step in bxt_set_cdclk(). + */ +} + +void intel_mpllb_disable(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + i915_reg_t enable_reg = (phy <= PHY_D ? + DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0)); + + /* + * 1. If the frequency will result in a change to the voltage + * requirement, follow the Display Voltage Frequency Switching - + * Sequence Before Frequency Change. + * + * We handle this step in bxt_set_cdclk(). + */ + + /* 2. Software programs DPLL_ENABLE [PLL Enable] to "0" */ + intel_uncore_rmw(&dev_priv->uncore, enable_reg, PLL_ENABLE, 0); + + /* + * 4. Software programs SNPS_PHY_MPLLB_DIV dp_mpllb_force_en to "0". + * This will allow the PLL to stop running. + */ + intel_uncore_rmw(&dev_priv->uncore, SNPS_PHY_MPLLB_DIV(phy), + SNPS_PHY_MPLLB_FORCE_EN, 0); + + /* + * 5. Software polls DPLL_ENABLE [PLL Lock] for PHY acknowledgment + * (dp_txX_ack) that the new transmitter setting request is completed. + */ + if (intel_de_wait_for_clear(dev_priv, enable_reg, PLL_LOCK, 5)) + DRM_ERROR("Port %c PLL not locked\n", phy_name(phy)); + + /* + * 6. If the frequency will result in a change to the voltage + * requirement, follow the Display Voltage Frequency Switching - + * Sequence After Frequency Change. + * + * We handle this step in bxt_set_cdclk(). + */ +} + +int intel_mpllb_calc_port_clock(struct intel_encoder *encoder, + const struct intel_mpllb_state *pll_state) +{ + unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1; + unsigned int multiplier, tx_clk_div, refclk; + bool frac_en; + + if (0) + refclk = 38400; + else + refclk = 100000; + + refclk >>= REG_FIELD_GET(SNPS_PHY_MPLLB_REF_CLK_DIV, pll_state->mpllb_div2) - 1; + + frac_en = REG_FIELD_GET(SNPS_PHY_MPLLB_FRACN_EN, pll_state->mpllb_fracn1); + + if (frac_en) { + frac_quot = REG_FIELD_GET(SNPS_PHY_MPLLB_FRACN_QUOT, pll_state->mpllb_fracn2); + frac_rem = REG_FIELD_GET(SNPS_PHY_MPLLB_FRACN_REM, pll_state->mpllb_fracn2); + frac_den = REG_FIELD_GET(SNPS_PHY_MPLLB_FRACN_DEN, pll_state->mpllb_fracn1); + } + + multiplier = REG_FIELD_GET(SNPS_PHY_MPLLB_MULTIPLIER, pll_state->mpllb_div2) / 2 + 16; + + tx_clk_div = REG_FIELD_GET(SNPS_PHY_MPLLB_TX_CLK_DIV, pll_state->mpllb_div); + + return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, (multiplier << 16) + frac_quot) + + DIV_ROUND_CLOSEST(refclk * frac_rem, frac_den), + 10 << (tx_clk_div + 16)); +} + +void intel_mpllb_readout_hw_state(struct intel_encoder *encoder, + struct intel_mpllb_state *pll_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + + pll_state->mpllb_cp = intel_de_read(dev_priv, SNPS_PHY_MPLLB_CP(phy)); + pll_state->mpllb_div = intel_de_read(dev_priv, SNPS_PHY_MPLLB_DIV(phy)); + pll_state->mpllb_div2 = intel_de_read(dev_priv, SNPS_PHY_MPLLB_DIV2(phy)); + pll_state->mpllb_sscen = intel_de_read(dev_priv, SNPS_PHY_MPLLB_SSCEN(phy)); + pll_state->mpllb_sscstep = intel_de_read(dev_priv, SNPS_PHY_MPLLB_SSCSTEP(phy)); + pll_state->mpllb_fracn1 = intel_de_read(dev_priv, SNPS_PHY_MPLLB_FRACN1(phy)); + pll_state->mpllb_fracn2 = intel_de_read(dev_priv, SNPS_PHY_MPLLB_FRACN2(phy)); + + /* + * REF_CONTROL is under firmware control and never programmed by the + * driver; we read it only for sanity checking purposes. The bspec + * only tells us the expected value for one field in this register, + * so we'll only read out those specific bits here. + */ + pll_state->ref_control = intel_de_read(dev_priv, SNPS_PHY_REF_CONTROL(phy)) & + SNPS_PHY_REF_CONTROL_REF_RANGE; + + /* + * MPLLB_DIV is programmed twice, once with the software-computed + * state, then again with the MPLLB_FORCE_EN bit added. Drop that + * extra bit during readout so that we return the actual expected + * software state. + */ + pll_state->mpllb_div &= ~SNPS_PHY_MPLLB_FORCE_EN; +} + +int intel_snps_phy_check_hdmi_link_rate(int clock) +{ + const struct intel_mpllb_state **tables = dg2_hdmi_tables; + int i; + + for (i = 0; tables[i]; i++) { + if (clock == tables[i]->clock) + return MODE_OK; + } + + return MODE_CLOCK_RANGE; +} diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.h b/drivers/gpu/drm/i915/display/intel_snps_phy.h new file mode 100644 index 000000000000..6261ff88ef5c --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __INTEL_SNPS_PHY_H__ +#define __INTEL_SNPS_PHY_H__ + +#include <linux/types.h> + +struct drm_i915_private; +struct intel_encoder; +struct intel_crtc_state; +struct intel_mpllb_state; +enum phy; + +void intel_snps_phy_wait_for_calibration(struct drm_i915_private *dev_priv); +void intel_snps_phy_update_psr_power_state(struct drm_i915_private *dev_priv, + enum phy phy, bool enable); + +int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state, + struct intel_encoder *encoder); +void intel_mpllb_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); +void intel_mpllb_disable(struct intel_encoder *encoder); +void intel_mpllb_readout_hw_state(struct intel_encoder *encoder, + struct intel_mpllb_state *pll_state); +int intel_mpllb_calc_port_clock(struct intel_encoder *encoder, + const struct intel_mpllb_state *pll_state); + +int intel_snps_phy_check_hdmi_link_rate(int clock); +void intel_snps_phy_ddi_vswing_sequence(struct intel_encoder *encoder, + u32 level); + +#endif /* __INTEL_SNPS_PHY_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index 4ae9a7455b23..08116f41da26 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -1856,7 +1856,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv, zpos = sprite + 1; drm_plane_create_zpos_immutable_property(&plane->base, zpos); - drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); + intel_plane_helper_add(plane); return plane; diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h index dbe24d7e7375..330077c2e588 100644 --- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h @@ -456,7 +456,7 @@ struct child_device_config { u16 dp_gpio_pin_num; /* 195 */ u8 dp_iboost_level:4; /* 196 */ u8 hdmi_iboost_level:4; /* 196 */ - u8 dp_max_link_rate:3; /* 216/230 CNL+ */ + u8 dp_max_link_rate:3; /* 216/230 GLK+ */ u8 dp_max_link_rate_reserved:5; /* 216/230 */ } __packed; diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 85749370508c..df3286aa6999 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -348,7 +348,10 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state) if (DISPLAY_VER(i915) >= 12) return true; - if ((DISPLAY_VER(i915) >= 11 || IS_CANNONLAKE(i915)) && (pipe != PIPE_A || (cpu_transcoder == TRANSCODER_EDP || cpu_transcoder == TRANSCODER_DSI_0 || cpu_transcoder == TRANSCODER_DSI_1))) + if (DISPLAY_VER(i915) >= 11 && + (pipe != PIPE_A || cpu_transcoder == TRANSCODER_EDP || + cpu_transcoder == TRANSCODER_DSI_0 || + cpu_transcoder == TRANSCODER_DSI_1)) return true; return false; diff --git a/drivers/gpu/drm/i915/display/skl_scaler.c b/drivers/gpu/drm/i915/display/skl_scaler.c index 911a113ee006..37eabeff8197 100644 --- a/drivers/gpu/drm/i915/display/skl_scaler.c +++ b/drivers/gpu/drm/i915/display/skl_scaler.c @@ -294,12 +294,12 @@ int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, return 0; } -static int cnl_coef_tap(int i) +static int glk_coef_tap(int i) { return i % 7; } -static u16 cnl_nearest_filter_coef(int t) +static u16 glk_nearest_filter_coef(int t) { return t == 3 ? 0x0800 : 0x3000; } @@ -341,29 +341,29 @@ static u16 cnl_nearest_filter_coef(int t) * */ -static void cnl_program_nearest_filter_coefs(struct drm_i915_private *dev_priv, +static void glk_program_nearest_filter_coefs(struct drm_i915_private *dev_priv, enum pipe pipe, int id, int set) { int i; - intel_de_write_fw(dev_priv, CNL_PS_COEF_INDEX_SET(pipe, id, set), + intel_de_write_fw(dev_priv, GLK_PS_COEF_INDEX_SET(pipe, id, set), PS_COEE_INDEX_AUTO_INC); for (i = 0; i < 17 * 7; i += 2) { u32 tmp; int t; - t = cnl_coef_tap(i); - tmp = cnl_nearest_filter_coef(t); + t = glk_coef_tap(i); + tmp = glk_nearest_filter_coef(t); - t = cnl_coef_tap(i + 1); - tmp |= cnl_nearest_filter_coef(t) << 16; + t = glk_coef_tap(i + 1); + tmp |= glk_nearest_filter_coef(t) << 16; - intel_de_write_fw(dev_priv, CNL_PS_COEF_DATA_SET(pipe, id, set), + intel_de_write_fw(dev_priv, GLK_PS_COEF_DATA_SET(pipe, id, set), tmp); } - intel_de_write_fw(dev_priv, CNL_PS_COEF_INDEX_SET(pipe, id, set), 0); + intel_de_write_fw(dev_priv, GLK_PS_COEF_INDEX_SET(pipe, id, set), 0); } static u32 skl_scaler_get_filter_select(enum drm_scaling_filter filter, int set) @@ -386,7 +386,7 @@ static void skl_scaler_setup_filter(struct drm_i915_private *dev_priv, enum pipe case DRM_SCALING_FILTER_DEFAULT: break; case DRM_SCALING_FILTER_NEAREST_NEIGHBOR: - cnl_program_nearest_filter_coefs(dev_priv, pipe, id, set); + glk_program_nearest_filter_coefs(dev_priv, pipe, id, set); break; default: MISSING_CASE(filter); diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 92a4fd508e92..724e7b04f3b6 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -835,7 +835,7 @@ static u32 skl_plane_ctl_rotate(unsigned int rotate) return 0; } -static u32 cnl_plane_ctl_flip(unsigned int reflect) +static u32 icl_plane_ctl_flip(unsigned int reflect) { switch (reflect) { case 0: @@ -917,8 +917,8 @@ static u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, plane_ctl |= skl_plane_ctl_tiling(fb->modifier); plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK); - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) - plane_ctl |= cnl_plane_ctl_flip(rotation & + if (DISPLAY_VER(dev_priv) >= 11) + plane_ctl |= icl_plane_ctl_flip(rotation & DRM_MODE_REFLECT_MASK); if (key->flags & I915_SET_COLORKEY_DESTINATION) @@ -926,7 +926,7 @@ static u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, else if (key->flags & I915_SET_COLORKEY_SOURCE) plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE; - /* Wa_22012358565:adlp */ + /* Wa_22012358565:adl-p */ if (DISPLAY_VER(dev_priv) == 13) plane_ctl |= adlp_plane_ctl_arb_slots(plane_state); @@ -1270,7 +1270,7 @@ static int skl_plane_check_dst_coordinates(const struct intel_crtc_state *crtc_s int pipe_src_w = crtc_state->pipe_src_w; /* - * Display WA #1175: cnl,glk + * Display WA #1175: glk * Planes other than the cursor may cause FIFO underflow and display * corruption if starting less than 4 pixels from the right edge of * the screen. @@ -1828,7 +1828,7 @@ static bool skl_plane_has_ccs(struct drm_i915_private *dev_priv, if (plane_id == PLANE_CURSOR) return false; - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) return true; if (IS_GEMINILAKE(dev_priv)) @@ -1910,11 +1910,11 @@ static bool gen12_plane_supports_mc_ccs(struct drm_i915_private *dev_priv, { /* Wa_14010477008:tgl[a0..c0],rkl[all],dg1[all] */ if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv) || - IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) + IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_D0)) return false; /* Wa_22011186057 */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) return false; return plane_id < PLANE_SPRITE4; @@ -1938,7 +1938,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: /* Wa_22011186057 */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) return false; break; default: @@ -1995,7 +1995,7 @@ static const u64 *gen12_get_plane_modifiers(struct drm_i915_private *dev_priv, enum plane_id plane_id) { /* Wa_22011186057 */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) return adlp_step_a_plane_format_modifiers; else if (gen12_plane_supports_mc_ccs(dev_priv, plane_id)) return gen12_plane_format_modifiers_mc_ccs; @@ -2144,7 +2144,7 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270; - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) supported_rotations |= DRM_MODE_REFLECT_X; drm_plane_create_rotation_property(&plane->base, @@ -2174,12 +2174,12 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, if (DISPLAY_VER(dev_priv) >= 12) drm_plane_enable_fb_damage_clips(&plane->base); - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 11) drm_plane_create_scaling_filter_property(&plane->base, BIT(DRM_SCALING_FILTER_DEFAULT) | BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); - drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); + intel_plane_helper_add(plane); return plane; @@ -2295,7 +2295,7 @@ skl_get_initial_plane_config(struct intel_crtc *crtc, break; } - if ((DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) && val & PLANE_CTL_FLIP_HORIZONTAL) + if (DISPLAY_VER(dev_priv) >= 11 && val & PLANE_CTL_FLIP_HORIZONTAL) plane_config->rotation |= DRM_MODE_REFLECT_X; /* 90/270 degree rotation would require extra work */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c index daf9284ef1f5..f0435c6feb68 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c @@ -24,13 +24,11 @@ static void __do_clflush(struct drm_i915_gem_object *obj) i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); } -static int clflush_work(struct dma_fence_work *base) +static void clflush_work(struct dma_fence_work *base) { struct clflush *clflush = container_of(base, typeof(*clflush), base); __do_clflush(clflush->obj); - - return 0; } static void clflush_release(struct dma_fence_work *base) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c deleted file mode 100644 index 44821d94544f..000000000000 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c +++ /dev/null @@ -1,355 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include "i915_drv.h" -#include "gt/intel_context.h" -#include "gt/intel_engine_pm.h" -#include "i915_gem_client_blt.h" -#include "i915_gem_object_blt.h" - -struct i915_sleeve { - struct i915_vma *vma; - struct drm_i915_gem_object *obj; - struct sg_table *pages; - struct i915_page_sizes page_sizes; -}; - -static int vma_set_pages(struct i915_vma *vma) -{ - struct i915_sleeve *sleeve = vma->private; - - vma->pages = sleeve->pages; - vma->page_sizes = sleeve->page_sizes; - - return 0; -} - -static void vma_clear_pages(struct i915_vma *vma) -{ - GEM_BUG_ON(!vma->pages); - vma->pages = NULL; -} - -static void vma_bind(struct i915_address_space *vm, - struct i915_vm_pt_stash *stash, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) -{ - vm->vma_ops.bind_vma(vm, stash, vma, cache_level, flags); -} - -static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) -{ - vm->vma_ops.unbind_vma(vm, vma); -} - -static const struct i915_vma_ops proxy_vma_ops = { - .set_pages = vma_set_pages, - .clear_pages = vma_clear_pages, - .bind_vma = vma_bind, - .unbind_vma = vma_unbind, -}; - -static struct i915_sleeve *create_sleeve(struct i915_address_space *vm, - struct drm_i915_gem_object *obj, - struct sg_table *pages, - struct i915_page_sizes *page_sizes) -{ - struct i915_sleeve *sleeve; - struct i915_vma *vma; - int err; - - sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL); - if (!sleeve) - return ERR_PTR(-ENOMEM); - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_free; - } - - vma->private = sleeve; - vma->ops = &proxy_vma_ops; - - sleeve->vma = vma; - sleeve->pages = pages; - sleeve->page_sizes = *page_sizes; - - return sleeve; - -err_free: - kfree(sleeve); - return ERR_PTR(err); -} - -static void destroy_sleeve(struct i915_sleeve *sleeve) -{ - kfree(sleeve); -} - -struct clear_pages_work { - struct dma_fence dma; - struct dma_fence_cb cb; - struct i915_sw_fence wait; - struct work_struct work; - struct irq_work irq_work; - struct i915_sleeve *sleeve; - struct intel_context *ce; - u32 value; -}; - -static const char *clear_pages_work_driver_name(struct dma_fence *fence) -{ - return DRIVER_NAME; -} - -static const char *clear_pages_work_timeline_name(struct dma_fence *fence) -{ - return "clear"; -} - -static void clear_pages_work_release(struct dma_fence *fence) -{ - struct clear_pages_work *w = container_of(fence, typeof(*w), dma); - - destroy_sleeve(w->sleeve); - - i915_sw_fence_fini(&w->wait); - - BUILD_BUG_ON(offsetof(typeof(*w), dma)); - dma_fence_free(&w->dma); -} - -static const struct dma_fence_ops clear_pages_work_ops = { - .get_driver_name = clear_pages_work_driver_name, - .get_timeline_name = clear_pages_work_timeline_name, - .release = clear_pages_work_release, -}; - -static void clear_pages_signal_irq_worker(struct irq_work *work) -{ - struct clear_pages_work *w = container_of(work, typeof(*w), irq_work); - - dma_fence_signal(&w->dma); - dma_fence_put(&w->dma); -} - -static void clear_pages_dma_fence_cb(struct dma_fence *fence, - struct dma_fence_cb *cb) -{ - struct clear_pages_work *w = container_of(cb, typeof(*w), cb); - - if (fence->error) - dma_fence_set_error(&w->dma, fence->error); - - /* - * Push the signalling of the fence into yet another worker to avoid - * the nightmare locking around the fence spinlock. - */ - irq_work_queue(&w->irq_work); -} - -static void clear_pages_worker(struct work_struct *work) -{ - struct clear_pages_work *w = container_of(work, typeof(*w), work); - struct drm_i915_gem_object *obj = w->sleeve->vma->obj; - struct i915_vma *vma = w->sleeve->vma; - struct i915_gem_ww_ctx ww; - struct i915_request *rq; - struct i915_vma *batch; - int err = w->dma.error; - - if (unlikely(err)) - goto out_signal; - - if (obj->cache_dirty) { - if (i915_gem_object_has_struct_page(obj)) - drm_clflush_sg(w->sleeve->pages); - obj->cache_dirty = false; - } - obj->read_domains = I915_GEM_GPU_DOMAINS; - obj->write_domain = 0; - - i915_gem_ww_ctx_init(&ww, false); - intel_engine_pm_get(w->ce->engine); -retry: - err = intel_context_pin_ww(w->ce, &ww); - if (err) - goto out_signal; - - batch = intel_emit_vma_fill_blt(w->ce, vma, &ww, w->value); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_ctx; - } - - rq = i915_request_create(w->ce); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_batch; - } - - /* There's no way the fence has signalled */ - if (dma_fence_add_callback(&rq->fence, &w->cb, - clear_pages_dma_fence_cb)) - GEM_BUG_ON(1); - - err = intel_emit_vma_mark_active(batch, rq); - if (unlikely(err)) - goto out_request; - - /* - * w->dma is already exported via (vma|obj)->resv we need only - * keep track of the GPU activity within this vma/request, and - * propagate the signal from the request to w->dma. - */ - err = __i915_vma_move_to_active(vma, rq); - if (err) - goto out_request; - - if (rq->engine->emit_init_breadcrumb) { - err = rq->engine->emit_init_breadcrumb(rq); - if (unlikely(err)) - goto out_request; - } - - err = rq->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); -out_request: - if (unlikely(err)) { - i915_request_set_error_once(rq, err); - err = 0; - } - - i915_request_add(rq); -out_batch: - intel_emit_vma_release(w->ce, batch); -out_ctx: - intel_context_unpin(w->ce); -out_signal: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - - i915_vma_unpin(w->sleeve->vma); - intel_engine_pm_put(w->ce->engine); - - if (unlikely(err)) { - dma_fence_set_error(&w->dma, err); - dma_fence_signal(&w->dma); - dma_fence_put(&w->dma); - } -} - -static int pin_wait_clear_pages_work(struct clear_pages_work *w, - struct intel_context *ce) -{ - struct i915_vma *vma = w->sleeve->vma; - struct i915_gem_ww_ctx ww; - int err; - - i915_gem_ww_ctx_init(&ww, false); -retry: - err = i915_gem_object_lock(vma->obj, &ww); - if (err) - goto out; - - err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); - if (unlikely(err)) - goto out; - - err = i915_sw_fence_await_reservation(&w->wait, - vma->obj->base.resv, NULL, - true, 0, I915_FENCE_GFP); - if (err) - goto err_unpin_vma; - - dma_resv_add_excl_fence(vma->obj->base.resv, &w->dma); - -err_unpin_vma: - if (err) - i915_vma_unpin(vma); -out: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - return err; -} - -static int __i915_sw_fence_call -clear_pages_work_notify(struct i915_sw_fence *fence, - enum i915_sw_fence_notify state) -{ - struct clear_pages_work *w = container_of(fence, typeof(*w), wait); - - switch (state) { - case FENCE_COMPLETE: - schedule_work(&w->work); - break; - - case FENCE_FREE: - dma_fence_put(&w->dma); - break; - } - - return NOTIFY_DONE; -} - -static DEFINE_SPINLOCK(fence_lock); - -/* XXX: better name please */ -int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, - struct intel_context *ce, - struct sg_table *pages, - struct i915_page_sizes *page_sizes, - u32 value) -{ - struct clear_pages_work *work; - struct i915_sleeve *sleeve; - int err; - - sleeve = create_sleeve(ce->vm, obj, pages, page_sizes); - if (IS_ERR(sleeve)) - return PTR_ERR(sleeve); - - work = kmalloc(sizeof(*work), GFP_KERNEL); - if (!work) { - destroy_sleeve(sleeve); - return -ENOMEM; - } - - work->value = value; - work->sleeve = sleeve; - work->ce = ce; - - INIT_WORK(&work->work, clear_pages_worker); - - init_irq_work(&work->irq_work, clear_pages_signal_irq_worker); - - dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0); - i915_sw_fence_init(&work->wait, clear_pages_work_notify); - - err = pin_wait_clear_pages_work(work, ce); - if (err < 0) - dma_fence_set_error(&work->dma, err); - - dma_fence_get(&work->dma); - i915_sw_fence_commit(&work->wait); - - return err; -} - -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/i915_gem_client_blt.c" -#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.h b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.h deleted file mode 100644 index 3dbd28c22ff5..000000000000 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ -#ifndef __I915_GEM_CLIENT_BLT_H__ -#define __I915_GEM_CLIENT_BLT_H__ - -#include <linux/types.h> - -struct drm_i915_gem_object; -struct i915_page_sizes; -struct intel_context; -struct sg_table; - -int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, - struct intel_context *ce, - struct sg_table *pages, - struct i915_page_sizes *page_sizes, - u32 value); - -#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 7720b8c22c81..cff72679ad7c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -67,35 +67,32 @@ #include <linux/log2.h> #include <linux/nospec.h> +#include <drm/drm_syncobj.h> + #include "gt/gen6_ppgtt.h" #include "gt/intel_context.h" #include "gt/intel_context_param.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_engine_user.h" -#include "gt/intel_execlists_submission.h" /* virtual_engine */ #include "gt/intel_gpu_commands.h" #include "gt/intel_ring.h" #include "i915_gem_context.h" -#include "i915_globals.h" #include "i915_trace.h" #include "i915_user_extensions.h" #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 -static struct i915_global_gem_context { - struct i915_global base; - struct kmem_cache *slab_luts; -} global; +static struct kmem_cache *slab_luts; struct i915_lut_handle *i915_lut_handle_alloc(void) { - return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); + return kmem_cache_alloc(slab_luts, GFP_KERNEL); } void i915_lut_handle_free(struct i915_lut_handle *lut) { - return kmem_cache_free(global.slab_luts, lut); + return kmem_cache_free(slab_luts, lut); } static void lut_close(struct i915_gem_context *ctx) @@ -167,6 +164,577 @@ lookup_user_engine(struct i915_gem_context *ctx, return i915_gem_context_get_engine(ctx, idx); } +static int validate_priority(struct drm_i915_private *i915, + const struct drm_i915_gem_context_param *args) +{ + s64 priority = args->value; + + if (args->size) + return -EINVAL; + + if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) + return -ENODEV; + + if (priority > I915_CONTEXT_MAX_USER_PRIORITY || + priority < I915_CONTEXT_MIN_USER_PRIORITY) + return -EINVAL; + + if (priority > I915_CONTEXT_DEFAULT_PRIORITY && + !capable(CAP_SYS_NICE)) + return -EPERM; + + return 0; +} + +static void proto_context_close(struct i915_gem_proto_context *pc) +{ + int i; + + if (pc->vm) + i915_vm_put(pc->vm); + if (pc->user_engines) { + for (i = 0; i < pc->num_user_engines; i++) + kfree(pc->user_engines[i].siblings); + kfree(pc->user_engines); + } + kfree(pc); +} + +static int proto_context_set_persistence(struct drm_i915_private *i915, + struct i915_gem_proto_context *pc, + bool persist) +{ + if (persist) { + /* + * Only contexts that are short-lived [that will expire or be + * reset] are allowed to survive past termination. We require + * hangcheck to ensure that the persistent requests are healthy. + */ + if (!i915->params.enable_hangcheck) + return -EINVAL; + + pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); + } else { + /* To cancel a context we use "preempt-to-idle" */ + if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) + return -ENODEV; + + /* + * If the cancel fails, we then need to reset, cleanly! + * + * If the per-engine reset fails, all hope is lost! We resort + * to a full GPU reset in that unlikely case, but realistically + * if the engine could not reset, the full reset does not fare + * much better. The damage has been done. + * + * However, if we cannot reset an engine by itself, we cannot + * cleanup a hanging persistent context without causing + * colateral damage, and we should not pretend we can by + * exposing the interface. + */ + if (!intel_has_reset_engine(&i915->gt)) + return -ENODEV; + + pc->user_flags &= ~BIT(UCONTEXT_PERSISTENCE); + } + + return 0; +} + +static struct i915_gem_proto_context * +proto_context_create(struct drm_i915_private *i915, unsigned int flags) +{ + struct i915_gem_proto_context *pc, *err; + + pc = kzalloc(sizeof(*pc), GFP_KERNEL); + if (!pc) + return ERR_PTR(-ENOMEM); + + pc->num_user_engines = -1; + pc->user_engines = NULL; + pc->user_flags = BIT(UCONTEXT_BANNABLE) | + BIT(UCONTEXT_RECOVERABLE); + if (i915->params.enable_hangcheck) + pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); + pc->sched.priority = I915_PRIORITY_NORMAL; + + if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { + if (!HAS_EXECLISTS(i915)) { + err = ERR_PTR(-EINVAL); + goto proto_close; + } + pc->single_timeline = true; + } + + return pc; + +proto_close: + proto_context_close(pc); + return err; +} + +static int proto_context_register_locked(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + u32 *id) +{ + int ret; + void *old; + + lockdep_assert_held(&fpriv->proto_context_lock); + + ret = xa_alloc(&fpriv->context_xa, id, NULL, xa_limit_32b, GFP_KERNEL); + if (ret) + return ret; + + old = xa_store(&fpriv->proto_context_xa, *id, pc, GFP_KERNEL); + if (xa_is_err(old)) { + xa_erase(&fpriv->context_xa, *id); + return xa_err(old); + } + WARN_ON(old); + + return 0; +} + +static int proto_context_register(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + u32 *id) +{ + int ret; + + mutex_lock(&fpriv->proto_context_lock); + ret = proto_context_register_locked(fpriv, pc, id); + mutex_unlock(&fpriv->proto_context_lock); + + return ret; +} + +static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + const struct drm_i915_gem_context_param *args) +{ + struct drm_i915_private *i915 = fpriv->dev_priv; + struct i915_address_space *vm; + + if (args->size) + return -EINVAL; + + if (!HAS_FULL_PPGTT(i915)) + return -ENODEV; + + if (upper_32_bits(args->value)) + return -ENOENT; + + vm = i915_gem_vm_lookup(fpriv, args->value); + if (!vm) + return -ENOENT; + + if (pc->vm) + i915_vm_put(pc->vm); + pc->vm = vm; + + return 0; +} + +struct set_proto_ctx_engines { + struct drm_i915_private *i915; + unsigned num_engines; + struct i915_gem_proto_engine *engines; +}; + +static int +set_proto_ctx_engines_balance(struct i915_user_extension __user *base, + void *data) +{ + struct i915_context_engines_load_balance __user *ext = + container_of_user(base, typeof(*ext), base); + const struct set_proto_ctx_engines *set = data; + struct drm_i915_private *i915 = set->i915; + struct intel_engine_cs **siblings; + u16 num_siblings, idx; + unsigned int n; + int err; + + if (!HAS_EXECLISTS(i915)) + return -ENODEV; + + if (get_user(idx, &ext->engine_index)) + return -EFAULT; + + if (idx >= set->num_engines) { + drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", + idx, set->num_engines); + return -EINVAL; + } + + idx = array_index_nospec(idx, set->num_engines); + if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_INVALID) { + drm_dbg(&i915->drm, + "Invalid placement[%d], already occupied\n", idx); + return -EEXIST; + } + + if (get_user(num_siblings, &ext->num_siblings)) + return -EFAULT; + + err = check_user_mbz(&ext->flags); + if (err) + return err; + + err = check_user_mbz(&ext->mbz64); + if (err) + return err; + + if (num_siblings == 0) + return 0; + + siblings = kmalloc_array(num_siblings, sizeof(*siblings), GFP_KERNEL); + if (!siblings) + return -ENOMEM; + + for (n = 0; n < num_siblings; n++) { + struct i915_engine_class_instance ci; + + if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { + err = -EFAULT; + goto err_siblings; + } + + siblings[n] = intel_engine_lookup_user(i915, + ci.engine_class, + ci.engine_instance); + if (!siblings[n]) { + drm_dbg(&i915->drm, + "Invalid sibling[%d]: { class:%d, inst:%d }\n", + n, ci.engine_class, ci.engine_instance); + err = -EINVAL; + goto err_siblings; + } + } + + if (num_siblings == 1) { + set->engines[idx].type = I915_GEM_ENGINE_TYPE_PHYSICAL; + set->engines[idx].engine = siblings[0]; + kfree(siblings); + } else { + set->engines[idx].type = I915_GEM_ENGINE_TYPE_BALANCED; + set->engines[idx].num_siblings = num_siblings; + set->engines[idx].siblings = siblings; + } + + return 0; + +err_siblings: + kfree(siblings); + + return err; +} + +static int +set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) +{ + struct i915_context_engines_bond __user *ext = + container_of_user(base, typeof(*ext), base); + const struct set_proto_ctx_engines *set = data; + struct drm_i915_private *i915 = set->i915; + struct i915_engine_class_instance ci; + struct intel_engine_cs *master; + u16 idx, num_bonds; + int err, n; + + if (get_user(idx, &ext->virtual_index)) + return -EFAULT; + + if (idx >= set->num_engines) { + drm_dbg(&i915->drm, + "Invalid index for virtual engine: %d >= %d\n", + idx, set->num_engines); + return -EINVAL; + } + + idx = array_index_nospec(idx, set->num_engines); + if (set->engines[idx].type == I915_GEM_ENGINE_TYPE_INVALID) { + drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); + return -EINVAL; + } + + if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_PHYSICAL) { + drm_dbg(&i915->drm, + "Bonding with virtual engines not allowed\n"); + return -EINVAL; + } + + err = check_user_mbz(&ext->flags); + if (err) + return err; + + for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { + err = check_user_mbz(&ext->mbz64[n]); + if (err) + return err; + } + + if (copy_from_user(&ci, &ext->master, sizeof(ci))) + return -EFAULT; + + master = intel_engine_lookup_user(i915, + ci.engine_class, + ci.engine_instance); + if (!master) { + drm_dbg(&i915->drm, + "Unrecognised master engine: { class:%u, instance:%u }\n", + ci.engine_class, ci.engine_instance); + return -EINVAL; + } + + if (intel_engine_uses_guc(master)) { + DRM_DEBUG("bonding extension not supported with GuC submission"); + return -ENODEV; + } + + if (get_user(num_bonds, &ext->num_bonds)) + return -EFAULT; + + for (n = 0; n < num_bonds; n++) { + struct intel_engine_cs *bond; + + if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) + return -EFAULT; + + bond = intel_engine_lookup_user(i915, + ci.engine_class, + ci.engine_instance); + if (!bond) { + drm_dbg(&i915->drm, + "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", + n, ci.engine_class, ci.engine_instance); + return -EINVAL; + } + } + + return 0; +} + +static const i915_user_extension_fn set_proto_ctx_engines_extensions[] = { + [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_proto_ctx_engines_balance, + [I915_CONTEXT_ENGINES_EXT_BOND] = set_proto_ctx_engines_bond, +}; + +static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + const struct drm_i915_gem_context_param *args) +{ + struct drm_i915_private *i915 = fpriv->dev_priv; + struct set_proto_ctx_engines set = { .i915 = i915 }; + struct i915_context_param_engines __user *user = + u64_to_user_ptr(args->value); + unsigned int n; + u64 extensions; + int err; + + if (pc->num_user_engines >= 0) { + drm_dbg(&i915->drm, "Cannot set engines twice"); + return -EINVAL; + } + + if (args->size < sizeof(*user) || + !IS_ALIGNED(args->size - sizeof(*user), sizeof(*user->engines))) { + drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", + args->size); + return -EINVAL; + } + + set.num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); + /* RING_MASK has no shift so we can use it directly here */ + if (set.num_engines > I915_EXEC_RING_MASK + 1) + return -EINVAL; + + set.engines = kmalloc_array(set.num_engines, sizeof(*set.engines), GFP_KERNEL); + if (!set.engines) + return -ENOMEM; + + for (n = 0; n < set.num_engines; n++) { + struct i915_engine_class_instance ci; + struct intel_engine_cs *engine; + + if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { + kfree(set.engines); + return -EFAULT; + } + + memset(&set.engines[n], 0, sizeof(set.engines[n])); + + if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && + ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) + continue; + + engine = intel_engine_lookup_user(i915, + ci.engine_class, + ci.engine_instance); + if (!engine) { + drm_dbg(&i915->drm, + "Invalid engine[%d]: { class:%d, instance:%d }\n", + n, ci.engine_class, ci.engine_instance); + kfree(set.engines); + return -ENOENT; + } + + set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL; + set.engines[n].engine = engine; + } + + err = -EFAULT; + if (!get_user(extensions, &user->extensions)) + err = i915_user_extensions(u64_to_user_ptr(extensions), + set_proto_ctx_engines_extensions, + ARRAY_SIZE(set_proto_ctx_engines_extensions), + &set); + if (err) { + kfree(set.engines); + return err; + } + + pc->num_user_engines = set.num_engines; + pc->user_engines = set.engines; + + return 0; +} + +static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + struct drm_i915_gem_context_param *args) +{ + struct drm_i915_private *i915 = fpriv->dev_priv; + struct drm_i915_gem_context_param_sseu user_sseu; + struct intel_sseu *sseu; + int ret; + + if (args->size < sizeof(user_sseu)) + return -EINVAL; + + if (GRAPHICS_VER(i915) != 11) + return -ENODEV; + + if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), + sizeof(user_sseu))) + return -EFAULT; + + if (user_sseu.rsvd) + return -EINVAL; + + if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) + return -EINVAL; + + if (!!(user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) != (pc->num_user_engines >= 0)) + return -EINVAL; + + if (pc->num_user_engines >= 0) { + int idx = user_sseu.engine.engine_instance; + struct i915_gem_proto_engine *pe; + + if (idx >= pc->num_user_engines) + return -EINVAL; + + pe = &pc->user_engines[idx]; + + /* Only render engine supports RPCS configuration. */ + if (pe->engine->class != RENDER_CLASS) + return -EINVAL; + + sseu = &pe->sseu; + } else { + /* Only render engine supports RPCS configuration. */ + if (user_sseu.engine.engine_class != I915_ENGINE_CLASS_RENDER) + return -EINVAL; + + /* There is only one render engine */ + if (user_sseu.engine.engine_instance != 0) + return -EINVAL; + + sseu = &pc->legacy_rcs_sseu; + } + + ret = i915_gem_user_to_context_sseu(&i915->gt, &user_sseu, sseu); + if (ret) + return ret; + + args->size = sizeof(user_sseu); + + return 0; +} + +static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, + struct i915_gem_proto_context *pc, + struct drm_i915_gem_context_param *args) +{ + int ret = 0; + + switch (args->param) { + case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: + if (args->size) + ret = -EINVAL; + else if (args->value) + pc->user_flags |= BIT(UCONTEXT_NO_ERROR_CAPTURE); + else + pc->user_flags &= ~BIT(UCONTEXT_NO_ERROR_CAPTURE); + break; + + case I915_CONTEXT_PARAM_BANNABLE: + if (args->size) + ret = -EINVAL; + else if (!capable(CAP_SYS_ADMIN) && !args->value) + ret = -EPERM; + else if (args->value) + pc->user_flags |= BIT(UCONTEXT_BANNABLE); + else + pc->user_flags &= ~BIT(UCONTEXT_BANNABLE); + break; + + case I915_CONTEXT_PARAM_RECOVERABLE: + if (args->size) + ret = -EINVAL; + else if (args->value) + pc->user_flags |= BIT(UCONTEXT_RECOVERABLE); + else + pc->user_flags &= ~BIT(UCONTEXT_RECOVERABLE); + break; + + case I915_CONTEXT_PARAM_PRIORITY: + ret = validate_priority(fpriv->dev_priv, args); + if (!ret) + pc->sched.priority = args->value; + break; + + case I915_CONTEXT_PARAM_SSEU: + ret = set_proto_ctx_sseu(fpriv, pc, args); + break; + + case I915_CONTEXT_PARAM_VM: + ret = set_proto_ctx_vm(fpriv, pc, args); + break; + + case I915_CONTEXT_PARAM_ENGINES: + ret = set_proto_ctx_engines(fpriv, pc, args); + break; + + case I915_CONTEXT_PARAM_PERSISTENCE: + if (args->size) + ret = -EINVAL; + ret = proto_context_set_persistence(fpriv->dev_priv, pc, + args->value); + break; + + case I915_CONTEXT_PARAM_NO_ZEROMAP: + case I915_CONTEXT_PARAM_BAN_PERIOD: + case I915_CONTEXT_PARAM_RINGSIZE: + default: + ret = -EINVAL; + break; + } + + return ret; +} + static struct i915_address_space * context_get_vm_rcu(struct i915_gem_context *ctx) { @@ -205,14 +773,16 @@ context_get_vm_rcu(struct i915_gem_context *ctx) } while (1); } -static void intel_context_set_gem(struct intel_context *ce, - struct i915_gem_context *ctx) +static int intel_context_set_gem(struct intel_context *ce, + struct i915_gem_context *ctx, + struct intel_sseu sseu) { + int ret = 0; + GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); RCU_INIT_POINTER(ce->gem_context, ctx); - if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) - ce->ring = __intel_context_ring_size(SZ_16K); + ce->ring_size = SZ_16K; if (rcu_access_pointer(ctx->vm)) { struct i915_address_space *vm; @@ -225,15 +795,23 @@ static void intel_context_set_gem(struct intel_context *ce, ce->vm = vm; } - GEM_BUG_ON(ce->timeline); - if (ctx->timeline) - ce->timeline = intel_timeline_get(ctx->timeline); - if (ctx->sched.priority >= I915_PRIORITY_NORMAL && - intel_engine_has_timeslices(ce->engine)) + intel_engine_has_timeslices(ce->engine) && + intel_engine_has_semaphores(ce->engine)) __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); - intel_context_set_watchdog_us(ce, ctx->watchdog.timeout_us); + if (IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) && + ctx->i915->params.request_timeout_ms) { + unsigned int timeout_ms = ctx->i915->params.request_timeout_ms; + + intel_context_set_watchdog_us(ce, (u64)timeout_ms * 1000); + } + + /* A valid SSEU has no zero fields */ + if (sseu.slice_mask && !WARN_ON(ce->engine->class != RENDER_CLASS)) + ret = intel_context_reconfigure_sseu(ce, sseu); + + return ret; } static void __free_engines(struct i915_gem_engines *e, unsigned int count) @@ -301,11 +879,12 @@ static struct i915_gem_engines *alloc_engines(unsigned int count) return e; } -static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) +static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx, + struct intel_sseu rcs_sseu) { const struct intel_gt *gt = &ctx->i915->gt; struct intel_engine_cs *engine; - struct i915_gem_engines *e; + struct i915_gem_engines *e, *err; enum intel_engine_id id; e = alloc_engines(I915_NUM_ENGINES); @@ -314,6 +893,8 @@ static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) for_each_engine(engine, gt, id) { struct intel_context *ce; + struct intel_sseu sseu = {}; + int ret; if (engine->legacy_idx == INVALID_ENGINE) continue; @@ -323,18 +904,79 @@ static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) ce = intel_context_create(engine); if (IS_ERR(ce)) { - __free_engines(e, e->num_engines + 1); - return ERR_CAST(ce); + err = ERR_CAST(ce); + goto free_engines; } - intel_context_set_gem(ce, ctx); - e->engines[engine->legacy_idx] = ce; - e->num_engines = max(e->num_engines, engine->legacy_idx); + e->num_engines = max(e->num_engines, engine->legacy_idx + 1); + + if (engine->class == RENDER_CLASS) + sseu = rcs_sseu; + + ret = intel_context_set_gem(ce, ctx, sseu); + if (ret) { + err = ERR_PTR(ret); + goto free_engines; + } + + } + + return e; + +free_engines: + free_engines(e); + return err; +} + +static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, + unsigned int num_engines, + struct i915_gem_proto_engine *pe) +{ + struct i915_gem_engines *e, *err; + unsigned int n; + + e = alloc_engines(num_engines); + for (n = 0; n < num_engines; n++) { + struct intel_context *ce; + int ret; + + switch (pe[n].type) { + case I915_GEM_ENGINE_TYPE_PHYSICAL: + ce = intel_context_create(pe[n].engine); + break; + + case I915_GEM_ENGINE_TYPE_BALANCED: + ce = intel_engine_create_virtual(pe[n].siblings, + pe[n].num_siblings); + break; + + case I915_GEM_ENGINE_TYPE_INVALID: + default: + GEM_WARN_ON(pe[n].type != I915_GEM_ENGINE_TYPE_INVALID); + continue; + } + + if (IS_ERR(ce)) { + err = ERR_CAST(ce); + goto free_engines; + } + + e->engines[n] = ce; + + ret = intel_context_set_gem(ce, ctx, pe->sseu); + if (ret) { + err = ERR_PTR(ret); + goto free_engines; + } } - e->num_engines++; + e->num_engines = num_engines; return e; + +free_engines: + free_engines(e); + return err; } void i915_gem_context_release(struct kref *ref) @@ -347,9 +989,6 @@ void i915_gem_context_release(struct kref *ref) mutex_destroy(&ctx->engines_mutex); mutex_destroy(&ctx->lut_mutex); - if (ctx->timeline) - intel_timeline_put(ctx->timeline); - put_pid(ctx->pid); mutex_destroy(&ctx->mutex); @@ -441,7 +1080,7 @@ static void kill_engines(struct i915_gem_engines *engines, bool ban) for_each_gem_engine(ce, engines, it) { struct intel_engine_cs *engine; - if (ban && intel_context_set_banned(ce)) + if (ban && intel_context_ban(ce, NULL)) continue; /* @@ -566,6 +1205,9 @@ static void context_close(struct i915_gem_context *ctx) if (vm) i915_vm_close(vm); + if (ctx->syncobj) + drm_syncobj_put(ctx->syncobj); + ctx->file_priv = ERR_PTR(-EBADF); /* @@ -635,57 +1277,6 @@ static int __context_set_persistence(struct i915_gem_context *ctx, bool state) return 0; } -static struct i915_gem_context * -__create_context(struct drm_i915_private *i915) -{ - struct i915_gem_context *ctx; - struct i915_gem_engines *e; - int err; - int i; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return ERR_PTR(-ENOMEM); - - kref_init(&ctx->ref); - ctx->i915 = i915; - ctx->sched.priority = I915_PRIORITY_NORMAL; - mutex_init(&ctx->mutex); - INIT_LIST_HEAD(&ctx->link); - - spin_lock_init(&ctx->stale.lock); - INIT_LIST_HEAD(&ctx->stale.engines); - - mutex_init(&ctx->engines_mutex); - e = default_engines(ctx); - if (IS_ERR(e)) { - err = PTR_ERR(e); - goto err_free; - } - RCU_INIT_POINTER(ctx->engines, e); - - INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); - mutex_init(&ctx->lut_mutex); - - /* NB: Mark all slices as needing a remap so that when the context first - * loads it will restore whatever remap state already exists. If there - * is no remap info, it will be a NOP. */ - ctx->remap_slice = ALL_L3_SLICES(i915); - - i915_gem_context_set_bannable(ctx); - i915_gem_context_set_recoverable(ctx); - __context_set_persistence(ctx, true /* cgroup hook? */); - - for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) - ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; - - return ctx; - -err_free: - kfree(ctx); - return ERR_PTR(err); -} - static inline struct i915_gem_engines * __context_engines_await(const struct i915_gem_context *ctx, bool *user_engines) @@ -714,168 +1305,112 @@ __context_engines_await(const struct i915_gem_context *ctx, return engines; } -static int +static void context_apply_all(struct i915_gem_context *ctx, - int (*fn)(struct intel_context *ce, void *data), + void (*fn)(struct intel_context *ce, void *data), void *data) { struct i915_gem_engines_iter it; struct i915_gem_engines *e; struct intel_context *ce; - int err = 0; e = __context_engines_await(ctx, NULL); - for_each_gem_engine(ce, e, it) { - err = fn(ce, data); - if (err) - break; - } + for_each_gem_engine(ce, e, it) + fn(ce, data); i915_sw_fence_complete(&e->fence); - - return err; -} - -static int __apply_ppgtt(struct intel_context *ce, void *vm) -{ - i915_vm_put(ce->vm); - ce->vm = i915_vm_get(vm); - return 0; -} - -static struct i915_address_space * -__set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) -{ - struct i915_address_space *old; - - old = rcu_replace_pointer(ctx->vm, - i915_vm_open(vm), - lockdep_is_held(&ctx->mutex)); - GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); - - context_apply_all(ctx, __apply_ppgtt, vm); - - return old; -} - -static void __assign_ppgtt(struct i915_gem_context *ctx, - struct i915_address_space *vm) -{ - if (vm == rcu_access_pointer(ctx->vm)) - return; - - vm = __set_ppgtt(ctx, vm); - if (vm) - i915_vm_close(vm); -} - -static void __set_timeline(struct intel_timeline **dst, - struct intel_timeline *src) -{ - struct intel_timeline *old = *dst; - - *dst = src ? intel_timeline_get(src) : NULL; - - if (old) - intel_timeline_put(old); -} - -static int __apply_timeline(struct intel_context *ce, void *timeline) -{ - __set_timeline(&ce->timeline, timeline); - return 0; -} - -static void __assign_timeline(struct i915_gem_context *ctx, - struct intel_timeline *timeline) -{ - __set_timeline(&ctx->timeline, timeline); - context_apply_all(ctx, __apply_timeline, timeline); -} - -static int __apply_watchdog(struct intel_context *ce, void *timeout_us) -{ - return intel_context_set_watchdog_us(ce, (uintptr_t)timeout_us); -} - -static int -__set_watchdog(struct i915_gem_context *ctx, unsigned long timeout_us) -{ - int ret; - - ret = context_apply_all(ctx, __apply_watchdog, - (void *)(uintptr_t)timeout_us); - if (!ret) - ctx->watchdog.timeout_us = timeout_us; - - return ret; -} - -static void __set_default_fence_expiry(struct i915_gem_context *ctx) -{ - struct drm_i915_private *i915 = ctx->i915; - int ret; - - if (!IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) || - !i915->params.request_timeout_ms) - return; - - /* Default expiry for user fences. */ - ret = __set_watchdog(ctx, i915->params.request_timeout_ms * 1000); - if (ret) - drm_notice(&i915->drm, - "Failed to configure default fence expiry! (%d)", - ret); } static struct i915_gem_context * -i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) +i915_gem_create_context(struct drm_i915_private *i915, + const struct i915_gem_proto_context *pc) { struct i915_gem_context *ctx; + struct i915_address_space *vm = NULL; + struct i915_gem_engines *e; + int err; + int i; - if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && - !HAS_EXECLISTS(i915)) - return ERR_PTR(-EINVAL); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); - ctx = __create_context(i915); - if (IS_ERR(ctx)) - return ctx; + kref_init(&ctx->ref); + ctx->i915 = i915; + ctx->sched = pc->sched; + mutex_init(&ctx->mutex); + INIT_LIST_HEAD(&ctx->link); - if (HAS_FULL_PPGTT(i915)) { + spin_lock_init(&ctx->stale.lock); + INIT_LIST_HEAD(&ctx->stale.engines); + + if (pc->vm) { + vm = i915_vm_get(pc->vm); + } else if (HAS_FULL_PPGTT(i915)) { struct i915_ppgtt *ppgtt; ppgtt = i915_ppgtt_create(&i915->gt); if (IS_ERR(ppgtt)) { drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); - context_close(ctx); - return ERR_CAST(ppgtt); + err = PTR_ERR(ppgtt); + goto err_ctx; } + vm = &ppgtt->vm; + } + if (vm) { + RCU_INIT_POINTER(ctx->vm, i915_vm_open(vm)); - mutex_lock(&ctx->mutex); - __assign_ppgtt(ctx, &ppgtt->vm); - mutex_unlock(&ctx->mutex); + /* i915_vm_open() takes a reference */ + i915_vm_put(vm); + } - i915_vm_put(&ppgtt->vm); + mutex_init(&ctx->engines_mutex); + if (pc->num_user_engines >= 0) { + i915_gem_context_set_user_engines(ctx); + e = user_engines(ctx, pc->num_user_engines, pc->user_engines); + } else { + i915_gem_context_clear_user_engines(ctx); + e = default_engines(ctx, pc->legacy_rcs_sseu); + } + if (IS_ERR(e)) { + err = PTR_ERR(e); + goto err_vm; } + RCU_INIT_POINTER(ctx->engines, e); - if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { - struct intel_timeline *timeline; + INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); + mutex_init(&ctx->lut_mutex); - timeline = intel_timeline_create(&i915->gt); - if (IS_ERR(timeline)) { - context_close(ctx); - return ERR_CAST(timeline); - } + /* NB: Mark all slices as needing a remap so that when the context first + * loads it will restore whatever remap state already exists. If there + * is no remap info, it will be a NOP. */ + ctx->remap_slice = ALL_L3_SLICES(i915); - __assign_timeline(ctx, timeline); - intel_timeline_put(timeline); - } + ctx->user_flags = pc->user_flags; - __set_default_fence_expiry(ctx); + for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) + ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; + + if (pc->single_timeline) { + err = drm_syncobj_create(&ctx->syncobj, + DRM_SYNCOBJ_CREATE_SIGNALED, + NULL); + if (err) + goto err_engines; + } trace_i915_context_create(ctx); return ctx; + +err_engines: + free_engines(e); +err_vm: + if (ctx->vm) + i915_vm_close(ctx->vm); +err_ctx: + kfree(ctx); + return ERR_PTR(err); } static void init_contexts(struct i915_gem_contexts *gc) @@ -889,83 +1424,83 @@ void i915_gem_init__contexts(struct drm_i915_private *i915) init_contexts(&i915->gem.contexts); } -static int gem_context_register(struct i915_gem_context *ctx, - struct drm_i915_file_private *fpriv, - u32 *id) +static void gem_context_register(struct i915_gem_context *ctx, + struct drm_i915_file_private *fpriv, + u32 id) { struct drm_i915_private *i915 = ctx->i915; - struct i915_address_space *vm; - int ret; + void *old; ctx->file_priv = fpriv; - mutex_lock(&ctx->mutex); - vm = i915_gem_context_vm(ctx); - if (vm) - WRITE_ONCE(vm->file, fpriv); /* XXX */ - mutex_unlock(&ctx->mutex); - ctx->pid = get_task_pid(current, PIDTYPE_PID); snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", current->comm, pid_nr(ctx->pid)); /* And finally expose ourselves to userspace via the idr */ - ret = xa_alloc(&fpriv->context_xa, id, ctx, xa_limit_32b, GFP_KERNEL); - if (ret) - goto err_pid; + old = xa_store(&fpriv->context_xa, id, ctx, GFP_KERNEL); + WARN_ON(old); spin_lock(&i915->gem.contexts.lock); list_add_tail(&ctx->link, &i915->gem.contexts.list); spin_unlock(&i915->gem.contexts.lock); - - return 0; - -err_pid: - put_pid(fetch_and_zero(&ctx->pid)); - return ret; } int i915_gem_context_open(struct drm_i915_private *i915, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; + struct i915_gem_proto_context *pc; struct i915_gem_context *ctx; int err; - u32 id; - xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC); + mutex_init(&file_priv->proto_context_lock); + xa_init_flags(&file_priv->proto_context_xa, XA_FLAGS_ALLOC); + + /* 0 reserved for the default context */ + xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC1); /* 0 reserved for invalid/unassigned ppgtt */ xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1); - ctx = i915_gem_create_context(i915, 0); + pc = proto_context_create(i915, 0); + if (IS_ERR(pc)) { + err = PTR_ERR(pc); + goto err; + } + + ctx = i915_gem_create_context(i915, pc); + proto_context_close(pc); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto err; } - err = gem_context_register(ctx, file_priv, &id); - if (err < 0) - goto err_ctx; + gem_context_register(ctx, file_priv, 0); - GEM_BUG_ON(id); return 0; -err_ctx: - context_close(ctx); err: xa_destroy(&file_priv->vm_xa); xa_destroy(&file_priv->context_xa); + xa_destroy(&file_priv->proto_context_xa); + mutex_destroy(&file_priv->proto_context_lock); return err; } void i915_gem_context_close(struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; + struct i915_gem_proto_context *pc; struct i915_address_space *vm; struct i915_gem_context *ctx; unsigned long idx; + xa_for_each(&file_priv->proto_context_xa, idx, pc) + proto_context_close(pc); + xa_destroy(&file_priv->proto_context_xa); + mutex_destroy(&file_priv->proto_context_lock); + xa_for_each(&file_priv->context_xa, idx, ctx) context_close(ctx); xa_destroy(&file_priv->context_xa); @@ -995,8 +1530,6 @@ int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); - ppgtt->vm.file = file_priv; - if (args->extensions) { err = i915_user_extensions(u64_to_user_ptr(args->extensions), NULL, 0, @@ -1040,120 +1573,6 @@ int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, return 0; } -struct context_barrier_task { - struct i915_active base; - void (*task)(void *data); - void *data; -}; - -static void cb_retire(struct i915_active *base) -{ - struct context_barrier_task *cb = container_of(base, typeof(*cb), base); - - if (cb->task) - cb->task(cb->data); - - i915_active_fini(&cb->base); - kfree(cb); -} - -I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); -static int context_barrier_task(struct i915_gem_context *ctx, - intel_engine_mask_t engines, - bool (*skip)(struct intel_context *ce, void *data), - int (*pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data), - int (*emit)(struct i915_request *rq, void *data), - void (*task)(void *data), - void *data) -{ - struct context_barrier_task *cb; - struct i915_gem_engines_iter it; - struct i915_gem_engines *e; - struct i915_gem_ww_ctx ww; - struct intel_context *ce; - int err = 0; - - GEM_BUG_ON(!task); - - cb = kmalloc(sizeof(*cb), GFP_KERNEL); - if (!cb) - return -ENOMEM; - - i915_active_init(&cb->base, NULL, cb_retire, 0); - err = i915_active_acquire(&cb->base); - if (err) { - kfree(cb); - return err; - } - - e = __context_engines_await(ctx, NULL); - if (!e) { - i915_active_release(&cb->base); - return -ENOENT; - } - - for_each_gem_engine(ce, e, it) { - struct i915_request *rq; - - if (I915_SELFTEST_ONLY(context_barrier_inject_fault & - ce->engine->mask)) { - err = -ENXIO; - break; - } - - if (!(ce->engine->mask & engines)) - continue; - - if (skip && skip(ce, data)) - continue; - - i915_gem_ww_ctx_init(&ww, true); -retry: - err = intel_context_pin_ww(ce, &ww); - if (err) - goto err; - - if (pin) - err = pin(ce, &ww, data); - if (err) - goto err_unpin; - - rq = i915_request_create(ce); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_unpin; - } - - err = 0; - if (emit) - err = emit(rq, data); - if (err == 0) - err = i915_active_add_request(&cb->base, rq); - - i915_request_add(rq); -err_unpin: - intel_context_unpin(ce); -err: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - - if (err) - break; - } - i915_sw_fence_complete(&e->fence); - - cb->task = err ? NULL : task; /* caller needs to unwind instead */ - cb->data = data; - - i915_active_release(&cb->base); - - return err; -} - static int get_ppgtt(struct drm_i915_file_private *file_priv, struct i915_gem_context *ctx, struct drm_i915_gem_context_param *args) @@ -1186,211 +1605,6 @@ err_put: return err; } -static void set_ppgtt_barrier(void *data) -{ - struct i915_address_space *old = data; - - if (GRAPHICS_VER(old->i915) < 8) - gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); - - i915_vm_close(old); -} - -static int pin_ppgtt_update(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data) -{ - struct i915_address_space *vm = ce->vm; - - if (!HAS_LOGICAL_RING_CONTEXTS(vm->i915)) - /* ppGTT is not part of the legacy context image */ - return gen6_ppgtt_pin(i915_vm_to_ppgtt(vm), ww); - - return 0; -} - -static int emit_ppgtt_update(struct i915_request *rq, void *data) -{ - struct i915_address_space *vm = rq->context->vm; - struct intel_engine_cs *engine = rq->engine; - u32 base = engine->mmio_base; - u32 *cs; - int i; - - if (i915_vm_is_4lvl(vm)) { - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - const dma_addr_t pd_daddr = px_dma(ppgtt->pd); - - cs = intel_ring_begin(rq, 6); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(2); - - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); - *cs++ = upper_32_bits(pd_daddr); - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); - *cs++ = lower_32_bits(pd_daddr); - - *cs++ = MI_NOOP; - intel_ring_advance(rq, cs); - } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - int err; - - /* Magic required to prevent forcewake errors! */ - err = engine->emit_flush(rq, EMIT_INVALIDATE); - if (err) - return err; - - cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; - for (i = GEN8_3LVL_PDPES; i--; ) { - const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); - - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); - *cs++ = upper_32_bits(pd_daddr); - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); - *cs++ = lower_32_bits(pd_daddr); - } - *cs++ = MI_NOOP; - intel_ring_advance(rq, cs); - } - - return 0; -} - -static bool skip_ppgtt_update(struct intel_context *ce, void *data) -{ - if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) - return !ce->state; - else - return !atomic_read(&ce->pin_count); -} - -static int set_ppgtt(struct drm_i915_file_private *file_priv, - struct i915_gem_context *ctx, - struct drm_i915_gem_context_param *args) -{ - struct i915_address_space *vm, *old; - int err; - - if (args->size) - return -EINVAL; - - if (!rcu_access_pointer(ctx->vm)) - return -ENODEV; - - if (upper_32_bits(args->value)) - return -ENOENT; - - rcu_read_lock(); - vm = xa_load(&file_priv->vm_xa, args->value); - if (vm && !kref_get_unless_zero(&vm->ref)) - vm = NULL; - rcu_read_unlock(); - if (!vm) - return -ENOENT; - - err = mutex_lock_interruptible(&ctx->mutex); - if (err) - goto out; - - if (i915_gem_context_is_closed(ctx)) { - err = -ENOENT; - goto unlock; - } - - if (vm == rcu_access_pointer(ctx->vm)) - goto unlock; - - old = __set_ppgtt(ctx, vm); - - /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ - lut_close(ctx); - - /* - * We need to flush any requests using the current ppgtt before - * we release it as the requests do not hold a reference themselves, - * only indirectly through the context. - */ - err = context_barrier_task(ctx, ALL_ENGINES, - skip_ppgtt_update, - pin_ppgtt_update, - emit_ppgtt_update, - set_ppgtt_barrier, - old); - if (err) { - i915_vm_close(__set_ppgtt(ctx, old)); - i915_vm_close(old); - lut_close(ctx); /* force a rebuild of the old obj:vma cache */ - } - -unlock: - mutex_unlock(&ctx->mutex); -out: - i915_vm_put(vm); - return err; -} - -static int __apply_ringsize(struct intel_context *ce, void *sz) -{ - return intel_context_set_ring_size(ce, (unsigned long)sz); -} - -static int set_ringsize(struct i915_gem_context *ctx, - struct drm_i915_gem_context_param *args) -{ - if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) - return -ENODEV; - - if (args->size) - return -EINVAL; - - if (!IS_ALIGNED(args->value, I915_GTT_PAGE_SIZE)) - return -EINVAL; - - if (args->value < I915_GTT_PAGE_SIZE) - return -EINVAL; - - if (args->value > 128 * I915_GTT_PAGE_SIZE) - return -EINVAL; - - return context_apply_all(ctx, - __apply_ringsize, - __intel_context_ring_size(args->value)); -} - -static int __get_ringsize(struct intel_context *ce, void *arg) -{ - long sz; - - sz = intel_context_get_ring_size(ce); - GEM_BUG_ON(sz > INT_MAX); - - return sz; /* stop on first engine */ -} - -static int get_ringsize(struct i915_gem_context *ctx, - struct drm_i915_gem_context_param *args) -{ - int sz; - - if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) - return -ENODEV; - - if (args->size) - return -EINVAL; - - sz = context_apply_all(ctx, __get_ringsize, NULL); - if (sz < 0) - return sz; - - args->value = sz; - return 0; -} - int i915_gem_user_to_context_sseu(struct intel_gt *gt, const struct drm_i915_gem_context_param_sseu *user, @@ -1545,382 +1759,6 @@ out_ce: return ret; } -struct set_engines { - struct i915_gem_context *ctx; - struct i915_gem_engines *engines; -}; - -static int -set_engines__load_balance(struct i915_user_extension __user *base, void *data) -{ - struct i915_context_engines_load_balance __user *ext = - container_of_user(base, typeof(*ext), base); - const struct set_engines *set = data; - struct drm_i915_private *i915 = set->ctx->i915; - struct intel_engine_cs *stack[16]; - struct intel_engine_cs **siblings; - struct intel_context *ce; - u16 num_siblings, idx; - unsigned int n; - int err; - - if (!HAS_EXECLISTS(i915)) - return -ENODEV; - - if (intel_uc_uses_guc_submission(&i915->gt.uc)) - return -ENODEV; /* not implement yet */ - - if (get_user(idx, &ext->engine_index)) - return -EFAULT; - - if (idx >= set->engines->num_engines) { - drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", - idx, set->engines->num_engines); - return -EINVAL; - } - - idx = array_index_nospec(idx, set->engines->num_engines); - if (set->engines->engines[idx]) { - drm_dbg(&i915->drm, - "Invalid placement[%d], already occupied\n", idx); - return -EEXIST; - } - - if (get_user(num_siblings, &ext->num_siblings)) - return -EFAULT; - - err = check_user_mbz(&ext->flags); - if (err) - return err; - - err = check_user_mbz(&ext->mbz64); - if (err) - return err; - - siblings = stack; - if (num_siblings > ARRAY_SIZE(stack)) { - siblings = kmalloc_array(num_siblings, - sizeof(*siblings), - GFP_KERNEL); - if (!siblings) - return -ENOMEM; - } - - for (n = 0; n < num_siblings; n++) { - struct i915_engine_class_instance ci; - - if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { - err = -EFAULT; - goto out_siblings; - } - - siblings[n] = intel_engine_lookup_user(i915, - ci.engine_class, - ci.engine_instance); - if (!siblings[n]) { - drm_dbg(&i915->drm, - "Invalid sibling[%d]: { class:%d, inst:%d }\n", - n, ci.engine_class, ci.engine_instance); - err = -EINVAL; - goto out_siblings; - } - } - - ce = intel_execlists_create_virtual(siblings, n); - if (IS_ERR(ce)) { - err = PTR_ERR(ce); - goto out_siblings; - } - - intel_context_set_gem(ce, set->ctx); - - if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { - intel_context_put(ce); - err = -EEXIST; - goto out_siblings; - } - -out_siblings: - if (siblings != stack) - kfree(siblings); - - return err; -} - -static int -set_engines__bond(struct i915_user_extension __user *base, void *data) -{ - struct i915_context_engines_bond __user *ext = - container_of_user(base, typeof(*ext), base); - const struct set_engines *set = data; - struct drm_i915_private *i915 = set->ctx->i915; - struct i915_engine_class_instance ci; - struct intel_engine_cs *virtual; - struct intel_engine_cs *master; - u16 idx, num_bonds; - int err, n; - - if (get_user(idx, &ext->virtual_index)) - return -EFAULT; - - if (idx >= set->engines->num_engines) { - drm_dbg(&i915->drm, - "Invalid index for virtual engine: %d >= %d\n", - idx, set->engines->num_engines); - return -EINVAL; - } - - idx = array_index_nospec(idx, set->engines->num_engines); - if (!set->engines->engines[idx]) { - drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); - return -EINVAL; - } - virtual = set->engines->engines[idx]->engine; - - err = check_user_mbz(&ext->flags); - if (err) - return err; - - for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { - err = check_user_mbz(&ext->mbz64[n]); - if (err) - return err; - } - - if (copy_from_user(&ci, &ext->master, sizeof(ci))) - return -EFAULT; - - master = intel_engine_lookup_user(i915, - ci.engine_class, ci.engine_instance); - if (!master) { - drm_dbg(&i915->drm, - "Unrecognised master engine: { class:%u, instance:%u }\n", - ci.engine_class, ci.engine_instance); - return -EINVAL; - } - - if (get_user(num_bonds, &ext->num_bonds)) - return -EFAULT; - - for (n = 0; n < num_bonds; n++) { - struct intel_engine_cs *bond; - - if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) - return -EFAULT; - - bond = intel_engine_lookup_user(i915, - ci.engine_class, - ci.engine_instance); - if (!bond) { - drm_dbg(&i915->drm, - "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", - n, ci.engine_class, ci.engine_instance); - return -EINVAL; - } - - /* - * A non-virtual engine has no siblings to choose between; and - * a submit fence will always be directed to the one engine. - */ - if (intel_engine_is_virtual(virtual)) { - err = intel_virtual_engine_attach_bond(virtual, - master, - bond); - if (err) - return err; - } - } - - return 0; -} - -static const i915_user_extension_fn set_engines__extensions[] = { - [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, - [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, -}; - -static int -set_engines(struct i915_gem_context *ctx, - const struct drm_i915_gem_context_param *args) -{ - struct drm_i915_private *i915 = ctx->i915; - struct i915_context_param_engines __user *user = - u64_to_user_ptr(args->value); - struct set_engines set = { .ctx = ctx }; - unsigned int num_engines, n; - u64 extensions; - int err; - - if (!args->size) { /* switch back to legacy user_ring_map */ - if (!i915_gem_context_user_engines(ctx)) - return 0; - - set.engines = default_engines(ctx); - if (IS_ERR(set.engines)) - return PTR_ERR(set.engines); - - goto replace; - } - - BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); - if (args->size < sizeof(*user) || - !IS_ALIGNED(args->size, sizeof(*user->engines))) { - drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", - args->size); - return -EINVAL; - } - - /* - * Note that I915_EXEC_RING_MASK limits execbuf to only using the - * first 64 engines defined here. - */ - num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); - set.engines = alloc_engines(num_engines); - if (!set.engines) - return -ENOMEM; - - for (n = 0; n < num_engines; n++) { - struct i915_engine_class_instance ci; - struct intel_engine_cs *engine; - struct intel_context *ce; - - if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { - __free_engines(set.engines, n); - return -EFAULT; - } - - if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && - ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { - set.engines->engines[n] = NULL; - continue; - } - - engine = intel_engine_lookup_user(ctx->i915, - ci.engine_class, - ci.engine_instance); - if (!engine) { - drm_dbg(&i915->drm, - "Invalid engine[%d]: { class:%d, instance:%d }\n", - n, ci.engine_class, ci.engine_instance); - __free_engines(set.engines, n); - return -ENOENT; - } - - ce = intel_context_create(engine); - if (IS_ERR(ce)) { - __free_engines(set.engines, n); - return PTR_ERR(ce); - } - - intel_context_set_gem(ce, ctx); - - set.engines->engines[n] = ce; - } - set.engines->num_engines = num_engines; - - err = -EFAULT; - if (!get_user(extensions, &user->extensions)) - err = i915_user_extensions(u64_to_user_ptr(extensions), - set_engines__extensions, - ARRAY_SIZE(set_engines__extensions), - &set); - if (err) { - free_engines(set.engines); - return err; - } - -replace: - mutex_lock(&ctx->engines_mutex); - if (i915_gem_context_is_closed(ctx)) { - mutex_unlock(&ctx->engines_mutex); - free_engines(set.engines); - return -ENOENT; - } - if (args->size) - i915_gem_context_set_user_engines(ctx); - else - i915_gem_context_clear_user_engines(ctx); - set.engines = rcu_replace_pointer(ctx->engines, set.engines, 1); - mutex_unlock(&ctx->engines_mutex); - - /* Keep track of old engine sets for kill_context() */ - engines_idle_release(ctx, set.engines); - - return 0; -} - -static int -get_engines(struct i915_gem_context *ctx, - struct drm_i915_gem_context_param *args) -{ - struct i915_context_param_engines __user *user; - struct i915_gem_engines *e; - size_t n, count, size; - bool user_engines; - int err = 0; - - e = __context_engines_await(ctx, &user_engines); - if (!e) - return -ENOENT; - - if (!user_engines) { - i915_sw_fence_complete(&e->fence); - args->size = 0; - return 0; - } - - count = e->num_engines; - - /* Be paranoid in case we have an impedance mismatch */ - if (!check_struct_size(user, engines, count, &size)) { - err = -EINVAL; - goto err_free; - } - if (overflows_type(size, args->size)) { - err = -EINVAL; - goto err_free; - } - - if (!args->size) { - args->size = size; - goto err_free; - } - - if (args->size < size) { - err = -EINVAL; - goto err_free; - } - - user = u64_to_user_ptr(args->value); - if (put_user(0, &user->extensions)) { - err = -EFAULT; - goto err_free; - } - - for (n = 0; n < count; n++) { - struct i915_engine_class_instance ci = { - .engine_class = I915_ENGINE_CLASS_INVALID, - .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, - }; - - if (e->engines[n]) { - ci.engine_class = e->engines[n]->engine->uabi_class; - ci.engine_instance = e->engines[n]->engine->uabi_instance; - } - - if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { - err = -EFAULT; - goto err_free; - } - } - - args->size = size; - -err_free: - i915_sw_fence_complete(&e->fence); - return err; -} - static int set_persistence(struct i915_gem_context *ctx, const struct drm_i915_gem_context_param *args) @@ -1931,41 +1769,30 @@ set_persistence(struct i915_gem_context *ctx, return __context_set_persistence(ctx, args->value); } -static int __apply_priority(struct intel_context *ce, void *arg) +static void __apply_priority(struct intel_context *ce, void *arg) { struct i915_gem_context *ctx = arg; if (!intel_engine_has_timeslices(ce->engine)) - return 0; + return; - if (ctx->sched.priority >= I915_PRIORITY_NORMAL) + if (ctx->sched.priority >= I915_PRIORITY_NORMAL && + intel_engine_has_semaphores(ce->engine)) intel_context_set_use_semaphores(ce); else intel_context_clear_use_semaphores(ce); - - return 0; } static int set_priority(struct i915_gem_context *ctx, const struct drm_i915_gem_context_param *args) { - s64 priority = args->value; - - if (args->size) - return -EINVAL; - - if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) - return -ENODEV; - - if (priority > I915_CONTEXT_MAX_USER_PRIORITY || - priority < I915_CONTEXT_MIN_USER_PRIORITY) - return -EINVAL; + int err; - if (priority > I915_CONTEXT_DEFAULT_PRIORITY && - !capable(CAP_SYS_NICE)) - return -EPERM; + err = validate_priority(ctx->i915, args); + if (err) + return err; - ctx->sched.priority = priority; + ctx->sched.priority = args->value; context_apply_all(ctx, __apply_priority, ctx); return 0; @@ -1978,15 +1805,6 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, int ret = 0; switch (args->param) { - case I915_CONTEXT_PARAM_NO_ZEROMAP: - if (args->size) - ret = -EINVAL; - else if (args->value) - set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); - else - clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); - break; - case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: if (args->size) ret = -EINVAL; @@ -2024,23 +1842,15 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, ret = set_sseu(ctx, args); break; - case I915_CONTEXT_PARAM_VM: - ret = set_ppgtt(fpriv, ctx, args); - break; - - case I915_CONTEXT_PARAM_ENGINES: - ret = set_engines(ctx, args); - break; - case I915_CONTEXT_PARAM_PERSISTENCE: ret = set_persistence(ctx, args); break; - case I915_CONTEXT_PARAM_RINGSIZE: - ret = set_ringsize(ctx, args); - break; - + case I915_CONTEXT_PARAM_NO_ZEROMAP: case I915_CONTEXT_PARAM_BAN_PERIOD: + case I915_CONTEXT_PARAM_RINGSIZE: + case I915_CONTEXT_PARAM_VM: + case I915_CONTEXT_PARAM_ENGINES: default: ret = -EINVAL; break; @@ -2050,7 +1860,7 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, } struct create_ext { - struct i915_gem_context *ctx; + struct i915_gem_proto_context *pc; struct drm_i915_file_private *fpriv; }; @@ -2065,233 +1875,84 @@ static int create_setparam(struct i915_user_extension __user *ext, void *data) if (local.param.ctx_id) return -EINVAL; - return ctx_setparam(arg->fpriv, arg->ctx, &local.param); + return set_proto_ctx_param(arg->fpriv, arg->pc, &local.param); } -static int copy_ring_size(struct intel_context *dst, - struct intel_context *src) +static int invalid_ext(struct i915_user_extension __user *ext, void *data) { - long sz; - - sz = intel_context_get_ring_size(src); - if (sz < 0) - return sz; - - return intel_context_set_ring_size(dst, sz); + return -EINVAL; } -static int clone_engines(struct i915_gem_context *dst, - struct i915_gem_context *src) -{ - struct i915_gem_engines *clone, *e; - bool user_engines; - unsigned long n; - - e = __context_engines_await(src, &user_engines); - if (!e) - return -ENOENT; - - clone = alloc_engines(e->num_engines); - if (!clone) - goto err_unlock; - - for (n = 0; n < e->num_engines; n++) { - struct intel_engine_cs *engine; - - if (!e->engines[n]) { - clone->engines[n] = NULL; - continue; - } - engine = e->engines[n]->engine; - - /* - * Virtual engines are singletons; they can only exist - * inside a single context, because they embed their - * HW context... As each virtual context implies a single - * timeline (each engine can only dequeue a single request - * at any time), it would be surprising for two contexts - * to use the same engine. So let's create a copy of - * the virtual engine instead. - */ - if (intel_engine_is_virtual(engine)) - clone->engines[n] = - intel_execlists_clone_virtual(engine); - else - clone->engines[n] = intel_context_create(engine); - if (IS_ERR_OR_NULL(clone->engines[n])) { - __free_engines(clone, n); - goto err_unlock; - } - - intel_context_set_gem(clone->engines[n], dst); - - /* Copy across the preferred ringsize */ - if (copy_ring_size(clone->engines[n], e->engines[n])) { - __free_engines(clone, n + 1); - goto err_unlock; - } - } - clone->num_engines = n; - i915_sw_fence_complete(&e->fence); - - /* Serialised by constructor */ - engines_idle_release(dst, rcu_replace_pointer(dst->engines, clone, 1)); - if (user_engines) - i915_gem_context_set_user_engines(dst); - else - i915_gem_context_clear_user_engines(dst); - return 0; - -err_unlock: - i915_sw_fence_complete(&e->fence); - return -ENOMEM; -} - -static int clone_flags(struct i915_gem_context *dst, - struct i915_gem_context *src) -{ - dst->user_flags = src->user_flags; - return 0; -} - -static int clone_schedattr(struct i915_gem_context *dst, - struct i915_gem_context *src) -{ - dst->sched = src->sched; - return 0; -} - -static int clone_sseu(struct i915_gem_context *dst, - struct i915_gem_context *src) -{ - struct i915_gem_engines *e = i915_gem_context_lock_engines(src); - struct i915_gem_engines *clone; - unsigned long n; - int err; - - /* no locking required; sole access under constructor*/ - clone = __context_engines_static(dst); - if (e->num_engines != clone->num_engines) { - err = -EINVAL; - goto unlock; - } - - for (n = 0; n < e->num_engines; n++) { - struct intel_context *ce = e->engines[n]; - - if (clone->engines[n]->engine->class != ce->engine->class) { - /* Must have compatible engine maps! */ - err = -EINVAL; - goto unlock; - } - - /* serialises with set_sseu */ - err = intel_context_lock_pinned(ce); - if (err) - goto unlock; - - clone->engines[n]->sseu = ce->sseu; - intel_context_unlock_pinned(ce); - } - - err = 0; -unlock: - i915_gem_context_unlock_engines(src); - return err; -} +static const i915_user_extension_fn create_extensions[] = { + [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, + [I915_CONTEXT_CREATE_EXT_CLONE] = invalid_ext, +}; -static int clone_timeline(struct i915_gem_context *dst, - struct i915_gem_context *src) +static bool client_is_banned(struct drm_i915_file_private *file_priv) { - if (src->timeline) - __assign_timeline(dst, src->timeline); - - return 0; + return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; } -static int clone_vm(struct i915_gem_context *dst, - struct i915_gem_context *src) +static inline struct i915_gem_context * +__context_lookup(struct drm_i915_file_private *file_priv, u32 id) { - struct i915_address_space *vm; - int err = 0; - - if (!rcu_access_pointer(src->vm)) - return 0; + struct i915_gem_context *ctx; rcu_read_lock(); - vm = context_get_vm_rcu(src); + ctx = xa_load(&file_priv->context_xa, id); + if (ctx && !kref_get_unless_zero(&ctx->ref)) + ctx = NULL; rcu_read_unlock(); - if (!mutex_lock_interruptible(&dst->mutex)) { - __assign_ppgtt(dst, vm); - mutex_unlock(&dst->mutex); - } else { - err = -EINTR; - } - - i915_vm_put(vm); - return err; + return ctx; } -static int create_clone(struct i915_user_extension __user *ext, void *data) -{ - static int (* const fn[])(struct i915_gem_context *dst, - struct i915_gem_context *src) = { -#define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y - MAP(ENGINES, clone_engines), - MAP(FLAGS, clone_flags), - MAP(SCHEDATTR, clone_schedattr), - MAP(SSEU, clone_sseu), - MAP(TIMELINE, clone_timeline), - MAP(VM, clone_vm), -#undef MAP - }; - struct drm_i915_gem_context_create_ext_clone local; - const struct create_ext *arg = data; - struct i915_gem_context *dst = arg->ctx; - struct i915_gem_context *src; - int err, bit; +static struct i915_gem_context * +finalize_create_context_locked(struct drm_i915_file_private *file_priv, + struct i915_gem_proto_context *pc, u32 id) +{ + struct i915_gem_context *ctx; + void *old; - if (copy_from_user(&local, ext, sizeof(local))) - return -EFAULT; + lockdep_assert_held(&file_priv->proto_context_lock); - BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != - I915_CONTEXT_CLONE_UNKNOWN); + ctx = i915_gem_create_context(file_priv->dev_priv, pc); + if (IS_ERR(ctx)) + return ctx; - if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) - return -EINVAL; + gem_context_register(ctx, file_priv, id); - if (local.rsvd) - return -EINVAL; + old = xa_erase(&file_priv->proto_context_xa, id); + GEM_BUG_ON(old != pc); + proto_context_close(pc); - rcu_read_lock(); - src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); - rcu_read_unlock(); - if (!src) - return -ENOENT; + /* One for the xarray and one for the caller */ + return i915_gem_context_get(ctx); +} - GEM_BUG_ON(src == dst); +struct i915_gem_context * +i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) +{ + struct i915_gem_proto_context *pc; + struct i915_gem_context *ctx; - for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { - if (!(local.flags & BIT(bit))) - continue; + ctx = __context_lookup(file_priv, id); + if (ctx) + return ctx; - err = fn[bit](dst, src); - if (err) - return err; + mutex_lock(&file_priv->proto_context_lock); + /* Try one more time under the lock */ + ctx = __context_lookup(file_priv, id); + if (!ctx) { + pc = xa_load(&file_priv->proto_context_xa, id); + if (!pc) + ctx = ERR_PTR(-ENOENT); + else + ctx = finalize_create_context_locked(file_priv, pc, id); } + mutex_unlock(&file_priv->proto_context_lock); - return 0; -} - -static const i915_user_extension_fn create_extensions[] = { - [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, - [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, -}; - -static bool client_is_banned(struct drm_i915_file_private *file_priv) -{ - return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; + return ctx; } int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, @@ -2321,9 +1982,9 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, return -EIO; } - ext_data.ctx = i915_gem_create_context(i915, args->flags); - if (IS_ERR(ext_data.ctx)) - return PTR_ERR(ext_data.ctx); + ext_data.pc = proto_context_create(i915, args->flags); + if (IS_ERR(ext_data.pc)) + return PTR_ERR(ext_data.pc); if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { ret = i915_user_extensions(u64_to_user_ptr(args->extensions), @@ -2331,20 +1992,39 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, ARRAY_SIZE(create_extensions), &ext_data); if (ret) - goto err_ctx; + goto err_pc; } - ret = gem_context_register(ext_data.ctx, ext_data.fpriv, &id); - if (ret < 0) - goto err_ctx; + if (GRAPHICS_VER(i915) > 12) { + struct i915_gem_context *ctx; + + /* Get ourselves a context ID */ + ret = xa_alloc(&ext_data.fpriv->context_xa, &id, NULL, + xa_limit_32b, GFP_KERNEL); + if (ret) + goto err_pc; + + ctx = i915_gem_create_context(i915, ext_data.pc); + if (IS_ERR(ctx)) { + ret = PTR_ERR(ctx); + goto err_pc; + } + + proto_context_close(ext_data.pc); + gem_context_register(ctx, ext_data.fpriv, id); + } else { + ret = proto_context_register(ext_data.fpriv, ext_data.pc, &id); + if (ret < 0) + goto err_pc; + } args->ctx_id = id; drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id); return 0; -err_ctx: - context_close(ext_data.ctx); +err_pc: + proto_context_close(ext_data.pc); return ret; } @@ -2353,6 +2033,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_context_destroy *args = data; struct drm_i915_file_private *file_priv = file->driver_priv; + struct i915_gem_proto_context *pc; struct i915_gem_context *ctx; if (args->pad != 0) @@ -2361,11 +2042,24 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, if (!args->ctx_id) return -ENOENT; + /* We need to hold the proto-context lock here to prevent races + * with finalize_create_context_locked(). + */ + mutex_lock(&file_priv->proto_context_lock); ctx = xa_erase(&file_priv->context_xa, args->ctx_id); - if (!ctx) + pc = xa_erase(&file_priv->proto_context_xa, args->ctx_id); + mutex_unlock(&file_priv->proto_context_lock); + + if (!ctx && !pc) return -ENOENT; + GEM_WARN_ON(ctx && pc); + + if (pc) + proto_context_close(pc); + + if (ctx) + context_close(ctx); - context_close(ctx); return 0; } @@ -2433,15 +2127,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, int ret = 0; ctx = i915_gem_context_lookup(file_priv, args->ctx_id); - if (!ctx) - return -ENOENT; + if (IS_ERR(ctx)) + return PTR_ERR(ctx); switch (args->param) { - case I915_CONTEXT_PARAM_NO_ZEROMAP: - args->size = 0; - args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); - break; - case I915_CONTEXT_PARAM_GTT_SIZE: args->size = 0; rcu_read_lock(); @@ -2480,20 +2169,15 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, ret = get_ppgtt(file_priv, ctx, args); break; - case I915_CONTEXT_PARAM_ENGINES: - ret = get_engines(ctx, args); - break; - case I915_CONTEXT_PARAM_PERSISTENCE: args->size = 0; args->value = i915_gem_context_is_persistent(ctx); break; - case I915_CONTEXT_PARAM_RINGSIZE: - ret = get_ringsize(ctx, args); - break; - + case I915_CONTEXT_PARAM_NO_ZEROMAP: case I915_CONTEXT_PARAM_BAN_PERIOD: + case I915_CONTEXT_PARAM_ENGINES: + case I915_CONTEXT_PARAM_RINGSIZE: default: ret = -EINVAL; break; @@ -2508,16 +2192,32 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, { struct drm_i915_file_private *file_priv = file->driver_priv; struct drm_i915_gem_context_param *args = data; + struct i915_gem_proto_context *pc; struct i915_gem_context *ctx; - int ret; + int ret = 0; - ctx = i915_gem_context_lookup(file_priv, args->ctx_id); - if (!ctx) - return -ENOENT; + mutex_lock(&file_priv->proto_context_lock); + ctx = __context_lookup(file_priv, args->ctx_id); + if (!ctx) { + pc = xa_load(&file_priv->proto_context_xa, args->ctx_id); + if (pc) { + /* Contexts should be finalized inside + * GEM_CONTEXT_CREATE starting with graphics + * version 13. + */ + WARN_ON(GRAPHICS_VER(file_priv->dev_priv) > 12); + ret = set_proto_ctx_param(file_priv, pc, args); + } else { + ret = -ENOENT; + } + } + mutex_unlock(&file_priv->proto_context_lock); - ret = ctx_setparam(file_priv, ctx, args); + if (ctx) { + ret = ctx_setparam(file_priv, ctx, args); + i915_gem_context_put(ctx); + } - i915_gem_context_put(ctx); return ret; } @@ -2527,16 +2227,13 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_reset_stats *args = data; struct i915_gem_context *ctx; - int ret; if (args->flags || args->pad) return -EINVAL; - ret = -ENOENT; - rcu_read_lock(); - ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); - if (!ctx) - goto out; + ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); /* * We opt for unserialised reads here. This may result in tearing @@ -2553,10 +2250,8 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, args->batch_active = atomic_read(&ctx->guilty_count); args->batch_pending = atomic_read(&ctx->active_count); - ret = 0; -out: - rcu_read_unlock(); - return ret; + i915_gem_context_put(ctx); + return 0; } /* GEM context-engines iterator: for_each_gem_engine() */ @@ -2584,27 +2279,16 @@ i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) #include "selftests/i915_gem_context.c" #endif -static void i915_global_gem_context_shrink(void) +void i915_gem_context_module_exit(void) { - kmem_cache_shrink(global.slab_luts); + kmem_cache_destroy(slab_luts); } -static void i915_global_gem_context_exit(void) -{ - kmem_cache_destroy(global.slab_luts); -} - -static struct i915_global_gem_context global = { { - .shrink = i915_global_gem_context_shrink, - .exit = i915_global_gem_context_exit, -} }; - -int __init i915_global_gem_context_init(void) +int __init i915_gem_context_module_init(void) { - global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); - if (!global.slab_luts) + slab_luts = KMEM_CACHE(i915_lut_handle, 0); + if (!slab_luts) return -ENOMEM; - i915_global_register(&global.base); return 0; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index b5c908f3f4f2..18060536b0c2 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -133,6 +133,9 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +struct i915_gem_context * +i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id); + static inline struct i915_gem_context * i915_gem_context_get(struct i915_gem_context *ctx) { @@ -221,6 +224,9 @@ i915_gem_engines_iter_next(struct i915_gem_engines_iter *it); for (i915_gem_engines_iter_init(&(it), (engines)); \ ((ce) = i915_gem_engines_iter_next(&(it)));) +void i915_gem_context_module_exit(void); +int i915_gem_context_module_init(void); + struct i915_lut_handle *i915_lut_handle_alloc(void); void i915_lut_handle_free(struct i915_lut_handle *lut); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h index 340473aa70de..94c03a97cb77 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h @@ -30,33 +30,187 @@ struct i915_address_space; struct intel_timeline; struct intel_ring; +/** + * struct i915_gem_engines - A set of engines + */ struct i915_gem_engines { union { + /** @link: Link in i915_gem_context::stale::engines */ struct list_head link; + + /** @rcu: RCU to use when freeing */ struct rcu_head rcu; }; + + /** @fence: Fence used for delayed destruction of engines */ struct i915_sw_fence fence; + + /** @ctx: i915_gem_context backpointer */ struct i915_gem_context *ctx; + + /** @num_engines: Number of engines in this set */ unsigned int num_engines; + + /** @engines: Array of engines */ struct intel_context *engines[]; }; +/** + * struct i915_gem_engines_iter - Iterator for an i915_gem_engines set + */ struct i915_gem_engines_iter { + /** @idx: Index into i915_gem_engines::engines */ unsigned int idx; + + /** @engines: Engine set being iterated */ const struct i915_gem_engines *engines; }; /** + * enum i915_gem_engine_type - Describes the type of an i915_gem_proto_engine + */ +enum i915_gem_engine_type { + /** @I915_GEM_ENGINE_TYPE_INVALID: An invalid engine */ + I915_GEM_ENGINE_TYPE_INVALID = 0, + + /** @I915_GEM_ENGINE_TYPE_PHYSICAL: A single physical engine */ + I915_GEM_ENGINE_TYPE_PHYSICAL, + + /** @I915_GEM_ENGINE_TYPE_BALANCED: A load-balanced engine set */ + I915_GEM_ENGINE_TYPE_BALANCED, +}; + +/** + * struct i915_gem_proto_engine - prototype engine + * + * This struct describes an engine that a context may contain. Engines + * have three types: + * + * - I915_GEM_ENGINE_TYPE_INVALID: Invalid engines can be created but they + * show up as a NULL in i915_gem_engines::engines[i] and any attempt to + * use them by the user results in -EINVAL. They are also useful during + * proto-context construction because the client may create invalid + * engines and then set them up later as virtual engines. + * + * - I915_GEM_ENGINE_TYPE_PHYSICAL: A single physical engine, described by + * i915_gem_proto_engine::engine. + * + * - I915_GEM_ENGINE_TYPE_BALANCED: A load-balanced engine set, described + * i915_gem_proto_engine::num_siblings and i915_gem_proto_engine::siblings. + */ +struct i915_gem_proto_engine { + /** @type: Type of this engine */ + enum i915_gem_engine_type type; + + /** @engine: Engine, for physical */ + struct intel_engine_cs *engine; + + /** @num_siblings: Number of balanced siblings */ + unsigned int num_siblings; + + /** @siblings: Balanced siblings */ + struct intel_engine_cs **siblings; + + /** @sseu: Client-set SSEU parameters */ + struct intel_sseu sseu; +}; + +/** + * struct i915_gem_proto_context - prototype context + * + * The struct i915_gem_proto_context represents the creation parameters for + * a struct i915_gem_context. This is used to gather parameters provided + * either through creation flags or via SET_CONTEXT_PARAM so that, when we + * create the final i915_gem_context, those parameters can be immutable. + * + * The context uAPI allows for two methods of setting context parameters: + * SET_CONTEXT_PARAM and CONTEXT_CREATE_EXT_SETPARAM. The former is + * allowed to be called at any time while the later happens as part of + * GEM_CONTEXT_CREATE. When these were initially added, Currently, + * everything settable via one is settable via the other. While some + * params are fairly simple and setting them on a live context is harmless + * such the context priority, others are far trickier such as the VM or the + * set of engines. To avoid some truly nasty race conditions, we don't + * allow setting the VM or the set of engines on live contexts. + * + * The way we dealt with this without breaking older userspace that sets + * the VM or engine set via SET_CONTEXT_PARAM is to delay the creation of + * the actual context until after the client is done configuring it with + * SET_CONTEXT_PARAM. From the perspective of the client, it has the same + * u32 context ID the whole time. From the perspective of i915, however, + * it's an i915_gem_proto_context right up until the point where we attempt + * to do something which the proto-context can't handle at which point the + * real context gets created. + * + * This is accomplished via a little xarray dance. When GEM_CONTEXT_CREATE + * is called, we create a proto-context, reserve a slot in context_xa but + * leave it NULL, the proto-context in the corresponding slot in + * proto_context_xa. Then, whenever we go to look up a context, we first + * check context_xa. If it's there, we return the i915_gem_context and + * we're done. If it's not, we look in proto_context_xa and, if we find it + * there, we create the actual context and kill the proto-context. + * + * At the time we made this change (April, 2021), we did a fairly complete + * audit of existing userspace to ensure this wouldn't break anything: + * + * - Mesa/i965 didn't use the engines or VM APIs at all + * + * - Mesa/ANV used the engines API but via CONTEXT_CREATE_EXT_SETPARAM and + * didn't use the VM API. + * + * - Mesa/iris didn't use the engines or VM APIs at all + * + * - The open-source compute-runtime didn't yet use the engines API but + * did use the VM API via SET_CONTEXT_PARAM. However, CONTEXT_SETPARAM + * was always the second ioctl on that context, immediately following + * GEM_CONTEXT_CREATE. + * + * - The media driver sets engines and bonding/balancing via + * SET_CONTEXT_PARAM. However, CONTEXT_SETPARAM to set the VM was + * always the second ioctl on that context, immediately following + * GEM_CONTEXT_CREATE and setting engines immediately followed that. + * + * In order for this dance to work properly, any modification to an + * i915_gem_proto_context that is exposed to the client via + * drm_i915_file_private::proto_context_xa must be guarded by + * drm_i915_file_private::proto_context_lock. The exception is when a + * proto-context has not yet been exposed such as when handling + * CONTEXT_CREATE_SET_PARAM during GEM_CONTEXT_CREATE. + */ +struct i915_gem_proto_context { + /** @vm: See &i915_gem_context.vm */ + struct i915_address_space *vm; + + /** @user_flags: See &i915_gem_context.user_flags */ + unsigned long user_flags; + + /** @sched: See &i915_gem_context.sched */ + struct i915_sched_attr sched; + + /** @num_user_engines: Number of user-specified engines or -1 */ + int num_user_engines; + + /** @user_engines: User-specified engines */ + struct i915_gem_proto_engine *user_engines; + + /** @legacy_rcs_sseu: Client-set SSEU parameters for the legacy RCS */ + struct intel_sseu legacy_rcs_sseu; + + /** @single_timeline: See See &i915_gem_context.syncobj */ + bool single_timeline; +}; + +/** * struct i915_gem_context - client state * * The struct i915_gem_context represents the combined view of the driver and * logical hardware state for a particular client. */ struct i915_gem_context { - /** i915: i915 device backpointer */ + /** @i915: i915 device backpointer */ struct drm_i915_private *i915; - /** file_priv: owning file descriptor */ + /** @file_priv: owning file descriptor */ struct drm_i915_file_private *file_priv; /** @@ -81,9 +235,23 @@ struct i915_gem_context { * CONTEXT_USER_ENGINES flag is set). */ struct i915_gem_engines __rcu *engines; - struct mutex engines_mutex; /* guards writes to engines */ - struct intel_timeline *timeline; + /** @engines_mutex: guards writes to engines */ + struct mutex engines_mutex; + + /** + * @syncobj: Shared timeline syncobj + * + * When the SHARED_TIMELINE flag is set on context creation, we + * emulate a single timeline across all engines using this syncobj. + * For every execbuffer2 call, this syncobj is used as both an in- + * and out-fence. Unlike the real intel_timeline, this doesn't + * provide perfect atomic in-order guarantees if the client races + * with itself by calling execbuffer2 twice concurrently. However, + * if userspace races with itself, that's not likely to yield well- + * defined results anyway so we choose to not care. + */ + struct drm_syncobj *syncobj; /** * @vm: unique address space (GTT) @@ -106,7 +274,7 @@ struct i915_gem_context { */ struct pid *pid; - /** link: place with &drm_i915_private.context_list */ + /** @link: place with &drm_i915_private.context_list */ struct list_head link; /** @@ -129,7 +297,6 @@ struct i915_gem_context { * @user_flags: small set of booleans controlled by the user */ unsigned long user_flags; -#define UCONTEXT_NO_ZEROMAP 0 #define UCONTEXT_NO_ERROR_CAPTURE 1 #define UCONTEXT_BANNABLE 2 #define UCONTEXT_RECOVERABLE 3 @@ -142,11 +309,13 @@ struct i915_gem_context { #define CONTEXT_CLOSED 0 #define CONTEXT_USER_ENGINES 1 + /** @mutex: guards everything that isn't engines or handles_vma */ struct mutex mutex; + /** @sched: scheduler parameters */ struct i915_sched_attr sched; - /** guilty_count: How many times this context has caused a GPU hang. */ + /** @guilty_count: How many times this context has caused a GPU hang. */ atomic_t guilty_count; /** * @active_count: How many times this context was active during a GPU @@ -154,25 +323,23 @@ struct i915_gem_context { */ atomic_t active_count; - struct { - u64 timeout_us; - } watchdog; - /** * @hang_timestamp: The last time(s) this context caused a GPU hang */ unsigned long hang_timestamp[2]; #define CONTEXT_FAST_HANG_JIFFIES (120 * HZ) /* 3 hangs within 120s? Banned! */ - /** remap_slice: Bitmask of cache lines that need remapping */ + /** @remap_slice: Bitmask of cache lines that need remapping */ u8 remap_slice; /** - * handles_vma: rbtree to look up our context specific obj/vma for + * @handles_vma: rbtree to look up our context specific obj/vma for * the user handle. (user handles are per fd, but the binding is * per vm, which may be one per context or shared with the global GTT) */ struct radix_tree_root handles_vma; + + /** @lut_mutex: Locks handles_vma */ struct mutex lut_mutex; /** @@ -184,8 +351,11 @@ struct i915_gem_context { */ char name[TASK_COMM_LEN + 8]; + /** @stale: tracks stale engines to be destroyed */ struct { + /** @lock: guards engines */ spinlock_t lock; + /** @engines: list of stale engines */ struct list_head engines; } stale; }; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c index 93bf63bbaff1..23fee13a3384 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_create.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c @@ -11,13 +11,14 @@ #include "i915_trace.h" #include "i915_user_extensions.h" -static u32 object_max_page_size(struct drm_i915_gem_object *obj) +static u32 object_max_page_size(struct intel_memory_region **placements, + unsigned int n_placements) { u32 max_page_size = 0; int i; - for (i = 0; i < obj->mm.n_placements; i++) { - struct intel_memory_region *mr = obj->mm.placements[i]; + for (i = 0; i < n_placements; i++) { + struct intel_memory_region *mr = placements[i]; GEM_BUG_ON(!is_power_of_2(mr->min_page_size)); max_page_size = max_t(u32, max_page_size, mr->min_page_size); @@ -27,10 +28,13 @@ static u32 object_max_page_size(struct drm_i915_gem_object *obj) return max_page_size; } -static void object_set_placements(struct drm_i915_gem_object *obj, - struct intel_memory_region **placements, - unsigned int n_placements) +static int object_set_placements(struct drm_i915_gem_object *obj, + struct intel_memory_region **placements, + unsigned int n_placements) { + struct intel_memory_region **arr; + unsigned int i; + GEM_BUG_ON(!n_placements); /* @@ -44,9 +48,20 @@ static void object_set_placements(struct drm_i915_gem_object *obj, obj->mm.placements = &i915->mm.regions[mr->id]; obj->mm.n_placements = 1; } else { - obj->mm.placements = placements; + arr = kmalloc_array(n_placements, + sizeof(struct intel_memory_region *), + GFP_KERNEL); + if (!arr) + return -ENOMEM; + + for (i = 0; i < n_placements; i++) + arr[i] = placements[i]; + + obj->mm.placements = arr; obj->mm.n_placements = n_placements; } + + return 0; } static int i915_gem_publish(struct drm_i915_gem_object *obj, @@ -67,22 +82,46 @@ static int i915_gem_publish(struct drm_i915_gem_object *obj, return 0; } -static int -i915_gem_setup(struct drm_i915_gem_object *obj, u64 size) +/** + * Creates a new object using the same path as DRM_I915_GEM_CREATE_EXT + * @i915: i915 private + * @size: size of the buffer, in bytes + * @placements: possible placement regions, in priority order + * @n_placements: number of possible placement regions + * + * This function is exposed primarily for selftests and does very little + * error checking. It is assumed that the set of placement regions has + * already been verified to be valid. + */ +struct drm_i915_gem_object * +__i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, + struct intel_memory_region **placements, + unsigned int n_placements) { - struct intel_memory_region *mr = obj->mm.placements[0]; + struct intel_memory_region *mr = placements[0]; + struct drm_i915_gem_object *obj; unsigned int flags; int ret; - size = round_up(size, object_max_page_size(obj)); + i915_gem_flush_free_objects(i915); + + size = round_up(size, object_max_page_size(placements, n_placements)); if (size == 0) - return -EINVAL; + return ERR_PTR(-EINVAL); /* For most of the ABI (e.g. mmap) we think in system pages */ GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); if (i915_gem_object_size_2big(size)) - return -E2BIG; + return ERR_PTR(-E2BIG); + + obj = i915_gem_object_alloc(); + if (!obj) + return ERR_PTR(-ENOMEM); + + ret = object_set_placements(obj, placements, n_placements); + if (ret) + goto object_free; /* * I915_BO_ALLOC_USER will make sure the object is cleared before @@ -90,14 +129,20 @@ i915_gem_setup(struct drm_i915_gem_object *obj, u64 size) */ flags = I915_BO_ALLOC_USER; - ret = mr->ops->init_object(mr, obj, size, flags); + ret = mr->ops->init_object(mr, obj, size, 0, flags); if (ret) - return ret; + goto object_free; GEM_BUG_ON(size != obj->base.size); trace_i915_gem_object_create(obj); - return 0; + return obj; + +object_free: + if (obj->mm.n_placements > 1) + kfree(obj->mm.placements); + i915_gem_object_free(obj); + return ERR_PTR(ret); } int @@ -110,7 +155,6 @@ i915_gem_dumb_create(struct drm_file *file, enum intel_memory_type mem_type; int cpp = DIV_ROUND_UP(args->bpp, 8); u32 format; - int ret; switch (cpp) { case 1: @@ -143,22 +187,13 @@ i915_gem_dumb_create(struct drm_file *file, if (HAS_LMEM(to_i915(dev))) mem_type = INTEL_MEMORY_LOCAL; - obj = i915_gem_object_alloc(); - if (!obj) - return -ENOMEM; - mr = intel_memory_region_by_type(to_i915(dev), mem_type); - object_set_placements(obj, &mr, 1); - ret = i915_gem_setup(obj, args->size); - if (ret) - goto object_free; + obj = __i915_gem_object_create_user(to_i915(dev), args->size, &mr, 1); + if (IS_ERR(obj)) + return PTR_ERR(obj); return i915_gem_publish(obj, file, &args->size, &args->handle); - -object_free: - i915_gem_object_free(obj); - return ret; } /** @@ -175,31 +210,20 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_create *args = data; struct drm_i915_gem_object *obj; struct intel_memory_region *mr; - int ret; - - i915_gem_flush_free_objects(i915); - - obj = i915_gem_object_alloc(); - if (!obj) - return -ENOMEM; mr = intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); - object_set_placements(obj, &mr, 1); - ret = i915_gem_setup(obj, args->size); - if (ret) - goto object_free; + obj = __i915_gem_object_create_user(i915, args->size, &mr, 1); + if (IS_ERR(obj)) + return PTR_ERR(obj); return i915_gem_publish(obj, file, &args->size, &args->handle); - -object_free: - i915_gem_object_free(obj); - return ret; } struct create_ext { struct drm_i915_private *i915; - struct drm_i915_gem_object *vanilla_object; + struct intel_memory_region *placements[INTEL_REGION_UNKNOWN]; + unsigned int n_placements; }; static void repr_placements(char *buf, size_t size, @@ -230,8 +254,7 @@ static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, struct drm_i915_private *i915 = ext_data->i915; struct drm_i915_gem_memory_class_instance __user *uregions = u64_to_user_ptr(args->regions); - struct drm_i915_gem_object *obj = ext_data->vanilla_object; - struct intel_memory_region **placements; + struct intel_memory_region *placements[INTEL_REGION_UNKNOWN]; u32 mask; int i, ret = 0; @@ -245,6 +268,8 @@ static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, ret = -EINVAL; } + BUILD_BUG_ON(ARRAY_SIZE(i915->mm.regions) != ARRAY_SIZE(placements)); + BUILD_BUG_ON(ARRAY_SIZE(ext_data->placements) != ARRAY_SIZE(placements)); if (args->num_regions > ARRAY_SIZE(i915->mm.regions)) { drm_dbg(&i915->drm, "num_regions is too large\n"); ret = -EINVAL; @@ -253,21 +278,13 @@ static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, if (ret) return ret; - placements = kmalloc_array(args->num_regions, - sizeof(struct intel_memory_region *), - GFP_KERNEL); - if (!placements) - return -ENOMEM; - mask = 0; for (i = 0; i < args->num_regions; i++) { struct drm_i915_gem_memory_class_instance region; struct intel_memory_region *mr; - if (copy_from_user(®ion, uregions, sizeof(region))) { - ret = -EFAULT; - goto out_free; - } + if (copy_from_user(®ion, uregions, sizeof(region))) + return -EFAULT; mr = intel_memory_region_lookup(i915, region.memory_class, @@ -293,14 +310,14 @@ static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, ++uregions; } - if (obj->mm.placements) { + if (ext_data->n_placements) { ret = -EINVAL; goto out_dump; } - object_set_placements(obj, placements, args->num_regions); - if (args->num_regions == 1) - kfree(placements); + ext_data->n_placements = args->num_regions; + for (i = 0; i < args->num_regions; i++) + ext_data->placements[i] = placements[i]; return 0; @@ -308,11 +325,11 @@ out_dump: if (1) { char buf[256]; - if (obj->mm.placements) { + if (ext_data->n_placements) { repr_placements(buf, sizeof(buf), - obj->mm.placements, - obj->mm.n_placements); + ext_data->placements, + ext_data->n_placements); drm_dbg(&i915->drm, "Placements were already set in previous EXT. Existing placements: %s\n", buf); @@ -322,8 +339,6 @@ out_dump: drm_dbg(&i915->drm, "New placements(so far validated): %s\n", buf); } -out_free: - kfree(placements); return ret; } @@ -358,44 +373,30 @@ i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_create_ext *args = data; struct create_ext ext_data = { .i915 = i915 }; - struct intel_memory_region **placements_ext; struct drm_i915_gem_object *obj; int ret; if (args->flags) return -EINVAL; - i915_gem_flush_free_objects(i915); - - obj = i915_gem_object_alloc(); - if (!obj) - return -ENOMEM; - - ext_data.vanilla_object = obj; ret = i915_user_extensions(u64_to_user_ptr(args->extensions), create_extensions, ARRAY_SIZE(create_extensions), &ext_data); - placements_ext = obj->mm.placements; if (ret) - goto object_free; + return ret; - if (!placements_ext) { - struct intel_memory_region *mr = + if (!ext_data.n_placements) { + ext_data.placements[0] = intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); - - object_set_placements(obj, &mr, 1); + ext_data.n_placements = 1; } - ret = i915_gem_setup(obj, args->size); - if (ret) - goto object_free; + obj = __i915_gem_object_create_user(i915, args->size, + ext_data.placements, + ext_data.n_placements); + if (IS_ERR(obj)) + return PTR_ERR(obj); return i915_gem_publish(obj, file, &args->size, &args->handle); - -object_free: - if (obj->mm.n_placements > 1) - kfree(placements_ext); - i915_gem_object_free(obj); - return ret; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index 616c3a2f1baf..afa34111de02 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -12,6 +12,8 @@ #include "i915_gem_object.h" #include "i915_scatterlist.h" +I915_SELFTEST_DECLARE(static bool force_different_devices;) + static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf) { return to_intel_bo(buf->priv); @@ -25,15 +27,11 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme struct scatterlist *src, *dst; int ret, i; - ret = i915_gem_object_pin_pages_unlocked(obj); - if (ret) - goto err; - /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { ret = -ENOMEM; - goto err_unpin_pages; + goto err; } ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); @@ -58,8 +56,6 @@ err_free_sg: sg_free_table(st); err_free: kfree(st); -err_unpin_pages: - i915_gem_object_unpin_pages(obj); err: return ERR_PTR(ret); } @@ -68,13 +64,9 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, struct sg_table *sg, enum dma_data_direction dir) { - struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); - dma_unmap_sgtable(attachment->dev, sg, dir, DMA_ATTR_SKIP_CPU_SYNC); sg_free_table(sg); kfree(sg); - - i915_gem_object_unpin_pages(obj); } static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map) @@ -168,7 +160,46 @@ retry: return err; } +static int i915_gem_dmabuf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_i915_gem_object *obj = dma_buf_to_obj(dmabuf); + struct i915_gem_ww_ctx ww; + int err; + + if (!i915_gem_object_can_migrate(obj, INTEL_REGION_SMEM)) + return -EOPNOTSUPP; + + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(obj, &ww); + if (err) + continue; + + err = i915_gem_object_migrate(obj, &ww, INTEL_REGION_SMEM); + if (err) + continue; + + err = i915_gem_object_wait_migration(obj, 0); + if (err) + continue; + + err = i915_gem_object_pin_pages(obj); + } + + return err; +} + +static void i915_gem_dmabuf_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_i915_gem_object *obj = dma_buf_to_obj(dmabuf); + + i915_gem_object_unpin_pages(obj); +} + static const struct dma_buf_ops i915_dmabuf_ops = { + .attach = i915_gem_dmabuf_attach, + .detach = i915_gem_dmabuf_detach, .map_dma_buf = i915_gem_map_dma_buf, .unmap_dma_buf = i915_gem_unmap_dma_buf, .release = drm_gem_dmabuf_release, @@ -204,6 +235,8 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) struct sg_table *pages; unsigned int sg_page_sizes; + assert_object_held(obj); + pages = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL); if (IS_ERR(pages)) @@ -241,7 +274,8 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, if (dma_buf->ops == &i915_dmabuf_ops) { obj = dma_buf_to_obj(dma_buf); /* is it from our device? */ - if (obj->base.dev == dev) { + if (obj->base.dev == dev && + !I915_SELFTEST_ONLY(force_different_devices)) { /* * Importing dmabuf exported from out own gem increases * refcount on gem itself instead of f_count of dmabuf. diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c index 073822100da7..b684a62bf3b0 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c @@ -268,6 +268,9 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int err = 0; + if (IS_DGFX(to_i915(dev))) + return -ENODEV; + rcu_read_lock(); obj = i915_gem_object_lookup_rcu(file, args->handle); if (!obj) { @@ -303,6 +306,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, enum i915_cache_level level; int ret = 0; + if (IS_DGFX(i915)) + return -ENODEV; + switch (args->caching) { case I915_CACHING_NONE: level = I915_CACHE_NONE; @@ -375,7 +381,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret; - /* Frame buffer must be in LMEM (no migration yet) */ + /* Frame buffer must be in LMEM */ if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) return ERR_PTR(-EINVAL); @@ -484,6 +490,9 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, u32 write_domain = args->write_domain; int err; + if (IS_DGFX(to_i915(dev))) + return -ENODEV; + /* Only handle setting domains to types used by the CPU. */ if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) return -EINVAL; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 989ff064cf3a..1aa249908b64 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -277,18 +277,9 @@ struct i915_execbuffer { bool has_llc : 1; bool has_fence : 1; bool needs_unfenced : 1; - - struct i915_request *rq; - u32 *rq_cmd; - unsigned int rq_size; - struct intel_gt_buffer_pool_node *pool; } reloc_cache; - struct intel_gt_buffer_pool_node *reloc_pool; /** relocation pool for -EDEADLK handling */ - struct intel_context *reloc_context; - u64 invalid_flags; /** Set of execobj.flags that are invalid */ - u32 context_flags; /** Set of execobj.flags to insert from the ctx */ u64 batch_len; /** Length of batch within object */ u32 batch_start_offset; /** Location within object of batch */ @@ -539,9 +530,6 @@ eb_validate_vma(struct i915_execbuffer *eb, entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP; } - if (!(entry->flags & EXEC_OBJECT_PINNED)) - entry->flags |= eb->context_flags; - return 0; } @@ -741,17 +729,13 @@ static int eb_select_context(struct i915_execbuffer *eb) struct i915_gem_context *ctx; ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); - if (unlikely(!ctx)) - return -ENOENT; + if (unlikely(IS_ERR(ctx))) + return PTR_ERR(ctx); eb->gem_context = ctx; if (rcu_access_pointer(ctx->vm)) eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; - eb->context_flags = 0; - if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags)) - eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS; - return 0; } @@ -920,21 +904,38 @@ err: return err; } -static int eb_validate_vmas(struct i915_execbuffer *eb) +static int eb_lock_vmas(struct i915_execbuffer *eb) { unsigned int i; int err; - INIT_LIST_HEAD(&eb->unbound); - for (i = 0; i < eb->buffer_count; i++) { - struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; struct eb_vma *ev = &eb->vma[i]; struct i915_vma *vma = ev->vma; err = i915_gem_object_lock(vma->obj, &eb->ww); if (err) return err; + } + + return 0; +} + +static int eb_validate_vmas(struct i915_execbuffer *eb) +{ + unsigned int i; + int err; + + INIT_LIST_HEAD(&eb->unbound); + + err = eb_lock_vmas(eb); + if (err) + return err; + + for (i = 0; i < eb->buffer_count; i++) { + struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; + struct eb_vma *ev = &eb->vma[i]; + struct i915_vma *vma = ev->vma; err = eb_pin_vma(eb, entry, ev); if (err == -EDEADLK) @@ -992,7 +993,7 @@ eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) } } -static void eb_release_vmas(struct i915_execbuffer *eb, bool final, bool release_userptr) +static void eb_release_vmas(struct i915_execbuffer *eb, bool final) { const unsigned int count = eb->buffer_count; unsigned int i; @@ -1006,11 +1007,6 @@ static void eb_release_vmas(struct i915_execbuffer *eb, bool final, bool release eb_unreserve_vma(ev); - if (release_userptr && ev->flags & __EXEC_OBJECT_USERPTR_INIT) { - ev->flags &= ~__EXEC_OBJECT_USERPTR_INIT; - i915_gem_object_userptr_submit_fini(vma->obj); - } - if (final) i915_vma_put(vma); } @@ -1020,8 +1016,6 @@ static void eb_release_vmas(struct i915_execbuffer *eb, bool final, bool release static void eb_destroy(const struct i915_execbuffer *eb) { - GEM_BUG_ON(eb->reloc_cache.rq); - if (eb->lut_size > 0) kfree(eb->buckets); } @@ -1033,14 +1027,6 @@ relocation_target(const struct drm_i915_gem_relocation_entry *reloc, return gen8_canonical_addr((int)reloc->delta + target->node.start); } -static void reloc_cache_clear(struct reloc_cache *cache) -{ - cache->rq = NULL; - cache->rq_cmd = NULL; - cache->pool = NULL; - cache->rq_size = 0; -} - static void reloc_cache_init(struct reloc_cache *cache, struct drm_i915_private *i915) { @@ -1053,7 +1039,6 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->has_fence = cache->graphics_ver < 4; cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; cache->node.flags = 0; - reloc_cache_clear(cache); } static inline void *unmask_page(unsigned long p) @@ -1075,48 +1060,10 @@ static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) return &i915->ggtt; } -static void reloc_cache_put_pool(struct i915_execbuffer *eb, struct reloc_cache *cache) -{ - if (!cache->pool) - return; - - /* - * This is a bit nasty, normally we keep objects locked until the end - * of execbuffer, but we already submit this, and have to unlock before - * dropping the reference. Fortunately we can only hold 1 pool node at - * a time, so this should be harmless. - */ - i915_gem_ww_unlock_single(cache->pool->obj); - intel_gt_buffer_pool_put(cache->pool); - cache->pool = NULL; -} - -static void reloc_gpu_flush(struct i915_execbuffer *eb, struct reloc_cache *cache) -{ - struct drm_i915_gem_object *obj = cache->rq->batch->obj; - - GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); - cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END; - - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - - intel_gt_chipset_flush(cache->rq->engine->gt); - - i915_request_add(cache->rq); - reloc_cache_put_pool(eb, cache); - reloc_cache_clear(cache); - - eb->reloc_pool = NULL; -} - static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) { void *vaddr; - if (cache->rq) - reloc_gpu_flush(eb, cache); - if (!cache->vaddr) return; @@ -1298,295 +1245,6 @@ static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) *addr = value; } -static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) -{ - struct drm_i915_gem_object *obj = vma->obj; - int err; - - assert_vma_held(vma); - - if (obj->cache_dirty & ~obj->cache_coherent) - i915_gem_clflush_object(obj, 0); - obj->write_domain = 0; - - err = i915_request_await_object(rq, vma->obj, true); - if (err == 0) - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - - return err; -} - -static int __reloc_gpu_alloc(struct i915_execbuffer *eb, - struct intel_engine_cs *engine, - struct i915_vma *vma, - unsigned int len) -{ - struct reloc_cache *cache = &eb->reloc_cache; - struct intel_gt_buffer_pool_node *pool = eb->reloc_pool; - struct i915_request *rq; - struct i915_vma *batch; - u32 *cmd; - int err; - - if (!pool) { - pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE, - cache->has_llc ? - I915_MAP_WB : - I915_MAP_WC); - if (IS_ERR(pool)) - return PTR_ERR(pool); - } - eb->reloc_pool = NULL; - - err = i915_gem_object_lock(pool->obj, &eb->ww); - if (err) - goto err_pool; - - cmd = i915_gem_object_pin_map(pool->obj, pool->type); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err_pool; - } - intel_gt_buffer_pool_mark_used(pool); - - memset32(cmd, 0, pool->obj->base.size / sizeof(u32)); - - batch = i915_vma_instance(pool->obj, vma->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err_unmap; - } - - err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER | PIN_NONBLOCK); - if (err) - goto err_unmap; - - if (engine == eb->context->engine) { - rq = i915_request_create(eb->context); - } else { - struct intel_context *ce = eb->reloc_context; - - if (!ce) { - ce = intel_context_create(engine); - if (IS_ERR(ce)) { - err = PTR_ERR(ce); - goto err_unpin; - } - - i915_vm_put(ce->vm); - ce->vm = i915_vm_get(eb->context->vm); - eb->reloc_context = ce; - } - - err = intel_context_pin_ww(ce, &eb->ww); - if (err) - goto err_unpin; - - rq = i915_request_create(ce); - intel_context_unpin(ce); - } - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_unpin; - } - - err = intel_gt_buffer_pool_mark_active(pool, rq); - if (err) - goto err_request; - - err = reloc_move_to_gpu(rq, vma); - if (err) - goto err_request; - - err = eb->engine->emit_bb_start(rq, - batch->node.start, PAGE_SIZE, - cache->graphics_ver > 5 ? 0 : I915_DISPATCH_SECURE); - if (err) - goto skip_request; - - assert_vma_held(batch); - err = i915_request_await_object(rq, batch->obj, false); - if (err == 0) - err = i915_vma_move_to_active(batch, rq, 0); - if (err) - goto skip_request; - - rq->batch = batch; - i915_vma_unpin(batch); - - cache->rq = rq; - cache->rq_cmd = cmd; - cache->rq_size = 0; - cache->pool = pool; - - /* Return with batch mapping (cmd) still pinned */ - return 0; - -skip_request: - i915_request_set_error_once(rq, err); -err_request: - i915_request_add(rq); -err_unpin: - i915_vma_unpin(batch); -err_unmap: - i915_gem_object_unpin_map(pool->obj); -err_pool: - eb->reloc_pool = pool; - return err; -} - -static bool reloc_can_use_engine(const struct intel_engine_cs *engine) -{ - return engine->class != VIDEO_DECODE_CLASS || GRAPHICS_VER(engine->i915) != 6; -} - -static u32 *reloc_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - unsigned int len) -{ - struct reloc_cache *cache = &eb->reloc_cache; - u32 *cmd; - - if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1)) - reloc_gpu_flush(eb, cache); - - if (unlikely(!cache->rq)) { - int err; - struct intel_engine_cs *engine = eb->engine; - - /* If we need to copy for the cmdparser, we will stall anyway */ - if (eb_use_cmdparser(eb)) - return ERR_PTR(-EWOULDBLOCK); - - if (!reloc_can_use_engine(engine)) { - engine = engine->gt->engine_class[COPY_ENGINE_CLASS][0]; - if (!engine) - return ERR_PTR(-ENODEV); - } - - err = __reloc_gpu_alloc(eb, engine, vma, len); - if (unlikely(err)) - return ERR_PTR(err); - } - - cmd = cache->rq_cmd + cache->rq_size; - cache->rq_size += len; - - return cmd; -} - -static inline bool use_reloc_gpu(struct i915_vma *vma) -{ - if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) - return true; - - if (DBG_FORCE_RELOC) - return false; - - return !dma_resv_test_signaled(vma->resv, true); -} - -static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) -{ - struct page *page; - unsigned long addr; - - GEM_BUG_ON(vma->pages != vma->obj->mm.pages); - - page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT); - addr = PFN_PHYS(page_to_pfn(page)); - GEM_BUG_ON(overflows_type(addr, u32)); /* expected dma32 */ - - return addr + offset_in_page(offset); -} - -static int __reloc_entry_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - u64 offset, - u64 target_addr) -{ - const unsigned int ver = eb->reloc_cache.graphics_ver; - unsigned int len; - u32 *batch; - u64 addr; - - if (ver >= 8) - len = offset & 7 ? 8 : 5; - else if (ver >= 4) - len = 4; - else - len = 3; - - batch = reloc_gpu(eb, vma, len); - if (batch == ERR_PTR(-EDEADLK)) - return -EDEADLK; - else if (IS_ERR(batch)) - return false; - - addr = gen8_canonical_addr(vma->node.start + offset); - if (ver >= 8) { - if (offset & 7) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = lower_32_bits(target_addr); - - addr = gen8_canonical_addr(addr + 4); - - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = upper_32_bits(target_addr); - } else { - *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = lower_32_bits(target_addr); - *batch++ = upper_32_bits(target_addr); - } - } else if (ver >= 6) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = 0; - *batch++ = addr; - *batch++ = target_addr; - } else if (IS_I965G(eb->i915)) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = 0; - *batch++ = vma_phys_addr(vma, offset); - *batch++ = target_addr; - } else if (ver >= 4) { - *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *batch++ = 0; - *batch++ = addr; - *batch++ = target_addr; - } else if (ver >= 3 && - !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) { - *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *batch++ = addr; - *batch++ = target_addr; - } else { - *batch++ = MI_STORE_DWORD_IMM; - *batch++ = vma_phys_addr(vma, offset); - *batch++ = target_addr; - } - - return true; -} - -static int reloc_entry_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - u64 offset, - u64 target_addr) -{ - if (eb->reloc_cache.vaddr) - return false; - - if (!use_reloc_gpu(vma)) - return false; - - return __reloc_entry_gpu(eb, vma, offset, target_addr); -} - static u64 relocate_entry(struct i915_vma *vma, const struct drm_i915_gem_relocation_entry *reloc, @@ -1595,32 +1253,25 @@ relocate_entry(struct i915_vma *vma, { u64 target_addr = relocation_target(reloc, target); u64 offset = reloc->offset; - int reloc_gpu = reloc_entry_gpu(eb, vma, offset, target_addr); - - if (reloc_gpu < 0) - return reloc_gpu; - - if (!reloc_gpu) { - bool wide = eb->reloc_cache.use_64bit_reloc; - void *vaddr; + bool wide = eb->reloc_cache.use_64bit_reloc; + void *vaddr; repeat: - vaddr = reloc_vaddr(vma->obj, eb, - offset >> PAGE_SHIFT); - if (IS_ERR(vaddr)) - return PTR_ERR(vaddr); - - GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); - clflush_write32(vaddr + offset_in_page(offset), - lower_32_bits(target_addr), - eb->reloc_cache.vaddr); - - if (wide) { - offset += sizeof(u32); - target_addr >>= 32; - wide = false; - goto repeat; - } + vaddr = reloc_vaddr(vma->obj, eb, + offset >> PAGE_SHIFT); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); + clflush_write32(vaddr + offset_in_page(offset), + lower_32_bits(target_addr), + eb->reloc_cache.vaddr); + + if (wide) { + offset += sizeof(u32); + target_addr >>= 32; + wide = false; + goto repeat; } return target->node.start | UPDATE; @@ -1992,7 +1643,7 @@ repeat: } /* We may process another execbuffer during the unlock... */ - eb_release_vmas(eb, false, true); + eb_release_vmas(eb, false); i915_gem_ww_ctx_fini(&eb->ww); if (rq) { @@ -2061,9 +1712,7 @@ repeat_validate: list_for_each_entry(ev, &eb->relocs, reloc_link) { if (!have_copy) { - pagefault_disable(); err = eb_relocate_vma(eb, ev); - pagefault_enable(); if (err) break; } else { @@ -2096,7 +1745,7 @@ repeat_validate: err: if (err == -EDEADLK) { - eb_release_vmas(eb, false, false); + eb_release_vmas(eb, false); err = i915_gem_ww_ctx_backoff(&eb->ww); if (!err) goto repeat_validate; @@ -2193,7 +1842,7 @@ retry: err: if (err == -EDEADLK) { - eb_release_vmas(eb, false, false); + eb_release_vmas(eb, false); err = i915_gem_ww_ctx_backoff(&eb->ww); if (!err) goto retry; @@ -2270,7 +1919,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) #ifdef CONFIG_MMU_NOTIFIER if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) { - spin_lock(&eb->i915->mm.notifier_lock); + read_lock(&eb->i915->mm.notifier_lock); /* * count is always at least 1, otherwise __EXEC_USERPTR_USED @@ -2288,7 +1937,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) break; } - spin_unlock(&eb->i915->mm.notifier_lock); + read_unlock(&eb->i915->mm.notifier_lock); } #endif @@ -3156,8 +2805,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.exec = exec; eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); eb.vma[0].vma = NULL; - eb.reloc_pool = eb.batch_pool = NULL; - eb.reloc_context = NULL; + eb.batch_pool = NULL; eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; reloc_cache_init(&eb.reloc_cache, eb.i915); @@ -3232,7 +2880,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, err = eb_lookup_vmas(&eb); if (err) { - eb_release_vmas(&eb, true, true); + eb_release_vmas(&eb, true); goto err_engine; } @@ -3255,9 +2903,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, batch = eb.batch->vma; - /* All GPU relocation batches must be submitted prior to the user rq */ - GEM_BUG_ON(eb.reloc_cache.rq); - /* Allocate a request for this batch buffer nice and early. */ eb.request = i915_request_create(eb.context); if (IS_ERR(eb.request)) { @@ -3265,11 +2910,20 @@ i915_gem_do_execbuffer(struct drm_device *dev, goto err_vma; } + if (unlikely(eb.gem_context->syncobj)) { + struct dma_fence *fence; + + fence = drm_syncobj_fence_get(eb.gem_context->syncobj); + err = i915_request_await_dma_fence(eb.request, fence); + dma_fence_put(fence); + if (err) + goto err_ext; + } + if (in_fence) { if (args->flags & I915_EXEC_FENCE_SUBMIT) err = i915_request_await_execution(eb.request, - in_fence, - eb.engine->bond_execute); + in_fence); else err = i915_request_await_dma_fence(eb.request, in_fence); @@ -3322,10 +2976,16 @@ err_request: fput(out_fence->file); } } + + if (unlikely(eb.gem_context->syncobj)) { + drm_syncobj_replace_fence(eb.gem_context->syncobj, + &eb.request->fence); + } + i915_request_put(eb.request); err_vma: - eb_release_vmas(&eb, true, true); + eb_release_vmas(&eb, true); if (eb.trampoline) i915_vma_unpin(eb.trampoline); WARN_ON(err == -EDEADLK); @@ -3333,10 +2993,6 @@ err_vma: if (eb.batch_pool) intel_gt_buffer_pool_put(eb.batch_pool); - if (eb.reloc_pool) - intel_gt_buffer_pool_put(eb.reloc_pool); - if (eb.reloc_context) - intel_context_put(eb.reloc_context); err_engine: eb_put_engine(&eb); err_context: @@ -3450,7 +3106,3 @@ end:; kvfree(exec2_list); return err; } - -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/i915_gem_execbuffer.c" -#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index ce6b664b10aa..e5ae9c06510c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -42,7 +42,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) max_order = MAX_ORDER; #ifdef CONFIG_SWIOTLB - if (is_swiotlb_active()) { + if (is_swiotlb_active(obj->base.dev->dev)) { unsigned int max_segment; max_segment = swiotlb_max_segment(); @@ -177,8 +177,8 @@ i915_gem_object_create_internal(struct drm_i915_private *i915, return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, size); - i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class, - I915_BO_ALLOC_STRUCT_PAGE); + i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class, 0); + obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; /* * Mark the object as volatile, such that the pages are marked as diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index d539dffa1554..eb345305dc52 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -24,27 +24,6 @@ i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, } /** - * i915_gem_object_validates_to_lmem - Whether the object is resident in - * lmem when pages are present. - * @obj: The object to check. - * - * Migratable objects residency may change from under us if the object is - * not pinned or locked. This function is intended to be used to check whether - * the object can only reside in lmem when pages are present. - * - * Return: Whether the object is always resident in lmem when pages are - * present. - */ -bool i915_gem_object_validates_to_lmem(struct drm_i915_gem_object *obj) -{ - struct intel_memory_region *mr = READ_ONCE(obj->mm.region); - - return !i915_gem_object_migratable(obj) && - mr && (mr->type == INTEL_MEMORY_LOCAL || - mr->type == INTEL_MEMORY_STOLEN_LOCAL); -} - -/** * i915_gem_object_is_lmem - Whether the object is resident in * lmem * @obj: The object to check. @@ -71,11 +50,64 @@ bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) mr->type == INTEL_MEMORY_STOLEN_LOCAL); } +/** + * __i915_gem_object_is_lmem - Whether the object is resident in + * lmem while in the fence signaling critical path. + * @obj: The object to check. + * + * This function is intended to be called from within the fence signaling + * path where the fence keeps the object from being migrated. For example + * during gpu reset or similar. + * + * Return: Whether the object is resident in lmem. + */ +bool __i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) +{ + struct intel_memory_region *mr = READ_ONCE(obj->mm.region); + +#ifdef CONFIG_LOCKDEP + GEM_WARN_ON(dma_resv_test_signaled(obj->base.resv, true)); +#endif + return mr && (mr->type == INTEL_MEMORY_LOCAL || + mr->type == INTEL_MEMORY_STOLEN_LOCAL); +} + +/** + * __i915_gem_object_create_lmem_with_ps - Create lmem object and force the + * minimum page size for the backing pages. + * @i915: The i915 instance. + * @size: The size in bytes for the object. Note that we need to round the size + * up depending on the @page_size. The final object size can be fished out from + * the drm GEM object. + * @page_size: The requested minimum page size in bytes for this object. This is + * useful if we need something bigger than the regions min_page_size due to some + * hw restriction, or in some very specialised cases where it needs to be + * smaller, where the internal fragmentation cost is too great when rounding up + * the object size. + * @flags: The optional BO allocation flags. + * + * Note that this interface assumes you know what you are doing when forcing the + * @page_size. If this is smaller than the regions min_page_size then it can + * never be inserted into any GTT, otherwise it might lead to undefined + * behaviour. + * + * Return: The object pointer, which might be an ERR_PTR in the case of failure. + */ +struct drm_i915_gem_object * +__i915_gem_object_create_lmem_with_ps(struct drm_i915_private *i915, + resource_size_t size, + resource_size_t page_size, + unsigned int flags) +{ + return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_LMEM], + size, page_size, flags); +} + struct drm_i915_gem_object * i915_gem_object_create_lmem(struct drm_i915_private *i915, resource_size_t size, unsigned int flags) { return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_LMEM], - size, flags); + size, 0, flags); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index ea76fd11ccb0..4ee81fc66302 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -21,6 +21,13 @@ i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); +bool __i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); + +struct drm_i915_gem_object * +__i915_gem_object_create_lmem_with_ps(struct drm_i915_private *i915, + resource_size_t size, + resource_size_t page_size, + unsigned int flags); struct drm_i915_gem_object * i915_gem_object_create_lmem(struct drm_i915_private *i915, resource_size_t size, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 4f50a508c7a0..5130e8ed9564 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -645,7 +645,8 @@ mmap_offset_attach(struct drm_i915_gem_object *obj, goto insert; /* Attempt to reap some mmap space from dead objects */ - err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT); + err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT, + NULL); if (err) goto err; @@ -679,13 +680,19 @@ __assign_mmap_offset(struct drm_i915_gem_object *obj, return -ENODEV; if (obj->ops->mmap_offset) { + if (mmap_type != I915_MMAP_TYPE_FIXED) + return -ENODEV; + *offset = obj->ops->mmap_offset(obj); return 0; } + if (mmap_type == I915_MMAP_TYPE_FIXED) + return -ENODEV; + if (mmap_type != I915_MMAP_TYPE_GTT && !i915_gem_object_has_struct_page(obj) && - !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) + !i915_gem_object_has_iomem(obj)) return -ENODEV; mmo = mmap_offset_attach(obj, mmap_type, file); @@ -709,7 +716,12 @@ __assign_mmap_offset_handle(struct drm_file *file, if (!obj) return -ENOENT; + err = i915_gem_object_lock_interruptible(obj, NULL); + if (err) + goto out_put; err = __assign_mmap_offset(obj, mmap_type, offset, file); + i915_gem_object_unlock(obj); +out_put: i915_gem_object_put(obj); return err; } @@ -722,7 +734,9 @@ i915_gem_dumb_mmap_offset(struct drm_file *file, { enum i915_mmap_type mmap_type; - if (boot_cpu_has(X86_FEATURE_PAT)) + if (HAS_LMEM(to_i915(dev))) + mmap_type = I915_MMAP_TYPE_FIXED; + else if (boot_cpu_has(X86_FEATURE_PAT)) mmap_type = I915_MMAP_TYPE_WC; else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) return -ENODEV; @@ -793,6 +807,10 @@ i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, type = I915_MMAP_TYPE_UC; break; + case I915_MMAP_OFFSET_FIXED: + type = I915_MMAP_TYPE_FIXED; + break; + default: return -EINVAL; } @@ -933,10 +951,7 @@ int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) return PTR_ERR(anon); } - vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; - - if (i915_gem_object_has_iomem(obj)) - vma->vm_flags |= VM_IO; + vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; /* * We keep the ref on mmo->obj, not vm_file, but we require @@ -966,6 +981,9 @@ int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_ops = &vm_ops_cpu; break; + case I915_MMAP_TYPE_FIXED: + GEM_WARN_ON(1); + fallthrough; case I915_MMAP_TYPE_WB: vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); vma->vm_ops = &vm_ops_cpu; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index cf18c430d51f..6fb9afb65034 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -30,14 +30,10 @@ #include "i915_gem_context.h" #include "i915_gem_mman.h" #include "i915_gem_object.h" -#include "i915_globals.h" #include "i915_memcpy.h" #include "i915_trace.h" -static struct i915_global_object { - struct i915_global base; - struct kmem_cache *slab_objects; -} global; +static struct kmem_cache *slab_objects; static const struct drm_gem_object_funcs i915_gem_object_funcs; @@ -45,7 +41,7 @@ struct drm_i915_gem_object *i915_gem_object_alloc(void) { struct drm_i915_gem_object *obj; - obj = kmem_cache_zalloc(global.slab_objects, GFP_KERNEL); + obj = kmem_cache_zalloc(slab_objects, GFP_KERNEL); if (!obj) return NULL; obj->base.funcs = &i915_gem_object_funcs; @@ -55,7 +51,7 @@ struct drm_i915_gem_object *i915_gem_object_alloc(void) void i915_gem_object_free(struct drm_i915_gem_object *obj) { - return kmem_cache_free(global.slab_objects, obj); + return kmem_cache_free(slab_objects, obj); } void i915_gem_object_init(struct drm_i915_gem_object *obj, @@ -475,34 +471,200 @@ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj) return obj->mm.n_placements > 1; } -void i915_gem_init__objects(struct drm_i915_private *i915) +/** + * i915_gem_object_has_struct_page - Whether the object is page-backed + * @obj: The object to query. + * + * This function should only be called while the object is locked or pinned, + * otherwise the page backing may change under the caller. + * + * Return: True if page-backed, false otherwise. + */ +bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) { - INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); +#ifdef CONFIG_LOCKDEP + if (IS_DGFX(to_i915(obj->base.dev)) && + i915_gem_object_evictable((void __force *)obj)) + assert_object_held_shared(obj); +#endif + return obj->mem_flags & I915_BO_FLAG_STRUCT_PAGE; +} + +/** + * i915_gem_object_has_iomem - Whether the object is iomem-backed + * @obj: The object to query. + * + * This function should only be called while the object is locked or pinned, + * otherwise the iomem backing may change under the caller. + * + * Return: True if iomem-backed, false otherwise. + */ +bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj) +{ +#ifdef CONFIG_LOCKDEP + if (IS_DGFX(to_i915(obj->base.dev)) && + i915_gem_object_evictable((void __force *)obj)) + assert_object_held_shared(obj); +#endif + return obj->mem_flags & I915_BO_FLAG_IOMEM; +} + +/** + * i915_gem_object_can_migrate - Whether an object likely can be migrated + * + * @obj: The object to migrate + * @id: The region intended to migrate to + * + * Check whether the object backend supports migration to the + * given region. Note that pinning may affect the ability to migrate as + * returned by this function. + * + * This function is primarily intended as a helper for checking the + * possibility to migrate objects and might be slightly less permissive + * than i915_gem_object_migrate() when it comes to objects with the + * I915_BO_ALLOC_USER flag set. + * + * Return: true if migration is possible, false otherwise. + */ +bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, + enum intel_region_id id) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int num_allowed = obj->mm.n_placements; + struct intel_memory_region *mr; + unsigned int i; + + GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); + GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); + + mr = i915->mm.regions[id]; + if (!mr) + return false; + + if (obj->mm.region == mr) + return true; + + if (!i915_gem_object_evictable(obj)) + return false; + + if (!obj->ops->migrate) + return false; + + if (!(obj->flags & I915_BO_ALLOC_USER)) + return true; + + if (num_allowed == 0) + return false; + + for (i = 0; i < num_allowed; ++i) { + if (mr == obj->mm.placements[i]) + return true; + } + + return false; } -static void i915_global_objects_shrink(void) +/** + * i915_gem_object_migrate - Migrate an object to the desired region id + * @obj: The object to migrate. + * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may + * not be successful in evicting other objects to make room for this object. + * @id: The region id to migrate to. + * + * Attempt to migrate the object to the desired memory region. The + * object backend must support migration and the object may not be + * pinned, (explicitly pinned pages or pinned vmas). The object must + * be locked. + * On successful completion, the object will have pages pointing to + * memory in the new region, but an async migration task may not have + * completed yet, and to accomplish that, i915_gem_object_wait_migration() + * must be called. + * + * Note: the @ww parameter is not used yet, but included to make sure + * callers put some effort into obtaining a valid ww ctx if one is + * available. + * + * Return: 0 on success. Negative error code on failure. In particular may + * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance + * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and + * -EBUSY if the object is pinned. + */ +int i915_gem_object_migrate(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + enum intel_region_id id) { - kmem_cache_shrink(global.slab_objects); + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct intel_memory_region *mr; + + GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); + GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); + assert_object_held(obj); + + mr = i915->mm.regions[id]; + GEM_BUG_ON(!mr); + + if (!i915_gem_object_can_migrate(obj, id)) + return -EINVAL; + + if (!obj->ops->migrate) { + if (GEM_WARN_ON(obj->mm.region != mr)) + return -EINVAL; + return 0; + } + + return obj->ops->migrate(obj, mr); } -static void i915_global_objects_exit(void) +/** + * i915_gem_object_placement_possible - Check whether the object can be + * placed at certain memory type + * @obj: Pointer to the object + * @type: The memory type to check + * + * Return: True if the object can be placed in @type. False otherwise. + */ +bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, + enum intel_memory_type type) { - kmem_cache_destroy(global.slab_objects); + unsigned int i; + + if (!obj->mm.n_placements) { + switch (type) { + case INTEL_MEMORY_LOCAL: + return i915_gem_object_has_iomem(obj); + case INTEL_MEMORY_SYSTEM: + return i915_gem_object_has_pages(obj); + default: + /* Ignore stolen for now */ + GEM_BUG_ON(1); + return false; + } + } + + for (i = 0; i < obj->mm.n_placements; i++) { + if (obj->mm.placements[i]->type == type) + return true; + } + + return false; +} + +void i915_gem_init__objects(struct drm_i915_private *i915) +{ + INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); } -static struct i915_global_object global = { { - .shrink = i915_global_objects_shrink, - .exit = i915_global_objects_exit, -} }; +void i915_objects_module_exit(void) +{ + kmem_cache_destroy(slab_objects); +} -int __init i915_global_objects_init(void) +int __init i915_objects_module_init(void) { - global.slab_objects = - KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); - if (!global.slab_objects) + slab_objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); + if (!slab_objects) return -ENOMEM; - i915_global_register(&global.base); return 0; } @@ -515,6 +677,7 @@ static const struct drm_gem_object_funcs i915_gem_object_funcs = { #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/huge_gem_object.c" #include "selftests/huge_pages.c" +#include "selftests/i915_gem_migrate.c" #include "selftests/i915_gem_object.c" #include "selftests/i915_gem_coherency.c" #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index e9eecebf5c9d..48112b9d76df 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -12,10 +12,14 @@ #include <drm/drm_device.h> #include "display/intel_frontbuffer.h" +#include "intel_memory_region.h" #include "i915_gem_object_types.h" #include "i915_gem_gtt.h" +#include "i915_gem_ww.h" #include "i915_vma_types.h" +enum intel_region_id; + /* * XXX: There is a prevalence of the assumption that we fit the * object's page count inside a 32bit _signed_ variable. Let's document @@ -44,6 +48,9 @@ static inline bool i915_gem_object_size_2big(u64 size) void i915_gem_init__objects(struct drm_i915_private *i915); +void i915_objects_module_exit(void); +int i915_objects_module_init(void); + struct drm_i915_gem_object *i915_gem_object_alloc(void); void i915_gem_object_free(struct drm_i915_gem_object *obj); @@ -57,6 +64,10 @@ i915_gem_object_create_shmem(struct drm_i915_private *i915, struct drm_i915_gem_object * i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, const void *data, resource_size_t size); +struct drm_i915_gem_object * +__i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, + struct intel_memory_region **placements, + unsigned int n_placements); extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; @@ -147,7 +158,7 @@ i915_gem_object_put(struct drm_i915_gem_object *obj) /* * If more than one potential simultaneous locker, assert held. */ -static inline void assert_object_held_shared(struct drm_i915_gem_object *obj) +static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) { /* * Note mm list lookup is protected by @@ -169,13 +180,17 @@ static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, else ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); - if (!ret && ww) + if (!ret && ww) { + i915_gem_object_get(obj); list_add_tail(&obj->obj_link, &ww->obj_list); + } if (ret == -EALREADY) ret = 0; - if (ret == -EDEADLK) + if (ret == -EDEADLK) { + i915_gem_object_get(obj); ww->contended = obj; + } return ret; } @@ -261,17 +276,9 @@ i915_gem_object_type_has(const struct drm_i915_gem_object *obj, return obj->ops->flags & flags; } -static inline bool -i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) -{ - return obj->flags & I915_BO_ALLOC_STRUCT_PAGE; -} +bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); -static inline bool -i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj) -{ - return i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM); -} +bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); static inline bool i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) @@ -342,22 +349,22 @@ struct scatterlist * __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, struct i915_gem_object_page_iter *iter, unsigned int n, - unsigned int *offset, bool allow_alloc, bool dma); + unsigned int *offset, bool dma); static inline struct scatterlist * i915_gem_object_get_sg(struct drm_i915_gem_object *obj, unsigned int n, - unsigned int *offset, bool allow_alloc) + unsigned int *offset) { - return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset, allow_alloc, false); + return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset, false); } static inline struct scatterlist * i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, unsigned int n, - unsigned int *offset, bool allow_alloc) + unsigned int *offset) { - return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset, allow_alloc, true); + return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset, true); } struct page * @@ -598,7 +605,18 @@ bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); -bool i915_gem_object_validates_to_lmem(struct drm_i915_gem_object *obj); +int i915_gem_object_migrate(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + enum intel_region_id id); + +bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, + enum intel_region_id id); + +int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, + unsigned int flags); + +bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, + enum intel_memory_type type); #ifdef CONFIG_MMU_NOTIFIER static inline bool @@ -609,14 +627,12 @@ i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); -void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj); int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); #else static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } -static inline void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); } static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c deleted file mode 100644 index 3e28c68fda3e..000000000000 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c +++ /dev/null @@ -1,461 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include "i915_drv.h" -#include "gt/intel_context.h" -#include "gt/intel_engine_pm.h" -#include "gt/intel_gpu_commands.h" -#include "gt/intel_gt.h" -#include "gt/intel_gt_buffer_pool.h" -#include "gt/intel_ring.h" -#include "i915_gem_clflush.h" -#include "i915_gem_object_blt.h" - -struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, - struct i915_vma *vma, - struct i915_gem_ww_ctx *ww, - u32 value) -{ - struct drm_i915_private *i915 = ce->vm->i915; - const u32 block_size = SZ_8M; /* ~1ms at 8GiB/s preemption delay */ - struct intel_gt_buffer_pool_node *pool; - struct i915_vma *batch; - u64 offset; - u64 count; - u64 rem; - u32 size; - u32 *cmd; - int err; - - GEM_BUG_ON(intel_engine_is_virtual(ce->engine)); - intel_engine_pm_get(ce->engine); - - count = div_u64(round_up(vma->size, block_size), block_size); - size = (1 + 8 * count) * sizeof(u32); - size = round_up(size, PAGE_SIZE); - pool = intel_gt_get_buffer_pool(ce->engine->gt, size, I915_MAP_WC); - if (IS_ERR(pool)) { - err = PTR_ERR(pool); - goto out_pm; - } - - err = i915_gem_object_lock(pool->obj, ww); - if (err) - goto out_put; - - batch = i915_vma_instance(pool->obj, ce->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put; - } - - err = i915_vma_pin_ww(batch, ww, 0, 0, PIN_USER); - if (unlikely(err)) - goto out_put; - - /* we pinned the pool, mark it as such */ - intel_gt_buffer_pool_mark_used(pool); - - cmd = i915_gem_object_pin_map(pool->obj, pool->type); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto out_unpin; - } - - rem = vma->size; - offset = vma->node.start; - - do { - u32 size = min_t(u64, rem, block_size); - - GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); - - if (GRAPHICS_VER(i915) >= 8) { - *cmd++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2); - *cmd++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; - *cmd++ = 0; - *cmd++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cmd++ = lower_32_bits(offset); - *cmd++ = upper_32_bits(offset); - *cmd++ = value; - } else { - *cmd++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); - *cmd++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; - *cmd++ = 0; - *cmd++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cmd++ = offset; - *cmd++ = value; - } - - /* Allow ourselves to be preempted in between blocks. */ - *cmd++ = MI_ARB_CHECK; - - offset += size; - rem -= size; - } while (rem); - - *cmd = MI_BATCH_BUFFER_END; - - i915_gem_object_flush_map(pool->obj); - i915_gem_object_unpin_map(pool->obj); - - intel_gt_chipset_flush(ce->vm->gt); - - batch->private = pool; - return batch; - -out_unpin: - i915_vma_unpin(batch); -out_put: - intel_gt_buffer_pool_put(pool); -out_pm: - intel_engine_pm_put(ce->engine); - return ERR_PTR(err); -} - -int intel_emit_vma_mark_active(struct i915_vma *vma, struct i915_request *rq) -{ - int err; - - err = i915_request_await_object(rq, vma->obj, false); - if (err == 0) - err = i915_vma_move_to_active(vma, rq, 0); - if (unlikely(err)) - return err; - - return intel_gt_buffer_pool_mark_active(vma->private, rq); -} - -void intel_emit_vma_release(struct intel_context *ce, struct i915_vma *vma) -{ - i915_vma_unpin(vma); - intel_gt_buffer_pool_put(vma->private); - intel_engine_pm_put(ce->engine); -} - -static int -move_obj_to_gpu(struct drm_i915_gem_object *obj, - struct i915_request *rq, - bool write) -{ - if (obj->cache_dirty & ~obj->cache_coherent) - i915_gem_clflush_object(obj, 0); - - return i915_request_await_object(rq, obj, write); -} - -int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, - struct intel_context *ce, - u32 value) -{ - struct i915_gem_ww_ctx ww; - struct i915_request *rq; - struct i915_vma *batch; - struct i915_vma *vma; - int err; - - vma = i915_vma_instance(obj, ce->vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - i915_gem_ww_ctx_init(&ww, true); - intel_engine_pm_get(ce->engine); -retry: - err = i915_gem_object_lock(obj, &ww); - if (err) - goto out; - - err = intel_context_pin_ww(ce, &ww); - if (err) - goto out; - - err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); - if (err) - goto out_ctx; - - batch = intel_emit_vma_fill_blt(ce, vma, &ww, value); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_vma; - } - - rq = i915_request_create(ce); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_batch; - } - - err = intel_emit_vma_mark_active(batch, rq); - if (unlikely(err)) - goto out_request; - - err = move_obj_to_gpu(vma->obj, rq, true); - if (err == 0) - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (unlikely(err)) - goto out_request; - - if (ce->engine->emit_init_breadcrumb) - err = ce->engine->emit_init_breadcrumb(rq); - - if (likely(!err)) - err = ce->engine->emit_bb_start(rq, - batch->node.start, - batch->node.size, - 0); -out_request: - if (unlikely(err)) - i915_request_set_error_once(rq, err); - - i915_request_add(rq); -out_batch: - intel_emit_vma_release(ce, batch); -out_vma: - i915_vma_unpin(vma); -out_ctx: - intel_context_unpin(ce); -out: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - intel_engine_pm_put(ce->engine); - return err; -} - -/* Wa_1209644611:icl,ehl */ -static bool wa_1209644611_applies(struct drm_i915_private *i915, u32 size) -{ - u32 height = size >> PAGE_SHIFT; - - if (GRAPHICS_VER(i915) != 11) - return false; - - return height % 4 == 3 && height <= 8; -} - -struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, - struct i915_gem_ww_ctx *ww, - struct i915_vma *src, - struct i915_vma *dst) -{ - struct drm_i915_private *i915 = ce->vm->i915; - const u32 block_size = SZ_8M; /* ~1ms at 8GiB/s preemption delay */ - struct intel_gt_buffer_pool_node *pool; - struct i915_vma *batch; - u64 src_offset, dst_offset; - u64 count, rem; - u32 size, *cmd; - int err; - - GEM_BUG_ON(src->size != dst->size); - - GEM_BUG_ON(intel_engine_is_virtual(ce->engine)); - intel_engine_pm_get(ce->engine); - - count = div_u64(round_up(dst->size, block_size), block_size); - size = (1 + 11 * count) * sizeof(u32); - size = round_up(size, PAGE_SIZE); - pool = intel_gt_get_buffer_pool(ce->engine->gt, size, I915_MAP_WC); - if (IS_ERR(pool)) { - err = PTR_ERR(pool); - goto out_pm; - } - - err = i915_gem_object_lock(pool->obj, ww); - if (err) - goto out_put; - - batch = i915_vma_instance(pool->obj, ce->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put; - } - - err = i915_vma_pin_ww(batch, ww, 0, 0, PIN_USER); - if (unlikely(err)) - goto out_put; - - /* we pinned the pool, mark it as such */ - intel_gt_buffer_pool_mark_used(pool); - - cmd = i915_gem_object_pin_map(pool->obj, pool->type); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto out_unpin; - } - - rem = src->size; - src_offset = src->node.start; - dst_offset = dst->node.start; - - do { - size = min_t(u64, rem, block_size); - GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); - - if (GRAPHICS_VER(i915) >= 9 && - !wa_1209644611_applies(i915, size)) { - *cmd++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); - *cmd++ = BLT_DEPTH_32 | PAGE_SIZE; - *cmd++ = 0; - *cmd++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cmd++ = lower_32_bits(dst_offset); - *cmd++ = upper_32_bits(dst_offset); - *cmd++ = 0; - *cmd++ = PAGE_SIZE; - *cmd++ = lower_32_bits(src_offset); - *cmd++ = upper_32_bits(src_offset); - } else if (GRAPHICS_VER(i915) >= 8) { - *cmd++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); - *cmd++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; - *cmd++ = 0; - *cmd++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cmd++ = lower_32_bits(dst_offset); - *cmd++ = upper_32_bits(dst_offset); - *cmd++ = 0; - *cmd++ = PAGE_SIZE; - *cmd++ = lower_32_bits(src_offset); - *cmd++ = upper_32_bits(src_offset); - } else { - *cmd++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); - *cmd++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; - *cmd++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; - *cmd++ = dst_offset; - *cmd++ = PAGE_SIZE; - *cmd++ = src_offset; - } - - /* Allow ourselves to be preempted in between blocks. */ - *cmd++ = MI_ARB_CHECK; - - src_offset += size; - dst_offset += size; - rem -= size; - } while (rem); - - *cmd = MI_BATCH_BUFFER_END; - - i915_gem_object_flush_map(pool->obj); - i915_gem_object_unpin_map(pool->obj); - - intel_gt_chipset_flush(ce->vm->gt); - batch->private = pool; - return batch; - -out_unpin: - i915_vma_unpin(batch); -out_put: - intel_gt_buffer_pool_put(pool); -out_pm: - intel_engine_pm_put(ce->engine); - return ERR_PTR(err); -} - -int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, - struct drm_i915_gem_object *dst, - struct intel_context *ce) -{ - struct i915_address_space *vm = ce->vm; - struct i915_vma *vma[2], *batch; - struct i915_gem_ww_ctx ww; - struct i915_request *rq; - int err, i; - - vma[0] = i915_vma_instance(src, vm, NULL); - if (IS_ERR(vma[0])) - return PTR_ERR(vma[0]); - - vma[1] = i915_vma_instance(dst, vm, NULL); - if (IS_ERR(vma[1])) - return PTR_ERR(vma[1]); - - i915_gem_ww_ctx_init(&ww, true); - intel_engine_pm_get(ce->engine); -retry: - err = i915_gem_object_lock(src, &ww); - if (!err) - err = i915_gem_object_lock(dst, &ww); - if (!err) - err = intel_context_pin_ww(ce, &ww); - if (err) - goto out; - - err = i915_vma_pin_ww(vma[0], &ww, 0, 0, PIN_USER); - if (err) - goto out_ctx; - - err = i915_vma_pin_ww(vma[1], &ww, 0, 0, PIN_USER); - if (unlikely(err)) - goto out_unpin_src; - - batch = intel_emit_vma_copy_blt(ce, &ww, vma[0], vma[1]); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_unpin_dst; - } - - rq = i915_request_create(ce); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_batch; - } - - err = intel_emit_vma_mark_active(batch, rq); - if (unlikely(err)) - goto out_request; - - for (i = 0; i < ARRAY_SIZE(vma); i++) { - err = move_obj_to_gpu(vma[i]->obj, rq, i); - if (unlikely(err)) - goto out_request; - } - - for (i = 0; i < ARRAY_SIZE(vma); i++) { - unsigned int flags = i ? EXEC_OBJECT_WRITE : 0; - - err = i915_vma_move_to_active(vma[i], rq, flags); - if (unlikely(err)) - goto out_request; - } - - if (rq->engine->emit_init_breadcrumb) { - err = rq->engine->emit_init_breadcrumb(rq); - if (unlikely(err)) - goto out_request; - } - - err = rq->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); - -out_request: - if (unlikely(err)) - i915_request_set_error_once(rq, err); - - i915_request_add(rq); -out_batch: - intel_emit_vma_release(ce, batch); -out_unpin_dst: - i915_vma_unpin(vma[1]); -out_unpin_src: - i915_vma_unpin(vma[0]); -out_ctx: - intel_context_unpin(ce); -out: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - intel_engine_pm_put(ce->engine); - return err; -} - -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/i915_gem_object_blt.c" -#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h deleted file mode 100644 index 2409fdcccf0e..000000000000 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef __I915_GEM_OBJECT_BLT_H__ -#define __I915_GEM_OBJECT_BLT_H__ - -#include <linux/types.h> - -#include "gt/intel_context.h" -#include "gt/intel_engine_pm.h" -#include "i915_vma.h" - -struct drm_i915_gem_object; -struct i915_gem_ww_ctx; - -struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, - struct i915_vma *vma, - struct i915_gem_ww_ctx *ww, - u32 value); - -struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, - struct i915_gem_ww_ctx *ww, - struct i915_vma *src, - struct i915_vma *dst); - -int intel_emit_vma_mark_active(struct i915_vma *vma, struct i915_request *rq); -void intel_emit_vma_release(struct intel_context *ce, struct i915_vma *vma); - -int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, - struct intel_context *ce, - u32 value); - -int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, - struct drm_i915_gem_object *dst, - struct intel_context *ce); - -#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 2a23b77424b3..2471f36aaff3 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -18,6 +18,7 @@ struct drm_i915_gem_object; struct intel_fronbuffer; +struct intel_memory_region; /* * struct i915_lut_handle tracks the fast lookups from handle to vma used @@ -33,10 +34,9 @@ struct i915_lut_handle { struct drm_i915_gem_object_ops { unsigned int flags; -#define I915_GEM_OBJECT_HAS_IOMEM BIT(1) -#define I915_GEM_OBJECT_IS_SHRINKABLE BIT(2) -#define I915_GEM_OBJECT_IS_PROXY BIT(3) -#define I915_GEM_OBJECT_NO_MMAP BIT(4) +#define I915_GEM_OBJECT_IS_SHRINKABLE BIT(1) +#define I915_GEM_OBJECT_IS_PROXY BIT(2) +#define I915_GEM_OBJECT_NO_MMAP BIT(3) /* Interface between the GEM object and its backing storage. * get_pages() is called once prior to the use of the associated set @@ -78,12 +78,100 @@ struct drm_i915_gem_object_ops { * delayed_free - Override the default delayed free implementation */ void (*delayed_free)(struct drm_i915_gem_object *obj); + + /** + * migrate - Migrate object to a different region either for + * pinning or for as long as the object lock is held. + */ + int (*migrate)(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr); + void (*release)(struct drm_i915_gem_object *obj); const struct vm_operations_struct *mmap_ops; const char *name; /* friendly name for debug, e.g. lockdep classes */ }; +/** + * enum i915_cache_level - The supported GTT caching values for system memory + * pages. + * + * These translate to some special GTT PTE bits when binding pages into some + * address space. It also determines whether an object, or rather its pages are + * coherent with the GPU, when also reading or writing through the CPU cache + * with those pages. + * + * Userspace can also control this through struct drm_i915_gem_caching. + */ +enum i915_cache_level { + /** + * @I915_CACHE_NONE: + * + * GPU access is not coherent with the CPU cache. If the cache is dirty + * and we need the underlying pages to be coherent with some later GPU + * access then we need to manually flush the pages. + * + * On shared LLC platforms reads and writes through the CPU cache are + * still coherent even with this setting. See also + * &drm_i915_gem_object.cache_coherent for more details. Due to this we + * should only ever use uncached for scanout surfaces, otherwise we end + * up over-flushing in some places. + * + * This is the default on non-LLC platforms. + */ + I915_CACHE_NONE = 0, + /** + * @I915_CACHE_LLC: + * + * GPU access is coherent with the CPU cache. If the cache is dirty, + * then the GPU will ensure that access remains coherent, when both + * reading and writing through the CPU cache. GPU writes can dirty the + * CPU cache. + * + * Not used for scanout surfaces. + * + * Applies to both platforms with shared LLC(HAS_LLC), and snooping + * based platforms(HAS_SNOOP). + * + * This is the default on shared LLC platforms. The only exception is + * scanout objects, where the display engine is not coherent with the + * CPU cache. For such objects I915_CACHE_NONE or I915_CACHE_WT is + * automatically applied by the kernel in pin_for_display, if userspace + * has not done so already. + */ + I915_CACHE_LLC, + /** + * @I915_CACHE_L3_LLC: + * + * Explicitly enable the Gfx L3 cache, with coherent LLC. + * + * The Gfx L3 sits between the domain specific caches, e.g + * sampler/render caches, and the larger LLC. LLC is coherent with the + * GPU, but L3 is only visible to the GPU, so likely needs to be flushed + * when the workload completes. + * + * Not used for scanout surfaces. + * + * Only exposed on some gen7 + GGTT. More recent hardware has dropped + * this explicit setting, where it should now be enabled by default. + */ + I915_CACHE_L3_LLC, + /** + * @I915_CACHE_WT: + * + * Write-through. Used for scanout surfaces. + * + * The GPU can utilise the caches, while still having the display engine + * be coherent with GPU writes, as a result we don't need to flush the + * CPU caches when moving out of the render domain. This is the default + * setting chosen by the kernel, if supported by the HW, otherwise we + * fallback to I915_CACHE_NONE. On the CPU side writes through the CPU + * cache still need to be flushed, to remain coherent with the display + * engine. + */ + I915_CACHE_WT, +}; + enum i915_map_type { I915_MAP_WB = 0, I915_MAP_WC, @@ -97,6 +185,7 @@ enum i915_mmap_type { I915_MMAP_TYPE_WC, I915_MMAP_TYPE_WB, I915_MMAP_TYPE_UC, + I915_MMAP_TYPE_FIXED, }; struct i915_mmap_offset { @@ -201,25 +290,138 @@ struct drm_i915_gem_object { unsigned long flags; #define I915_BO_ALLOC_CONTIGUOUS BIT(0) #define I915_BO_ALLOC_VOLATILE BIT(1) -#define I915_BO_ALLOC_STRUCT_PAGE BIT(2) -#define I915_BO_ALLOC_CPU_CLEAR BIT(3) -#define I915_BO_ALLOC_USER BIT(4) +#define I915_BO_ALLOC_CPU_CLEAR BIT(2) +#define I915_BO_ALLOC_USER BIT(3) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ I915_BO_ALLOC_VOLATILE | \ - I915_BO_ALLOC_STRUCT_PAGE | \ I915_BO_ALLOC_CPU_CLEAR | \ I915_BO_ALLOC_USER) -#define I915_BO_READONLY BIT(5) -#define I915_TILING_QUIRK_BIT 6 /* unknown swizzling; do not release! */ +#define I915_BO_READONLY BIT(4) +#define I915_TILING_QUIRK_BIT 5 /* unknown swizzling; do not release! */ - /* - * Is the object to be mapped as read-only to the GPU - * Only honoured if hardware has relevant pte bit + /** + * @mem_flags - Mutable placement-related flags + * + * These are flags that indicate specifics of the memory region + * the object is currently in. As such they are only stable + * either under the object lock or if the object is pinned. + */ + unsigned int mem_flags; +#define I915_BO_FLAG_STRUCT_PAGE BIT(0) /* Object backed by struct pages */ +#define I915_BO_FLAG_IOMEM BIT(1) /* Object backed by IO memory */ + /** + * @cache_level: The desired GTT caching level. + * + * See enum i915_cache_level for possible values, along with what + * each does. */ unsigned int cache_level:3; - unsigned int cache_coherent:2; + /** + * @cache_coherent: + * + * Track whether the pages are coherent with the GPU if reading or + * writing through the CPU caches. The largely depends on the + * @cache_level setting. + * + * On platforms which don't have the shared LLC(HAS_SNOOP), like on Atom + * platforms, coherency must be explicitly requested with some special + * GTT caching bits(see enum i915_cache_level). When enabling coherency + * it does come at a performance and power cost on such platforms. On + * the flip side the kernel does not need to manually flush any buffers + * which need to be coherent with the GPU, if the object is not coherent + * i.e @cache_coherent is zero. + * + * On platforms that share the LLC with the CPU(HAS_LLC), all GT memory + * access will automatically snoop the CPU caches(even with CACHE_NONE). + * The one exception is when dealing with the display engine, like with + * scanout surfaces. To handle this the kernel will always flush the + * surface out of the CPU caches when preparing it for scanout. Also + * note that since scanout surfaces are only ever read by the display + * engine we only need to care about flushing any writes through the CPU + * cache, reads on the other hand will always be coherent. + * + * Something strange here is why @cache_coherent is not a simple + * boolean, i.e coherent vs non-coherent. The reasoning for this is back + * to the display engine not being fully coherent. As a result scanout + * surfaces will either be marked as I915_CACHE_NONE or I915_CACHE_WT. + * In the case of seeing I915_CACHE_NONE the kernel makes the assumption + * that this is likely a scanout surface, and will set @cache_coherent + * as only I915_BO_CACHE_COHERENT_FOR_READ, on platforms with the shared + * LLC. The kernel uses this to always flush writes through the CPU + * cache as early as possible, where it can, in effect keeping + * @cache_dirty clean, so we can potentially avoid stalling when + * flushing the surface just before doing the scanout. This does mean + * we might unnecessarily flush non-scanout objects in some places, but + * the default assumption is that all normal objects should be using + * I915_CACHE_LLC, at least on platforms with the shared LLC. + * + * Supported values: + * + * I915_BO_CACHE_COHERENT_FOR_READ: + * + * On shared LLC platforms, we use this for special scanout surfaces, + * where the display engine is not coherent with the CPU cache. As such + * we need to ensure we flush any writes before doing the scanout. As an + * optimisation we try to flush any writes as early as possible to avoid + * stalling later. + * + * Thus for scanout surfaces using I915_CACHE_NONE, on shared LLC + * platforms, we use: + * + * cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ + * + * While for normal objects that are fully coherent, including special + * scanout surfaces marked as I915_CACHE_WT, we use: + * + * cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ | + * I915_BO_CACHE_COHERENT_FOR_WRITE + * + * And then for objects that are not coherent at all we use: + * + * cache_coherent = 0 + * + * I915_BO_CACHE_COHERENT_FOR_WRITE: + * + * When writing through the CPU cache, the GPU is still coherent. Note + * that this also implies I915_BO_CACHE_COHERENT_FOR_READ. + */ #define I915_BO_CACHE_COHERENT_FOR_READ BIT(0) #define I915_BO_CACHE_COHERENT_FOR_WRITE BIT(1) + unsigned int cache_coherent:2; + + /** + * @cache_dirty: + * + * Track if we are we dirty with writes through the CPU cache for this + * object. As a result reading directly from main memory might yield + * stale data. + * + * This also ties into whether the kernel is tracking the object as + * coherent with the GPU, as per @cache_coherent, as it determines if + * flushing might be needed at various points. + * + * Another part of @cache_dirty is managing flushing when first + * acquiring the pages for system memory, at this point the pages are + * considered foreign, so the default assumption is that the cache is + * dirty, for example the page zeroing done by the kernel might leave + * writes though the CPU cache, or swapping-in, while the actual data in + * main memory is potentially stale. Note that this is a potential + * security issue when dealing with userspace objects and zeroing. Now, + * whether we actually need apply the big sledgehammer of flushing all + * the pages on acquire depends on if @cache_coherent is marked as + * I915_BO_CACHE_COHERENT_FOR_WRITE, i.e that the GPU will be coherent + * for both reads and writes though the CPU cache. + * + * Note that on shared LLC platforms we still apply the heavy flush for + * I915_CACHE_NONE objects, under the assumption that this is going to + * be used for scanout. + * + * Update: On some hardware there is now also the 'Bypass LLC' MOCS + * entry, which defeats our @cache_coherent tracking, since userspace + * can freely bypass the CPU cache when touching the pages with the GPU, + * where the kernel is completely unaware. On such platform we need + * apply the sledgehammer-on-acquire regardless of the @cache_coherent. + */ unsigned int cache_dirty:1; /** @@ -265,9 +467,10 @@ struct drm_i915_gem_object { struct intel_memory_region *region; /** - * Memory manager node allocated for this object. + * Memory manager resource allocated for this object. Only + * needed for the mock region. */ - void *st_mm_node; + struct ttm_resource *res; /** * Element within memory_region->objects or region->purgeable diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 086005c1c7ea..8eb1c3a6fc9c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -321,8 +321,7 @@ static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, dma_addr_t addr; void *vaddr; - if (type != I915_MAP_WC) - return ERR_PTR(-ENODEV); + GEM_BUG_ON(type != I915_MAP_WC); if (n_pfn > ARRAY_SIZE(stack)) { /* Too big for stack -- allocate temporary array instead */ @@ -351,7 +350,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, int err; if (!i915_gem_object_has_struct_page(obj) && - !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) + !i915_gem_object_has_iomem(obj)) return ERR_PTR(-ENXIO); assert_object_held(obj); @@ -374,6 +373,34 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, } GEM_BUG_ON(!i915_gem_object_has_pages(obj)); + /* + * For discrete our CPU mappings needs to be consistent in order to + * function correctly on !x86. When mapping things through TTM, we use + * the same rules to determine the caching type. + * + * The caching rules, starting from DG1: + * + * - If the object can be placed in device local-memory, then the + * pages should be allocated and mapped as write-combined only. + * + * - Everything else is always allocated and mapped as write-back, + * with the guarantee that everything is also coherent with the + * GPU. + * + * Internal users of lmem are already expected to get this right, so no + * fudging needed there. + */ + if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) { + if (type != I915_MAP_WC && !obj->mm.n_placements) { + ptr = ERR_PTR(-ENODEV); + goto err_unpin; + } + + type = I915_MAP_WC; + } else if (IS_DGFX(to_i915(obj->base.dev))) { + type = I915_MAP_WB; + } + ptr = page_unpack_bits(obj->mm.mapping, &has_type); if (ptr && has_type != type) { if (pinned) { @@ -467,7 +494,7 @@ __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, struct i915_gem_object_page_iter *iter, unsigned int n, unsigned int *offset, - bool allow_alloc, bool dma) + bool dma) { struct scatterlist *sg; unsigned int idx, count; @@ -489,9 +516,6 @@ __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, if (n < READ_ONCE(iter->sg_idx)) goto lookup; - if (!allow_alloc) - goto manual_lookup; - mutex_lock(&iter->lock); /* We prefer to reuse the last sg so that repeated lookup of this @@ -541,16 +565,7 @@ scan: if (unlikely(n < idx)) /* insertion completed by another thread */ goto lookup; - goto manual_walk; - -manual_lookup: - idx = 0; - sg = obj->mm.pages->sgl; - count = __sg_page_count(sg); - -manual_walk: - /* - * In case we failed to insert the entry into the radixtree, we need + /* In case we failed to insert the entry into the radixtree, we need * to look beyond the current sg. */ while (idx + count <= n) { @@ -597,7 +612,7 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); - sg = i915_gem_object_get_sg(obj, n, &offset, true); + sg = i915_gem_object_get_sg(obj, n, &offset); return nth_page(sg_page(sg), offset); } @@ -623,7 +638,7 @@ i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, struct scatterlist *sg; unsigned int offset; - sg = i915_gem_object_get_sg_dma(obj, n, &offset, true); + sg = i915_gem_object_get_sg_dma(obj, n, &offset); if (len) *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index be72ad0634ba..7986612f48fa 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -76,7 +76,7 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); /* We're no longer struct page backed */ - obj->flags &= ~I915_BO_ALLOC_STRUCT_PAGE; + obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE; __i915_gem_object_set_pages(obj, st, sg->length); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index d1f1840540dd..1f557b2178ed 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -13,11 +13,7 @@ void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj, { obj->mm.region = intel_memory_region_get(mem); - if (obj->base.size <= mem->min_page_size) - obj->flags |= I915_BO_ALLOC_CONTIGUOUS; - mutex_lock(&mem->objects.lock); - list_add(&obj->mm.region_link, &mem->objects.list); mutex_unlock(&mem->objects.lock); } @@ -36,9 +32,11 @@ void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj) struct drm_i915_gem_object * i915_gem_object_create_region(struct intel_memory_region *mem, resource_size_t size, + resource_size_t page_size, unsigned int flags) { struct drm_i915_gem_object *obj; + resource_size_t default_page_size; int err; /* @@ -52,7 +50,14 @@ i915_gem_object_create_region(struct intel_memory_region *mem, if (!mem) return ERR_PTR(-ENODEV); - size = round_up(size, mem->min_page_size); + default_page_size = mem->min_page_size; + if (page_size) + default_page_size = page_size; + + GEM_BUG_ON(!is_power_of_2_u64(default_page_size)); + GEM_BUG_ON(default_page_size < PAGE_SIZE); + + size = round_up(size, default_page_size); GEM_BUG_ON(!size); GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_MIN_ALIGNMENT)); @@ -64,7 +69,7 @@ i915_gem_object_create_region(struct intel_memory_region *mem, if (!obj) return ERR_PTR(-ENOMEM); - err = mem->ops->init_object(mem, obj, size, flags); + err = mem->ops->init_object(mem, obj, size, page_size, flags); if (err) goto err_object_free; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.h b/drivers/gpu/drm/i915/gem/i915_gem_region.h index 84fcb3297400..1008e580a89a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.h @@ -19,6 +19,7 @@ void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj); struct drm_i915_gem_object * i915_gem_object_create_region(struct intel_memory_region *mem, resource_size_t size, + resource_size_t page_size, unsigned int flags); #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 5d16c4462fda..11f072193f3b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -182,6 +182,24 @@ rebuild_st: if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_do_bit_17_swizzle(obj, st); + /* + * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it + * possible for userspace to bypass the GTT caching bits set by the + * kernel, as per the given object cache_level. This is troublesome + * since the heavy flush we apply when first gathering the pages is + * skipped if the kernel thinks the object is coherent with the GPU. As + * a result it might be possible to bypass the cache and read the + * contents of the page directly, which could be stale data. If it's + * just a case of userspace shooting themselves in the foot then so be + * it, but since i915 takes the stance of always zeroing memory before + * handing it to userspace, we need to prevent this. + * + * By setting cache_dirty here we make the clflush in set_pages + * unconditional on such platforms. + */ + if (IS_JSL_EHL(i915) && obj->flags & I915_BO_ALLOC_USER) + obj->cache_dirty = true; + __i915_gem_object_set_pages(obj, st, sg_page_sizes); return 0; @@ -302,6 +320,7 @@ void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_ struct pagevec pvec; struct page *page; + GEM_WARN_ON(IS_DGFX(to_i915(obj->base.dev))); __i915_gem_object_release_shmem(obj, pages, true); i915_gem_gtt_finish_pages(obj, pages); @@ -444,7 +463,7 @@ shmem_pread(struct drm_i915_gem_object *obj, static void shmem_release(struct drm_i915_gem_object *obj) { - if (obj->flags & I915_BO_ALLOC_STRUCT_PAGE) + if (i915_gem_object_has_struct_page(obj)) i915_gem_object_release_memory_region(obj); fput(obj->base.filp); @@ -489,6 +508,7 @@ static int __create_shmem(struct drm_i915_private *i915, static int shmem_object_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags) { static struct lock_class_key lock_class; @@ -513,9 +533,8 @@ static int shmem_object_init(struct intel_memory_region *mem, mapping_set_gfp_mask(mapping, mask); GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); - i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, - I915_BO_ALLOC_STRUCT_PAGE); - + i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0); + obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; @@ -548,7 +567,7 @@ i915_gem_object_create_shmem(struct drm_i915_private *i915, resource_size_t size) { return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], - size, 0); + size, 0, 0); } /* Allocate a new GEM object and fill it with the supplied data */ @@ -561,6 +580,7 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, resource_size_t offset; int err; + GEM_WARN_ON(IS_DGFX(dev_priv)); obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)); if (IS_ERR(obj)) return obj; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index b0c3a7dc60d1..ddd37ccb1362 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -447,7 +447,6 @@ static int i915_gem_init_stolen(struct intel_memory_region *mem) break; case 8: case 9: - case 10: if (IS_LP(i915)) chv_get_stolen_reserved(i915, uncore, &reserved_base, &reserved_size); @@ -670,6 +669,7 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, static int _i915_gem_object_stolen_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags) { struct drm_i915_private *i915 = mem->i915; @@ -708,7 +708,7 @@ struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_i915_private *i915, resource_size_t size) { - return i915_gem_object_create_region(i915->mm.stolen_region, size, 0); + return i915_gem_object_create_region(i915->mm.stolen_region, size, 0, 0); } static int init_stolen_smem(struct intel_memory_region *mem) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index e646aac9d7a4..f0a61a9474fc 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -15,6 +15,9 @@ #include "gem/i915_gem_ttm.h" #include "gem/i915_gem_mman.h" +#include "gt/intel_migrate.h" +#include "gt/intel_engine_pm.h" + #define I915_PL_LMEM0 TTM_PL_PRIV #define I915_PL_SYSTEM TTM_PL_SYSTEM #define I915_PL_STOLEN TTM_PL_VRAM @@ -24,6 +27,11 @@ #define I915_TTM_PRIO_NO_PAGES 1 #define I915_TTM_PRIO_HAS_PAGES 2 +/* + * Size of struct ttm_place vector in on-stack struct ttm_placement allocs + */ +#define I915_TTM_MAX_PLACEMENTS INTEL_REGION_UNKNOWN + /** * struct i915_ttm_tt - TTM page vector with additional private information * @ttm: The base TTM page vector. @@ -42,36 +50,123 @@ struct i915_ttm_tt { struct sg_table *cached_st; }; -static const struct ttm_place lmem0_sys_placement_flags[] = { - { - .fpfn = 0, - .lpfn = 0, - .mem_type = I915_PL_LMEM0, - .flags = 0, - }, { - .fpfn = 0, - .lpfn = 0, - .mem_type = I915_PL_SYSTEM, - .flags = 0, - } -}; - -static struct ttm_placement i915_lmem0_placement = { - .num_placement = 1, - .placement = &lmem0_sys_placement_flags[0], - .num_busy_placement = 1, - .busy_placement = &lmem0_sys_placement_flags[0], +static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = I915_PL_SYSTEM, + .flags = 0, }; static struct ttm_placement i915_sys_placement = { .num_placement = 1, - .placement = &lmem0_sys_placement_flags[1], + .placement = &sys_placement_flags, .num_busy_placement = 1, - .busy_placement = &lmem0_sys_placement_flags[1], + .busy_placement = &sys_placement_flags, }; +static int i915_ttm_err_to_gem(int err) +{ + /* Fastpath */ + if (likely(!err)) + return 0; + + switch (err) { + case -EBUSY: + /* + * TTM likes to convert -EDEADLK to -EBUSY, and wants us to + * restart the operation, since we don't record the contending + * lock. We use -EAGAIN to restart. + */ + return -EAGAIN; + case -ENOSPC: + /* + * Memory type / region is full, and we can't evict. + * Except possibly system, that returns -ENOMEM; + */ + return -ENXIO; + default: + break; + } + + return err; +} + +static bool gpu_binds_iomem(struct ttm_resource *mem) +{ + return mem->mem_type != TTM_PL_SYSTEM; +} + +static bool cpu_maps_iomem(struct ttm_resource *mem) +{ + /* Once / if we support GGTT, this is also false for cached ttm_tts */ + return mem->mem_type != TTM_PL_SYSTEM; +} + +static enum i915_cache_level +i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res, + struct ttm_tt *ttm) +{ + return ((HAS_LLC(i915) || HAS_SNOOP(i915)) && !gpu_binds_iomem(res) && + ttm->caching == ttm_cached) ? I915_CACHE_LLC : + I915_CACHE_NONE; +} + static void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj); +static enum ttm_caching +i915_ttm_select_tt_caching(const struct drm_i915_gem_object *obj) +{ + /* + * Objects only allowed in system get cached cpu-mappings. + * Other objects get WC mapping for now. Even if in system. + */ + if (obj->mm.region->type == INTEL_MEMORY_SYSTEM && + obj->mm.n_placements <= 1) + return ttm_cached; + + return ttm_write_combined; +} + +static void +i915_ttm_place_from_region(const struct intel_memory_region *mr, + struct ttm_place *place, + unsigned int flags) +{ + memset(place, 0, sizeof(*place)); + place->mem_type = intel_region_to_ttm_type(mr); + + if (flags & I915_BO_ALLOC_CONTIGUOUS) + place->flags = TTM_PL_FLAG_CONTIGUOUS; +} + +static void +i915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, + struct ttm_place *requested, + struct ttm_place *busy, + struct ttm_placement *placement) +{ + unsigned int num_allowed = obj->mm.n_placements; + unsigned int flags = obj->flags; + unsigned int i; + + placement->num_placement = 1; + i915_ttm_place_from_region(num_allowed ? obj->mm.placements[0] : + obj->mm.region, requested, flags); + + /* Cache this on object? */ + placement->num_busy_placement = num_allowed; + for (i = 0; i < placement->num_busy_placement; ++i) + i915_ttm_place_from_region(obj->mm.placements[i], busy + i, flags); + + if (num_allowed == 0) { + *busy = *requested; + placement->num_busy_placement = 1; + } + + placement->placement = requested; + placement->busy_placement = busy; +} + static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) { @@ -89,7 +184,8 @@ static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, man->use_tt) page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; - ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, ttm_write_combined); + ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, + i915_ttm_select_tt_caching(obj)); if (ret) { kfree(i915_tt); return NULL; @@ -118,6 +214,7 @@ static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) { struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); + ttm_tt_fini(ttm); kfree(i915_tt); } @@ -127,11 +224,7 @@ static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); /* Will do for now. Our pinned objects are still on TTM's LRU lists */ - if (!i915_gem_object_evictable(obj)) - return false; - - /* This isn't valid with a buddy allocator */ - return ttm_bo_eviction_valuable(bo, place); + return i915_gem_object_evictable(obj); } static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, @@ -174,6 +267,55 @@ static void i915_ttm_free_cached_io_st(struct drm_i915_gem_object *obj) obj->ttm.cached_io_st = NULL; } +static void +i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj) +{ + struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); + + if (cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) { + obj->write_domain = I915_GEM_DOMAIN_WC; + obj->read_domains = I915_GEM_DOMAIN_WC; + } else { + obj->write_domain = I915_GEM_DOMAIN_CPU; + obj->read_domains = I915_GEM_DOMAIN_CPU; + } +} + +static void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj) +{ + struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); + unsigned int cache_level; + unsigned int i; + + /* + * If object was moved to an allowable region, update the object + * region to consider it migrated. Note that if it's currently not + * in an allowable region, it's evicted and we don't update the + * object region. + */ + if (intel_region_to_ttm_type(obj->mm.region) != bo->resource->mem_type) { + for (i = 0; i < obj->mm.n_placements; ++i) { + struct intel_memory_region *mr = obj->mm.placements[i]; + + if (intel_region_to_ttm_type(mr) == bo->resource->mem_type && + mr != obj->mm.region) { + i915_gem_object_release_memory_region(obj); + i915_gem_object_init_memory_region(obj, mr); + break; + } + } + } + + obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM); + + obj->mem_flags |= cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM : + I915_BO_FLAG_STRUCT_PAGE; + + cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource, + bo->ttm); + i915_gem_object_set_cache_coherency(obj, cache_level); +} + static void i915_ttm_purge(struct drm_i915_gem_object *obj) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); @@ -189,8 +331,10 @@ static void i915_ttm_purge(struct drm_i915_gem_object *obj) /* TTM's purge interface. Note that we might be reentering. */ ret = ttm_bo_validate(bo, &place, &ctx); - if (!ret) { + obj->write_domain = 0; + obj->read_domains = 0; + i915_ttm_adjust_gem_after_move(obj); i915_ttm_free_cached_io_st(obj); obj->mm.madv = __I915_MADV_PURGED; } @@ -213,6 +357,7 @@ static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) if (likely(obj)) { /* This releases all gem object bindings to the backend. */ + i915_ttm_free_cached_io_st(obj); __i915_gem_free_object(obj); } } @@ -236,7 +381,6 @@ i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type) static struct sg_table *i915_ttm_tt_get_st(struct ttm_tt *ttm) { struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); - struct scatterlist *sg; struct sg_table *st; int ret; @@ -247,13 +391,13 @@ static struct sg_table *i915_ttm_tt_get_st(struct ttm_tt *ttm) if (!st) return ERR_PTR(-ENOMEM); - sg = __sg_alloc_table_from_pages - (st, ttm->pages, ttm->num_pages, 0, - (unsigned long)ttm->num_pages << PAGE_SHIFT, - i915_sg_segment_size(), NULL, 0, GFP_KERNEL); - if (IS_ERR(sg)) { + ret = sg_alloc_table_from_pages_segment(st, + ttm->pages, ttm->num_pages, + 0, (unsigned long)ttm->num_pages << PAGE_SHIFT, + i915_sg_segment_size(), GFP_KERNEL); + if (ret) { kfree(st); - return ERR_CAST(sg); + return ERR_PTR(ret); } ret = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); @@ -272,13 +416,75 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, struct ttm_resource *res) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); - struct ttm_resource_manager *man = - ttm_manager_type(bo->bdev, res->mem_type); - if (man->use_tt) + if (!gpu_binds_iomem(res)) return i915_ttm_tt_get_st(bo->ttm); - return intel_region_ttm_node_to_st(obj->mm.region, res); + /* + * If CPU mapping differs, we need to add the ttm_tt pages to + * the resulting st. Might make sense for GGTT. + */ + GEM_WARN_ON(!cpu_maps_iomem(res)); + return intel_region_ttm_resource_to_st(obj->mm.region, res); +} + +static int i915_ttm_accel_move(struct ttm_buffer_object *bo, + struct ttm_resource *dst_mem, + struct sg_table *dst_st) +{ + struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), + bdev); + struct ttm_resource_manager *src_man = + ttm_manager_type(bo->bdev, bo->resource->mem_type); + struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); + struct sg_table *src_st; + struct i915_request *rq; + struct ttm_tt *ttm = bo->ttm; + enum i915_cache_level src_level, dst_level; + int ret; + + if (!i915->gt.migrate.context) + return -EINVAL; + + dst_level = i915_ttm_cache_level(i915, dst_mem, ttm); + if (!ttm || !ttm_tt_is_populated(ttm)) { + if (bo->type == ttm_bo_type_kernel) + return -EINVAL; + + if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)) + return 0; + + intel_engine_pm_get(i915->gt.migrate.context->engine); + ret = intel_context_migrate_clear(i915->gt.migrate.context, NULL, + dst_st->sgl, dst_level, + gpu_binds_iomem(dst_mem), + 0, &rq); + + if (!ret && rq) { + i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); + i915_request_put(rq); + } + intel_engine_pm_put(i915->gt.migrate.context->engine); + } else { + src_st = src_man->use_tt ? i915_ttm_tt_get_st(ttm) : + obj->ttm.cached_io_st; + + src_level = i915_ttm_cache_level(i915, bo->resource, ttm); + intel_engine_pm_get(i915->gt.migrate.context->engine); + ret = intel_context_migrate_copy(i915->gt.migrate.context, + NULL, src_st->sgl, src_level, + gpu_binds_iomem(bo->resource), + dst_st->sgl, dst_level, + gpu_binds_iomem(dst_mem), + &rq); + if (!ret && rq) { + i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); + i915_request_put(rq); + } + intel_engine_pm_put(i915->gt.migrate.context->engine); + } + + return ret; } static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, @@ -289,8 +495,6 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); struct ttm_resource_manager *dst_man = ttm_manager_type(bo->bdev, dst_mem->mem_type); - struct ttm_resource_manager *src_man = - ttm_manager_type(bo->bdev, bo->resource->mem_type); struct intel_memory_region *dst_reg, *src_reg; union { struct ttm_kmap_iter_tt tt; @@ -331,34 +535,40 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, if (IS_ERR(dst_st)) return PTR_ERR(dst_st); - /* If we start mapping GGTT, we can no longer use man::use_tt here. */ - dst_iter = dst_man->use_tt ? - ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) : - ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap, - dst_st, dst_reg->region.start); - - src_iter = src_man->use_tt ? - ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) : - ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap, - obj->ttm.cached_io_st, - src_reg->region.start); - - ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter); + ret = i915_ttm_accel_move(bo, dst_mem, dst_st); + if (ret) { + /* If we start mapping GGTT, we can no longer use man::use_tt here. */ + dst_iter = !cpu_maps_iomem(dst_mem) ? + ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) : + ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap, + dst_st, dst_reg->region.start); + + src_iter = !cpu_maps_iomem(bo->resource) ? + ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) : + ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap, + obj->ttm.cached_io_st, + src_reg->region.start); + + ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter); + } + /* Below dst_mem becomes bo->resource. */ ttm_bo_move_sync_cleanup(bo, dst_mem); + i915_ttm_adjust_domains_after_move(obj); i915_ttm_free_cached_io_st(obj); - if (!dst_man->use_tt) { + if (gpu_binds_iomem(dst_mem) || cpu_maps_iomem(dst_mem)) { obj->ttm.cached_io_st = dst_st; obj->ttm.get_io_page.sg_pos = dst_st->sgl; obj->ttm.get_io_page.sg_idx = 0; } + i915_ttm_adjust_gem_after_move(obj); return 0; } static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) { - if (mem->mem_type < I915_PL_LMEM0) + if (!cpu_maps_iomem(mem)) return 0; mem->bus.caching = ttm_write_combined; @@ -377,7 +587,7 @@ static unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo, GEM_WARN_ON(bo->ttm); - sg = __i915_gem_object_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs, true, true); + sg = __i915_gem_object_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs, true); return ((base + sg_dma_address(sg)) >> PAGE_SHIFT) + ofs; } @@ -405,7 +615,8 @@ struct ttm_device_funcs *i915_ttm_driver(void) return &i915_ttm_bo_driver; } -static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) +static int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, + struct ttm_placement *placement) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); struct ttm_operation_ctx ctx = { @@ -413,25 +624,111 @@ static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) .no_wait_gpu = false, }; struct sg_table *st; + int real_num_busy; int ret; - /* Move to the requested placement. */ - ret = ttm_bo_validate(bo, &i915_lmem0_placement, &ctx); - if (ret) - return ret == -ENOSPC ? -ENXIO : ret; - - /* Object either has a page vector or is an iomem object */ - st = bo->ttm ? i915_ttm_tt_get_st(bo->ttm) : obj->ttm.cached_io_st; - if (IS_ERR(st)) - return PTR_ERR(st); + /* First try only the requested placement. No eviction. */ + real_num_busy = fetch_and_zero(&placement->num_busy_placement); + ret = ttm_bo_validate(bo, placement, &ctx); + if (ret) { + ret = i915_ttm_err_to_gem(ret); + /* + * Anything that wants to restart the operation gets to + * do that. + */ + if (ret == -EDEADLK || ret == -EINTR || ret == -ERESTARTSYS || + ret == -EAGAIN) + return ret; - __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl)); + /* + * If the initial attempt fails, allow all accepted placements, + * evicting if necessary. + */ + placement->num_busy_placement = real_num_busy; + ret = ttm_bo_validate(bo, placement, &ctx); + if (ret) + return i915_ttm_err_to_gem(ret); + } i915_ttm_adjust_lru(obj); + if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) { + ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx); + if (ret) + return ret; + + i915_ttm_adjust_domains_after_move(obj); + i915_ttm_adjust_gem_after_move(obj); + } + + if (!i915_gem_object_has_pages(obj)) { + /* Object either has a page vector or is an iomem object */ + st = bo->ttm ? i915_ttm_tt_get_st(bo->ttm) : obj->ttm.cached_io_st; + if (IS_ERR(st)) + return PTR_ERR(st); + + __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl)); + } return ret; } +static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) +{ + struct ttm_place requested, busy[I915_TTM_MAX_PLACEMENTS]; + struct ttm_placement placement; + + GEM_BUG_ON(obj->mm.n_placements > I915_TTM_MAX_PLACEMENTS); + + /* Move to the requested placement. */ + i915_ttm_placement_from_obj(obj, &requested, busy, &placement); + + return __i915_ttm_get_pages(obj, &placement); +} + +/** + * DOC: Migration vs eviction + * + * GEM migration may not be the same as TTM migration / eviction. If + * the TTM core decides to evict an object it may be evicted to a + * TTM memory type that is not in the object's allowable GEM regions, or + * in fact theoretically to a TTM memory type that doesn't correspond to + * a GEM memory region. In that case the object's GEM region is not + * updated, and the data is migrated back to the GEM region at + * get_pages time. TTM may however set up CPU ptes to the object even + * when it is evicted. + * Gem forced migration using the i915_ttm_migrate() op, is allowed even + * to regions that are not in the object's list of allowable placements. + */ +static int i915_ttm_migrate(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr) +{ + struct ttm_place requested; + struct ttm_placement placement; + int ret; + + i915_ttm_place_from_region(mr, &requested, obj->flags); + placement.num_placement = 1; + placement.num_busy_placement = 1; + placement.placement = &requested; + placement.busy_placement = &requested; + + ret = __i915_ttm_get_pages(obj, &placement); + if (ret) + return ret; + + /* + * Reinitialize the region bindings. This is primarily + * required for objects where the new region is not in + * its allowable placements. + */ + if (obj->mm.region != mr) { + i915_gem_object_release_memory_region(obj); + i915_gem_object_init_memory_region(obj, mr); + } + + return 0; +} + static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, struct sg_table *st) { @@ -560,15 +857,15 @@ static u64 i915_ttm_mmap_offset(struct drm_i915_gem_object *obj) return drm_vma_node_offset_addr(&obj->base.vma_node); } -const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { +static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { .name = "i915_gem_object_ttm", - .flags = I915_GEM_OBJECT_HAS_IOMEM, .get_pages = i915_ttm_get_pages, .put_pages = i915_ttm_put_pages, .truncate = i915_ttm_purge, .adjust_lru = i915_ttm_adjust_lru, .delayed_free = i915_ttm_delayed_free, + .migrate = i915_ttm_migrate, .mmap_offset = i915_ttm_mmap_offset, .mmap_ops = &vm_ops_ttm, }; @@ -595,37 +892,32 @@ void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) int __i915_gem_ttm_object_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags) { static struct lock_class_key lock_class; struct drm_i915_private *i915 = mem->i915; + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; enum ttm_bo_type bo_type; - size_t alignment = 0; int ret; - /* Adjust alignment to GPU- and CPU huge page sizes. */ - - if (mem->is_range_manager) { - if (size >= SZ_1G) - alignment = SZ_1G >> PAGE_SHIFT; - else if (size >= SZ_2M) - alignment = SZ_2M >> PAGE_SHIFT; - else if (size >= SZ_64K) - alignment = SZ_64K >> PAGE_SHIFT; - } - drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &i915_gem_ttm_obj_ops, &lock_class, flags); i915_gem_object_init_memory_region(obj, mem); i915_gem_object_make_unshrinkable(obj); - obj->read_domains = I915_GEM_DOMAIN_WC | I915_GEM_DOMAIN_GTT; - i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE); INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); mutex_init(&obj->ttm.get_io_page.lock); - bo_type = (obj->flags & I915_BO_ALLOC_USER) ? ttm_bo_type_device : ttm_bo_type_kernel; + obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); + + /* Forcing the page size is kernel internal only */ + GEM_BUG_ON(page_size && obj->mm.n_placements); + /* * If this function fails, it will call the destructor, but * our caller still owns the object. So no freeing in the @@ -633,14 +925,39 @@ int __i915_gem_ttm_object_init(struct intel_memory_region *mem, * Similarly, in delayed_destroy, we can't call ttm_bo_put() * until successful initialization. */ - obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); - ret = ttm_bo_init(&i915->bdev, i915_gem_to_ttm(obj), size, - bo_type, &i915_sys_placement, alignment, - true, NULL, NULL, i915_ttm_bo_destroy); + ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), size, + bo_type, &i915_sys_placement, + page_size >> PAGE_SHIFT, + &ctx, NULL, NULL, i915_ttm_bo_destroy); + if (ret) + return i915_ttm_err_to_gem(ret); - if (!ret) - obj->ttm.created = true; + obj->ttm.created = true; + i915_ttm_adjust_domains_after_move(obj); + i915_ttm_adjust_gem_after_move(obj); + i915_gem_object_unlock(obj); - /* i915 wants -ENXIO when out of memory region space. */ - return (ret == -ENOSPC) ? -ENXIO : ret; + return 0; +} + +static const struct intel_memory_region_ops ttm_system_region_ops = { + .init_object = __i915_gem_ttm_object_init, +}; + +struct intel_memory_region * +i915_gem_ttm_system_setup(struct drm_i915_private *i915, + u16 type, u16 instance) +{ + struct intel_memory_region *mr; + + mr = intel_memory_region_create(i915, 0, + totalram_pages() << PAGE_SHIFT, + PAGE_SIZE, 0, + type, instance, + &ttm_system_region_ops); + if (IS_ERR(mr)) + return mr; + + intel_memory_region_set_name(mr, "system-ttm"); + return mr; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h index b8d3dcbb50df..40927f67b6d9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h @@ -44,5 +44,6 @@ i915_ttm_to_gem(struct ttm_buffer_object *bo) int __i915_gem_ttm_object_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags); #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 7487bab11f0b..8ea0fa665e53 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -67,11 +67,11 @@ static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni, if (!mmu_notifier_range_blockable(range)) return false; - spin_lock(&i915->mm.notifier_lock); + write_lock(&i915->mm.notifier_lock); mmu_interval_set_seq(mni, cur_seq); - spin_unlock(&i915->mm.notifier_lock); + write_unlock(&i915->mm.notifier_lock); /* * We don't wait when the process is exiting. This is valid @@ -107,16 +107,15 @@ i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj) static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj) { - struct drm_i915_private *i915 = to_i915(obj->base.dev); struct page **pvec = NULL; - spin_lock(&i915->mm.notifier_lock); + assert_object_held_shared(obj); + if (!--obj->userptr.page_ref) { pvec = obj->userptr.pvec; obj->userptr.pvec = NULL; } GEM_BUG_ON(obj->userptr.page_ref < 0); - spin_unlock(&i915->mm.notifier_lock); if (pvec) { const unsigned long num_pages = obj->base.size >> PAGE_SHIFT; @@ -128,12 +127,10 @@ static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj) static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { - struct drm_i915_private *i915 = to_i915(obj->base.dev); const unsigned long num_pages = obj->base.size >> PAGE_SHIFT; unsigned int max_segment = i915_sg_segment_size(); struct sg_table *st; unsigned int sg_page_sizes; - struct scatterlist *sg; struct page **pvec; int ret; @@ -141,25 +138,20 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) if (!st) return -ENOMEM; - spin_lock(&i915->mm.notifier_lock); - if (GEM_WARN_ON(!obj->userptr.page_ref)) { - spin_unlock(&i915->mm.notifier_lock); - ret = -EFAULT; + if (!obj->userptr.page_ref) { + ret = -EAGAIN; goto err_free; } obj->userptr.page_ref++; pvec = obj->userptr.pvec; - spin_unlock(&i915->mm.notifier_lock); alloc_table: - sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0, - num_pages << PAGE_SHIFT, max_segment, - NULL, 0, GFP_KERNEL); - if (IS_ERR(sg)) { - ret = PTR_ERR(sg); + ret = sg_alloc_table_from_pages_segment(st, pvec, num_pages, 0, + num_pages << PAGE_SHIFT, + max_segment, GFP_KERNEL); + if (ret) goto err; - } ret = i915_gem_gtt_prepare_pages(obj, st); if (ret) { @@ -241,7 +233,7 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, i915_gem_object_userptr_drop_ref(obj); } -static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj, bool get_pages) +static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj) { struct sg_table *pages; int err; @@ -259,15 +251,11 @@ static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj, bool if (!IS_ERR_OR_NULL(pages)) i915_gem_userptr_put_pages(obj, pages); - if (get_pages) - err = ____i915_gem_object_get_pages(obj); - return err; } int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { - struct drm_i915_private *i915 = to_i915(obj->base.dev); const unsigned long num_pages = obj->base.size >> PAGE_SHIFT; struct page **pvec; unsigned int gup_flags = 0; @@ -277,39 +265,22 @@ int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) if (obj->userptr.notifier.mm != current->mm) return -EFAULT; + notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier); + ret = i915_gem_object_lock_interruptible(obj, NULL); if (ret) return ret; - /* optimistically try to preserve current pages while unlocked */ - if (i915_gem_object_has_pages(obj) && - !mmu_interval_check_retry(&obj->userptr.notifier, - obj->userptr.notifier_seq)) { - spin_lock(&i915->mm.notifier_lock); - if (obj->userptr.pvec && - !mmu_interval_read_retry(&obj->userptr.notifier, - obj->userptr.notifier_seq)) { - obj->userptr.page_ref++; - - /* We can keep using the current binding, this is the fastpath */ - ret = 1; - } - spin_unlock(&i915->mm.notifier_lock); + if (notifier_seq == obj->userptr.notifier_seq && obj->userptr.pvec) { + i915_gem_object_unlock(obj); + return 0; } - if (!ret) { - /* Make sure userptr is unbound for next attempt, so we don't use stale pages. */ - ret = i915_gem_object_userptr_unbind(obj, false); - } + ret = i915_gem_object_userptr_unbind(obj); i915_gem_object_unlock(obj); - if (ret < 0) + if (ret) return ret; - if (ret > 0) - return 0; - - notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier); - pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL); if (!pvec) return -ENOMEM; @@ -329,7 +300,9 @@ int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) } ret = 0; - spin_lock(&i915->mm.notifier_lock); + ret = i915_gem_object_lock_interruptible(obj, NULL); + if (ret) + goto out; if (mmu_interval_read_retry(&obj->userptr.notifier, !obj->userptr.page_ref ? notifier_seq : @@ -341,12 +314,14 @@ int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) if (!obj->userptr.page_ref++) { obj->userptr.pvec = pvec; obj->userptr.notifier_seq = notifier_seq; - pvec = NULL; + ret = ____i915_gem_object_get_pages(obj); } + obj->userptr.page_ref--; + out_unlock: - spin_unlock(&i915->mm.notifier_lock); + i915_gem_object_unlock(obj); out: if (pvec) { @@ -369,11 +344,6 @@ int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) return 0; } -void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj) -{ - i915_gem_object_userptr_drop_ref(obj); -} - int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { int err; @@ -396,7 +366,6 @@ int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) i915_gem_object_unlock(obj); } - i915_gem_object_userptr_submit_fini(obj); return err; } @@ -450,6 +419,34 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { #endif +static int +probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) +{ + const unsigned long end = addr + len; + struct vm_area_struct *vma; + int ret = -EFAULT; + + mmap_read_lock(mm); + for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { + /* Check for holes, note that we also update the addr below */ + if (vma->vm_start > addr) + break; + + if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + break; + + if (vma->vm_end >= end) { + ret = 0; + break; + } + + addr = vma->vm_end; + } + mmap_read_unlock(mm); + + return ret; +} + /* * Creates a new mm object that wraps some normal memory from the process * context - user memory. @@ -505,7 +502,8 @@ i915_gem_userptr_ioctl(struct drm_device *dev, } if (args->flags & ~(I915_USERPTR_READ_ONLY | - I915_USERPTR_UNSYNCHRONIZED)) + I915_USERPTR_UNSYNCHRONIZED | + I915_USERPTR_PROBE)) return -EINVAL; if (i915_gem_object_size_2big(args->user_size)) @@ -532,14 +530,24 @@ i915_gem_userptr_ioctl(struct drm_device *dev, return -ENODEV; } + if (args->flags & I915_USERPTR_PROBE) { + /* + * Check that the range pointed to represents real struct + * pages and not iomappings (at this moment in time!) + */ + ret = probe_range(current->mm, args->user_ptr, args->user_size); + if (ret) + return ret; + } + #ifdef CONFIG_MMU_NOTIFIER obj = i915_gem_object_alloc(); if (obj == NULL) return -ENOMEM; drm_gem_private_object_init(dev, &obj->base, args->user_size); - i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class, - I915_BO_ALLOC_STRUCT_PAGE); + i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class, 0); + obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); @@ -572,7 +580,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, int i915_gem_init_userptr(struct drm_i915_private *dev_priv) { #ifdef CONFIG_MMU_NOTIFIER - spin_lock_init(&dev_priv->mm.notifier_lock); + rwlock_init(&dev_priv->mm.notifier_lock); #endif return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c index 1e97520c62b2..f909aaa09d9c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c @@ -104,8 +104,8 @@ static void fence_set_priority(struct dma_fence *fence, engine = rq->engine; rcu_read_lock(); /* RCU serialisation for set-wedged protection */ - if (engine->schedule) - engine->schedule(rq, attr); + if (engine->sched_engine->schedule) + engine->sched_engine->schedule(rq, attr); rcu_read_unlock(); } @@ -290,3 +290,22 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) i915_gem_object_put(obj); return ret; } + +/** + * i915_gem_object_wait_migration - Sync an accelerated migration operation + * @obj: The migrating object. + * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE. + * + * Wait for any pending async migration operation on the object, + * whether it's explicitly (i915_gem_object_migrate()) or implicitly + * (swapin, initial clearing) initiated. + * + * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting. + */ +int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, + unsigned int flags) +{ + might_sleep(); + /* NOP for now. */ + return 0; +} diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c index 0c8ecfdf5405..f963b8e1e37b 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c @@ -114,8 +114,8 @@ huge_gem_object(struct drm_i915_private *i915, return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, dma_size); - i915_gem_object_init(obj, &huge_ops, &lock_class, - I915_BO_ALLOC_STRUCT_PAGE); + i915_gem_object_init(obj, &huge_ops, &lock_class, 0); + obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index dadd485bc52f..a094f3ce1a90 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -167,9 +167,8 @@ huge_pages_object(struct drm_i915_private *i915, return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, size); - i915_gem_object_init(obj, &huge_page_ops, &lock_class, - I915_BO_ALLOC_STRUCT_PAGE); - + i915_gem_object_init(obj, &huge_page_ops, &lock_class, 0); + obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE; i915_gem_object_set_volatile(obj); obj->write_domain = I915_GEM_DOMAIN_CPU; @@ -497,7 +496,8 @@ static int igt_mock_memory_region_huge_pages(void *arg) int i; for (i = 0; i < ARRAY_SIZE(flags); ++i) { - obj = i915_gem_object_create_region(mem, page_size, + obj = i915_gem_object_create_region(mem, + page_size, page_size, flags[i]); if (IS_ERR(obj)) { err = PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 176e6b22f87f..ecbcbb86ae1e 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -5,6 +5,7 @@ #include "i915_selftest.h" +#include "gt/intel_context.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" #include "gt/intel_gpu_commands.h" @@ -16,118 +17,6 @@ #include "huge_gem_object.h" #include "mock_context.h" -static int __igt_client_fill(struct intel_engine_cs *engine) -{ - struct intel_context *ce = engine->kernel_context; - struct drm_i915_gem_object *obj; - I915_RND_STATE(prng); - IGT_TIMEOUT(end); - u32 *vaddr; - int err = 0; - - intel_engine_pm_get(engine); - do { - const u32 max_block_size = S16_MAX * PAGE_SIZE; - u32 sz = min_t(u64, ce->vm->total >> 4, prandom_u32_state(&prng)); - u32 phys_sz = sz % (max_block_size + 1); - u32 val = prandom_u32_state(&prng); - u32 i; - - sz = round_up(sz, PAGE_SIZE); - phys_sz = round_up(phys_sz, PAGE_SIZE); - - pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__, - phys_sz, sz, val); - - obj = huge_gem_object(engine->i915, phys_sz, sz); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_flush; - } - - vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_put; - } - - /* - * XXX: The goal is move this to get_pages, so try to dirty the - * CPU cache first to check that we do the required clflush - * before scheduling the blt for !llc platforms. This matches - * some version of reality where at get_pages the pages - * themselves may not yet be coherent with the GPU(swap-in). If - * we are missing the flush then we should see the stale cache - * values after we do the set_to_cpu_domain and pick it up as a - * test failure. - */ - memset32(vaddr, val ^ 0xdeadbeaf, - huge_gem_object_phys_size(obj) / sizeof(u32)); - - if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) - obj->cache_dirty = true; - - err = i915_gem_schedule_fill_pages_blt(obj, ce, obj->mm.pages, - &obj->mm.page_sizes, - val); - if (err) - goto err_unpin; - - i915_gem_object_lock(obj, NULL); - err = i915_gem_object_set_to_cpu_domain(obj, false); - i915_gem_object_unlock(obj); - if (err) - goto err_unpin; - - for (i = 0; i < huge_gem_object_phys_size(obj) / sizeof(u32); ++i) { - if (vaddr[i] != val) { - pr_err("vaddr[%u]=%x, expected=%x\n", i, - vaddr[i], val); - err = -EINVAL; - goto err_unpin; - } - } - - i915_gem_object_unpin_map(obj); - i915_gem_object_put(obj); - } while (!time_after(jiffies, end)); - - goto err_flush; - -err_unpin: - i915_gem_object_unpin_map(obj); -err_put: - i915_gem_object_put(obj); -err_flush: - if (err == -ENOMEM) - err = 0; - intel_engine_pm_put(engine); - - return err; -} - -static int igt_client_fill(void *arg) -{ - int inst = 0; - - do { - struct intel_engine_cs *engine; - int err; - - engine = intel_engine_lookup_user(arg, - I915_ENGINE_CLASS_COPY, - inst++); - if (!engine) - return 0; - - err = __igt_client_fill(engine); - if (err == -ENOMEM) - err = 0; - if (err) - return err; - } while (1); -} - #define WIDTH 512 #define HEIGHT 32 @@ -693,7 +582,6 @@ static int igt_client_tiled_blits(void *arg) int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { - SUBTEST(igt_client_fill), SUBTEST(igt_client_tiled_blits), }; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index dbcfa28a9d91..8eb5050f8cb3 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -680,7 +680,7 @@ static int igt_ctx_exec(void *arg) struct i915_gem_context *ctx; struct intel_context *ce; - ctx = kernel_context(i915); + ctx = kernel_context(i915, NULL); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto out_file; @@ -813,16 +813,12 @@ static int igt_shared_ctx_exec(void *arg) struct i915_gem_context *ctx; struct intel_context *ce; - ctx = kernel_context(i915); + ctx = kernel_context(i915, ctx_vm(parent)); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto out_test; } - mutex_lock(&ctx->mutex); - __assign_ppgtt(ctx, ctx_vm(parent)); - mutex_unlock(&ctx->mutex); - ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); GEM_BUG_ON(IS_ERR(ce)); @@ -1875,125 +1871,6 @@ out_file: return err; } -static bool skip_unused_engines(struct intel_context *ce, void *data) -{ - return !ce->state; -} - -static void mock_barrier_task(void *data) -{ - unsigned int *counter = data; - - ++*counter; -} - -static int mock_context_barrier(void *arg) -{ -#undef pr_fmt -#define pr_fmt(x) "context_barrier_task():" # x - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx; - struct i915_request *rq; - unsigned int counter; - int err; - - /* - * The context barrier provides us with a callback after it emits - * a request; useful for retiring old state after loading new. - */ - - ctx = mock_context(i915, "mock"); - if (!ctx) - return -ENOMEM; - - counter = 0; - err = context_barrier_task(ctx, 0, NULL, NULL, NULL, - mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - if (counter == 0) { - pr_err("Did not retire immediately with 0 engines\n"); - err = -EINVAL; - goto out; - } - - counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines, - NULL, NULL, mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - if (counter == 0) { - pr_err("Did not retire immediately for all unused engines\n"); - err = -EINVAL; - goto out; - } - - rq = igt_request_alloc(ctx, i915->gt.engine[RCS0]); - if (IS_ERR(rq)) { - pr_err("Request allocation failed!\n"); - goto out; - } - i915_request_add(rq); - - counter = 0; - context_barrier_inject_fault = BIT(RCS0); - err = context_barrier_task(ctx, ALL_ENGINES, NULL, NULL, NULL, - mock_barrier_task, &counter); - context_barrier_inject_fault = 0; - if (err == -ENXIO) - err = 0; - else - pr_err("Did not hit fault injection!\n"); - if (counter != 0) { - pr_err("Invoked callback on error!\n"); - err = -EIO; - } - if (err) - goto out; - - counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, skip_unused_engines, - NULL, NULL, mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - mock_device_flush(i915); - if (counter == 0) { - pr_err("Did not retire on each active engines\n"); - err = -EINVAL; - goto out; - } - -out: - mock_context_close(ctx); - return err; -#undef pr_fmt -#define pr_fmt(x) x -} - -int i915_gem_context_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(mock_context_barrier), - }; - struct drm_i915_private *i915; - int err; - - i915 = mock_gem_device(); - if (!i915) - return -ENOMEM; - - err = i915_subtests(tests, i915); - - mock_destroy_device(i915); - return err; -} - int i915_gem_context_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c index dd74bc09ec88..ffae7df5e4d7 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c @@ -35,7 +35,7 @@ static int igt_dmabuf_export(void *arg) static int igt_dmabuf_import_self(void *arg) { struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; + struct drm_i915_gem_object *obj, *import_obj; struct drm_gem_object *import; struct dma_buf *dmabuf; int err; @@ -65,10 +65,19 @@ static int igt_dmabuf_import_self(void *arg) err = -EINVAL; goto out_import; } + import_obj = to_intel_bo(import); + + i915_gem_object_lock(import_obj, NULL); + err = __i915_gem_object_get_pages(import_obj); + i915_gem_object_unlock(import_obj); + if (err) { + pr_err("Same object dma-buf get_pages failed!\n"); + goto out_import; + } err = 0; out_import: - i915_gem_object_put(to_intel_bo(import)); + i915_gem_object_put(import_obj); out_dmabuf: dma_buf_put(dmabuf); out: @@ -76,6 +85,180 @@ out: return err; } +static int igt_dmabuf_import_same_driver_lmem(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_memory_region *lmem = i915->mm.regions[INTEL_REGION_LMEM]; + struct drm_i915_gem_object *obj; + struct drm_gem_object *import; + struct dma_buf *dmabuf; + int err; + + if (!lmem) + return 0; + + force_different_devices = true; + + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &lmem, 1); + if (IS_ERR(obj)) { + pr_err("__i915_gem_object_create_user failed with err=%ld\n", + PTR_ERR(dmabuf)); + err = PTR_ERR(obj); + goto out_ret; + } + + dmabuf = i915_gem_prime_export(&obj->base, 0); + if (IS_ERR(dmabuf)) { + pr_err("i915_gem_prime_export failed with err=%ld\n", + PTR_ERR(dmabuf)); + err = PTR_ERR(dmabuf); + goto out; + } + + /* + * We expect an import of an LMEM-only object to fail with + * -EOPNOTSUPP because it can't be migrated to SMEM. + */ + import = i915_gem_prime_import(&i915->drm, dmabuf); + if (!IS_ERR(import)) { + drm_gem_object_put(import); + pr_err("i915_gem_prime_import succeeded when it shouldn't have\n"); + err = -EINVAL; + } else if (PTR_ERR(import) != -EOPNOTSUPP) { + pr_err("i915_gem_prime_import failed with the wrong err=%ld\n", + PTR_ERR(import)); + err = PTR_ERR(import); + } + + dma_buf_put(dmabuf); +out: + i915_gem_object_put(obj); +out_ret: + force_different_devices = false; + return err; +} + +static int igt_dmabuf_import_same_driver(struct drm_i915_private *i915, + struct intel_memory_region **regions, + unsigned int num_regions) +{ + struct drm_i915_gem_object *obj, *import_obj; + struct drm_gem_object *import; + struct dma_buf *dmabuf; + struct dma_buf_attachment *import_attach; + struct sg_table *st; + long timeout; + int err; + + force_different_devices = true; + + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, + regions, num_regions); + if (IS_ERR(obj)) { + pr_err("__i915_gem_object_create_user failed with err=%ld\n", + PTR_ERR(dmabuf)); + err = PTR_ERR(obj); + goto out_ret; + } + + dmabuf = i915_gem_prime_export(&obj->base, 0); + if (IS_ERR(dmabuf)) { + pr_err("i915_gem_prime_export failed with err=%ld\n", + PTR_ERR(dmabuf)); + err = PTR_ERR(dmabuf); + goto out; + } + + import = i915_gem_prime_import(&i915->drm, dmabuf); + if (IS_ERR(import)) { + pr_err("i915_gem_prime_import failed with err=%ld\n", + PTR_ERR(import)); + err = PTR_ERR(import); + goto out_dmabuf; + } + + if (import == &obj->base) { + pr_err("i915_gem_prime_import reused gem object!\n"); + err = -EINVAL; + goto out_import; + } + + import_obj = to_intel_bo(import); + + i915_gem_object_lock(import_obj, NULL); + err = __i915_gem_object_get_pages(import_obj); + if (err) { + pr_err("Different objects dma-buf get_pages failed!\n"); + i915_gem_object_unlock(import_obj); + goto out_import; + } + + /* + * If the exported object is not in system memory, something + * weird is going on. TODO: When p2p is supported, this is no + * longer considered weird. + */ + if (obj->mm.region != i915->mm.regions[INTEL_REGION_SMEM]) { + pr_err("Exported dma-buf is not in system memory\n"); + err = -EINVAL; + } + + i915_gem_object_unlock(import_obj); + + /* Now try a fake an importer */ + import_attach = dma_buf_attach(dmabuf, obj->base.dev->dev); + if (IS_ERR(import_attach)) { + err = PTR_ERR(import_attach); + goto out_import; + } + + st = dma_buf_map_attachment(import_attach, DMA_BIDIRECTIONAL); + if (IS_ERR(st)) { + err = PTR_ERR(st); + goto out_detach; + } + + timeout = dma_resv_wait_timeout(dmabuf->resv, false, true, 5 * HZ); + if (!timeout) { + pr_err("dmabuf wait for exclusive fence timed out.\n"); + timeout = -ETIME; + } + err = timeout > 0 ? 0 : timeout; + dma_buf_unmap_attachment(import_attach, st, DMA_BIDIRECTIONAL); +out_detach: + dma_buf_detach(dmabuf, import_attach); +out_import: + i915_gem_object_put(import_obj); +out_dmabuf: + dma_buf_put(dmabuf); +out: + i915_gem_object_put(obj); +out_ret: + force_different_devices = false; + return err; +} + +static int igt_dmabuf_import_same_driver_smem(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_memory_region *smem = i915->mm.regions[INTEL_REGION_SMEM]; + + return igt_dmabuf_import_same_driver(i915, &smem, 1); +} + +static int igt_dmabuf_import_same_driver_lmem_smem(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_memory_region *regions[2]; + + if (!i915->mm.regions[INTEL_REGION_LMEM]) + return 0; + + regions[0] = i915->mm.regions[INTEL_REGION_LMEM]; + regions[1] = i915->mm.regions[INTEL_REGION_SMEM]; + return igt_dmabuf_import_same_driver(i915, regions, 2); +} + static int igt_dmabuf_import(void *arg) { struct drm_i915_private *i915 = arg; @@ -286,6 +469,9 @@ int i915_gem_dmabuf_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_dmabuf_export), + SUBTEST(igt_dmabuf_import_same_driver_lmem), + SUBTEST(igt_dmabuf_import_same_driver_smem), + SUBTEST(igt_dmabuf_import_same_driver_lmem_smem), }; return i915_subtests(tests, i915); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c new file mode 100644 index 000000000000..28a700f08b49 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020-2021 Intel Corporation + */ + +#include "gt/intel_migrate.h" + +static int igt_fill_check_buffer(struct drm_i915_gem_object *obj, + bool fill) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int i, count = obj->base.size / sizeof(u32); + enum i915_map_type map_type = + i915_coherent_map_type(i915, obj, false); + u32 *cur; + int err = 0; + + assert_object_held(obj); + cur = i915_gem_object_pin_map(obj, map_type); + if (IS_ERR(cur)) + return PTR_ERR(cur); + + if (fill) + for (i = 0; i < count; ++i) + *cur++ = i; + else + for (i = 0; i < count; ++i) + if (*cur++ != i) { + pr_err("Object content mismatch at location %d of %d\n", i, count); + err = -EINVAL; + break; + } + + i915_gem_object_unpin_map(obj); + + return err; +} + +static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src, + enum intel_region_id dst) +{ + struct drm_i915_private *i915 = gt->i915; + struct intel_memory_region *src_mr = i915->mm.regions[src]; + struct drm_i915_gem_object *obj; + struct i915_gem_ww_ctx ww; + int err = 0; + + GEM_BUG_ON(!src_mr); + + /* Switch object backing-store on create */ + obj = i915_gem_object_create_region(src_mr, PAGE_SIZE, 0, 0); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(obj, &ww); + if (err) + continue; + + err = igt_fill_check_buffer(obj, true); + if (err) + continue; + + err = i915_gem_object_migrate(obj, &ww, dst); + if (err) + continue; + + err = i915_gem_object_pin_pages(obj); + if (err) + continue; + + if (i915_gem_object_can_migrate(obj, src)) + err = -EINVAL; + + i915_gem_object_unpin_pages(obj); + err = i915_gem_object_wait_migration(obj, true); + if (err) + continue; + + err = igt_fill_check_buffer(obj, false); + } + i915_gem_object_put(obj); + + return err; +} + +static int igt_smem_create_migrate(void *arg) +{ + return igt_create_migrate(arg, INTEL_REGION_LMEM, INTEL_REGION_SMEM); +} + +static int igt_lmem_create_migrate(void *arg) +{ + return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM); +} + +static int igt_same_create_migrate(void *arg) +{ + return igt_create_migrate(arg, INTEL_REGION_LMEM, INTEL_REGION_LMEM); +} + +static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *obj) +{ + int err; + + err = i915_gem_object_lock(obj, ww); + if (err) + return err; + + if (i915_gem_object_is_lmem(obj)) { + err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM); + if (err) { + pr_err("Object failed migration to smem\n"); + if (err) + return err; + } + + if (i915_gem_object_is_lmem(obj)) { + pr_err("object still backed by lmem\n"); + err = -EINVAL; + } + + if (!i915_gem_object_has_struct_page(obj)) { + pr_err("object not backed by struct page\n"); + err = -EINVAL; + } + + } else { + err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM); + if (err) { + pr_err("Object failed migration to lmem\n"); + if (err) + return err; + } + + if (i915_gem_object_has_struct_page(obj)) { + pr_err("object still backed by struct page\n"); + err = -EINVAL; + } + + if (!i915_gem_object_is_lmem(obj)) { + pr_err("object not backed by lmem\n"); + err = -EINVAL; + } + } + + return err; +} + +static int igt_lmem_pages_migrate(void *arg) +{ + struct intel_gt *gt = arg; + struct drm_i915_private *i915 = gt->i915; + struct drm_i915_gem_object *obj; + struct i915_gem_ww_ctx ww; + struct i915_request *rq; + int err; + int i; + + /* From LMEM to shmem and back again */ + + obj = i915_gem_object_create_lmem(i915, SZ_2M, 0); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + /* Initial GPU fill, sync, CPU initialization. */ + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(obj, &ww); + if (err) + continue; + + err = ____i915_gem_object_get_pages(obj); + if (err) + continue; + + err = intel_migrate_clear(>->migrate, &ww, NULL, + obj->mm.pages->sgl, obj->cache_level, + i915_gem_object_is_lmem(obj), + 0xdeadbeaf, &rq); + if (rq) { + dma_resv_add_excl_fence(obj->base.resv, &rq->fence); + i915_request_put(rq); + } + if (err) + continue; + + err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, + 5 * HZ); + if (err) + continue; + + err = igt_fill_check_buffer(obj, true); + if (err) + continue; + } + if (err) + goto out_put; + + /* + * Migrate to and from smem without explicitly syncing. + * Finalize with data in smem for fast readout. + */ + for (i = 1; i <= 5; ++i) { + for_i915_gem_ww(&ww, err, true) + err = lmem_pages_migrate_one(&ww, obj); + if (err) + goto out_put; + } + + err = i915_gem_object_lock_interruptible(obj, NULL); + if (err) + goto out_put; + + /* Finally sync migration and check content. */ + err = i915_gem_object_wait_migration(obj, true); + if (err) + goto out_unlock; + + err = igt_fill_check_buffer(obj, false); + +out_unlock: + i915_gem_object_unlock(obj); +out_put: + i915_gem_object_put(obj); + + return err; +} + +int i915_gem_migrate_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_smem_create_migrate), + SUBTEST(igt_lmem_create_migrate), + SUBTEST(igt_same_create_migrate), + SUBTEST(igt_lmem_pages_migrate), + }; + + if (!HAS_LMEM(i915)) + return 0; + + return intel_gt_live_subtests(tests, &i915->gt); +} diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index bc32622a8c5f..b20f5621f62b 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -573,6 +573,14 @@ err: return 0; } +static enum i915_mmap_type default_mapping(struct drm_i915_private *i915) +{ + if (HAS_LMEM(i915)) + return I915_MMAP_TYPE_FIXED; + + return I915_MMAP_TYPE_GTT; +} + static bool assert_mmap_offset(struct drm_i915_private *i915, unsigned long size, int expected) @@ -585,7 +593,7 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, if (IS_ERR(obj)) return expected && expected == PTR_ERR(obj); - ret = __assign_mmap_offset(obj, I915_MMAP_TYPE_GTT, &offset, NULL); + ret = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); i915_gem_object_put(obj); return ret == expected; @@ -689,7 +697,7 @@ static int igt_mmap_offset_exhaustion(void *arg) goto out; } - err = __assign_mmap_offset(obj, I915_MMAP_TYPE_GTT, &offset, NULL); + err = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); if (err) { pr_err("Unable to insert object into reclaimed hole\n"); goto err_obj; @@ -831,34 +839,25 @@ static int wc_check(struct drm_i915_gem_object *obj) static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) { - if (type == I915_MMAP_TYPE_GTT && - !i915_ggtt_has_aperture(&to_i915(obj->base.dev)->ggtt)) - return false; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + bool no_map; - if (type != I915_MMAP_TYPE_GTT && - !i915_gem_object_has_struct_page(obj) && - !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) + if (HAS_LMEM(i915)) + return type == I915_MMAP_TYPE_FIXED; + else if (type == I915_MMAP_TYPE_FIXED) return false; - return true; -} - -static void object_set_placements(struct drm_i915_gem_object *obj, - struct intel_memory_region **placements, - unsigned int n_placements) -{ - GEM_BUG_ON(!n_placements); + if (type == I915_MMAP_TYPE_GTT && + !i915_ggtt_has_aperture(&to_i915(obj->base.dev)->ggtt)) + return false; - if (n_placements == 1) { - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct intel_memory_region *mr = placements[0]; + i915_gem_object_lock(obj, NULL); + no_map = (type != I915_MMAP_TYPE_GTT && + !i915_gem_object_has_struct_page(obj) && + !i915_gem_object_has_iomem(obj)); + i915_gem_object_unlock(obj); - obj->mm.placements = &i915->mm.regions[mr->id]; - obj->mm.n_placements = 1; - } else { - obj->mm.placements = placements; - obj->mm.n_placements = n_placements; - } + return !no_map; } #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) @@ -955,18 +954,18 @@ static int igt_mmap(void *arg) struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mr, sizes[i], I915_BO_ALLOC_USER); + obj = __i915_gem_object_create_user(i915, sizes[i], &mr, 1); if (obj == ERR_PTR(-ENODEV)) continue; if (IS_ERR(obj)) return PTR_ERR(obj); - object_set_placements(obj, &mr, 1); - err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); + if (err == 0) + err = __igt_mmap(i915, obj, I915_MMAP_TYPE_FIXED); i915_gem_object_put(obj); if (err) @@ -984,14 +983,21 @@ static const char *repr_mmap_type(enum i915_mmap_type type) case I915_MMAP_TYPE_WB: return "wb"; case I915_MMAP_TYPE_WC: return "wc"; case I915_MMAP_TYPE_UC: return "uc"; + case I915_MMAP_TYPE_FIXED: return "fixed"; default: return "unknown"; } } -static bool can_access(const struct drm_i915_gem_object *obj) +static bool can_access(struct drm_i915_gem_object *obj) { - return i915_gem_object_has_struct_page(obj) || - i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM); + bool access; + + i915_gem_object_lock(obj, NULL); + access = i915_gem_object_has_struct_page(obj) || + i915_gem_object_has_iomem(obj); + i915_gem_object_unlock(obj); + + return access; } static int __igt_mmap_access(struct drm_i915_private *i915, @@ -1075,15 +1081,13 @@ static int igt_mmap_access(void *arg) struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mr, PAGE_SIZE, I915_BO_ALLOC_USER); + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); if (obj == ERR_PTR(-ENODEV)) continue; if (IS_ERR(obj)) return PTR_ERR(obj); - object_set_placements(obj, &mr, 1); - err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); @@ -1091,6 +1095,8 @@ static int igt_mmap_access(void *arg) err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC); if (err == 0) err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC); + if (err == 0) + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_FIXED); i915_gem_object_put(obj); if (err) @@ -1220,18 +1226,18 @@ static int igt_mmap_gpu(void *arg) struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mr, PAGE_SIZE, I915_BO_ALLOC_USER); + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); if (obj == ERR_PTR(-ENODEV)) continue; if (IS_ERR(obj)) return PTR_ERR(obj); - object_set_placements(obj, &mr, 1); - err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); + if (err == 0) + err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_FIXED); i915_gem_object_put(obj); if (err) @@ -1375,18 +1381,18 @@ static int igt_mmap_revoke(void *arg) struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mr, PAGE_SIZE, I915_BO_ALLOC_USER); + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); if (obj == ERR_PTR(-ENODEV)) continue; if (IS_ERR(obj)) return PTR_ERR(obj); - object_set_placements(obj, &mr, 1); - err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); + if (err == 0) + err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_FIXED); i915_gem_object_put(obj); if (err) diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c deleted file mode 100644 index 8c335d1a8406..000000000000 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c +++ /dev/null @@ -1,597 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include <linux/sort.h> - -#include "gt/intel_gt.h" -#include "gt/intel_engine_user.h" - -#include "i915_selftest.h" - -#include "gem/i915_gem_context.h" -#include "selftests/igt_flush_test.h" -#include "selftests/i915_random.h" -#include "selftests/mock_drm.h" -#include "huge_gem_object.h" -#include "mock_context.h" - -static int wrap_ktime_compare(const void *A, const void *B) -{ - const ktime_t *a = A, *b = B; - - return ktime_compare(*a, *b); -} - -static int __perf_fill_blt(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - int inst = 0; - - do { - struct intel_engine_cs *engine; - ktime_t t[5]; - int pass; - int err; - - engine = intel_engine_lookup_user(i915, - I915_ENGINE_CLASS_COPY, - inst++); - if (!engine) - return 0; - - intel_engine_pm_get(engine); - for (pass = 0; pass < ARRAY_SIZE(t); pass++) { - struct intel_context *ce = engine->kernel_context; - ktime_t t0, t1; - - t0 = ktime_get(); - - err = i915_gem_object_fill_blt(obj, ce, 0); - if (err) - break; - - err = i915_gem_object_wait(obj, - I915_WAIT_ALL, - MAX_SCHEDULE_TIMEOUT); - if (err) - break; - - t1 = ktime_get(); - t[pass] = ktime_sub(t1, t0); - } - intel_engine_pm_put(engine); - if (err) - return err; - - sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); - pr_info("%s: blt %zd KiB fill: %lld MiB/s\n", - engine->name, - obj->base.size >> 10, - div64_u64(mul_u32_u32(4 * obj->base.size, - 1000 * 1000 * 1000), - t[1] + 2 * t[2] + t[3]) >> 20); - } while (1); -} - -static int perf_fill_blt(void *arg) -{ - struct drm_i915_private *i915 = arg; - static const unsigned long sizes[] = { - SZ_4K, - SZ_64K, - SZ_2M, - SZ_64M - }; - int i; - - for (i = 0; i < ARRAY_SIZE(sizes); i++) { - struct drm_i915_gem_object *obj; - int err; - - obj = i915_gem_object_create_internal(i915, sizes[i]); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = __perf_fill_blt(obj); - i915_gem_object_put(obj); - if (err) - return err; - } - - return 0; -} - -static int __perf_copy_blt(struct drm_i915_gem_object *src, - struct drm_i915_gem_object *dst) -{ - struct drm_i915_private *i915 = to_i915(src->base.dev); - int inst = 0; - - do { - struct intel_engine_cs *engine; - ktime_t t[5]; - int pass; - int err = 0; - - engine = intel_engine_lookup_user(i915, - I915_ENGINE_CLASS_COPY, - inst++); - if (!engine) - return 0; - - intel_engine_pm_get(engine); - for (pass = 0; pass < ARRAY_SIZE(t); pass++) { - struct intel_context *ce = engine->kernel_context; - ktime_t t0, t1; - - t0 = ktime_get(); - - err = i915_gem_object_copy_blt(src, dst, ce); - if (err) - break; - - err = i915_gem_object_wait(dst, - I915_WAIT_ALL, - MAX_SCHEDULE_TIMEOUT); - if (err) - break; - - t1 = ktime_get(); - t[pass] = ktime_sub(t1, t0); - } - intel_engine_pm_put(engine); - if (err) - return err; - - sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); - pr_info("%s: blt %zd KiB copy: %lld MiB/s\n", - engine->name, - src->base.size >> 10, - div64_u64(mul_u32_u32(4 * src->base.size, - 1000 * 1000 * 1000), - t[1] + 2 * t[2] + t[3]) >> 20); - } while (1); -} - -static int perf_copy_blt(void *arg) -{ - struct drm_i915_private *i915 = arg; - static const unsigned long sizes[] = { - SZ_4K, - SZ_64K, - SZ_2M, - SZ_64M - }; - int i; - - for (i = 0; i < ARRAY_SIZE(sizes); i++) { - struct drm_i915_gem_object *src, *dst; - int err; - - src = i915_gem_object_create_internal(i915, sizes[i]); - if (IS_ERR(src)) - return PTR_ERR(src); - - dst = i915_gem_object_create_internal(i915, sizes[i]); - if (IS_ERR(dst)) { - err = PTR_ERR(dst); - goto err_src; - } - - err = __perf_copy_blt(src, dst); - - i915_gem_object_put(dst); -err_src: - i915_gem_object_put(src); - if (err) - return err; - } - - return 0; -} - -struct igt_thread_arg { - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - struct file *file; - struct rnd_state prng; - unsigned int n_cpus; -}; - -static int igt_fill_blt_thread(void *arg) -{ - struct igt_thread_arg *thread = arg; - struct intel_engine_cs *engine = thread->engine; - struct rnd_state *prng = &thread->prng; - struct drm_i915_gem_object *obj; - struct i915_gem_context *ctx; - struct intel_context *ce; - unsigned int prio; - IGT_TIMEOUT(end); - u64 total, max; - int err; - - ctx = thread->ctx; - if (!ctx) { - ctx = live_context_for_engine(engine, thread->file); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - prio = i915_prandom_u32_max_state(I915_PRIORITY_MAX, prng); - ctx->sched.priority = prio; - } - - ce = i915_gem_context_get_engine(ctx, 0); - GEM_BUG_ON(IS_ERR(ce)); - - /* - * If we have a tiny shared address space, like for the GGTT - * then we can't be too greedy. - */ - max = ce->vm->total; - if (i915_is_ggtt(ce->vm) || thread->ctx) - max = div_u64(max, thread->n_cpus); - max >>= 4; - - total = PAGE_SIZE; - do { - /* Aim to keep the runtime under reasonable bounds! */ - const u32 max_phys_size = SZ_64K; - u32 val = prandom_u32_state(prng); - u32 phys_sz; - u32 sz; - u32 *vaddr; - u32 i; - - total = min(total, max); - sz = i915_prandom_u32_max_state(total, prng) + 1; - phys_sz = sz % max_phys_size + 1; - - sz = round_up(sz, PAGE_SIZE); - phys_sz = round_up(phys_sz, PAGE_SIZE); - phys_sz = min(phys_sz, sz); - - pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__, - phys_sz, sz, val); - - obj = huge_gem_object(engine->i915, phys_sz, sz); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_flush; - } - - vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_put; - } - - /* - * Make sure the potentially async clflush does its job, if - * required. - */ - memset32(vaddr, val ^ 0xdeadbeaf, - huge_gem_object_phys_size(obj) / sizeof(u32)); - - if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) - obj->cache_dirty = true; - - err = i915_gem_object_fill_blt(obj, ce, val); - if (err) - goto err_unpin; - - err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT); - if (err) - goto err_unpin; - - for (i = 0; i < huge_gem_object_phys_size(obj) / sizeof(u32); i += 17) { - if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) - drm_clflush_virt_range(&vaddr[i], sizeof(vaddr[i])); - - if (vaddr[i] != val) { - pr_err("vaddr[%u]=%x, expected=%x\n", i, - vaddr[i], val); - err = -EINVAL; - goto err_unpin; - } - } - - i915_gem_object_unpin_map(obj); - i915_gem_object_put(obj); - - total <<= 1; - } while (!time_after(jiffies, end)); - - goto err_flush; - -err_unpin: - i915_gem_object_unpin_map(obj); -err_put: - i915_gem_object_put(obj); -err_flush: - if (err == -ENOMEM) - err = 0; - - intel_context_put(ce); - return err; -} - -static int igt_copy_blt_thread(void *arg) -{ - struct igt_thread_arg *thread = arg; - struct intel_engine_cs *engine = thread->engine; - struct rnd_state *prng = &thread->prng; - struct drm_i915_gem_object *src, *dst; - struct i915_gem_context *ctx; - struct intel_context *ce; - unsigned int prio; - IGT_TIMEOUT(end); - u64 total, max; - int err; - - ctx = thread->ctx; - if (!ctx) { - ctx = live_context_for_engine(engine, thread->file); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - prio = i915_prandom_u32_max_state(I915_PRIORITY_MAX, prng); - ctx->sched.priority = prio; - } - - ce = i915_gem_context_get_engine(ctx, 0); - GEM_BUG_ON(IS_ERR(ce)); - - /* - * If we have a tiny shared address space, like for the GGTT - * then we can't be too greedy. - */ - max = ce->vm->total; - if (i915_is_ggtt(ce->vm) || thread->ctx) - max = div_u64(max, thread->n_cpus); - max >>= 4; - - total = PAGE_SIZE; - do { - /* Aim to keep the runtime under reasonable bounds! */ - const u32 max_phys_size = SZ_64K; - u32 val = prandom_u32_state(prng); - u32 phys_sz; - u32 sz; - u32 *vaddr; - u32 i; - - total = min(total, max); - sz = i915_prandom_u32_max_state(total, prng) + 1; - phys_sz = sz % max_phys_size + 1; - - sz = round_up(sz, PAGE_SIZE); - phys_sz = round_up(phys_sz, PAGE_SIZE); - phys_sz = min(phys_sz, sz); - - pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__, - phys_sz, sz, val); - - src = huge_gem_object(engine->i915, phys_sz, sz); - if (IS_ERR(src)) { - err = PTR_ERR(src); - goto err_flush; - } - - vaddr = i915_gem_object_pin_map_unlocked(src, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_put_src; - } - - memset32(vaddr, val, - huge_gem_object_phys_size(src) / sizeof(u32)); - - i915_gem_object_unpin_map(src); - - if (!(src->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) - src->cache_dirty = true; - - dst = huge_gem_object(engine->i915, phys_sz, sz); - if (IS_ERR(dst)) { - err = PTR_ERR(dst); - goto err_put_src; - } - - vaddr = i915_gem_object_pin_map_unlocked(dst, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_put_dst; - } - - memset32(vaddr, val ^ 0xdeadbeaf, - huge_gem_object_phys_size(dst) / sizeof(u32)); - - if (!(dst->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) - dst->cache_dirty = true; - - err = i915_gem_object_copy_blt(src, dst, ce); - if (err) - goto err_unpin; - - err = i915_gem_object_wait(dst, 0, MAX_SCHEDULE_TIMEOUT); - if (err) - goto err_unpin; - - for (i = 0; i < huge_gem_object_phys_size(dst) / sizeof(u32); i += 17) { - if (!(dst->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) - drm_clflush_virt_range(&vaddr[i], sizeof(vaddr[i])); - - if (vaddr[i] != val) { - pr_err("vaddr[%u]=%x, expected=%x\n", i, - vaddr[i], val); - err = -EINVAL; - goto err_unpin; - } - } - - i915_gem_object_unpin_map(dst); - - i915_gem_object_put(src); - i915_gem_object_put(dst); - - total <<= 1; - } while (!time_after(jiffies, end)); - - goto err_flush; - -err_unpin: - i915_gem_object_unpin_map(dst); -err_put_dst: - i915_gem_object_put(dst); -err_put_src: - i915_gem_object_put(src); -err_flush: - if (err == -ENOMEM) - err = 0; - - intel_context_put(ce); - return err; -} - -static int igt_threaded_blt(struct intel_engine_cs *engine, - int (*blt_fn)(void *arg), - unsigned int flags) -#define SINGLE_CTX BIT(0) -{ - struct igt_thread_arg *thread; - struct task_struct **tsk; - unsigned int n_cpus, i; - I915_RND_STATE(prng); - int err = 0; - - n_cpus = num_online_cpus() + 1; - - tsk = kcalloc(n_cpus, sizeof(struct task_struct *), GFP_KERNEL); - if (!tsk) - return 0; - - thread = kcalloc(n_cpus, sizeof(struct igt_thread_arg), GFP_KERNEL); - if (!thread) - goto out_tsk; - - thread[0].file = mock_file(engine->i915); - if (IS_ERR(thread[0].file)) { - err = PTR_ERR(thread[0].file); - goto out_thread; - } - - if (flags & SINGLE_CTX) { - thread[0].ctx = live_context_for_engine(engine, thread[0].file); - if (IS_ERR(thread[0].ctx)) { - err = PTR_ERR(thread[0].ctx); - goto out_file; - } - } - - for (i = 0; i < n_cpus; ++i) { - thread[i].engine = engine; - thread[i].file = thread[0].file; - thread[i].ctx = thread[0].ctx; - thread[i].n_cpus = n_cpus; - thread[i].prng = - I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng)); - - tsk[i] = kthread_run(blt_fn, &thread[i], "igt/blt-%d", i); - if (IS_ERR(tsk[i])) { - err = PTR_ERR(tsk[i]); - break; - } - - get_task_struct(tsk[i]); - } - - yield(); /* start all threads before we kthread_stop() */ - - for (i = 0; i < n_cpus; ++i) { - int status; - - if (IS_ERR_OR_NULL(tsk[i])) - continue; - - status = kthread_stop(tsk[i]); - if (status && !err) - err = status; - - put_task_struct(tsk[i]); - } - -out_file: - fput(thread[0].file); -out_thread: - kfree(thread); -out_tsk: - kfree(tsk); - return err; -} - -static int test_copy_engines(struct drm_i915_private *i915, - int (*fn)(void *arg), - unsigned int flags) -{ - struct intel_engine_cs *engine; - int ret; - - for_each_uabi_class_engine(engine, I915_ENGINE_CLASS_COPY, i915) { - ret = igt_threaded_blt(engine, fn, flags); - if (ret) - return ret; - } - - return 0; -} - -static int igt_fill_blt(void *arg) -{ - return test_copy_engines(arg, igt_fill_blt_thread, 0); -} - -static int igt_fill_blt_ctx0(void *arg) -{ - return test_copy_engines(arg, igt_fill_blt_thread, SINGLE_CTX); -} - -static int igt_copy_blt(void *arg) -{ - return test_copy_engines(arg, igt_copy_blt_thread, 0); -} - -static int igt_copy_blt_ctx0(void *arg) -{ - return test_copy_engines(arg, igt_copy_blt_thread, SINGLE_CTX); -} - -int i915_gem_object_blt_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_fill_blt), - SUBTEST(igt_fill_blt_ctx0), - SUBTEST(igt_copy_blt), - SUBTEST(igt_copy_blt_ctx0), - }; - - if (intel_gt_is_wedged(&i915->gt)) - return 0; - - return i915_live_subtests(tests, i915); -} - -int i915_gem_object_blt_perf_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(perf_fill_blt), - SUBTEST(perf_copy_blt), - }; - - if (intel_gt_is_wedged(&i915->gt)) - return 0; - - return i915_live_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c index 3a6ce87f8b52..d43d8dae0f69 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c @@ -25,13 +25,14 @@ static int mock_phys_object(void *arg) goto out; } + i915_gem_object_lock(obj, NULL); if (!i915_gem_object_has_struct_page(obj)) { + i915_gem_object_unlock(obj); err = -EINVAL; pr_err("shmem has no struct page\n"); goto out_obj; } - i915_gem_object_lock(obj, NULL); err = i915_gem_object_attach_phys(obj, PAGE_SIZE); i915_gem_object_unlock(obj); if (err) { diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.c b/drivers/gpu/drm/i915/gem/selftests/mock_context.c index 51b5a3421b40..fee070df1c97 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.c @@ -14,6 +14,7 @@ mock_context(struct drm_i915_private *i915, { struct i915_gem_context *ctx; struct i915_gem_engines *e; + struct intel_sseu null_sseu = {}; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -30,15 +31,6 @@ mock_context(struct drm_i915_private *i915, i915_gem_context_set_persistence(ctx); - mutex_init(&ctx->engines_mutex); - e = default_engines(ctx); - if (IS_ERR(e)) - goto err_free; - RCU_INIT_POINTER(ctx->engines, e); - - INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); - mutex_init(&ctx->lut_mutex); - if (name) { struct i915_ppgtt *ppgtt; @@ -46,25 +38,29 @@ mock_context(struct drm_i915_private *i915, ppgtt = mock_ppgtt(i915, name); if (!ppgtt) - goto err_put; - - mutex_lock(&ctx->mutex); - __set_ppgtt(ctx, &ppgtt->vm); - mutex_unlock(&ctx->mutex); + goto err_free; + ctx->vm = i915_vm_open(&ppgtt->vm); i915_vm_put(&ppgtt->vm); } + mutex_init(&ctx->engines_mutex); + e = default_engines(ctx, null_sseu); + if (IS_ERR(e)) + goto err_vm; + RCU_INIT_POINTER(ctx->engines, e); + + INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); + mutex_init(&ctx->lut_mutex); + return ctx; +err_vm: + if (ctx->vm) + i915_vm_close(ctx->vm); err_free: kfree(ctx); return NULL; - -err_put: - i915_gem_context_set_closed(ctx); - i915_gem_context_put(ctx); - return NULL; } void mock_context_close(struct i915_gem_context *ctx) @@ -80,20 +76,29 @@ void mock_init_contexts(struct drm_i915_private *i915) struct i915_gem_context * live_context(struct drm_i915_private *i915, struct file *file) { + struct drm_i915_file_private *fpriv = to_drm_file(file)->driver_priv; + struct i915_gem_proto_context *pc; struct i915_gem_context *ctx; int err; u32 id; - ctx = i915_gem_create_context(i915, 0); + pc = proto_context_create(i915, 0); + if (IS_ERR(pc)) + return ERR_CAST(pc); + + ctx = i915_gem_create_context(i915, pc); + proto_context_close(pc); if (IS_ERR(ctx)) return ctx; i915_gem_context_set_no_error_capture(ctx); - err = gem_context_register(ctx, to_drm_file(file)->driver_priv, &id); + err = xa_alloc(&fpriv->context_xa, &id, NULL, xa_limit_32b, GFP_KERNEL); if (err < 0) goto err_ctx; + gem_context_register(ctx, fpriv, id); + return ctx; err_ctx: @@ -106,6 +111,7 @@ live_context_for_engine(struct intel_engine_cs *engine, struct file *file) { struct i915_gem_engines *engines; struct i915_gem_context *ctx; + struct intel_sseu null_sseu = {}; struct intel_context *ce; engines = alloc_engines(1); @@ -124,7 +130,7 @@ live_context_for_engine(struct intel_engine_cs *engine, struct file *file) return ERR_CAST(ce); } - intel_context_set_gem(ce, ctx); + intel_context_set_gem(ce, ctx, null_sseu); engines->engines[0] = ce; engines->num_engines = 1; @@ -139,11 +145,24 @@ live_context_for_engine(struct intel_engine_cs *engine, struct file *file) } struct i915_gem_context * -kernel_context(struct drm_i915_private *i915) +kernel_context(struct drm_i915_private *i915, + struct i915_address_space *vm) { struct i915_gem_context *ctx; + struct i915_gem_proto_context *pc; + + pc = proto_context_create(i915, 0); + if (IS_ERR(pc)) + return ERR_CAST(pc); + + if (vm) { + if (pc->vm) + i915_vm_put(pc->vm); + pc->vm = i915_vm_get(vm); + } - ctx = i915_gem_create_context(i915, 0); + ctx = i915_gem_create_context(i915, pc); + proto_context_close(pc); if (IS_ERR(ctx)) return ctx; diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.h b/drivers/gpu/drm/i915/gem/selftests/mock_context.h index 2a6121d33352..7a02fd9b5866 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.h +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.h @@ -10,6 +10,7 @@ struct file; struct drm_i915_private; struct intel_engine_cs; +struct i915_address_space; void mock_init_contexts(struct drm_i915_private *i915); @@ -25,7 +26,8 @@ live_context(struct drm_i915_private *i915, struct file *file); struct i915_gem_context * live_context_for_engine(struct intel_engine_cs *engine, struct file *file); -struct i915_gem_context *kernel_context(struct drm_i915_private *i915); +struct i915_gem_context *kernel_context(struct drm_i915_private *i915, + struct i915_address_space *vm); void kernel_context_close(struct i915_gem_context *ctx); #endif /* !__MOCK_CONTEXT_H */ diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c b/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c index 4270b5a34a83..d6f5836396f8 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c @@ -437,20 +437,20 @@ static int frequency_show(struct seq_file *m, void *unused) max_freq = (IS_GEN9_LP(i915) ? rp_state_cap >> 0 : rp_state_cap >> 16) & 0xff; max_freq *= (IS_GEN9_BC(i915) || - GRAPHICS_VER(i915) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; max_freq *= (IS_GEN9_BC(i915) || - GRAPHICS_VER(i915) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (IS_GEN9_LP(i915) ? rp_state_cap >> 16 : rp_state_cap >> 0) & 0xff; max_freq *= (IS_GEN9_BC(i915) || - GRAPHICS_VER(i915) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); seq_printf(m, "Max overclocked frequency: %dMHz\n", @@ -500,7 +500,7 @@ static int llc_show(struct seq_file *m, void *data) min_gpu_freq = rps->min_freq; max_gpu_freq = rps->max_freq; - if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 10) { + if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { /* Convert GT frequency to 50 HZ units */ min_gpu_freq /= GEN9_FREQ_SCALER; max_gpu_freq /= GEN9_FREQ_SCALER; @@ -518,7 +518,7 @@ static int llc_show(struct seq_file *m, void *data) intel_gpu_freq(rps, (gpu_freq * (IS_GEN9_BC(i915) || - GRAPHICS_VER(i915) >= 10 ? + GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1))), ((ia_freq >> 0) & 0xff) * 100, ((ia_freq >> 8) & 0xff) * 100); diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c index 94e0a5669f90..461844dffd7e 100644 --- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c @@ -42,7 +42,7 @@ int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode) vf_flush_wa = true; /* WaForGAMHang:kbl */ - if (IS_KBL_GT_STEP(rq->engine->i915, 0, STEP_B0)) + if (IS_KBL_GT_STEP(rq->engine->i915, 0, STEP_C0)) dc_flush_wa = true; } @@ -208,7 +208,7 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) flags |= PIPE_CONTROL_FLUSH_L3; flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; - /* Wa_1409600907:tgl */ + /* Wa_1409600907:tgl,adl-p */ flags |= PIPE_CONTROL_DEPTH_STALL; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; @@ -279,7 +279,7 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) if (mode & EMIT_INVALIDATE) aux_inv = rq->engine->mask & ~BIT(BCS0); if (aux_inv) - cmd += 2 * hweight8(aux_inv) + 2; + cmd += 2 * hweight32(aux_inv) + 2; cs = intel_ring_begin(rq, cmd); if (IS_ERR(cs)) @@ -313,9 +313,8 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) struct intel_engine_cs *engine; unsigned int tmp; - *cs++ = MI_LOAD_REGISTER_IMM(hweight8(aux_inv)); - for_each_engine_masked(engine, rq->engine->gt, - aux_inv, tmp) { + *cs++ = MI_LOAD_REGISTER_IMM(hweight32(aux_inv)); + for_each_engine_masked(engine, rq->engine->gt, aux_inv, tmp) { *cs++ = i915_mmio_reg_offset(aux_inv_reg(engine)); *cs++ = AUX_INV; } @@ -506,7 +505,8 @@ gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) *cs++ = MI_USER_INTERRUPT; *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; - if (intel_engine_has_semaphores(rq->engine)) + if (intel_engine_has_semaphores(rq->engine) && + !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) cs = emit_preempt_busywait(rq, cs); rq->tail = intel_ring_offset(rq, cs); @@ -598,7 +598,8 @@ gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) *cs++ = MI_USER_INTERRUPT; *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; - if (intel_engine_has_semaphores(rq->engine)) + if (intel_engine_has_semaphores(rq->engine) && + !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) cs = gen12_emit_preempt_busywait(rq, cs); rq->tail = intel_ring_offset(rq, cs); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index da4f5eb43ac2..6e0e52eeb87a 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -358,6 +358,54 @@ static void gen8_ppgtt_alloc(struct i915_address_space *vm, &start, start + length, vm->top); } +static void __gen8_ppgtt_foreach(struct i915_address_space *vm, + struct i915_page_directory *pd, + u64 *start, u64 end, int lvl, + void (*fn)(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data), + void *data) +{ + unsigned int idx, len; + + len = gen8_pd_range(*start, end, lvl--, &idx); + + spin_lock(&pd->lock); + do { + struct i915_page_table *pt = pd->entry[idx]; + + atomic_inc(&pt->used); + spin_unlock(&pd->lock); + + if (lvl) { + __gen8_ppgtt_foreach(vm, as_pd(pt), start, end, lvl, + fn, data); + } else { + fn(vm, pt, data); + *start += gen8_pt_count(*start, end); + } + + spin_lock(&pd->lock); + atomic_dec(&pt->used); + } while (idx++, --len); + spin_unlock(&pd->lock); +} + +static void gen8_ppgtt_foreach(struct i915_address_space *vm, + u64 start, u64 length, + void (*fn)(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data), + void *data) +{ + start >>= GEN8_PTE_SHIFT; + length >>= GEN8_PTE_SHIFT; + + __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, + &start, start + length, vm->top, + fn, data); +} + static __always_inline u64 gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, struct i915_page_directory *pdp, @@ -552,6 +600,24 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm, } } +static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, + dma_addr_t addr, + u64 offset, + enum i915_cache_level level, + u32 flags) +{ + u64 idx = offset >> GEN8_PTE_SHIFT; + struct i915_page_directory * const pdp = + gen8_pdp_for_page_index(vm, idx); + struct i915_page_directory *pd = + i915_pd_entry(pdp, gen8_pd_index(idx, 2)); + gen8_pte_t *vaddr; + + vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); + clflush_cache_range(&vaddr[gen8_pd_index(idx, 0)], sizeof(*vaddr)); +} + static int gen8_init_scratch(struct i915_address_space *vm) { u32 pte_flags; @@ -731,8 +797,10 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; ppgtt->vm.insert_entries = gen8_ppgtt_insert; + ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; ppgtt->vm.clear_range = gen8_ppgtt_clear; + ppgtt->vm.foreach = gen8_ppgtt_foreach; ppgtt->vm.pte_encode = gen8_pte_encode; diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c index 38cc42783dfb..209cf265bf74 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c @@ -15,28 +15,14 @@ #include "intel_gt_pm.h" #include "intel_gt_requests.h" -static bool irq_enable(struct intel_engine_cs *engine) +static bool irq_enable(struct intel_breadcrumbs *b) { - if (!engine->irq_enable) - return false; - - /* Caller disables interrupts */ - spin_lock(&engine->gt->irq_lock); - engine->irq_enable(engine); - spin_unlock(&engine->gt->irq_lock); - - return true; + return intel_engine_irq_enable(b->irq_engine); } -static void irq_disable(struct intel_engine_cs *engine) +static void irq_disable(struct intel_breadcrumbs *b) { - if (!engine->irq_disable) - return; - - /* Caller disables interrupts */ - spin_lock(&engine->gt->irq_lock); - engine->irq_disable(engine); - spin_unlock(&engine->gt->irq_lock); + intel_engine_irq_disable(b->irq_engine); } static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) @@ -57,7 +43,7 @@ static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) WRITE_ONCE(b->irq_armed, true); /* Requests may have completed before we could enable the interrupt. */ - if (!b->irq_enabled++ && irq_enable(b->irq_engine)) + if (!b->irq_enabled++ && b->irq_enable(b)) irq_work_queue(&b->irq_work); } @@ -76,7 +62,7 @@ static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) { GEM_BUG_ON(!b->irq_enabled); if (!--b->irq_enabled) - irq_disable(b->irq_engine); + b->irq_disable(b); WRITE_ONCE(b->irq_armed, false); intel_gt_pm_put_async(b->irq_engine->gt); @@ -259,6 +245,9 @@ static void signal_irq_work(struct irq_work *work) llist_entry(signal, typeof(*rq), signal_node); struct list_head cb_list; + if (rq->engine->sched_engine->retire_inflight_request_prio) + rq->engine->sched_engine->retire_inflight_request_prio(rq); + spin_lock(&rq->lock); list_replace(&rq->fence.cb_list, &cb_list); __dma_fence_signal__timestamp(&rq->fence, timestamp); @@ -281,7 +270,7 @@ intel_breadcrumbs_create(struct intel_engine_cs *irq_engine) if (!b) return NULL; - b->irq_engine = irq_engine; + kref_init(&b->ref); spin_lock_init(&b->signalers_lock); INIT_LIST_HEAD(&b->signalers); @@ -290,6 +279,10 @@ intel_breadcrumbs_create(struct intel_engine_cs *irq_engine) spin_lock_init(&b->irq_lock); init_irq_work(&b->irq_work, signal_irq_work); + b->irq_engine = irq_engine; + b->irq_enable = irq_enable; + b->irq_disable = irq_disable; + return b; } @@ -303,9 +296,9 @@ void intel_breadcrumbs_reset(struct intel_breadcrumbs *b) spin_lock_irqsave(&b->irq_lock, flags); if (b->irq_enabled) - irq_enable(b->irq_engine); + b->irq_enable(b); else - irq_disable(b->irq_engine); + b->irq_disable(b); spin_unlock_irqrestore(&b->irq_lock, flags); } @@ -325,11 +318,14 @@ void __intel_breadcrumbs_park(struct intel_breadcrumbs *b) } } -void intel_breadcrumbs_free(struct intel_breadcrumbs *b) +void intel_breadcrumbs_free(struct kref *kref) { + struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref); + irq_work_sync(&b->irq_work); GEM_BUG_ON(!list_empty(&b->signalers)); GEM_BUG_ON(b->irq_armed); + kfree(b); } diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h index 3ce5ce270b04..be0d4f379a85 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.h @@ -9,7 +9,7 @@ #include <linux/atomic.h> #include <linux/irq_work.h> -#include "intel_engine_types.h" +#include "intel_breadcrumbs_types.h" struct drm_printer; struct i915_request; @@ -17,7 +17,7 @@ struct intel_breadcrumbs; struct intel_breadcrumbs * intel_breadcrumbs_create(struct intel_engine_cs *irq_engine); -void intel_breadcrumbs_free(struct intel_breadcrumbs *b); +void intel_breadcrumbs_free(struct kref *kref); void intel_breadcrumbs_reset(struct intel_breadcrumbs *b); void __intel_breadcrumbs_park(struct intel_breadcrumbs *b); @@ -48,4 +48,16 @@ void i915_request_cancel_breadcrumb(struct i915_request *request); void intel_context_remove_breadcrumbs(struct intel_context *ce, struct intel_breadcrumbs *b); +static inline struct intel_breadcrumbs * +intel_breadcrumbs_get(struct intel_breadcrumbs *b) +{ + kref_get(&b->ref); + return b; +} + +static inline void intel_breadcrumbs_put(struct intel_breadcrumbs *b) +{ + kref_put(&b->ref, intel_breadcrumbs_free); +} + #endif /* __INTEL_BREADCRUMBS__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h index 3a084ce8ff5e..72dfd3748c4c 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h @@ -7,10 +7,13 @@ #define __INTEL_BREADCRUMBS_TYPES__ #include <linux/irq_work.h> +#include <linux/kref.h> #include <linux/list.h> #include <linux/spinlock.h> #include <linux/types.h> +#include "intel_engine_types.h" + /* * Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the @@ -29,6 +32,7 @@ * the overhead of waking that client is much preferred. */ struct intel_breadcrumbs { + struct kref ref; atomic_t active; spinlock_t signalers_lock; /* protects the list of signalers */ @@ -42,7 +46,10 @@ struct intel_breadcrumbs { bool irq_armed; /* Not all breadcrumbs are attached to physical HW */ + intel_engine_mask_t engine_mask; struct intel_engine_cs *irq_engine; + bool (*irq_enable)(struct intel_breadcrumbs *b); + void (*irq_disable)(struct intel_breadcrumbs *b); }; #endif /* __INTEL_BREADCRUMBS_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 4033184f13b9..745e84c72c90 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -7,28 +7,26 @@ #include "gem/i915_gem_pm.h" #include "i915_drv.h" -#include "i915_globals.h" +#include "i915_trace.h" #include "intel_context.h" #include "intel_engine.h" #include "intel_engine_pm.h" #include "intel_ring.h" -static struct i915_global_context { - struct i915_global base; - struct kmem_cache *slab_ce; -} global; +static struct kmem_cache *slab_ce; static struct intel_context *intel_context_alloc(void) { - return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); + return kmem_cache_zalloc(slab_ce, GFP_KERNEL); } static void rcu_context_free(struct rcu_head *rcu) { struct intel_context *ce = container_of(rcu, typeof(*ce), rcu); - kmem_cache_free(global.slab_ce, ce); + trace_intel_context_free(ce); + kmem_cache_free(slab_ce, ce); } void intel_context_free(struct intel_context *ce) @@ -46,6 +44,7 @@ intel_context_create(struct intel_engine_cs *engine) return ERR_PTR(-ENOMEM); intel_context_init(ce, engine); + trace_intel_context_create(ce); return ce; } @@ -80,7 +79,7 @@ static int intel_context_active_acquire(struct intel_context *ce) __i915_active_acquire(&ce->active); - if (intel_context_is_barrier(ce)) + if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine)) return 0; /* Preallocate tracking nodes */ @@ -268,6 +267,8 @@ int __intel_context_do_pin_ww(struct intel_context *ce, GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ + trace_intel_context_do_pin(ce); + err_unlock: mutex_unlock(&ce->pin_mutex); err_post_unpin: @@ -306,9 +307,9 @@ retry: return err; } -void intel_context_unpin(struct intel_context *ce) +void __intel_context_do_unpin(struct intel_context *ce, int sub) { - if (!atomic_dec_and_test(&ce->pin_count)) + if (!atomic_sub_and_test(sub, &ce->pin_count)) return; CE_TRACE(ce, "unpin\n"); @@ -323,6 +324,7 @@ void intel_context_unpin(struct intel_context *ce) */ intel_context_get(ce); intel_context_active_release(ce); + trace_intel_context_do_unpin(ce); intel_context_put(ce); } @@ -360,6 +362,12 @@ static int __intel_context_active(struct i915_active *active) return 0; } +static int sw_fence_dummy_notify(struct i915_sw_fence *sf, + enum i915_sw_fence_notify state) +{ + return NOTIFY_DONE; +} + void intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) { @@ -371,7 +379,8 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) ce->engine = engine; ce->ops = engine->cops; ce->sseu = engine->sseu; - ce->ring = __intel_context_ring_size(SZ_4K); + ce->ring = NULL; + ce->ring_size = SZ_4K; ewma_runtime_init(&ce->runtime.avg); @@ -383,6 +392,22 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) mutex_init(&ce->pin_mutex); + spin_lock_init(&ce->guc_state.lock); + INIT_LIST_HEAD(&ce->guc_state.fences); + + spin_lock_init(&ce->guc_active.lock); + INIT_LIST_HEAD(&ce->guc_active.requests); + + ce->guc_id = GUC_INVALID_LRC_ID; + INIT_LIST_HEAD(&ce->guc_id_link); + + /* + * Initialize fence to be complete as this is expected to be complete + * unless there is a pending schedule disable outstanding. + */ + i915_sw_fence_init(&ce->guc_blocked, sw_fence_dummy_notify); + i915_sw_fence_commit(&ce->guc_blocked); + i915_active_init(&ce->active, __intel_context_active, __intel_context_retire, 0); } @@ -397,28 +422,17 @@ void intel_context_fini(struct intel_context *ce) i915_active_fini(&ce->active); } -static void i915_global_context_shrink(void) -{ - kmem_cache_shrink(global.slab_ce); -} - -static void i915_global_context_exit(void) +void i915_context_module_exit(void) { - kmem_cache_destroy(global.slab_ce); + kmem_cache_destroy(slab_ce); } -static struct i915_global_context global = { { - .shrink = i915_global_context_shrink, - .exit = i915_global_context_exit, -} }; - -int __init i915_global_context_init(void) +int __init i915_context_module_init(void) { - global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); - if (!global.slab_ce) + slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); + if (!slab_ce) return -ENOMEM; - i915_global_register(&global.base); return 0; } @@ -499,6 +513,26 @@ retry: return rq; } +struct i915_request *intel_context_find_active_request(struct intel_context *ce) +{ + struct i915_request *rq, *active = NULL; + unsigned long flags; + + GEM_BUG_ON(!intel_engine_uses_guc(ce->engine)); + + spin_lock_irqsave(&ce->guc_active.lock, flags); + list_for_each_entry_reverse(rq, &ce->guc_active.requests, + sched.link) { + if (i915_request_completed(rq)) + break; + + active = rq; + } + spin_unlock_irqrestore(&ce->guc_active.lock, flags); + + return active; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_context.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index f83a73a2b39f..c41098950746 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -16,6 +16,7 @@ #include "intel_engine_types.h" #include "intel_ring_types.h" #include "intel_timeline_types.h" +#include "i915_trace.h" #define CE_TRACE(ce, fmt, ...) do { \ const struct intel_context *ce__ = (ce); \ @@ -30,6 +31,9 @@ void intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine); void intel_context_fini(struct intel_context *ce); +void i915_context_module_exit(void); +int i915_context_module_init(void); + struct intel_context * intel_context_create(struct intel_engine_cs *engine); @@ -69,6 +73,13 @@ intel_context_is_pinned(struct intel_context *ce) return atomic_read(&ce->pin_count); } +static inline void intel_context_cancel_request(struct intel_context *ce, + struct i915_request *rq) +{ + GEM_BUG_ON(!ce->ops->cancel_request); + return ce->ops->cancel_request(ce, rq); +} + /** * intel_context_unlock_pinned - Releases the earlier locking of 'pinned' status * @ce - the context @@ -113,7 +124,32 @@ static inline void __intel_context_pin(struct intel_context *ce) atomic_inc(&ce->pin_count); } -void intel_context_unpin(struct intel_context *ce); +void __intel_context_do_unpin(struct intel_context *ce, int sub); + +static inline void intel_context_sched_disable_unpin(struct intel_context *ce) +{ + __intel_context_do_unpin(ce, 2); +} + +static inline void intel_context_unpin(struct intel_context *ce) +{ + if (!ce->ops->sched_disable) { + __intel_context_do_unpin(ce, 1); + } else { + /* + * Move ownership of this pin to the scheduling disable which is + * an async operation. When that operation completes the above + * intel_context_sched_disable_unpin is called potentially + * unpinning the context. + */ + while (!atomic_add_unless(&ce->pin_count, -1, 1)) { + if (atomic_cmpxchg(&ce->pin_count, 1, 2) == 1) { + ce->ops->sched_disable(ce); + break; + } + } + } +} void intel_context_enter_engine(struct intel_context *ce); void intel_context_exit_engine(struct intel_context *ce); @@ -175,10 +211,8 @@ int intel_context_prepare_remote_request(struct intel_context *ce, struct i915_request *intel_context_create_request(struct intel_context *ce); -static inline struct intel_ring *__intel_context_ring_size(u64 sz) -{ - return u64_to_ptr(struct intel_ring, sz); -} +struct i915_request * +intel_context_find_active_request(struct intel_context *ce); static inline bool intel_context_is_barrier(const struct intel_context *ce) { @@ -220,6 +254,18 @@ static inline bool intel_context_set_banned(struct intel_context *ce) return test_and_set_bit(CONTEXT_BANNED, &ce->flags); } +static inline bool intel_context_ban(struct intel_context *ce, + struct i915_request *rq) +{ + bool ret = intel_context_set_banned(ce); + + trace_intel_context_ban(ce); + if (ce->ops->ban) + ce->ops->ban(ce, rq); + + return ret; +} + static inline bool intel_context_force_single_submission(const struct intel_context *ce) { diff --git a/drivers/gpu/drm/i915/gt/intel_context_param.c b/drivers/gpu/drm/i915/gt/intel_context_param.c deleted file mode 100644 index 65dcd090245d..000000000000 --- a/drivers/gpu/drm/i915/gt/intel_context_param.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include "i915_active.h" -#include "intel_context.h" -#include "intel_context_param.h" -#include "intel_ring.h" - -int intel_context_set_ring_size(struct intel_context *ce, long sz) -{ - int err; - - if (intel_context_lock_pinned(ce)) - return -EINTR; - - err = i915_active_wait(&ce->active); - if (err < 0) - goto unlock; - - if (intel_context_is_pinned(ce)) { - err = -EBUSY; /* In active use, come back later! */ - goto unlock; - } - - if (test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { - struct intel_ring *ring; - - /* Replace the existing ringbuffer */ - ring = intel_engine_create_ring(ce->engine, sz); - if (IS_ERR(ring)) { - err = PTR_ERR(ring); - goto unlock; - } - - intel_ring_put(ce->ring); - ce->ring = ring; - - /* Context image will be updated on next pin */ - } else { - ce->ring = __intel_context_ring_size(sz); - } - -unlock: - intel_context_unlock_pinned(ce); - return err; -} - -long intel_context_get_ring_size(struct intel_context *ce) -{ - long sz = (unsigned long)READ_ONCE(ce->ring); - - if (test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { - if (intel_context_lock_pinned(ce)) - return -EINTR; - - sz = ce->ring->size; - intel_context_unlock_pinned(ce); - } - - return sz; -} diff --git a/drivers/gpu/drm/i915/gt/intel_context_param.h b/drivers/gpu/drm/i915/gt/intel_context_param.h index 3ecacc675f41..0c69cb42d075 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_param.h +++ b/drivers/gpu/drm/i915/gt/intel_context_param.h @@ -10,14 +10,10 @@ #include "intel_context.h" -int intel_context_set_ring_size(struct intel_context *ce, long sz); -long intel_context_get_ring_size(struct intel_context *ce); - -static inline int +static inline void intel_context_set_watchdog_us(struct intel_context *ce, u64 timeout_us) { ce->watchdog.timeout_us = timeout_us; - return 0; } #endif /* INTEL_CONTEXT_PARAM_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index ed8c447a7346..e54351a170e2 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -13,12 +13,14 @@ #include <linux/types.h> #include "i915_active_types.h" +#include "i915_sw_fence.h" #include "i915_utils.h" #include "intel_engine_types.h" #include "intel_sseu.h" -#define CONTEXT_REDZONE POISON_INUSE +#include "uc/intel_guc_fwif.h" +#define CONTEXT_REDZONE POISON_INUSE DECLARE_EWMA(runtime, 3, 8); struct i915_gem_context; @@ -35,16 +37,29 @@ struct intel_context_ops { int (*alloc)(struct intel_context *ce); + void (*ban)(struct intel_context *ce, struct i915_request *rq); + int (*pre_pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void **vaddr); int (*pin)(struct intel_context *ce, void *vaddr); void (*unpin)(struct intel_context *ce); void (*post_unpin)(struct intel_context *ce); + void (*cancel_request)(struct intel_context *ce, + struct i915_request *rq); + void (*enter)(struct intel_context *ce); void (*exit)(struct intel_context *ce); + void (*sched_disable)(struct intel_context *ce); + void (*reset)(struct intel_context *ce); void (*destroy)(struct kref *kref); + + /* virtual engine/context interface */ + struct intel_context *(*create_virtual)(struct intel_engine_cs **engine, + unsigned int count); + struct intel_engine_cs *(*get_sibling)(struct intel_engine_cs *engine, + unsigned int sibling); }; struct intel_context { @@ -82,6 +97,7 @@ struct intel_context { spinlock_t signal_lock; /* protects signals, the list of requests */ struct i915_vma *state; + u32 ring_size; struct intel_ring *ring; struct intel_timeline *timeline; @@ -95,6 +111,7 @@ struct intel_context { #define CONTEXT_BANNED 6 #define CONTEXT_FORCE_SINGLE_SUBMISSION 7 #define CONTEXT_NOPREEMPT 8 +#define CONTEXT_LRCA_DIRTY 9 struct { u64 timeout_us; @@ -136,6 +153,51 @@ struct intel_context { struct intel_sseu sseu; u8 wa_bb_page; /* if set, page num reserved for context workarounds */ + + struct { + /** lock: protects everything in guc_state */ + spinlock_t lock; + /** + * sched_state: scheduling state of this context using GuC + * submission + */ + u16 sched_state; + /* + * fences: maintains of list of requests that have a submit + * fence related to GuC submission + */ + struct list_head fences; + } guc_state; + + struct { + /** lock: protects everything in guc_active */ + spinlock_t lock; + /** requests: active requests on this context */ + struct list_head requests; + } guc_active; + + /* GuC scheduling state flags that do not require a lock. */ + atomic_t guc_sched_state_no_lock; + + /* GuC LRC descriptor ID */ + u16 guc_id; + + /* GuC LRC descriptor reference count */ + atomic_t guc_id_ref; + + /* + * GuC ID link - in list when unpinned but guc_id still valid in GuC + */ + struct list_head guc_id_link; + + /* GuC context blocked fence */ + struct i915_sw_fence guc_blocked; + + /* + * GuC priority management + */ + u8 guc_prio; + u32 guc_prio_count[GUC_CLIENT_PRIORITY_NUM]; }; #endif /* __INTEL_CONTEXT_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 8d9184920c51..87579affb952 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -19,7 +19,9 @@ #include "intel_workarounds.h" struct drm_printer; +struct intel_context; struct intel_gt; +struct lock_class_key; /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, * but keeps the logic simple. Indeed, the whole purpose of this macro is just @@ -123,20 +125,6 @@ execlists_active(const struct intel_engine_execlists *execlists) return active; } -static inline void -execlists_active_lock_bh(struct intel_engine_execlists *execlists) -{ - local_bh_disable(); /* prevent local softirq and lock recursion */ - tasklet_lock(&execlists->tasklet); -} - -static inline void -execlists_active_unlock_bh(struct intel_engine_execlists *execlists) -{ - tasklet_unlock(&execlists->tasklet); - local_bh_enable(); /* restore softirq, and kick ksoftirqd! */ -} - struct i915_request * execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists); @@ -186,11 +174,12 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) #define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32)) #define I915_GEM_HWS_SEQNO 0x40 #define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32)) +#define I915_GEM_HWS_MIGRATE (0x42 * sizeof(u32)) #define I915_GEM_HWS_SCRATCH 0x80 #define I915_HWS_CSB_BUF0_INDEX 0x10 #define I915_HWS_CSB_WRITE_INDEX 0x1f -#define CNL_HWS_CSB_WRITE_INDEX 0x2f +#define ICL_HWS_CSB_WRITE_INDEX 0x2f void intel_engine_stop(struct intel_engine_cs *engine); void intel_engine_cleanup(struct intel_engine_cs *engine); @@ -223,6 +212,9 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, void intel_engine_init_execlists(struct intel_engine_cs *engine); +bool intel_engine_irq_enable(struct intel_engine_cs *engine); +void intel_engine_irq_disable(struct intel_engine_cs *engine); + static inline void __intel_engine_reset(struct intel_engine_cs *engine, bool stalled) { @@ -248,17 +240,27 @@ __printf(3, 4) void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m, const char *header, ...); +void intel_engine_dump_active_requests(struct list_head *requests, + struct i915_request *hung_rq, + struct drm_printer *m); ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now); struct i915_request * -intel_engine_find_active_request(struct intel_engine_cs *engine); +intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine); u32 intel_engine_context_size(struct intel_gt *gt, u8 class); +struct intel_context * +intel_engine_create_pinned_context(struct intel_engine_cs *engine, + struct i915_address_space *vm, + unsigned int ring_size, + unsigned int hwsp, + struct lock_class_key *key, + const char *name); + +void intel_engine_destroy_pinned_context(struct intel_context *ce); -void intel_engine_init_active(struct intel_engine_cs *engine, - unsigned int subclass); #define ENGINE_PHYSICAL 0 #define ENGINE_MOCK 1 #define ENGINE_VIRTUAL 2 @@ -277,13 +279,60 @@ intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) return intel_engine_has_preemption(engine); } +struct intel_context * +intel_engine_create_virtual(struct intel_engine_cs **siblings, + unsigned int count); + +static inline bool +intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine) +{ + /* + * For non-GuC submission we expect the back-end to look at the + * heartbeat status of the actual physical engine that the work + * has been (or is being) scheduled on, so we should only reach + * here with GuC submission enabled. + */ + GEM_BUG_ON(!intel_engine_uses_guc(engine)); + + return intel_guc_virtual_engine_has_heartbeat(engine); +} + static inline bool intel_engine_has_heartbeat(const struct intel_engine_cs *engine) { if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) return false; - return READ_ONCE(engine->props.heartbeat_interval_ms); + if (intel_engine_is_virtual(engine)) + return intel_virtual_engine_has_heartbeat(engine); + else + return READ_ONCE(engine->props.heartbeat_interval_ms); +} + +static inline struct intel_engine_cs * +intel_engine_get_sibling(struct intel_engine_cs *engine, unsigned int sibling) +{ + GEM_BUG_ON(!intel_engine_is_virtual(engine)); + return engine->cops->get_sibling(engine, sibling); +} + +static inline void +intel_engine_set_hung_context(struct intel_engine_cs *engine, + struct intel_context *ce) +{ + engine->hung_ce = ce; +} + +static inline void +intel_engine_clear_hung_context(struct intel_engine_cs *engine) +{ + intel_engine_set_hung_context(engine, NULL); +} + +static inline struct intel_context * +intel_engine_get_hung_context(struct intel_engine_cs *engine) +{ + return engine->hung_ce; } #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 7f03df236613..0d9105a31d84 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -35,14 +35,12 @@ #define DEFAULT_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE) #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) -#define GEN10_LR_CONTEXT_RENDER_SIZE (18 * PAGE_SIZE) #define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE) #define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE) #define MAX_MMIO_BASES 3 struct engine_info { - unsigned int hw_id; u8 class; u8 instance; /* mmio bases table *must* be sorted in reverse graphics_ver order */ @@ -54,7 +52,6 @@ struct engine_info { static const struct engine_info intel_engines[] = { [RCS0] = { - .hw_id = RCS0_HW, .class = RENDER_CLASS, .instance = 0, .mmio_bases = { @@ -62,7 +59,6 @@ static const struct engine_info intel_engines[] = { }, }, [BCS0] = { - .hw_id = BCS0_HW, .class = COPY_ENGINE_CLASS, .instance = 0, .mmio_bases = { @@ -70,7 +66,6 @@ static const struct engine_info intel_engines[] = { }, }, [VCS0] = { - .hw_id = VCS0_HW, .class = VIDEO_DECODE_CLASS, .instance = 0, .mmio_bases = { @@ -80,7 +75,6 @@ static const struct engine_info intel_engines[] = { }, }, [VCS1] = { - .hw_id = VCS1_HW, .class = VIDEO_DECODE_CLASS, .instance = 1, .mmio_bases = { @@ -89,7 +83,6 @@ static const struct engine_info intel_engines[] = { }, }, [VCS2] = { - .hw_id = VCS2_HW, .class = VIDEO_DECODE_CLASS, .instance = 2, .mmio_bases = { @@ -97,15 +90,41 @@ static const struct engine_info intel_engines[] = { }, }, [VCS3] = { - .hw_id = VCS3_HW, .class = VIDEO_DECODE_CLASS, .instance = 3, .mmio_bases = { { .graphics_ver = 11, .base = GEN11_BSD4_RING_BASE } }, }, + [VCS4] = { + .class = VIDEO_DECODE_CLASS, + .instance = 4, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD5_RING_BASE } + }, + }, + [VCS5] = { + .class = VIDEO_DECODE_CLASS, + .instance = 5, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD6_RING_BASE } + }, + }, + [VCS6] = { + .class = VIDEO_DECODE_CLASS, + .instance = 6, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD7_RING_BASE } + }, + }, + [VCS7] = { + .class = VIDEO_DECODE_CLASS, + .instance = 7, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD8_RING_BASE } + }, + }, [VECS0] = { - .hw_id = VECS0_HW, .class = VIDEO_ENHANCEMENT_CLASS, .instance = 0, .mmio_bases = { @@ -114,13 +133,26 @@ static const struct engine_info intel_engines[] = { }, }, [VECS1] = { - .hw_id = VECS1_HW, .class = VIDEO_ENHANCEMENT_CLASS, .instance = 1, .mmio_bases = { { .graphics_ver = 11, .base = GEN11_VEBOX2_RING_BASE } }, }, + [VECS2] = { + .class = VIDEO_ENHANCEMENT_CLASS, + .instance = 2, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_VEBOX3_RING_BASE } + }, + }, + [VECS3] = { + .class = VIDEO_ENHANCEMENT_CLASS, + .instance = 3, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE } + }, + }, }; /** @@ -153,8 +185,6 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) case 12: case 11: return GEN11_LR_CONTEXT_RENDER_SIZE; - case 10: - return GEN10_LR_CONTEXT_RENDER_SIZE; case 9: return GEN9_LR_CONTEXT_RENDER_SIZE; case 8: @@ -269,6 +299,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH)); BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH)); + BUILD_BUG_ON(I915_MAX_VCS > (MAX_ENGINE_INSTANCE + 1)); + BUILD_BUG_ON(I915_MAX_VECS > (MAX_ENGINE_INSTANCE + 1)); if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))) return -EINVAL; @@ -294,7 +326,6 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->i915 = i915; engine->gt = gt; engine->uncore = gt->uncore; - engine->hw_id = info->hw_id; guc_class = engine_class_to_guc_class(info->class); engine->guc_id = MAKE_GUC_ID(guc_class, info->instance); engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); @@ -328,9 +359,6 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) if (engine->context_size) DRIVER_CAPS(i915)->has_logical_contexts = true; - /* Nothing to do here, execute in order of dependencies */ - engine->schedule = NULL; - ewma__engine_latency_init(&engine->latency); seqcount_init(&engine->stats.lock); @@ -445,6 +473,28 @@ void intel_engines_free(struct intel_gt *gt) } } +static +bool gen11_vdbox_has_sfc(struct drm_i915_private *i915, + unsigned int physical_vdbox, + unsigned int logical_vdbox, u16 vdbox_mask) +{ + /* + * In Gen11, only even numbered logical VDBOXes are hooked + * up to an SFC (Scaler & Format Converter) unit. + * In Gen12, Even numbered physical instance always are connected + * to an SFC. Odd numbered physical instances have SFC only if + * previous even instance is fused off. + */ + if (GRAPHICS_VER(i915) == 12) + return (physical_vdbox % 2 == 0) || + !(BIT(physical_vdbox - 1) & vdbox_mask); + else if (GRAPHICS_VER(i915) == 11) + return logical_vdbox % 2 == 0; + + MISSING_CASE(GRAPHICS_VER(i915)); + return false; +} + /* * Determine which engines are fused off in our particular hardware. * Note that we have a catch-22 situation where we need to be able to access @@ -471,7 +521,14 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) if (GRAPHICS_VER(i915) < 11) return info->engine_mask; - media_fuse = ~intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); + /* + * On newer platforms the fusing register is called 'enable' and has + * enable semantics, while on older platforms it is called 'disable' + * and bits have disable semantices. + */ + media_fuse = intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); + if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) + media_fuse = ~media_fuse; vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> @@ -489,13 +546,9 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) continue; } - /* - * In Gen11, only even numbered logical VDBOXes are - * hooked up to an SFC (Scaler & Format Converter) unit. - * In TGL each VDBOX has access to an SFC. - */ - if (GRAPHICS_VER(i915) >= 12 || logical_vdbox++ % 2 == 0) + if (gen11_vdbox_has_sfc(i915, i, logical_vdbox, vdbox_mask)) gt->info.vdbox_sfc_access |= BIT(i); + logical_vdbox++; } drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n", vdbox_mask, VDBOX_MASK(gt)); @@ -585,9 +638,6 @@ void intel_engine_init_execlists(struct intel_engine_cs *engine) memset(execlists->pending, 0, sizeof(execlists->pending)); execlists->active = memset(execlists->inflight, 0, sizeof(execlists->inflight)); - - execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; } static void cleanup_status_page(struct intel_engine_cs *engine) @@ -714,11 +764,17 @@ static int engine_setup_common(struct intel_engine_cs *engine) goto err_status; } + engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL); + if (!engine->sched_engine) { + err = -ENOMEM; + goto err_sched_engine; + } + engine->sched_engine->private_data = engine; + err = intel_engine_init_cmd_parser(engine); if (err) goto err_cmd_parser; - intel_engine_init_active(engine, ENGINE_PHYSICAL); intel_engine_init_execlists(engine); intel_engine_init__pm(engine); intel_engine_init_retire(engine); @@ -737,7 +793,9 @@ static int engine_setup_common(struct intel_engine_cs *engine) return 0; err_cmd_parser: - intel_breadcrumbs_free(engine->breadcrumbs); + i915_sched_engine_put(engine->sched_engine); +err_sched_engine: + intel_breadcrumbs_put(engine->breadcrumbs); err_status: cleanup_status_page(engine); return err; @@ -775,11 +833,11 @@ static int measure_breadcrumb_dw(struct intel_context *ce) frame->rq.ring = &frame->ring; mutex_lock(&ce->timeline->mutex); - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); mutex_unlock(&ce->timeline->mutex); GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */ @@ -788,33 +846,13 @@ static int measure_breadcrumb_dw(struct intel_context *ce) return dw; } -void -intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass) -{ - INIT_LIST_HEAD(&engine->active.requests); - INIT_LIST_HEAD(&engine->active.hold); - - spin_lock_init(&engine->active.lock); - lockdep_set_subclass(&engine->active.lock, subclass); - - /* - * Due to an interesting quirk in lockdep's internal debug tracking, - * after setting a subclass we must ensure the lock is used. Otherwise, - * nr_unused_locks is incremented once too often. - */ -#ifdef CONFIG_DEBUG_LOCK_ALLOC - local_irq_disable(); - lock_map_acquire(&engine->active.lock.dep_map); - lock_map_release(&engine->active.lock.dep_map); - local_irq_enable(); -#endif -} - -static struct intel_context * -create_pinned_context(struct intel_engine_cs *engine, - unsigned int hwsp, - struct lock_class_key *key, - const char *name) +struct intel_context * +intel_engine_create_pinned_context(struct intel_engine_cs *engine, + struct i915_address_space *vm, + unsigned int ring_size, + unsigned int hwsp, + struct lock_class_key *key, + const char *name) { struct intel_context *ce; int err; @@ -825,6 +863,11 @@ create_pinned_context(struct intel_engine_cs *engine, __set_bit(CONTEXT_BARRIER_BIT, &ce->flags); ce->timeline = page_pack_bits(NULL, hwsp); + ce->ring = NULL; + ce->ring_size = ring_size; + + i915_vm_put(ce->vm); + ce->vm = i915_vm_get(vm); err = intel_context_pin(ce); /* perma-pin so it is always available */ if (err) { @@ -843,7 +886,7 @@ create_pinned_context(struct intel_engine_cs *engine, return ce; } -static void destroy_pinned_context(struct intel_context *ce) +void intel_engine_destroy_pinned_context(struct intel_context *ce) { struct intel_engine_cs *engine = ce->engine; struct i915_vma *hwsp = engine->status_page.vma; @@ -863,8 +906,9 @@ create_kernel_context(struct intel_engine_cs *engine) { static struct lock_class_key kernel; - return create_pinned_context(engine, I915_GEM_HWS_SEQNO_ADDR, - &kernel, "kernel_context"); + return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K, + I915_GEM_HWS_SEQNO_ADDR, + &kernel, "kernel_context"); } /** @@ -907,7 +951,7 @@ static int engine_init_common(struct intel_engine_cs *engine) return 0; err_context: - destroy_pinned_context(ce); + intel_engine_destroy_pinned_context(ce); return ret; } @@ -957,10 +1001,10 @@ int intel_engines_init(struct intel_gt *gt) */ void intel_engine_cleanup_common(struct intel_engine_cs *engine) { - GEM_BUG_ON(!list_empty(&engine->active.requests)); - tasklet_kill(&engine->execlists.tasklet); /* flush the callback */ + GEM_BUG_ON(!list_empty(&engine->sched_engine->requests)); - intel_breadcrumbs_free(engine->breadcrumbs); + i915_sched_engine_put(engine->sched_engine); + intel_breadcrumbs_put(engine->breadcrumbs); intel_engine_fini_retire(engine); intel_engine_cleanup_cmd_parser(engine); @@ -969,7 +1013,7 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) fput(engine->default_state); if (engine->kernel_context) - destroy_pinned_context(engine->kernel_context); + intel_engine_destroy_pinned_context(engine->kernel_context); GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); cleanup_status_page(engine); @@ -1105,45 +1149,8 @@ static u32 read_subslice_reg(const struct intel_engine_cs *engine, int slice, int subslice, i915_reg_t reg) { - struct drm_i915_private *i915 = engine->i915; - struct intel_uncore *uncore = engine->uncore; - u32 mcr_mask, mcr_ss, mcr, old_mcr, val; - enum forcewake_domains fw_domains; - - if (GRAPHICS_VER(i915) >= 11) { - mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; - mcr_ss = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); - } else { - mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; - mcr_ss = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); - } - - fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, - FW_REG_READ); - fw_domains |= intel_uncore_forcewake_for_reg(uncore, - GEN8_MCR_SELECTOR, - FW_REG_READ | FW_REG_WRITE); - - spin_lock_irq(&uncore->lock); - intel_uncore_forcewake_get__locked(uncore, fw_domains); - - old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); - - mcr &= ~mcr_mask; - mcr |= mcr_ss; - intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); - - val = intel_uncore_read_fw(uncore, reg); - - mcr &= ~mcr_mask; - mcr |= old_mcr & mcr_mask; - - intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); - - intel_uncore_forcewake_put__locked(uncore, fw_domains); - spin_unlock_irq(&uncore->lock); - - return val; + return intel_uncore_read_with_mcr_steering(engine->uncore, reg, + slice, subslice); } /* NB: please notice the memset */ @@ -1243,7 +1250,7 @@ static bool ring_is_idle(struct intel_engine_cs *engine) void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync) { - struct tasklet_struct *t = &engine->execlists.tasklet; + struct tasklet_struct *t = &engine->sched_engine->tasklet; if (!t->callback) return; @@ -1283,7 +1290,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) intel_engine_flush_submission(engine); /* ELSP is empty, but there are ready requests? E.g. after reset */ - if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) + if (!i915_sched_engine_is_empty(engine->sched_engine)) return false; /* Ring stopped? */ @@ -1314,6 +1321,30 @@ bool intel_engines_are_idle(struct intel_gt *gt) return true; } +bool intel_engine_irq_enable(struct intel_engine_cs *engine) +{ + if (!engine->irq_enable) + return false; + + /* Caller disables interrupts */ + spin_lock(&engine->gt->irq_lock); + engine->irq_enable(engine); + spin_unlock(&engine->gt->irq_lock); + + return true; +} + +void intel_engine_irq_disable(struct intel_engine_cs *engine) +{ + if (!engine->irq_disable) + return; + + /* Caller disables interrupts */ + spin_lock(&engine->gt->irq_lock); + engine->irq_disable(engine); + spin_unlock(&engine->gt->irq_lock); +} + void intel_engines_reset_default_submission(struct intel_gt *gt) { struct intel_engine_cs *engine; @@ -1349,7 +1380,7 @@ static struct intel_timeline *get_timeline(struct i915_request *rq) struct intel_timeline *tl; /* - * Even though we are holding the engine->active.lock here, there + * Even though we are holding the engine->sched_engine->lock here, there * is no control over the submission queue per-se and we are * inspecting the active state at a random point in time, with an * unknown queue. Play safe and make sure the timeline remains valid. @@ -1504,8 +1535,8 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n", yesno(test_bit(TASKLET_STATE_SCHED, - &engine->execlists.tasklet.state)), - enableddisabled(!atomic_read(&engine->execlists.tasklet.count)), + &engine->sched_engine->tasklet.state)), + enableddisabled(!atomic_read(&engine->sched_engine->tasklet.count)), repr_timer(&engine->execlists.preempt), repr_timer(&engine->execlists.timer)); @@ -1529,7 +1560,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, idx, hws[idx * 2], hws[idx * 2 + 1]); } - execlists_active_lock_bh(execlists); + i915_sched_engine_active_lock_bh(engine->sched_engine); rcu_read_lock(); for (port = execlists->active; (rq = *port); port++) { char hdr[160]; @@ -1560,7 +1591,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, i915_request_show(m, rq, hdr, 0); } rcu_read_unlock(); - execlists_active_unlock_bh(execlists); + i915_sched_engine_active_unlock_bh(engine->sched_engine); } else if (GRAPHICS_VER(dev_priv) > 6) { drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", ENGINE_READ(engine, RING_PP_DIR_BASE)); @@ -1650,6 +1681,98 @@ static void print_properties(struct intel_engine_cs *engine, read_ul(&engine->defaults, p->offset)); } +static void engine_dump_request(struct i915_request *rq, struct drm_printer *m, const char *msg) +{ + struct intel_timeline *tl = get_timeline(rq); + + i915_request_show(m, rq, msg, 0); + + drm_printf(m, "\t\tring->start: 0x%08x\n", + i915_ggtt_offset(rq->ring->vma)); + drm_printf(m, "\t\tring->head: 0x%08x\n", + rq->ring->head); + drm_printf(m, "\t\tring->tail: 0x%08x\n", + rq->ring->tail); + drm_printf(m, "\t\tring->emit: 0x%08x\n", + rq->ring->emit); + drm_printf(m, "\t\tring->space: 0x%08x\n", + rq->ring->space); + + if (tl) { + drm_printf(m, "\t\tring->hwsp: 0x%08x\n", + tl->hwsp_offset); + intel_timeline_put(tl); + } + + print_request_ring(m, rq); + + if (rq->context->lrc_reg_state) { + drm_printf(m, "Logical Ring Context:\n"); + hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE); + } +} + +void intel_engine_dump_active_requests(struct list_head *requests, + struct i915_request *hung_rq, + struct drm_printer *m) +{ + struct i915_request *rq; + const char *msg; + enum i915_request_state state; + + list_for_each_entry(rq, requests, sched.link) { + if (rq == hung_rq) + continue; + + state = i915_test_request_state(rq); + if (state < I915_REQUEST_QUEUED) + continue; + + if (state == I915_REQUEST_ACTIVE) + msg = "\t\tactive on engine"; + else + msg = "\t\tactive in queue"; + + engine_dump_request(rq, m, msg); + } +} + +static void engine_dump_active_requests(struct intel_engine_cs *engine, struct drm_printer *m) +{ + struct i915_request *hung_rq = NULL; + struct intel_context *ce; + bool guc; + + /* + * No need for an engine->irq_seqno_barrier() before the seqno reads. + * The GPU is still running so requests are still executing and any + * hardware reads will be out of date by the time they are reported. + * But the intention here is just to report an instantaneous snapshot + * so that's fine. + */ + lockdep_assert_held(&engine->sched_engine->lock); + + drm_printf(m, "\tRequests:\n"); + + guc = intel_uc_uses_guc_submission(&engine->gt->uc); + if (guc) { + ce = intel_engine_get_hung_context(engine); + if (ce) + hung_rq = intel_context_find_active_request(ce); + } else { + hung_rq = intel_engine_execlist_find_hung_request(engine); + } + + if (hung_rq) + engine_dump_request(hung_rq, m, "\t\thung"); + + if (guc) + intel_guc_dump_active_requests(engine, hung_rq, m); + else + intel_engine_dump_active_requests(&engine->sched_engine->requests, + hung_rq, m); +} + void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m, const char *header, ...) @@ -1694,41 +1817,12 @@ void intel_engine_dump(struct intel_engine_cs *engine, i915_reset_count(error)); print_properties(engine, m); - drm_printf(m, "\tRequests:\n"); + spin_lock_irqsave(&engine->sched_engine->lock, flags); + engine_dump_active_requests(engine, m); - spin_lock_irqsave(&engine->active.lock, flags); - rq = intel_engine_find_active_request(engine); - if (rq) { - struct intel_timeline *tl = get_timeline(rq); - - i915_request_show(m, rq, "\t\tactive ", 0); - - drm_printf(m, "\t\tring->start: 0x%08x\n", - i915_ggtt_offset(rq->ring->vma)); - drm_printf(m, "\t\tring->head: 0x%08x\n", - rq->ring->head); - drm_printf(m, "\t\tring->tail: 0x%08x\n", - rq->ring->tail); - drm_printf(m, "\t\tring->emit: 0x%08x\n", - rq->ring->emit); - drm_printf(m, "\t\tring->space: 0x%08x\n", - rq->ring->space); - - if (tl) { - drm_printf(m, "\t\tring->hwsp: 0x%08x\n", - tl->hwsp_offset); - intel_timeline_put(tl); - } - - print_request_ring(m, rq); - - if (rq->context->lrc_reg_state) { - drm_printf(m, "Logical Ring Context:\n"); - hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE); - } - } - drm_printf(m, "\tOn hold?: %lu\n", list_count(&engine->active.hold)); - spin_unlock_irqrestore(&engine->active.lock, flags); + drm_printf(m, "\tOn hold?: %lu\n", + list_count(&engine->sched_engine->hold)); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); @@ -1785,19 +1879,33 @@ ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now) return total; } -static bool match_ring(struct i915_request *rq) +struct intel_context * +intel_engine_create_virtual(struct intel_engine_cs **siblings, + unsigned int count) { - u32 ring = ENGINE_READ(rq->engine, RING_START); + if (count == 0) + return ERR_PTR(-EINVAL); + + if (count == 1) + return intel_context_create(siblings[0]); - return ring == i915_ggtt_offset(rq->ring->vma); + GEM_BUG_ON(!siblings[0]->cops->create_virtual); + return siblings[0]->cops->create_virtual(siblings, count); } struct i915_request * -intel_engine_find_active_request(struct intel_engine_cs *engine) +intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine) { struct i915_request *request, *active = NULL; /* + * This search does not work in GuC submission mode. However, the GuC + * will report the hanging context directly to the driver itself. So + * the driver should never get here when in GuC mode. + */ + GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc)); + + /* * We are called by the error capture, reset and to dump engine * state at random points in time. In particular, note that neither is * crucially ordered with an interrupt. After a hang, the GPU is dead @@ -1808,7 +1916,7 @@ intel_engine_find_active_request(struct intel_engine_cs *engine) * At all other times, we must assume the GPU is still running, but * we only care about the snapshot of this moment. */ - lockdep_assert_held(&engine->active.lock); + lockdep_assert_held(&engine->sched_engine->lock); rcu_read_lock(); request = execlists_active(&engine->execlists); @@ -1826,15 +1934,9 @@ intel_engine_find_active_request(struct intel_engine_cs *engine) if (active) return active; - list_for_each_entry(request, &engine->active.requests, sched.link) { - if (__i915_request_is_complete(request)) - continue; - - if (!__i915_request_has_started(request)) - continue; - - /* More than one preemptible request may match! */ - if (!match_ring(request)) + list_for_each_entry(request, &engine->sched_engine->requests, + sched.link) { + if (i915_test_request_state(request) != I915_REQUEST_ACTIVE) continue; active = request; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index b99ac41695f3..74775ae961b2 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -70,12 +70,38 @@ static void show_heartbeat(const struct i915_request *rq, { struct drm_printer p = drm_debug_printer("heartbeat"); - intel_engine_dump(engine, &p, - "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n", - engine->name, - rq->fence.context, - rq->fence.seqno, - rq->sched.attr.priority); + if (!rq) { + intel_engine_dump(engine, &p, + "%s heartbeat not ticking\n", + engine->name); + } else { + intel_engine_dump(engine, &p, + "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n", + engine->name, + rq->fence.context, + rq->fence.seqno, + rq->sched.attr.priority); + } +} + +static void +reset_engine(struct intel_engine_cs *engine, struct i915_request *rq) +{ + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + show_heartbeat(rq, engine); + + if (intel_engine_uses_guc(engine)) + /* + * GuC itself is toast or GuC's hang detection + * is disabled. Either way, need to find the + * hang culprit manually. + */ + intel_guc_find_hung_context(engine); + + intel_gt_handle_error(engine->gt, engine->mask, + I915_ERROR_CAPTURE, + "stopped heartbeat on %s", + engine->name); } static void heartbeat(struct work_struct *wrk) @@ -102,6 +128,11 @@ static void heartbeat(struct work_struct *wrk) if (intel_gt_is_wedged(engine->gt)) goto out; + if (i915_sched_engine_disabled(engine->sched_engine)) { + reset_engine(engine, engine->heartbeat.systole); + goto out; + } + if (engine->heartbeat.systole) { long delay = READ_ONCE(engine->props.heartbeat_interval_ms); @@ -121,7 +152,7 @@ static void heartbeat(struct work_struct *wrk) * but all other contexts, including the kernel * context are stuck waiting for the signal. */ - } else if (engine->schedule && + } else if (engine->sched_engine->schedule && rq->sched.attr.priority < I915_PRIORITY_BARRIER) { /* * Gradually raise the priority of the heartbeat to @@ -136,16 +167,10 @@ static void heartbeat(struct work_struct *wrk) attr.priority = I915_PRIORITY_BARRIER; local_bh_disable(); - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); local_bh_enable(); } else { - if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) - show_heartbeat(rq, engine); - - intel_gt_handle_error(engine->gt, engine->mask, - I915_ERROR_CAPTURE, - "stopped heartbeat on %s", - engine->name); + reset_engine(engine, rq); } rq->emitted_jiffies = jiffies; @@ -194,6 +219,25 @@ void intel_engine_park_heartbeat(struct intel_engine_cs *engine) i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); } +void intel_gt_unpark_heartbeats(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, gt, id) + if (intel_engine_pm_is_awake(engine)) + intel_engine_unpark_heartbeat(engine); +} + +void intel_gt_park_heartbeats(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, gt, id) + intel_engine_park_heartbeat(engine); +} + void intel_engine_init_heartbeat(struct intel_engine_cs *engine) { INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.h b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.h index a488ea3e84a3..5da6d809a87a 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.h @@ -7,6 +7,7 @@ #define INTEL_ENGINE_HEARTBEAT_H struct intel_engine_cs; +struct intel_gt; void intel_engine_init_heartbeat(struct intel_engine_cs *engine); @@ -16,6 +17,9 @@ int intel_engine_set_heartbeat(struct intel_engine_cs *engine, void intel_engine_park_heartbeat(struct intel_engine_cs *engine); void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine); +void intel_gt_park_heartbeats(struct intel_gt *gt); +void intel_gt_unpark_heartbeats(struct intel_gt *gt); + int intel_engine_pulse(struct intel_engine_cs *engine); int intel_engine_flush_barriers(struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 47f4397095e5..1f07ac4e0672 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -275,13 +275,11 @@ static int __engine_park(struct intel_wakeref *wf) intel_breadcrumbs_park(engine->breadcrumbs); /* Must be reset upon idling, or we may miss the busy wakeup. */ - GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); + GEM_BUG_ON(engine->sched_engine->queue_priority_hint != INT_MIN); if (engine->park) engine->park(engine); - engine->execlists.no_priolist = false; - /* While gt calls i915_vma_parked(), we have to break the lock cycle */ intel_gt_pm_put_async(engine->gt); return 0; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index e113f93b3274..ed91bcff20eb 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -21,32 +21,20 @@ #include "i915_pmu.h" #include "i915_priolist_types.h" #include "i915_selftest.h" -#include "intel_breadcrumbs_types.h" #include "intel_sseu.h" #include "intel_timeline_types.h" #include "intel_uncore.h" #include "intel_wakeref.h" #include "intel_workarounds_types.h" -/* Legacy HW Engine ID */ - -#define RCS0_HW 0 -#define VCS0_HW 1 -#define BCS0_HW 2 -#define VECS0_HW 3 -#define VCS1_HW 4 -#define VCS2_HW 6 -#define VCS3_HW 7 -#define VECS1_HW 12 - -/* Gen11+ HW Engine class + instance */ +/* HW Engine class + instance */ #define RENDER_CLASS 0 #define VIDEO_DECODE_CLASS 1 #define VIDEO_ENHANCEMENT_CLASS 2 #define COPY_ENGINE_CLASS 3 #define OTHER_CLASS 4 #define MAX_ENGINE_CLASS 4 -#define MAX_ENGINE_INSTANCE 3 +#define MAX_ENGINE_INSTANCE 7 #define I915_MAX_SLICES 3 #define I915_MAX_SUBSLICES 8 @@ -59,11 +47,13 @@ struct drm_i915_reg_table; struct i915_gem_context; struct i915_request; struct i915_sched_attr; +struct i915_sched_engine; struct intel_gt; struct intel_ring; struct intel_uncore; +struct intel_breadcrumbs; -typedef u8 intel_engine_mask_t; +typedef u32 intel_engine_mask_t; #define ALL_ENGINES ((intel_engine_mask_t)~0ul) struct intel_hw_status_page { @@ -100,8 +90,8 @@ struct i915_ctx_workarounds { struct i915_vma *vma; }; -#define I915_MAX_VCS 4 -#define I915_MAX_VECS 2 +#define I915_MAX_VCS 8 +#define I915_MAX_VECS 4 /* * Engine IDs definitions. @@ -114,9 +104,15 @@ enum intel_engine_id { VCS1, VCS2, VCS3, + VCS4, + VCS5, + VCS6, + VCS7, #define _VCS(n) (VCS0 + (n)) VECS0, VECS1, + VECS2, + VECS3, #define _VECS(n) (VECS0 + (n)) I915_NUM_ENGINES #define INVALID_ENGINE ((enum intel_engine_id)-1) @@ -138,11 +134,6 @@ struct st_preempt_hang { */ struct intel_engine_execlists { /** - * @tasklet: softirq tasklet for bottom handler - */ - struct tasklet_struct tasklet; - - /** * @timer: kick the current context if its timeslice expires */ struct timer_list timer; @@ -153,11 +144,6 @@ struct intel_engine_execlists { struct timer_list preempt; /** - * @default_priolist: priority list for I915_PRIORITY_NORMAL - */ - struct i915_priolist default_priolist; - - /** * @ccid: identifier for contexts submitted to this engine */ u32 ccid; @@ -192,11 +178,6 @@ struct intel_engine_execlists { u32 reset_ccid; /** - * @no_priolist: priority lists disabled - */ - bool no_priolist; - - /** * @submit_reg: gen-specific execlist submission register * set to the ExecList Submission Port (elsp) register pre-Gen11 and to * the ExecList Submission Queue Contents register array for Gen11+ @@ -238,23 +219,10 @@ struct intel_engine_execlists { unsigned int port_mask; /** - * @queue_priority_hint: Highest pending priority. - * - * When we add requests into the queue, or adjust the priority of - * executing requests, we compute the maximum priority of those - * pending requests. We can then use this value to determine if - * we need to preempt the executing requests to service the queue. - * However, since the we may have recorded the priority of an inflight - * request we wanted to preempt but since completed, at the time of - * dequeuing the priority hint may no longer may match the highest - * available request priority. + * @virtual: Queue of requets on a virtual engine, sorted by priority. + * Each RB entry is a struct i915_priolist containing a list of requests + * of the same priority. */ - int queue_priority_hint; - - /** - * @queue: queue of requests, in priority lists - */ - struct rb_root_cached queue; struct rb_root_cached virtual; /** @@ -295,7 +263,6 @@ struct intel_engine_cs { enum intel_engine_id id; enum intel_engine_id legacy_idx; - unsigned int hw_id; unsigned int guc_id; intel_engine_mask_t mask; @@ -326,15 +293,13 @@ struct intel_engine_cs { struct intel_sseu sseu; - struct { - spinlock_t lock; - struct list_head requests; - struct list_head hold; /* ready requests, but on hold */ - } active; + struct i915_sched_engine *sched_engine; /* keep a request in reserve for a [pm] barrier under oom */ struct i915_request *request_pool; + struct intel_context *hung_ce; + struct llist_head barrier_tasks; struct intel_context *kernel_context; /* pinned */ @@ -419,6 +384,8 @@ struct intel_engine_cs { void (*park)(struct intel_engine_cs *engine); void (*unpark)(struct intel_engine_cs *engine); + void (*bump_serial)(struct intel_engine_cs *engine); + void (*set_default_submission)(struct intel_engine_cs *engine); const struct intel_context_ops *cops; @@ -447,22 +414,13 @@ struct intel_engine_cs { */ void (*submit_request)(struct i915_request *rq); - /* - * Called on signaling of a SUBMIT_FENCE, passing along the signaling - * request down to the bonded pairs. - */ - void (*bond_execute)(struct i915_request *rq, - struct dma_fence *signal); + void (*release)(struct intel_engine_cs *engine); /* - * Call when the priority on a request has changed and it and its - * dependencies may need rescheduling. Note the request itself may - * not be ready to run! + * Add / remove request from engine active tracking */ - void (*schedule)(struct i915_request *request, - const struct i915_sched_attr *attr); - - void (*release)(struct intel_engine_cs *engine); + void (*add_active_request)(struct i915_request *rq); + void (*remove_active_request)(struct i915_request *rq); struct intel_engine_execlists execlists; @@ -485,6 +443,7 @@ struct intel_engine_cs { #define I915_ENGINE_IS_VIRTUAL BIT(5) #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) +#define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8) unsigned int flags; /* diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index 3cca7ea2d6ea..8f8bea08e734 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -11,6 +11,7 @@ #include "intel_engine.h" #include "intel_engine_user.h" #include "intel_gt.h" +#include "uc/intel_guc_submission.h" struct intel_engine_cs * intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance) @@ -108,13 +109,16 @@ static void set_scheduler_caps(struct drm_i915_private *i915) for_each_uabi_engine(engine, i915) { /* all engines must agree! */ int i; - if (engine->schedule) + if (engine->sched_engine->schedule) enabled |= (I915_SCHEDULER_CAP_ENABLED | I915_SCHEDULER_CAP_PRIORITY); else disabled |= (I915_SCHEDULER_CAP_ENABLED | I915_SCHEDULER_CAP_PRIORITY); + if (intel_uc_uses_guc_submission(&i915->gt.uc)) + enabled |= I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP; + for (i = 0; i < ARRAY_SIZE(map); i++) { if (engine->flags & BIT(map[i].engine)) enabled |= BIT(map[i].sched); diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index fc77592d88a9..de5f9c86b9a4 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -114,6 +114,7 @@ #include "gen8_engine_cs.h" #include "intel_breadcrumbs.h" #include "intel_context.h" +#include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" #include "intel_engine_stats.h" #include "intel_execlists_submission.h" @@ -153,6 +154,12 @@ #define GEN12_CSB_CTX_VALID(csb_dw) \ (FIELD_GET(GEN12_CSB_SW_CTX_ID_MASK, csb_dw) != GEN12_IDLE_CTX_ID) +#define XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE BIT(1) /* upper csb dword */ +#define XEHP_CSB_SW_CTX_ID_MASK GENMASK(31, 10) +#define XEHP_IDLE_CTX_ID 0xFFFF +#define XEHP_CSB_CTX_VALID(csb_dw) \ + (FIELD_GET(XEHP_CSB_SW_CTX_ID_MASK, csb_dw) != XEHP_IDLE_CTX_ID) + /* Typical size of the average request (2 pipecontrols and a MI_BB) */ #define EXECLISTS_REQUEST_SIZE 64 /* bytes */ @@ -182,18 +189,6 @@ struct virtual_engine { int prio; } nodes[I915_NUM_ENGINES]; - /* - * Keep track of bonded pairs -- restrictions upon on our selection - * of physical engines any particular request may be submitted to. - * If we receive a submit-fence from a master engine, we will only - * use one of sibling_mask physical engines. - */ - struct ve_bond { - const struct intel_engine_cs *master; - intel_engine_mask_t sibling_mask; - } *bonds; - unsigned int num_bonds; - /* And finally, which physical engines this virtual engine maps onto. */ unsigned int num_siblings; struct intel_engine_cs *siblings[]; @@ -205,6 +200,9 @@ static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine) return container_of(engine, struct virtual_engine, base); } +static struct intel_context * +execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count); + static struct i915_request * __active_request(const struct intel_timeline * const tl, struct i915_request *rq, @@ -273,11 +271,11 @@ static int effective_prio(const struct i915_request *rq) return prio; } -static int queue_prio(const struct intel_engine_execlists *execlists) +static int queue_prio(const struct i915_sched_engine *sched_engine) { struct rb_node *rb; - rb = rb_first_cached(&execlists->queue); + rb = rb_first_cached(&sched_engine->queue); if (!rb) return INT_MIN; @@ -318,14 +316,14 @@ static bool need_preempt(const struct intel_engine_cs *engine, * to preserve FIFO ordering of dependencies. */ last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1); - if (engine->execlists.queue_priority_hint <= last_prio) + if (engine->sched_engine->queue_priority_hint <= last_prio) return false; /* * Check against the first request in ELSP[1], it will, thanks to the * power of PI, be the highest priority of that context. */ - if (!list_is_last(&rq->sched.link, &engine->active.requests) && + if (!list_is_last(&rq->sched.link, &engine->sched_engine->requests) && rq_prio(list_next_entry(rq, sched.link)) > last_prio) return true; @@ -340,7 +338,7 @@ static bool need_preempt(const struct intel_engine_cs *engine, * context, it's priority would not exceed ELSP[0] aka last_prio. */ return max(virtual_prio(&engine->execlists), - queue_prio(&engine->execlists)) > last_prio; + queue_prio(engine->sched_engine)) > last_prio; } __maybe_unused static bool @@ -367,10 +365,10 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) struct list_head *pl; int prio = I915_PRIORITY_INVALID; - lockdep_assert_held(&engine->active.lock); + lockdep_assert_held(&engine->sched_engine->lock); list_for_each_entry_safe_reverse(rq, rn, - &engine->active.requests, + &engine->sched_engine->requests, sched.link) { if (__i915_request_is_complete(rq)) { list_del_init(&rq->sched.link); @@ -382,9 +380,10 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); if (rq_prio(rq) != prio) { prio = rq_prio(rq); - pl = i915_sched_lookup_priolist(engine, prio); + pl = i915_sched_lookup_priolist(engine->sched_engine, + prio); } - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); + GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine)); list_move(&rq->sched.link, pl); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); @@ -489,6 +488,16 @@ __execlists_schedule_in(struct i915_request *rq) /* Use a fixed tag for OA and friends */ GEM_BUG_ON(ce->tag <= BITS_PER_LONG); ce->lrc.ccid = ce->tag; + } else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) { + /* We don't need a strict matching tag, just different values */ + unsigned int tag = ffs(READ_ONCE(engine->context_tag)); + + GEM_BUG_ON(tag == 0 || tag >= BITS_PER_LONG); + clear_bit(tag - 1, &engine->context_tag); + ce->lrc.ccid = tag << (XEHP_SW_CTX_ID_SHIFT - 32); + + BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID); + } else { /* We don't need a strict matching tag, just different values */ unsigned int tag = __ffs(engine->context_tag); @@ -534,13 +543,13 @@ resubmit_virtual_request(struct i915_request *rq, struct virtual_engine *ve) { struct intel_engine_cs *engine = rq->engine; - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); WRITE_ONCE(rq->engine, &ve->base); ve->base.submit_request(rq); - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); } static void kick_siblings(struct i915_request *rq, struct intel_context *ce) @@ -569,7 +578,7 @@ static void kick_siblings(struct i915_request *rq, struct intel_context *ce) resubmit_virtual_request(rq, ve); if (READ_ONCE(ve->request)) - tasklet_hi_schedule(&ve->base.execlists.tasklet); + tasklet_hi_schedule(&ve->base.sched_engine->tasklet); } static void __execlists_schedule_out(struct i915_request * const rq, @@ -579,7 +588,7 @@ static void __execlists_schedule_out(struct i915_request * const rq, unsigned int ccid; /* - * NB process_csb() is not under the engine->active.lock and hence + * NB process_csb() is not under the engine->sched_engine->lock and hence * schedule_out can race with schedule_in meaning that we should * refrain from doing non-trivial work here. */ @@ -599,8 +608,14 @@ static void __execlists_schedule_out(struct i915_request * const rq, intel_engine_add_retire(engine, ce->timeline); ccid = ce->lrc.ccid; - ccid >>= GEN11_SW_CTX_ID_SHIFT - 32; - ccid &= GEN12_MAX_CONTEXT_HW_ID; + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) { + ccid >>= XEHP_SW_CTX_ID_SHIFT - 32; + ccid &= XEHP_MAX_CONTEXT_HW_ID; + } else { + ccid >>= GEN11_SW_CTX_ID_SHIFT - 32; + ccid &= GEN12_MAX_CONTEXT_HW_ID; + } + if (ccid < BITS_PER_LONG) { GEM_BUG_ON(ccid == 0); GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag)); @@ -738,9 +753,9 @@ trace_ports(const struct intel_engine_execlists *execlists, } static bool -reset_in_progress(const struct intel_engine_execlists *execlists) +reset_in_progress(const struct intel_engine_cs *engine) { - return unlikely(!__tasklet_is_enabled(&execlists->tasklet)); + return unlikely(!__tasklet_is_enabled(&engine->sched_engine->tasklet)); } static __maybe_unused noinline bool @@ -756,7 +771,7 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, trace_ports(execlists, msg, execlists->pending); /* We may be messing around with the lists during reset, lalala */ - if (reset_in_progress(execlists)) + if (reset_in_progress(engine)) return true; if (!execlists->pending[0]) { @@ -1096,7 +1111,8 @@ static void defer_active(struct intel_engine_cs *engine) if (!rq) return; - defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq))); + defer_request(rq, i915_sched_lookup_priolist(engine->sched_engine, + rq_prio(rq))); } static bool @@ -1133,13 +1149,14 @@ static bool needs_timeslice(const struct intel_engine_cs *engine, return false; /* If ELSP[1] is occupied, always check to see if worth slicing */ - if (!list_is_last_rcu(&rq->sched.link, &engine->active.requests)) { + if (!list_is_last_rcu(&rq->sched.link, + &engine->sched_engine->requests)) { ENGINE_TRACE(engine, "timeslice required for second inflight context\n"); return true; } /* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */ - if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) { + if (!i915_sched_engine_is_empty(engine->sched_engine)) { ENGINE_TRACE(engine, "timeslice required for queue\n"); return true; } @@ -1187,7 +1204,7 @@ static void start_timeslice(struct intel_engine_cs *engine) * its timeslice, so recheck. */ if (!timer_pending(&el->timer)) - tasklet_hi_schedule(&el->tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); return; } @@ -1236,6 +1253,7 @@ static bool completed(const struct i915_request *rq) static void execlists_dequeue(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; + struct i915_sched_engine * const sched_engine = engine->sched_engine; struct i915_request **port = execlists->pending; struct i915_request ** const last_port = port + execlists->port_mask; struct i915_request *last, * const *active; @@ -1265,7 +1283,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * and context switches) submission. */ - spin_lock(&engine->active.lock); + spin_lock(&sched_engine->lock); /* * If the queue is higher priority than the last @@ -1287,7 +1305,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) last->fence.context, last->fence.seqno, last->sched.attr.priority, - execlists->queue_priority_hint); + sched_engine->queue_priority_hint); record_preemption(execlists); /* @@ -1313,7 +1331,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) yesno(timer_expired(&execlists->timer)), last->fence.context, last->fence.seqno, rq_prio(last), - execlists->queue_priority_hint, + sched_engine->queue_priority_hint, yesno(timeslice_yield(execlists, last))); /* @@ -1365,7 +1383,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * Even if ELSP[1] is occupied and not worthy * of timeslices, our queue might be. */ - spin_unlock(&engine->active.lock); + spin_unlock(&sched_engine->lock); return; } } @@ -1375,7 +1393,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) while ((ve = first_virtual_engine(engine))) { struct i915_request *rq; - spin_lock(&ve->base.active.lock); + spin_lock(&ve->base.sched_engine->lock); rq = ve->request; if (unlikely(!virtual_matches(ve, rq, engine))) @@ -1384,14 +1402,14 @@ static void execlists_dequeue(struct intel_engine_cs *engine) GEM_BUG_ON(rq->engine != &ve->base); GEM_BUG_ON(rq->context != &ve->context); - if (unlikely(rq_prio(rq) < queue_prio(execlists))) { - spin_unlock(&ve->base.active.lock); + if (unlikely(rq_prio(rq) < queue_prio(sched_engine))) { + spin_unlock(&ve->base.sched_engine->lock); break; } if (last && !can_merge_rq(last, rq)) { - spin_unlock(&ve->base.active.lock); - spin_unlock(&engine->active.lock); + spin_unlock(&ve->base.sched_engine->lock); + spin_unlock(&engine->sched_engine->lock); return; /* leave this for another sibling */ } @@ -1405,7 +1423,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) yesno(engine != ve->siblings[0])); WRITE_ONCE(ve->request, NULL); - WRITE_ONCE(ve->base.execlists.queue_priority_hint, INT_MIN); + WRITE_ONCE(ve->base.sched_engine->queue_priority_hint, INT_MIN); rb = &ve->nodes[engine->id].rb; rb_erase_cached(rb, &execlists->virtual); @@ -1437,7 +1455,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) i915_request_put(rq); unlock: - spin_unlock(&ve->base.active.lock); + spin_unlock(&ve->base.sched_engine->lock); /* * Hmm, we have a bunch of virtual engine requests, @@ -1450,7 +1468,7 @@ unlock: break; } - while ((rb = rb_first_cached(&execlists->queue))) { + while ((rb = rb_first_cached(&sched_engine->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; @@ -1529,7 +1547,7 @@ unlock: } } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &sched_engine->queue); i915_priolist_free(p); } done: @@ -1551,8 +1569,9 @@ done: * request triggering preemption on the next dequeue (or subsequent * interrupt for secondary ports). */ - execlists->queue_priority_hint = queue_prio(execlists); - spin_unlock(&engine->active.lock); + sched_engine->queue_priority_hint = queue_prio(sched_engine); + i915_sched_engine_reset_on_empty(sched_engine); + spin_unlock(&sched_engine->lock); /* * We can skip poking the HW if we ended up with exactly the same set @@ -1655,13 +1674,24 @@ static void invalidate_csb_entries(const u64 *first, const u64 *last) * bits 44-46: reserved * bits 47-57: sw context id of the lrc the GT switched away from * bits 58-63: sw counter of the lrc the GT switched away from + * + * Xe_HP csb shuffles things around compared to TGL: + * + * bits 0-3: context switch detail (same possible values as TGL) + * bits 4-9: engine instance + * bits 10-25: sw context id of the lrc the GT switched to + * bits 26-31: sw counter of the lrc the GT switched to + * bit 32: semaphore wait mode (poll or signal), Only valid when + * switch detail is set to "wait on semaphore" + * bit 33: switched to new queue + * bits 34-41: wait detail (for switch detail 1 to 4) + * bits 42-57: sw context id of the lrc the GT switched away from + * bits 58-63: sw counter of the lrc the GT switched away from */ -static bool gen12_csb_parse(const u64 csb) +static inline bool +__gen12_csb_parse(bool ctx_to_valid, bool ctx_away_valid, bool new_queue, + u8 switch_detail) { - bool ctx_away_valid = GEN12_CSB_CTX_VALID(upper_32_bits(csb)); - bool new_queue = - lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE; - /* * The context switch detail is not guaranteed to be 5 when a preemption * occurs, so we can't just check for that. The check below works for @@ -1670,7 +1700,7 @@ static bool gen12_csb_parse(const u64 csb) * would require some extra handling, but we don't support that. */ if (!ctx_away_valid || new_queue) { - GEM_BUG_ON(!GEN12_CSB_CTX_VALID(lower_32_bits(csb))); + GEM_BUG_ON(!ctx_to_valid); return true; } @@ -1679,10 +1709,26 @@ static bool gen12_csb_parse(const u64 csb) * context switch on an unsuccessful wait instruction since we always * use polling mode. */ - GEM_BUG_ON(GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb))); + GEM_BUG_ON(switch_detail); return false; } +static bool xehp_csb_parse(const u64 csb) +{ + return __gen12_csb_parse(XEHP_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */ + XEHP_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */ + upper_32_bits(csb) & XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE, + GEN12_CTX_SWITCH_DETAIL(lower_32_bits(csb))); +} + +static bool gen12_csb_parse(const u64 csb) +{ + return __gen12_csb_parse(GEN12_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */ + GEN12_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */ + lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE, + GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb))); +} + static bool gen8_csb_parse(const u64 csb) { return csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED); @@ -1767,8 +1813,8 @@ process_csb(struct intel_engine_cs *engine, struct i915_request **inactive) * access. Either we are inside the tasklet, or the tasklet is disabled * and we assume that is only inside the reset paths and so serialised. */ - GEM_BUG_ON(!tasklet_is_locked(&execlists->tasklet) && - !reset_in_progress(execlists)); + GEM_BUG_ON(!tasklet_is_locked(&engine->sched_engine->tasklet) && + !reset_in_progress(engine)); /* * Note that csb_write, csb_status may be either in HWSP or mmio. @@ -1847,7 +1893,9 @@ process_csb(struct intel_engine_cs *engine, struct i915_request **inactive) ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n", head, upper_32_bits(csb), lower_32_bits(csb)); - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + promote = xehp_csb_parse(csb); + else if (GRAPHICS_VER(engine->i915) >= 12) promote = gen12_csb_parse(csb); else promote = gen8_csb_parse(csb); @@ -1979,7 +2027,8 @@ static void __execlists_hold(struct i915_request *rq) __i915_request_unsubmit(rq); clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); - list_move_tail(&rq->sched.link, &rq->engine->active.hold); + list_move_tail(&rq->sched.link, + &rq->engine->sched_engine->hold); i915_request_set_hold(rq); RQ_TRACE(rq, "on hold\n"); @@ -2016,7 +2065,7 @@ static bool execlists_hold(struct intel_engine_cs *engine, if (i915_request_on_hold(rq)) return false; - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); if (__i915_request_is_complete(rq)) { /* too late! */ rq = NULL; @@ -2032,10 +2081,10 @@ static bool execlists_hold(struct intel_engine_cs *engine, GEM_BUG_ON(i915_request_on_hold(rq)); GEM_BUG_ON(rq->engine != engine); __execlists_hold(rq); - GEM_BUG_ON(list_empty(&engine->active.hold)); + GEM_BUG_ON(list_empty(&engine->sched_engine->hold)); unlock: - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); return rq; } @@ -2079,7 +2128,7 @@ static void __execlists_unhold(struct i915_request *rq) i915_request_clear_hold(rq); list_move_tail(&rq->sched.link, - i915_sched_lookup_priolist(rq->engine, + i915_sched_lookup_priolist(rq->engine->sched_engine, rq_prio(rq))); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); @@ -2115,7 +2164,7 @@ static void __execlists_unhold(struct i915_request *rq) static void execlists_unhold(struct intel_engine_cs *engine, struct i915_request *rq) { - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); /* * Move this request back to the priority queue, and all of its @@ -2123,12 +2172,12 @@ static void execlists_unhold(struct intel_engine_cs *engine, */ __execlists_unhold(rq); - if (rq_prio(rq) > engine->execlists.queue_priority_hint) { - engine->execlists.queue_priority_hint = rq_prio(rq); - tasklet_hi_schedule(&engine->execlists.tasklet); + if (rq_prio(rq) > engine->sched_engine->queue_priority_hint) { + engine->sched_engine->queue_priority_hint = rq_prio(rq); + tasklet_hi_schedule(&engine->sched_engine->tasklet); } - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); } struct execlists_capture { @@ -2258,13 +2307,13 @@ static void execlists_capture(struct intel_engine_cs *engine) if (!cap) return; - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); cap->rq = active_context(engine, active_ccid(engine)); if (cap->rq) { cap->rq = active_request(cap->rq->context->timeline, cap->rq); cap->rq = i915_request_get_rcu(cap->rq); } - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); if (!cap->rq) goto err_free; @@ -2316,13 +2365,13 @@ static void execlists_reset(struct intel_engine_cs *engine, const char *msg) ENGINE_TRACE(engine, "reset for %s\n", msg); /* Mark this tasklet as disabled to avoid waiting for it to complete */ - tasklet_disable_nosync(&engine->execlists.tasklet); + tasklet_disable_nosync(&engine->sched_engine->tasklet); ring_set_paused(engine, 1); /* Freeze the current request in place */ execlists_capture(engine); intel_engine_reset(engine, msg); - tasklet_enable(&engine->execlists.tasklet); + tasklet_enable(&engine->sched_engine->tasklet); clear_and_wake_up_bit(bit, lock); } @@ -2345,8 +2394,9 @@ static bool preempt_timeout(const struct intel_engine_cs *const engine) */ static void execlists_submission_tasklet(struct tasklet_struct *t) { - struct intel_engine_cs * const engine = - from_tasklet(engine, t, execlists.tasklet); + struct i915_sched_engine *sched_engine = + from_tasklet(sched_engine, t, tasklet); + struct intel_engine_cs * const engine = sched_engine->private_data; struct i915_request *post[2 * EXECLIST_MAX_PORTS]; struct i915_request **inactive; @@ -2421,13 +2471,16 @@ static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir) intel_engine_signal_breadcrumbs(engine); if (tasklet) - tasklet_hi_schedule(&engine->execlists.tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); } static void __execlists_kick(struct intel_engine_execlists *execlists) { + struct intel_engine_cs *engine = + container_of(execlists, typeof(*engine), execlists); + /* Kick the tasklet for some interrupt coalescing and reset handling */ - tasklet_hi_schedule(&execlists->tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); } #define execlists_kick(t, member) \ @@ -2448,19 +2501,20 @@ static void queue_request(struct intel_engine_cs *engine, { GEM_BUG_ON(!list_empty(&rq->sched.link)); list_add_tail(&rq->sched.link, - i915_sched_lookup_priolist(engine, rq_prio(rq))); + i915_sched_lookup_priolist(engine->sched_engine, + rq_prio(rq))); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); } static bool submit_queue(struct intel_engine_cs *engine, const struct i915_request *rq) { - struct intel_engine_execlists *execlists = &engine->execlists; + struct i915_sched_engine *sched_engine = engine->sched_engine; - if (rq_prio(rq) <= execlists->queue_priority_hint) + if (rq_prio(rq) <= sched_engine->queue_priority_hint) return false; - execlists->queue_priority_hint = rq_prio(rq); + sched_engine->queue_priority_hint = rq_prio(rq); return true; } @@ -2468,7 +2522,7 @@ static bool ancestor_on_hold(const struct intel_engine_cs *engine, const struct i915_request *rq) { GEM_BUG_ON(i915_request_on_hold(rq)); - return !list_empty(&engine->active.hold) && hold_request(rq); + return !list_empty(&engine->sched_engine->hold) && hold_request(rq); } static void execlists_submit_request(struct i915_request *request) @@ -2477,23 +2531,24 @@ static void execlists_submit_request(struct i915_request *request) unsigned long flags; /* Will be called from irq-context when using foreign fences. */ - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); if (unlikely(ancestor_on_hold(engine, request))) { RQ_TRACE(request, "ancestor on hold\n"); - list_add_tail(&request->sched.link, &engine->active.hold); + list_add_tail(&request->sched.link, + &engine->sched_engine->hold); i915_request_set_hold(request); } else { queue_request(engine, request); - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); + GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine)); GEM_BUG_ON(list_empty(&request->sched.link)); if (submit_queue(engine, request)) __execlists_kick(&engine->execlists); } - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } static int @@ -2533,11 +2588,26 @@ static int execlists_context_alloc(struct intel_context *ce) return lrc_alloc(ce, ce->engine); } +static void execlists_context_cancel_request(struct intel_context *ce, + struct i915_request *rq) +{ + struct intel_engine_cs *engine = NULL; + + i915_request_active_engine(rq, &engine); + + if (engine && intel_engine_pulse(engine)) + intel_gt_handle_error(engine->gt, engine->mask, 0, + "request cancellation by %s", + current->comm); +} + static const struct intel_context_ops execlists_context_ops = { .flags = COPS_HAS_INFLIGHT, .alloc = execlists_context_alloc, + .cancel_request = execlists_context_cancel_request, + .pre_pin = execlists_context_pre_pin, .pin = execlists_context_pin, .unpin = lrc_unpin, @@ -2548,6 +2618,8 @@ static const struct intel_context_ops execlists_context_ops = { .reset = lrc_reset, .destroy = lrc_destroy, + + .create_virtual = execlists_create_virtual, }; static int emit_pdps(struct i915_request *rq) @@ -2800,10 +2872,8 @@ static int execlists_resume(struct intel_engine_cs *engine) static void execlists_reset_prepare(struct intel_engine_cs *engine) { - struct intel_engine_execlists * const execlists = &engine->execlists; - ENGINE_TRACE(engine, "depth<-%d\n", - atomic_read(&execlists->tasklet.count)); + atomic_read(&engine->sched_engine->tasklet.count)); /* * Prevent request submission to the hardware until we have @@ -2814,8 +2884,8 @@ static void execlists_reset_prepare(struct intel_engine_cs *engine) * Turning off the execlists->tasklet until the reset is over * prevents the race. */ - __tasklet_disable_sync_once(&execlists->tasklet); - GEM_BUG_ON(!reset_in_progress(execlists)); + __tasklet_disable_sync_once(&engine->sched_engine->tasklet); + GEM_BUG_ON(!reset_in_progress(engine)); /* * We stop engines, otherwise we might get failed reset and a @@ -2957,24 +3027,26 @@ static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled) /* Push back any incomplete requests for replay after the reset. */ rcu_read_lock(); - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); __unwind_incomplete_requests(engine); - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); rcu_read_unlock(); } static void nop_submission_tasklet(struct tasklet_struct *t) { - struct intel_engine_cs * const engine = - from_tasklet(engine, t, execlists.tasklet); + struct i915_sched_engine *sched_engine = + from_tasklet(sched_engine, t, tasklet); + struct intel_engine_cs * const engine = sched_engine->private_data; /* The driver is wedged; don't process any more events. */ - WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN); + WRITE_ONCE(engine->sched_engine->queue_priority_hint, INT_MIN); } static void execlists_reset_cancel(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; + struct i915_sched_engine * const sched_engine = engine->sched_engine; struct i915_request *rq, *rn; struct rb_node *rb; unsigned long flags; @@ -2998,15 +3070,15 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) execlists_reset_csb(engine, true); rcu_read_lock(); - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); /* Mark all executing requests as skipped. */ - list_for_each_entry(rq, &engine->active.requests, sched.link) + list_for_each_entry(rq, &engine->sched_engine->requests, sched.link) i915_request_put(i915_request_mark_eio(rq)); intel_engine_signal_breadcrumbs(engine); /* Flush the queued requests to the timeline list (for retiring). */ - while ((rb = rb_first_cached(&execlists->queue))) { + while ((rb = rb_first_cached(&sched_engine->queue))) { struct i915_priolist *p = to_priolist(rb); priolist_for_each_request_consume(rq, rn, p) { @@ -3016,12 +3088,12 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) } } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &sched_engine->queue); i915_priolist_free(p); } /* On-hold requests will be flushed to timeline upon their release */ - list_for_each_entry(rq, &engine->active.hold, sched.link) + list_for_each_entry(rq, &sched_engine->hold, sched.link) i915_request_put(i915_request_mark_eio(rq)); /* Cancel all attached virtual engines */ @@ -3032,7 +3104,7 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) rb_erase_cached(rb, &execlists->virtual); RB_CLEAR_NODE(rb); - spin_lock(&ve->base.active.lock); + spin_lock(&ve->base.sched_engine->lock); rq = fetch_and_zero(&ve->request); if (rq) { if (i915_request_mark_eio(rq)) { @@ -3042,20 +3114,20 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) } i915_request_put(rq); - ve->base.execlists.queue_priority_hint = INT_MIN; + ve->base.sched_engine->queue_priority_hint = INT_MIN; } - spin_unlock(&ve->base.active.lock); + spin_unlock(&ve->base.sched_engine->lock); } /* Remaining _unready_ requests will be nop'ed when submitted */ - execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; + sched_engine->queue_priority_hint = INT_MIN; + sched_engine->queue = RB_ROOT_CACHED; - GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet)); - execlists->tasklet.callback = nop_submission_tasklet; + GEM_BUG_ON(__tasklet_is_enabled(&engine->sched_engine->tasklet)); + engine->sched_engine->tasklet.callback = nop_submission_tasklet; - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); rcu_read_unlock(); } @@ -3073,14 +3145,14 @@ static void execlists_reset_finish(struct intel_engine_cs *engine) * reset as the next level of recovery, and as a final resort we * will declare the device wedged. */ - GEM_BUG_ON(!reset_in_progress(execlists)); + GEM_BUG_ON(!reset_in_progress(engine)); /* And kick in case we missed a new request submission. */ - if (__tasklet_enable(&execlists->tasklet)) + if (__tasklet_enable(&engine->sched_engine->tasklet)) __execlists_kick(execlists); ENGINE_TRACE(engine, "depth->%d\n", - atomic_read(&execlists->tasklet.count)); + atomic_read(&engine->sched_engine->tasklet.count)); } static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) @@ -3101,6 +3173,42 @@ static void execlists_park(struct intel_engine_cs *engine) cancel_timer(&engine->execlists.preempt); } +static void add_to_engine(struct i915_request *rq) +{ + lockdep_assert_held(&rq->engine->sched_engine->lock); + list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); +} + +static void remove_from_engine(struct i915_request *rq) +{ + struct intel_engine_cs *engine, *locked; + + /* + * Virtual engines complicate acquiring the engine timeline lock, + * as their rq->engine pointer is not stable until under that + * engine lock. The simple ploy we use is to take the lock then + * check that the rq still belongs to the newly locked engine. + */ + locked = READ_ONCE(rq->engine); + spin_lock_irq(&locked->sched_engine->lock); + while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { + spin_unlock(&locked->sched_engine->lock); + spin_lock(&engine->sched_engine->lock); + locked = engine; + } + list_del_init(&rq->sched.link); + + clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags); + + /* Prevent further __await_execution() registering a cb, then flush */ + set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); + + spin_unlock_irq(&locked->sched_engine->lock); + + i915_request_notify_execute_cb_imm(rq); +} + static bool can_preempt(struct intel_engine_cs *engine) { if (GRAPHICS_VER(engine->i915) > 8) @@ -3110,11 +3218,62 @@ static bool can_preempt(struct intel_engine_cs *engine) return engine->class != RENDER_CLASS; } +static void kick_execlists(const struct i915_request *rq, int prio) +{ + struct intel_engine_cs *engine = rq->engine; + struct i915_sched_engine *sched_engine = engine->sched_engine; + const struct i915_request *inflight; + + /* + * We only need to kick the tasklet once for the high priority + * new context we add into the queue. + */ + if (prio <= sched_engine->queue_priority_hint) + return; + + rcu_read_lock(); + + /* Nothing currently active? We're overdue for a submission! */ + inflight = execlists_active(&engine->execlists); + if (!inflight) + goto unlock; + + /* + * If we are already the currently executing context, don't + * bother evaluating if we should preempt ourselves. + */ + if (inflight->context == rq->context) + goto unlock; + + ENGINE_TRACE(engine, + "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n", + prio, + rq->fence.context, rq->fence.seqno, + inflight->fence.context, inflight->fence.seqno, + inflight->sched.attr.priority); + + sched_engine->queue_priority_hint = prio; + + /* + * Allow preemption of low -> normal -> high, but we do + * not allow low priority tasks to preempt other low priority + * tasks under the impression that latency for low priority + * tasks does not matter (as much as background throughput), + * so kiss. + */ + if (prio >= max(I915_PRIORITY_NORMAL, rq_prio(inflight))) + tasklet_hi_schedule(&sched_engine->tasklet); + +unlock: + rcu_read_unlock(); +} + static void execlists_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = execlists_submit_request; - engine->schedule = i915_schedule; - engine->execlists.tasklet.callback = execlists_submission_tasklet; + engine->sched_engine->schedule = i915_schedule; + engine->sched_engine->kick_backend = kick_execlists; + engine->sched_engine->tasklet.callback = execlists_submission_tasklet; } static void execlists_shutdown(struct intel_engine_cs *engine) @@ -3122,7 +3281,7 @@ static void execlists_shutdown(struct intel_engine_cs *engine) /* Synchronise with residual timers and any softirq they raise */ del_timer_sync(&engine->execlists.timer); del_timer_sync(&engine->execlists.preempt); - tasklet_kill(&engine->execlists.tasklet); + tasklet_kill(&engine->sched_engine->tasklet); } static void execlists_release(struct intel_engine_cs *engine) @@ -3144,6 +3303,8 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &execlists_context_ops; engine->request_alloc = execlists_request_alloc; + engine->add_active_request = add_to_engine; + engine->remove_active_request = remove_from_engine; engine->reset.prepare = execlists_reset_prepare; engine->reset.rewind = execlists_reset_rewind; @@ -3238,7 +3399,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) struct intel_uncore *uncore = engine->uncore; u32 base = engine->mmio_base; - tasklet_setup(&engine->execlists.tasklet, execlists_submission_tasklet); + tasklet_setup(&engine->sched_engine->tasklet, execlists_submission_tasklet); timer_setup(&engine->execlists.timer, execlists_timeslice, 0); timer_setup(&engine->execlists.preempt, execlists_preempt, 0); @@ -3255,6 +3416,10 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base)); execlists->ctrl_reg = uncore->regs + i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base)); + + engine->fw_domain = intel_uncore_forcewake_for_reg(engine->uncore, + RING_EXECLIST_CONTROL(engine->mmio_base), + FW_REG_WRITE); } else { execlists->submit_reg = uncore->regs + i915_mmio_reg_offset(RING_ELSP(base)); @@ -3272,7 +3437,8 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) execlists->csb_size = GEN11_CSB_ENTRIES; engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0); - if (GRAPHICS_VER(engine->i915) >= 11) { + if (GRAPHICS_VER(engine->i915) >= 11 && + GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 50)) { execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32); execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32); } @@ -3286,7 +3452,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) static struct list_head *virtual_queue(struct virtual_engine *ve) { - return &ve->base.execlists.default_priolist.requests; + return &ve->base.sched_engine->default_priolist.requests; } static void rcu_virtual_context_destroy(struct work_struct *wrk) @@ -3301,7 +3467,7 @@ static void rcu_virtual_context_destroy(struct work_struct *wrk) if (unlikely(ve->request)) { struct i915_request *old; - spin_lock_irq(&ve->base.active.lock); + spin_lock_irq(&ve->base.sched_engine->lock); old = fetch_and_zero(&ve->request); if (old) { @@ -3310,7 +3476,7 @@ static void rcu_virtual_context_destroy(struct work_struct *wrk) i915_request_put(old); } - spin_unlock_irq(&ve->base.active.lock); + spin_unlock_irq(&ve->base.sched_engine->lock); } /* @@ -3320,7 +3486,7 @@ static void rcu_virtual_context_destroy(struct work_struct *wrk) * rbtrees as in the case it is running in parallel, it may reinsert * the rb_node into a sibling. */ - tasklet_kill(&ve->base.execlists.tasklet); + tasklet_kill(&ve->base.sched_engine->tasklet); /* Decouple ourselves from the siblings, no more access allowed. */ for (n = 0; n < ve->num_siblings; n++) { @@ -3330,24 +3496,26 @@ static void rcu_virtual_context_destroy(struct work_struct *wrk) if (RB_EMPTY_NODE(node)) continue; - spin_lock_irq(&sibling->active.lock); + spin_lock_irq(&sibling->sched_engine->lock); - /* Detachment is lazily performed in the execlists tasklet */ + /* Detachment is lazily performed in the sched_engine->tasklet */ if (!RB_EMPTY_NODE(node)) rb_erase_cached(node, &sibling->execlists.virtual); - spin_unlock_irq(&sibling->active.lock); + spin_unlock_irq(&sibling->sched_engine->lock); } - GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet)); + GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.sched_engine->tasklet)); GEM_BUG_ON(!list_empty(virtual_queue(ve))); lrc_fini(&ve->context); intel_context_fini(&ve->context); - intel_breadcrumbs_free(ve->base.breadcrumbs); + if (ve->base.breadcrumbs) + intel_breadcrumbs_put(ve->base.breadcrumbs); + if (ve->base.sched_engine) + i915_sched_engine_put(ve->base.sched_engine); intel_engine_free_request_pool(&ve->base); - kfree(ve->bonds); kfree(ve); } @@ -3440,11 +3608,24 @@ static void virtual_context_exit(struct intel_context *ce) intel_engine_pm_put(ve->siblings[n]); } +static struct intel_engine_cs * +virtual_get_sibling(struct intel_engine_cs *engine, unsigned int sibling) +{ + struct virtual_engine *ve = to_virtual_engine(engine); + + if (sibling >= ve->num_siblings) + return NULL; + + return ve->siblings[sibling]; +} + static const struct intel_context_ops virtual_context_ops = { .flags = COPS_HAS_INFLIGHT, .alloc = virtual_context_alloc, + .cancel_request = execlists_context_cancel_request, + .pre_pin = virtual_context_pre_pin, .pin = virtual_context_pin, .unpin = lrc_unpin, @@ -3454,6 +3635,8 @@ static const struct intel_context_ops virtual_context_ops = { .exit = virtual_context_exit, .destroy = virtual_context_destroy, + + .get_sibling = virtual_get_sibling, }; static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve) @@ -3475,16 +3658,18 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve) ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n", rq->fence.context, rq->fence.seqno, - mask, ve->base.execlists.queue_priority_hint); + mask, ve->base.sched_engine->queue_priority_hint); return mask; } static void virtual_submission_tasklet(struct tasklet_struct *t) { + struct i915_sched_engine *sched_engine = + from_tasklet(sched_engine, t, tasklet); struct virtual_engine * const ve = - from_tasklet(ve, t, base.execlists.tasklet); - const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint); + (struct virtual_engine *)sched_engine->private_data; + const int prio = READ_ONCE(sched_engine->queue_priority_hint); intel_engine_mask_t mask; unsigned int n; @@ -3503,7 +3688,7 @@ static void virtual_submission_tasklet(struct tasklet_struct *t) if (!READ_ONCE(ve->request)) break; /* already handled by a sibling's tasklet */ - spin_lock_irq(&sibling->active.lock); + spin_lock_irq(&sibling->sched_engine->lock); if (unlikely(!(mask & sibling->mask))) { if (!RB_EMPTY_NODE(&node->rb)) { @@ -3552,11 +3737,11 @@ static void virtual_submission_tasklet(struct tasklet_struct *t) submit_engine: GEM_BUG_ON(RB_EMPTY_NODE(&node->rb)); node->prio = prio; - if (first && prio > sibling->execlists.queue_priority_hint) - tasklet_hi_schedule(&sibling->execlists.tasklet); + if (first && prio > sibling->sched_engine->queue_priority_hint) + tasklet_hi_schedule(&sibling->sched_engine->tasklet); unlock_engine: - spin_unlock_irq(&sibling->active.lock); + spin_unlock_irq(&sibling->sched_engine->lock); if (intel_context_inflight(&ve->context)) break; @@ -3574,7 +3759,7 @@ static void virtual_submit_request(struct i915_request *rq) GEM_BUG_ON(ve->base.submit_request != virtual_submit_request); - spin_lock_irqsave(&ve->base.active.lock, flags); + spin_lock_irqsave(&ve->base.sched_engine->lock, flags); /* By the time we resubmit a request, it may be completed */ if (__i915_request_is_complete(rq)) { @@ -3588,68 +3773,25 @@ static void virtual_submit_request(struct i915_request *rq) i915_request_put(ve->request); } - ve->base.execlists.queue_priority_hint = rq_prio(rq); + ve->base.sched_engine->queue_priority_hint = rq_prio(rq); ve->request = i915_request_get(rq); GEM_BUG_ON(!list_empty(virtual_queue(ve))); list_move_tail(&rq->sched.link, virtual_queue(ve)); - tasklet_hi_schedule(&ve->base.execlists.tasklet); + tasklet_hi_schedule(&ve->base.sched_engine->tasklet); unlock: - spin_unlock_irqrestore(&ve->base.active.lock, flags); + spin_unlock_irqrestore(&ve->base.sched_engine->lock, flags); } -static struct ve_bond * -virtual_find_bond(struct virtual_engine *ve, - const struct intel_engine_cs *master) -{ - int i; - - for (i = 0; i < ve->num_bonds; i++) { - if (ve->bonds[i].master == master) - return &ve->bonds[i]; - } - - return NULL; -} - -static void -virtual_bond_execute(struct i915_request *rq, struct dma_fence *signal) -{ - struct virtual_engine *ve = to_virtual_engine(rq->engine); - intel_engine_mask_t allowed, exec; - struct ve_bond *bond; - - allowed = ~to_request(signal)->engine->mask; - - bond = virtual_find_bond(ve, to_request(signal)->engine); - if (bond) - allowed &= bond->sibling_mask; - - /* Restrict the bonded request to run on only the available engines */ - exec = READ_ONCE(rq->execution_mask); - while (!try_cmpxchg(&rq->execution_mask, &exec, exec & allowed)) - ; - - /* Prevent the master from being re-run on the bonded engines */ - to_request(signal)->execution_mask &= ~allowed; -} - -struct intel_context * -intel_execlists_create_virtual(struct intel_engine_cs **siblings, - unsigned int count) +static struct intel_context * +execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count) { struct virtual_engine *ve; unsigned int n; int err; - if (count == 0) - return ERR_PTR(-EINVAL); - - if (count == 1) - return intel_context_create(siblings[0]); - ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL); if (!ve) return ERR_PTR(-ENOMEM); @@ -3681,19 +3823,24 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings, snprintf(ve->base.name, sizeof(ve->base.name), "virtual"); - intel_engine_init_active(&ve->base, ENGINE_VIRTUAL); intel_engine_init_execlists(&ve->base); + ve->base.sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL); + if (!ve->base.sched_engine) { + err = -ENOMEM; + goto err_put; + } + ve->base.sched_engine->private_data = &ve->base; + ve->base.cops = &virtual_context_ops; ve->base.request_alloc = execlists_request_alloc; - ve->base.schedule = i915_schedule; + ve->base.sched_engine->schedule = i915_schedule; + ve->base.sched_engine->kick_backend = kick_execlists; ve->base.submit_request = virtual_submit_request; - ve->base.bond_execute = virtual_bond_execute; INIT_LIST_HEAD(virtual_queue(ve)); - ve->base.execlists.queue_priority_hint = INT_MIN; - tasklet_setup(&ve->base.execlists.tasklet, virtual_submission_tasklet); + tasklet_setup(&ve->base.sched_engine->tasklet, virtual_submission_tasklet); intel_context_init(&ve->context, &ve->base); @@ -3721,7 +3868,7 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings, * layering if we handle cloning of the requests and * submitting a copy into each backend. */ - if (sibling->execlists.tasklet.callback != + if (sibling->sched_engine->tasklet.callback != execlists_submission_tasklet) { err = -ENODEV; goto err_put; @@ -3756,6 +3903,8 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings, "v%dx%d", ve->base.class, count); ve->base.context_size = sibling->context_size; + ve->base.add_active_request = sibling->add_active_request; + ve->base.remove_active_request = sibling->remove_active_request; ve->base.emit_bb_start = sibling->emit_bb_start; ve->base.emit_flush = sibling->emit_flush; ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb; @@ -3776,70 +3925,6 @@ err_put: return ERR_PTR(err); } -struct intel_context * -intel_execlists_clone_virtual(struct intel_engine_cs *src) -{ - struct virtual_engine *se = to_virtual_engine(src); - struct intel_context *dst; - - dst = intel_execlists_create_virtual(se->siblings, - se->num_siblings); - if (IS_ERR(dst)) - return dst; - - if (se->num_bonds) { - struct virtual_engine *de = to_virtual_engine(dst->engine); - - de->bonds = kmemdup(se->bonds, - sizeof(*se->bonds) * se->num_bonds, - GFP_KERNEL); - if (!de->bonds) { - intel_context_put(dst); - return ERR_PTR(-ENOMEM); - } - - de->num_bonds = se->num_bonds; - } - - return dst; -} - -int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, - const struct intel_engine_cs *master, - const struct intel_engine_cs *sibling) -{ - struct virtual_engine *ve = to_virtual_engine(engine); - struct ve_bond *bond; - int n; - - /* Sanity check the sibling is part of the virtual engine */ - for (n = 0; n < ve->num_siblings; n++) - if (sibling == ve->siblings[n]) - break; - if (n == ve->num_siblings) - return -EINVAL; - - bond = virtual_find_bond(ve, master); - if (bond) { - bond->sibling_mask |= sibling->mask; - return 0; - } - - bond = krealloc(ve->bonds, - sizeof(*bond) * (ve->num_bonds + 1), - GFP_KERNEL); - if (!bond) - return -ENOMEM; - - bond[ve->num_bonds].master = master; - bond[ve->num_bonds].sibling_mask = sibling->mask; - - ve->bonds = bond; - ve->num_bonds++; - - return 0; -} - void intel_execlists_show_requests(struct intel_engine_cs *engine, struct drm_printer *m, void (*show_request)(struct drm_printer *m, @@ -3849,16 +3934,17 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, unsigned int max) { const struct intel_engine_execlists *execlists = &engine->execlists; + struct i915_sched_engine *sched_engine = engine->sched_engine; struct i915_request *rq, *last; unsigned long flags; unsigned int count; struct rb_node *rb; - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&sched_engine->lock, flags); last = NULL; count = 0; - list_for_each_entry(rq, &engine->active.requests, sched.link) { + list_for_each_entry(rq, &sched_engine->requests, sched.link) { if (count++ < max - 1) show_request(m, rq, "\t\t", 0); else @@ -3873,13 +3959,13 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, show_request(m, last, "\t\t", 0); } - if (execlists->queue_priority_hint != INT_MIN) + if (sched_engine->queue_priority_hint != INT_MIN) drm_printf(m, "\t\tQueue priority hint: %d\n", - READ_ONCE(execlists->queue_priority_hint)); + READ_ONCE(sched_engine->queue_priority_hint)); last = NULL; count = 0; - for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); priolist_for_each_request(rq, p) { @@ -3921,7 +4007,7 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, show_request(m, last, "\t\t", 0); } - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&sched_engine->lock, flags); } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h index 4ca9b475e252..a1aa92c983a5 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h @@ -32,15 +32,7 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, int indent), unsigned int max); -struct intel_context * -intel_execlists_create_virtual(struct intel_engine_cs **siblings, - unsigned int count); - -struct intel_context * -intel_execlists_clone_virtual(struct intel_engine_cs *src); - -int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, - const struct intel_engine_cs *master, - const struct intel_engine_cs *sibling); +bool +intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine); #endif /* __INTEL_EXECLISTS_SUBMISSION_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 20e46b843324..de3ac58fceec 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -826,13 +826,13 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; /* - * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range + * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range * will be dropped. For WC mappings in general we have 64 byte burst * writes when the WC buffer is flushed, so we can't use it, but have to * resort to an uncached mapping. The WC issue is easily caught by the * readback check when writing GTT PTE entries. */ - if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 10) + if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 11) ggtt->gsm = ioremap(phys_addr, size); else ggtt->gsm = ioremap_wc(phys_addr, size); @@ -1494,7 +1494,7 @@ intel_partial_pages(const struct i915_ggtt_view *view, if (ret) goto err_sg_alloc; - iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset, true); + iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset); GEM_BUG_ON(!iter); sg = st->sgl; diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h index 2694dbb9967e..1c3af0fc0456 100644 --- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h @@ -123,8 +123,10 @@ #define MI_SEMAPHORE_SAD_NEQ_SDD (5 << 12) #define MI_SEMAPHORE_TOKEN_MASK REG_GENMASK(9, 5) #define MI_SEMAPHORE_TOKEN_SHIFT 5 +#define MI_STORE_DATA_IMM MI_INSTR(0x20, 0) #define MI_STORE_DWORD_IMM MI_INSTR(0x20, 1) #define MI_STORE_DWORD_IMM_GEN4 MI_INSTR(0x20, 2) +#define MI_STORE_QWORD_IMM_GEN8 (MI_INSTR(0x20, 3) | REG_BIT(21)) #define MI_MEM_VIRTUAL (1 << 22) /* 945,g33,965 */ #define MI_USE_GGTT (1 << 22) /* g4x+ */ #define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 2161bf01ef8b..62d40c986642 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -13,6 +13,7 @@ #include "intel_gt_clock_utils.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" +#include "intel_migrate.h" #include "intel_mocs.h" #include "intel_rc6.h" #include "intel_renderstate.h" @@ -40,8 +41,8 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) intel_gt_init_timelines(gt); intel_gt_pm_init_early(gt); - intel_rps_init_early(>->rps); intel_uc_init_early(>->uc); + intel_rps_init_early(>->rps); } int intel_gt_probe_lmem(struct intel_gt *gt) @@ -83,13 +84,73 @@ void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt) gt->ggtt = ggtt; } +static const struct intel_mmio_range icl_l3bank_steering_table[] = { + { 0x00B100, 0x00B3FF }, + {}, +}; + +static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = { + { 0x004000, 0x004AFF }, + { 0x00C800, 0x00CFFF }, + { 0x00DD00, 0x00DDFF }, + { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ + {}, +}; + +static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = { + { 0x00B000, 0x00B0FF }, + { 0x00D800, 0x00D8FF }, + {}, +}; + +static const struct intel_mmio_range dg2_lncf_steering_table[] = { + { 0x00B000, 0x00B0FF }, + { 0x00D880, 0x00D8FF }, + {}, +}; + +static u16 slicemask(struct intel_gt *gt, int count) +{ + u64 dss_mask = intel_sseu_get_subslices(>->info.sseu, 0); + + return intel_slicemask_from_dssmask(dss_mask, count); +} + int intel_gt_init_mmio(struct intel_gt *gt) { + struct drm_i915_private *i915 = gt->i915; + intel_gt_init_clock_frequency(gt); intel_uc_init_mmio(>->uc); intel_sseu_info_init(gt); + /* + * An mslice is unavailable only if both the meml3 for the slice is + * disabled *and* all of the DSS in the slice (quadrant) are disabled. + */ + if (HAS_MSLICES(i915)) + gt->info.mslice_mask = + slicemask(gt, GEN_DSS_PER_MSLICE) | + (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & + GEN12_MEML3_EN_MASK); + + if (IS_DG2(i915)) { + gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; + gt->steering_table[LNCF] = dg2_lncf_steering_table; + } else if (IS_XEHPSDV(i915)) { + gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; + gt->steering_table[LNCF] = xehpsdv_lncf_steering_table; + } else if (GRAPHICS_VER(i915) >= 11 && + GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) { + gt->steering_table[L3BANK] = icl_l3bank_steering_table; + gt->info.l3bank_mask = + ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & + GEN10_L3BANK_MASK; + } else if (HAS_MSLICES(i915)) { + MISSING_CASE(INTEL_INFO(i915)->platform); + } + return intel_engines_init_mmio(gt); } @@ -192,7 +253,7 @@ static void clear_register(struct intel_uncore *uncore, i915_reg_t reg) intel_uncore_rmw(uncore, reg, 0, 0); } -static void gen8_clear_engine_error_register(struct intel_engine_cs *engine) +static void gen6_clear_engine_error_register(struct intel_engine_cs *engine) { GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0); GEN6_RING_FAULT_REG_POSTING_READ(engine); @@ -238,7 +299,7 @@ intel_gt_clear_error_registers(struct intel_gt *gt, enum intel_engine_id id; for_each_engine_masked(engine, gt, engine_mask, id) - gen8_clear_engine_error_register(engine); + gen6_clear_engine_error_register(engine); } } @@ -572,6 +633,25 @@ static void __intel_gt_disable(struct intel_gt *gt) GEM_BUG_ON(intel_gt_pm_is_awake(gt)); } +int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) +{ + long remaining_timeout; + + /* If the device is asleep, we have no requests outstanding */ + if (!intel_gt_pm_is_awake(gt)) + return 0; + + while ((timeout = intel_gt_retire_requests_timeout(gt, timeout, + &remaining_timeout)) > 0) { + cond_resched(); + if (signal_pending(current)) + return -EINTR; + } + + return timeout ? timeout : intel_uc_wait_for_idle(>->uc, + remaining_timeout); +} + int intel_gt_init(struct intel_gt *gt) { int err; @@ -622,10 +702,14 @@ int intel_gt_init(struct intel_gt *gt) if (err) goto err_gt; + intel_uc_init_late(>->uc); + err = i915_inject_probe_error(gt->i915, -EIO); if (err) goto err_gt; + intel_migrate_init(>->migrate, gt); + goto out_fw; err_gt: __intel_gt_disable(gt); @@ -649,6 +733,7 @@ void intel_gt_driver_remove(struct intel_gt *gt) { __intel_gt_disable(gt); + intel_migrate_fini(>->migrate); intel_uc_driver_remove(>->uc); intel_engines_release(gt); @@ -697,6 +782,112 @@ void intel_gt_driver_late_release(struct intel_gt *gt) intel_engines_free(gt); } +/** + * intel_gt_reg_needs_read_steering - determine whether a register read + * requires explicit steering + * @gt: GT structure + * @reg: the register to check steering requirements for + * @type: type of multicast steering to check + * + * Determines whether @reg needs explicit steering of a specific type for + * reads. + * + * Returns false if @reg does not belong to a register range of the given + * steering type, or if the default (subslice-based) steering IDs are suitable + * for @type steering too. + */ +static bool intel_gt_reg_needs_read_steering(struct intel_gt *gt, + i915_reg_t reg, + enum intel_steering_type type) +{ + const u32 offset = i915_mmio_reg_offset(reg); + const struct intel_mmio_range *entry; + + if (likely(!intel_gt_needs_read_steering(gt, type))) + return false; + + for (entry = gt->steering_table[type]; entry->end; entry++) { + if (offset >= entry->start && offset <= entry->end) + return true; + } + + return false; +} + +/** + * intel_gt_get_valid_steering - determines valid IDs for a class of MCR steering + * @gt: GT structure + * @type: multicast register type + * @sliceid: Slice ID returned + * @subsliceid: Subslice ID returned + * + * Determines sliceid and subsliceid values that will steer reads + * of a specific multicast register class to a valid value. + */ +static void intel_gt_get_valid_steering(struct intel_gt *gt, + enum intel_steering_type type, + u8 *sliceid, u8 *subsliceid) +{ + switch (type) { + case L3BANK: + GEM_DEBUG_WARN_ON(!gt->info.l3bank_mask); /* should be impossible! */ + + *sliceid = 0; /* unused */ + *subsliceid = __ffs(gt->info.l3bank_mask); + break; + case MSLICE: + GEM_DEBUG_WARN_ON(!gt->info.mslice_mask); /* should be impossible! */ + + *sliceid = __ffs(gt->info.mslice_mask); + *subsliceid = 0; /* unused */ + break; + case LNCF: + GEM_DEBUG_WARN_ON(!gt->info.mslice_mask); /* should be impossible! */ + + /* + * An LNCF is always present if its mslice is present, so we + * can safely just steer to LNCF 0 in all cases. + */ + *sliceid = __ffs(gt->info.mslice_mask) << 1; + *subsliceid = 0; /* unused */ + break; + default: + MISSING_CASE(type); + *sliceid = 0; + *subsliceid = 0; + } +} + +/** + * intel_gt_read_register_fw - reads a GT register with support for multicast + * @gt: GT structure + * @reg: register to read + * + * This function will read a GT register. If the register is a multicast + * register, the read will be steered to a valid instance (i.e., one that + * isn't fused off or powered down by power gating). + * + * Returns the value from a valid instance of @reg. + */ +u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg) +{ + int type; + u8 sliceid, subsliceid; + + for (type = 0; type < NUM_STEERING_TYPES; type++) { + if (intel_gt_reg_needs_read_steering(gt, reg, type)) { + intel_gt_get_valid_steering(gt, type, &sliceid, + &subsliceid); + return intel_uncore_read_with_mcr_steering_fw(gt->uncore, + reg, + sliceid, + subsliceid); + } + } + + return intel_uncore_read_fw(gt->uncore, reg); +} + void intel_gt_info_print(const struct intel_gt_info *info, struct drm_printer *p) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h index 7ec395cace69..74e771871a9b 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt.h @@ -48,6 +48,8 @@ void intel_gt_driver_release(struct intel_gt *gt); void intel_gt_driver_late_release(struct intel_gt *gt); +int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout); + void intel_gt_check_and_clear_faults(struct intel_gt *gt); void intel_gt_clear_error_registers(struct intel_gt *gt, intel_engine_mask_t engine_mask); @@ -75,6 +77,14 @@ static inline bool intel_gt_is_wedged(const struct intel_gt *gt) return unlikely(test_bit(I915_WEDGED, >->reset.flags)); } +static inline bool intel_gt_needs_read_steering(struct intel_gt *gt, + enum intel_steering_type type) +{ + return gt->steering_table[type]; +} + +u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg); + void intel_gt_info_print(const struct intel_gt_info *info, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c index 9f0e729d2d15..3513d6f90747 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c @@ -24,8 +24,8 @@ static u32 read_reference_ts_freq(struct intel_uncore *uncore) return base_freq + frac_freq; } -static u32 gen10_get_crystal_clock_freq(struct intel_uncore *uncore, - u32 rpm_config_reg) +static u32 gen9_get_crystal_clock_freq(struct intel_uncore *uncore, + u32 rpm_config_reg) { u32 f19_2_mhz = 19200000; u32 f24_mhz = 24000000; @@ -128,10 +128,10 @@ static u32 read_clock_frequency(struct intel_uncore *uncore) } else { u32 c0 = intel_uncore_read(uncore, RPM_CONFIG0); - if (GRAPHICS_VER(uncore->i915) <= 10) - freq = gen10_get_crystal_clock_freq(uncore, c0); - else + if (GRAPHICS_VER(uncore->i915) >= 11) freq = gen11_get_crystal_clock_freq(uncore, c0); + else + freq = gen9_get_crystal_clock_freq(uncore, c0); /* * Now figure out how the command stream's timestamp diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index c13462274fe8..b2de83be4d97 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -184,7 +184,13 @@ void gen11_gt_irq_reset(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0); + if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) + intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0); + if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) + intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0); + if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) + intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); @@ -218,8 +224,13 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask); intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask); intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask); + if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) + intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask); + if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) + intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask); intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask); - + if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) + intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask); /* * RPS interrupts will get enabled/disabled on demand when RPS itself * is enabled/disabled. diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index aef3084e8b16..dea8e2479897 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -6,7 +6,6 @@ #include <linux/suspend.h> #include "i915_drv.h" -#include "i915_globals.h" #include "i915_params.h" #include "intel_context.h" #include "intel_engine_pm.h" @@ -67,8 +66,6 @@ static int __gt_unpark(struct intel_wakeref *wf) GT_TRACE(gt, "\n"); - i915_globals_unpark(); - /* * It seems that the DMC likes to transition between the DC states a lot * when there are no connected displays (no active power domains) during @@ -116,8 +113,6 @@ static int __gt_park(struct intel_wakeref *wf) GEM_BUG_ON(!wakeref); intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); - i915_globals_park(); - return 0; } @@ -174,8 +169,6 @@ static void gt_sanitize(struct intel_gt *gt, bool force) if (intel_gt_is_wedged(gt)) intel_gt_unset_wedged(gt); - intel_uc_sanitize(>->uc); - for_each_engine(engine, gt, id) if (engine->reset.prepare) engine->reset.prepare(engine); @@ -191,6 +184,8 @@ static void gt_sanitize(struct intel_gt *gt, bool force) __intel_engine_reset(engine, false); } + intel_uc_reset(>->uc, false); + for_each_engine(engine, gt, id) if (engine->reset.finish) engine->reset.finish(engine); @@ -243,6 +238,8 @@ int intel_gt_resume(struct intel_gt *gt) goto err_wedged; } + intel_uc_reset_finish(>->uc); + intel_rps_enable(>->rps); intel_llc_enable(>->llc); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.c b/drivers/gpu/drm/i915/gt/intel_gt_requests.c index 647eca9d867a..edb881d75630 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_requests.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.c @@ -130,7 +130,8 @@ void intel_engine_fini_retire(struct intel_engine_cs *engine) GEM_BUG_ON(engine->retire); } -long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout) +long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout, + long *remaining_timeout) { struct intel_gt_timelines *timelines = >->timelines; struct intel_timeline *tl, *tn; @@ -195,22 +196,10 @@ out_active: spin_lock(&timelines->lock); if (flush_submission(gt, timeout)) /* Wait, there's more! */ active_count++; - return active_count ? timeout : 0; -} - -int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) -{ - /* If the device is asleep, we have no requests outstanding */ - if (!intel_gt_pm_is_awake(gt)) - return 0; - - while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) { - cond_resched(); - if (signal_pending(current)) - return -EINTR; - } + if (remaining_timeout) + *remaining_timeout = timeout; - return timeout; + return active_count ? timeout : 0; } static void retire_work_handler(struct work_struct *work) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.h b/drivers/gpu/drm/i915/gt/intel_gt_requests.h index fcc30a6e4fe9..d2969f68dd64 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_requests.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.h @@ -6,14 +6,17 @@ #ifndef INTEL_GT_REQUESTS_H #define INTEL_GT_REQUESTS_H +#include <linux/stddef.h> + struct intel_engine_cs; struct intel_gt; struct intel_timeline; -long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout); +long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout, + long *remaining_timeout); static inline void intel_gt_retire_requests(struct intel_gt *gt) { - intel_gt_retire_requests_timeout(gt, 0); + intel_gt_retire_requests_timeout(gt, 0, NULL); } void intel_engine_init_retire(struct intel_engine_cs *engine); @@ -21,8 +24,6 @@ void intel_engine_add_retire(struct intel_engine_cs *engine, struct intel_timeline *tl); void intel_engine_fini_retire(struct intel_engine_cs *engine); -int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout); - void intel_gt_init_requests(struct intel_gt *gt); void intel_gt_park_requests(struct intel_gt *gt); void intel_gt_unpark_requests(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index fecfacf551d5..a81e21bf1bd1 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -24,6 +24,7 @@ #include "intel_reset_types.h" #include "intel_rc6_types.h" #include "intel_rps_types.h" +#include "intel_migrate_types.h" #include "intel_wakeref.h" struct drm_i915_private; @@ -31,6 +32,33 @@ struct i915_ggtt; struct intel_engine_cs; struct intel_uncore; +struct intel_mmio_range { + u32 start; + u32 end; +}; + +/* + * The hardware has multiple kinds of multicast register ranges that need + * special register steering (and future platforms are expected to add + * additional types). + * + * During driver startup, we initialize the steering control register to + * direct reads to a slice/subslice that are valid for the 'subslice' class + * of multicast registers. If another type of steering does not have any + * overlap in valid steering targets with 'subslice' style registers, we will + * need to explicitly re-steer reads of registers of the other type. + * + * Only the replication types that may need additional non-default steering + * are listed here. + */ +enum intel_steering_type { + L3BANK, + MSLICE, + LNCF, + + NUM_STEERING_TYPES +}; + enum intel_submission_method { INTEL_SUBMISSION_RING, INTEL_SUBMISSION_ELSP, @@ -145,8 +173,15 @@ struct intel_gt { struct i915_vma *scratch; + struct intel_migrate migrate; + + const struct intel_mmio_range *steering_table[NUM_STEERING_TYPES]; + struct intel_gt_info { intel_engine_mask_t engine_mask; + + u32 l3bank_mask; + u8 num_engines; /* Media engine access to SFC per instance */ @@ -154,6 +189,8 @@ struct intel_gt { /* Slice/subslice/EU info */ struct sseu_dev_info sseu; + + unsigned long mslice_mask; } info; }; diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 084ea65d59c0..e137dd32b5b8 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -16,7 +16,19 @@ struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz) { struct drm_i915_gem_object *obj; - obj = i915_gem_object_create_lmem(vm->i915, sz, 0); + /* + * To avoid severe over-allocation when dealing with min_page_size + * restrictions, we override that behaviour here by allowing an object + * size and page layout which can be smaller. In practice this should be + * totally fine, since GTT paging structures are not typically inserted + * into the GTT. + * + * Note that we also hit this path for the scratch page, and for this + * case it might need to be 64K, but that should work fine here since we + * used the passed in size for the page size, which should ensure it + * also has the same alignment. + */ + obj = __i915_gem_object_create_lmem_with_ps(vm->i915, sz, sz, 0); /* * Ensure all paging structures for this vm share the same dma-resv * object underneath, with the idea that one object_lock() will lock @@ -414,7 +426,7 @@ static void tgl_setup_private_ppat(struct intel_uncore *uncore) intel_uncore_write(uncore, GEN12_PAT_INDEX(7), GEN8_PPAT_WB); } -static void cnl_setup_private_ppat(struct intel_uncore *uncore) +static void icl_setup_private_ppat(struct intel_uncore *uncore) { intel_uncore_write(uncore, GEN10_PAT_INDEX(0), @@ -514,8 +526,8 @@ void setup_private_pat(struct intel_uncore *uncore) if (GRAPHICS_VER(i915) >= 12) tgl_setup_private_ppat(uncore); - else if (GRAPHICS_VER(i915) >= 10) - cnl_setup_private_ppat(uncore); + else if (GRAPHICS_VER(i915) >= 11) + icl_setup_private_ppat(uncore); else if (IS_CHERRYVIEW(i915) || IS_GEN9_LP(i915)) chv_setup_private_ppat(uncore); else diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index edea95b97c36..bc7153018ebd 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -140,7 +140,6 @@ typedef u64 gen8_pte_t; enum i915_cache_level; -struct drm_i915_file_private; struct drm_i915_gem_object; struct i915_fence_reg; struct i915_vma; @@ -220,16 +219,6 @@ struct i915_address_space { struct intel_gt *gt; struct drm_i915_private *i915; struct device *dma; - /* - * Every address space belongs to a struct file - except for the global - * GTT that is owned by the driver (and so @file is set to NULL). In - * principle, no information should leak from one context to another - * (or between files/processes etc) unless explicitly shared by the - * owner. Tracking the owner is important in order to free up per-file - * objects along with the file, to aide resource tracking, and to - * assign blame. - */ - struct drm_i915_file_private *file; u64 total; /* size addr space maps (ex. 2GB for ggtt) */ u64 reserved; /* size addr space reserved */ @@ -296,6 +285,13 @@ struct i915_address_space { u32 flags); void (*cleanup)(struct i915_address_space *vm); + void (*foreach)(struct i915_address_space *vm, + u64 start, u64 length, + void (*fn)(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data), + void *data); + struct i915_vma_ops vma_ops; I915_SELFTEST_DECLARE(struct fault_attr fault_attr); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index a27bac0a4bfb..bb4af4977920 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -70,7 +70,7 @@ static void set_offsets(u32 *regs, if (close) { /* Close the batch; used mainly by live_lrc_layout() */ *regs = MI_BATCH_BUFFER_END; - if (GRAPHICS_VER(engine->i915) >= 10) + if (GRAPHICS_VER(engine->i915) >= 11) *regs |= BIT(0); } } @@ -484,6 +484,47 @@ static const u8 gen12_rcs_offsets[] = { END }; +static const u8 xehp_rcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + END +}; + #undef END #undef REG16 #undef REG @@ -502,7 +543,9 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine) !intel_engine_has_relative_mmio(engine)); if (engine->class == RENDER_CLASS) { - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + return xehp_rcs_offsets; + else if (GRAPHICS_VER(engine->i915) >= 12) return gen12_rcs_offsets; else if (GRAPHICS_VER(engine->i915) >= 11) return gen11_rcs_offsets; @@ -522,7 +565,9 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine) static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) { - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + return 0x70; + else if (GRAPHICS_VER(engine->i915) >= 12) return 0x60; else if (GRAPHICS_VER(engine->i915) >= 9) return 0x54; @@ -534,7 +579,9 @@ static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) static int lrc_ring_gpr0(const struct intel_engine_cs *engine) { - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + return 0x84; + else if (GRAPHICS_VER(engine->i915) >= 12) return 0x74; else if (GRAPHICS_VER(engine->i915) >= 9) return 0x68; @@ -578,10 +625,16 @@ static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine) static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine) { - if (engine->class != RENDER_CLASS) - return -1; - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + /* + * Note that the CSFE context has a dummy slot for CMD_BUF_CCTL + * simply to match the RCS context image layout. + */ + return 0xc6; + else if (engine->class != RENDER_CLASS) + return -1; + else if (GRAPHICS_VER(engine->i915) >= 12) return 0xb6; else if (GRAPHICS_VER(engine->i915) >= 11) return 0xaa; @@ -600,8 +653,6 @@ lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine) return GEN12_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; case 11: return GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - case 10: - return GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; case 9: return GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; case 8: @@ -845,7 +896,7 @@ int lrc_alloc(struct intel_context *ce, struct intel_engine_cs *engine) if (IS_ERR(vma)) return PTR_ERR(vma); - ring = intel_engine_create_ring(engine, (unsigned long)ce->ring); + ring = intel_engine_create_ring(engine, ce->ring_size); if (IS_ERR(ring)) { err = PTR_ERR(ring); goto err_vma; @@ -1101,6 +1152,14 @@ setup_indirect_ctx_bb(const struct intel_context *ce, * bits 55-60: SW counter * bits 61-63: engine class * + * On Xe_HP, the upper dword of the descriptor has a new format: + * + * bits 32-37: virtual function number + * bit 38: mbz, reserved for use by hardware + * bits 39-54: SW context ID + * bits 55-57: reserved + * bits 58-63: SW counter + * * engine info, SW context ID and SW counter need to form a unique number * (Context ID) per lrc. */ @@ -1387,40 +1446,6 @@ static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) return batch; } -static u32 * -gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) -{ - int i; - - /* - * WaPipeControlBefore3DStateSamplePattern: cnl - * - * Ensure the engine is idle prior to programming a - * 3DSTATE_SAMPLE_PATTERN during a context restore. - */ - batch = gen8_emit_pipe_control(batch, - PIPE_CONTROL_CS_STALL, - 0); - /* - * WaPipeControlBefore3DStateSamplePattern says we need 4 dwords for - * the PIPE_CONTROL followed by 12 dwords of 0x0, so 16 dwords in - * total. However, a PIPE_CONTROL is 6 dwords long, not 4, which is - * confusing. Since gen8_emit_pipe_control() already advances the - * batch by 6 dwords, we advance the other 10 here, completing a - * cacheline. It's not clear if the workaround requires this padding - * before other commands, or if it's just the regular padding we would - * already have for the workaround bb, so leave it here for now. - */ - for (i = 0; i < 10; i++) - *batch++ = MI_NOOP; - - /* Pad to end of cacheline */ - while ((unsigned long)batch % CACHELINE_BYTES) - *batch++ = MI_NOOP; - - return batch; -} - #define CTX_WA_BB_SIZE (PAGE_SIZE) static int lrc_create_wa_ctx(struct intel_engine_cs *engine) @@ -1473,10 +1498,6 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) case 12: case 11: return; - case 10: - wa_bb_fn[0] = gen10_init_indirectctx_bb; - wa_bb_fn[1] = NULL; - break; case 9: wa_bb_fn[0] = gen9_init_indirectctx_bb; wa_bb_fn[1] = NULL; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h index 41e5350a7a05..f785d0ed238f 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h @@ -87,9 +87,10 @@ #define GEN11_CSB_WRITE_PTR_MASK (GEN11_CSB_PTR_MASK << 0) #define MAX_CONTEXT_HW_ID (1 << 21) /* exclusive */ -#define MAX_GUC_CONTEXT_HW_ID (1 << 20) /* exclusive */ #define GEN11_MAX_CONTEXT_HW_ID (1 << 11) /* exclusive */ /* in Gen12 ID 0x7FF is reserved to indicate idle */ #define GEN12_MAX_CONTEXT_HW_ID (GEN11_MAX_CONTEXT_HW_ID - 1) +/* in Xe_HP ID 0xFFFF is reserved to indicate "invalid context" */ +#define XEHP_MAX_CONTEXT_HW_ID 0xFFFF #endif /* _INTEL_LRC_REG_H_ */ diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c new file mode 100644 index 000000000000..1dac21aa7e5c --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_context.h" +#include "intel_gpu_commands.h" +#include "intel_gt.h" +#include "intel_gtt.h" +#include "intel_migrate.h" +#include "intel_ring.h" + +struct insert_pte_data { + u64 offset; + bool is_lmem; +}; + +#define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */ + +static bool engine_supports_migration(struct intel_engine_cs *engine) +{ + if (!engine) + return false; + + /* + * We need the ability to prevent aribtration (MI_ARB_ON_OFF), + * the ability to write PTE using inline data (MI_STORE_DATA) + * and of course the ability to do the block transfer (blits). + */ + GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS); + + return true; +} + +static void insert_pte(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data) +{ + struct insert_pte_data *d = data; + + vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, + d->is_lmem ? PTE_LM : 0); + d->offset += PAGE_SIZE; +} + +static struct i915_address_space *migrate_vm(struct intel_gt *gt) +{ + struct i915_vm_pt_stash stash = {}; + struct i915_ppgtt *vm; + int err; + int i; + + /* + * We construct a very special VM for use by all migration contexts, + * it is kept pinned so that it can be used at any time. As we need + * to pre-allocate the page directories for the migration VM, this + * limits us to only using a small number of prepared vma. + * + * To be able to pipeline and reschedule migration operations while + * avoiding unnecessary contention on the vm itself, the PTE updates + * are inline with the blits. All the blits use the same fixed + * addresses, with the backing store redirection being updated on the + * fly. Only 2 implicit vma are used for all migration operations. + * + * We lay the ppGTT out as: + * + * [0, CHUNK_SZ) -> first object + * [CHUNK_SZ, 2 * CHUNK_SZ) -> second object + * [2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE + * + * By exposing the dma addresses of the page directories themselves + * within the ppGTT, we are then able to rewrite the PTE prior to use. + * But the PTE update and subsequent migration operation must be atomic, + * i.e. within the same non-preemptible window so that we do not switch + * to another migration context that overwrites the PTE. + * + * TODO: Add support for huge LMEM PTEs + */ + + vm = i915_ppgtt_create(gt); + if (IS_ERR(vm)) + return ERR_CAST(vm); + + if (!vm->vm.allocate_va_range || !vm->vm.foreach) { + err = -ENODEV; + goto err_vm; + } + + /* + * Each engine instance is assigned its own chunk in the VM, so + * that we can run multiple instances concurrently + */ + for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { + struct intel_engine_cs *engine; + u64 base = (u64)i << 32; + struct insert_pte_data d = {}; + struct i915_gem_ww_ctx ww; + u64 sz; + + engine = gt->engine_class[COPY_ENGINE_CLASS][i]; + if (!engine_supports_migration(engine)) + continue; + + /* + * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need + * 4x2 page directories for source/destination. + */ + sz = 2 * CHUNK_SZ; + d.offset = base + sz; + + /* + * We need another page directory setup so that we can write + * the 8x512 PTE in each chunk. + */ + sz += (sz >> 12) * sizeof(u64); + + err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz); + if (err) + goto err_vm; + + for_i915_gem_ww(&ww, err, true) { + err = i915_vm_lock_objects(&vm->vm, &ww); + if (err) + continue; + err = i915_vm_map_pt_stash(&vm->vm, &stash); + if (err) + continue; + + vm->vm.allocate_va_range(&vm->vm, &stash, base, sz); + } + i915_vm_free_pt_stash(&vm->vm, &stash); + if (err) + goto err_vm; + + /* Now allow the GPU to rewrite the PTE via its own ppGTT */ + d.is_lmem = i915_gem_object_is_lmem(vm->vm.scratch[0]); + vm->vm.foreach(&vm->vm, base, base + sz, insert_pte, &d); + } + + return &vm->vm; + +err_vm: + i915_vm_put(&vm->vm); + return ERR_PTR(err); +} + +static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + int i; + + for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { + engine = gt->engine_class[COPY_ENGINE_CLASS][i]; + if (engine_supports_migration(engine)) + return engine; + } + + return NULL; +} + +static struct intel_context *pinned_context(struct intel_gt *gt) +{ + static struct lock_class_key key; + struct intel_engine_cs *engine; + struct i915_address_space *vm; + struct intel_context *ce; + + engine = first_copy_engine(gt); + if (!engine) + return ERR_PTR(-ENODEV); + + vm = migrate_vm(gt); + if (IS_ERR(vm)) + return ERR_CAST(vm); + + ce = intel_engine_create_pinned_context(engine, vm, SZ_512K, + I915_GEM_HWS_MIGRATE, + &key, "migrate"); + i915_vm_put(vm); + return ce; +} + +int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt) +{ + struct intel_context *ce; + + memset(m, 0, sizeof(*m)); + + ce = pinned_context(gt); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + m->context = ce; + return 0; +} + +static int random_index(unsigned int max) +{ + return upper_32_bits(mul_u32_u32(get_random_u32(), max)); +} + +static struct intel_context *__migrate_engines(struct intel_gt *gt) +{ + struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE]; + struct intel_engine_cs *engine; + unsigned int count, i; + + count = 0; + for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { + engine = gt->engine_class[COPY_ENGINE_CLASS][i]; + if (engine_supports_migration(engine)) + engines[count++] = engine; + } + + return intel_context_create(engines[random_index(count)]); +} + +struct intel_context *intel_migrate_create_context(struct intel_migrate *m) +{ + struct intel_context *ce; + + /* + * We randomly distribute contexts across the engines upon constrction, + * as they all share the same pinned vm, and so in order to allow + * multiple blits to run in parallel, we must construct each blit + * to use a different range of the vm for its GTT. This has to be + * known at construction, so we can not use the late greedy load + * balancing of the virtual-engine. + */ + ce = __migrate_engines(m->context->engine->gt); + if (IS_ERR(ce)) + return ce; + + ce->ring = NULL; + ce->ring_size = SZ_256K; + + i915_vm_put(ce->vm); + ce->vm = i915_vm_get(m->context->vm); + + return ce; +} + +static inline struct sgt_dma sg_sgt(struct scatterlist *sg) +{ + dma_addr_t addr = sg_dma_address(sg); + + return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) }; +} + +static int emit_no_arbitration(struct i915_request *rq) +{ + u32 *cs; + + cs = intel_ring_begin(rq, 2); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Explicitly disable preemption for this request. */ + *cs++ = MI_ARB_ON_OFF; + *cs++ = MI_NOOP; + intel_ring_advance(rq, cs); + + return 0; +} + +static int emit_pte(struct i915_request *rq, + struct sgt_dma *it, + enum i915_cache_level cache_level, + bool is_lmem, + u64 offset, + int length) +{ + const u64 encode = rq->context->vm->pte_encode(0, cache_level, + is_lmem ? PTE_LM : 0); + struct intel_ring *ring = rq->ring; + int total = 0; + u32 *hdr, *cs; + int pkt; + + GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8); + + /* Compute the page directory offset for the target address range */ + offset += (u64)rq->engine->instance << 32; + offset >>= 12; + offset *= sizeof(u64); + offset += 2 * CHUNK_SZ; + + cs = intel_ring_begin(rq, 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Pack as many PTE updates as possible into a single MI command */ + pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); + pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); + + hdr = cs; + *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */ + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); + + do { + if (cs - hdr >= pkt) { + *hdr += cs - hdr - 2; + *cs++ = MI_NOOP; + + ring->emit = (void *)cs - ring->vaddr; + intel_ring_advance(rq, cs); + intel_ring_update_space(ring); + + cs = intel_ring_begin(rq, 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); + pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); + + hdr = cs; + *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); + } + + *cs++ = lower_32_bits(encode | it->dma); + *cs++ = upper_32_bits(encode | it->dma); + + offset += 8; + total += I915_GTT_PAGE_SIZE; + + it->dma += I915_GTT_PAGE_SIZE; + if (it->dma >= it->max) { + it->sg = __sg_next(it->sg); + if (!it->sg || sg_dma_len(it->sg) == 0) + break; + + it->dma = sg_dma_address(it->sg); + it->max = it->dma + sg_dma_len(it->sg); + } + } while (total < length); + + *hdr += cs - hdr - 2; + *cs++ = MI_NOOP; + + ring->emit = (void *)cs - ring->vaddr; + intel_ring_advance(rq, cs); + intel_ring_update_space(ring); + + return total; +} + +static bool wa_1209644611_applies(int ver, u32 size) +{ + u32 height = size >> PAGE_SHIFT; + + if (ver != 11) + return false; + + return height % 4 == 3 && height <= 8; +} + +static int emit_copy(struct i915_request *rq, int size) +{ + const int ver = GRAPHICS_VER(rq->engine->i915); + u32 instance = rq->engine->instance; + u32 *cs; + + cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + if (ver >= 9 && !wa_1209644611_applies(ver, size)) { + *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); + *cs++ = BLT_DEPTH_32 | PAGE_SIZE; + *cs++ = 0; + *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; + *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = instance; + *cs++ = 0; + *cs++ = PAGE_SIZE; + *cs++ = 0; /* src offset */ + *cs++ = instance; + } else if (ver >= 8) { + *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); + *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; + *cs++ = 0; + *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; + *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = instance; + *cs++ = 0; + *cs++ = PAGE_SIZE; + *cs++ = 0; /* src offset */ + *cs++ = instance; + } else { + GEM_BUG_ON(instance); + *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); + *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; + *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; + *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = PAGE_SIZE; + *cs++ = 0; /* src offset */ + } + + intel_ring_advance(rq, cs); + return 0; +} + +int +intel_context_migrate_copy(struct intel_context *ce, + struct dma_fence *await, + struct scatterlist *src, + enum i915_cache_level src_cache_level, + bool src_is_lmem, + struct scatterlist *dst, + enum i915_cache_level dst_cache_level, + bool dst_is_lmem, + struct i915_request **out) +{ + struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst); + struct i915_request *rq; + int err; + + GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); + *out = NULL; + + GEM_BUG_ON(ce->ring->size < SZ_64K); + + do { + int len; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_ce; + } + + if (await) { + err = i915_request_await_dma_fence(rq, await); + if (err) + goto out_rq; + + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (err) + goto out_rq; + } + + await = NULL; + } + + /* The PTE updates + copy must not be interrupted. */ + err = emit_no_arbitration(rq); + if (err) + goto out_rq; + + len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, 0, + CHUNK_SZ); + if (len <= 0) { + err = len; + goto out_rq; + } + + err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem, + CHUNK_SZ, len); + if (err < 0) + goto out_rq; + if (err < len) { + err = -EINVAL; + goto out_rq; + } + + err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); + if (err) + goto out_rq; + + err = emit_copy(rq, len); + + /* Arbitration is re-enabled between requests. */ +out_rq: + if (*out) + i915_request_put(*out); + *out = i915_request_get(rq); + i915_request_add(rq); + if (err || !it_src.sg || !sg_dma_len(it_src.sg)) + break; + + cond_resched(); + } while (1); + +out_ce: + return err; +} + +static int emit_clear(struct i915_request *rq, int size, u32 value) +{ + const int ver = GRAPHICS_VER(rq->engine->i915); + u32 instance = rq->engine->instance; + u32 *cs; + + GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); + + cs = intel_ring_begin(rq, ver >= 8 ? 8 : 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + if (ver >= 8) { + *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2); + *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; + *cs++ = 0; + *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; + *cs++ = 0; /* offset */ + *cs++ = instance; + *cs++ = value; + *cs++ = MI_NOOP; + } else { + GEM_BUG_ON(instance); + *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); + *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; + *cs++ = 0; + *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; + *cs++ = 0; + *cs++ = value; + } + + intel_ring_advance(rq, cs); + return 0; +} + +int +intel_context_migrate_clear(struct intel_context *ce, + struct dma_fence *await, + struct scatterlist *sg, + enum i915_cache_level cache_level, + bool is_lmem, + u32 value, + struct i915_request **out) +{ + struct sgt_dma it = sg_sgt(sg); + struct i915_request *rq; + int err; + + GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); + *out = NULL; + + GEM_BUG_ON(ce->ring->size < SZ_64K); + + do { + int len; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_ce; + } + + if (await) { + err = i915_request_await_dma_fence(rq, await); + if (err) + goto out_rq; + + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (err) + goto out_rq; + } + + await = NULL; + } + + /* The PTE updates + clear must not be interrupted. */ + err = emit_no_arbitration(rq); + if (err) + goto out_rq; + + len = emit_pte(rq, &it, cache_level, is_lmem, 0, CHUNK_SZ); + if (len <= 0) { + err = len; + goto out_rq; + } + + err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); + if (err) + goto out_rq; + + err = emit_clear(rq, len, value); + + /* Arbitration is re-enabled between requests. */ +out_rq: + if (*out) + i915_request_put(*out); + *out = i915_request_get(rq); + i915_request_add(rq); + if (err || !it.sg || !sg_dma_len(it.sg)) + break; + + cond_resched(); + } while (1); + +out_ce: + return err; +} + +int intel_migrate_copy(struct intel_migrate *m, + struct i915_gem_ww_ctx *ww, + struct dma_fence *await, + struct scatterlist *src, + enum i915_cache_level src_cache_level, + bool src_is_lmem, + struct scatterlist *dst, + enum i915_cache_level dst_cache_level, + bool dst_is_lmem, + struct i915_request **out) +{ + struct intel_context *ce; + int err; + + *out = NULL; + if (!m->context) + return -ENODEV; + + ce = intel_migrate_create_context(m); + if (IS_ERR(ce)) + ce = intel_context_get(m->context); + GEM_BUG_ON(IS_ERR(ce)); + + err = intel_context_pin_ww(ce, ww); + if (err) + goto out; + + err = intel_context_migrate_copy(ce, await, + src, src_cache_level, src_is_lmem, + dst, dst_cache_level, dst_is_lmem, + out); + + intel_context_unpin(ce); +out: + intel_context_put(ce); + return err; +} + +int +intel_migrate_clear(struct intel_migrate *m, + struct i915_gem_ww_ctx *ww, + struct dma_fence *await, + struct scatterlist *sg, + enum i915_cache_level cache_level, + bool is_lmem, + u32 value, + struct i915_request **out) +{ + struct intel_context *ce; + int err; + + *out = NULL; + if (!m->context) + return -ENODEV; + + ce = intel_migrate_create_context(m); + if (IS_ERR(ce)) + ce = intel_context_get(m->context); + GEM_BUG_ON(IS_ERR(ce)); + + err = intel_context_pin_ww(ce, ww); + if (err) + goto out; + + err = intel_context_migrate_clear(ce, await, sg, cache_level, + is_lmem, value, out); + + intel_context_unpin(ce); +out: + intel_context_put(ce); + return err; +} + +void intel_migrate_fini(struct intel_migrate *m) +{ + struct intel_context *ce; + + ce = fetch_and_zero(&m->context); + if (!ce) + return; + + intel_engine_destroy_pinned_context(ce); +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftest_migrate.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.h b/drivers/gpu/drm/i915/gt/intel_migrate.h new file mode 100644 index 000000000000..4e18e755a00b --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_migrate.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __INTEL_MIGRATE__ +#define __INTEL_MIGRATE__ + +#include <linux/types.h> + +#include "intel_migrate_types.h" + +struct dma_fence; +struct i915_request; +struct i915_gem_ww_ctx; +struct intel_gt; +struct scatterlist; +enum i915_cache_level; + +int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt); + +struct intel_context *intel_migrate_create_context(struct intel_migrate *m); + +int intel_migrate_copy(struct intel_migrate *m, + struct i915_gem_ww_ctx *ww, + struct dma_fence *await, + struct scatterlist *src, + enum i915_cache_level src_cache_level, + bool src_is_lmem, + struct scatterlist *dst, + enum i915_cache_level dst_cache_level, + bool dst_is_lmem, + struct i915_request **out); + +int intel_context_migrate_copy(struct intel_context *ce, + struct dma_fence *await, + struct scatterlist *src, + enum i915_cache_level src_cache_level, + bool src_is_lmem, + struct scatterlist *dst, + enum i915_cache_level dst_cache_level, + bool dst_is_lmem, + struct i915_request **out); + +int +intel_migrate_clear(struct intel_migrate *m, + struct i915_gem_ww_ctx *ww, + struct dma_fence *await, + struct scatterlist *sg, + enum i915_cache_level cache_level, + bool is_lmem, + u32 value, + struct i915_request **out); +int +intel_context_migrate_clear(struct intel_context *ce, + struct dma_fence *await, + struct scatterlist *sg, + enum i915_cache_level cache_level, + bool is_lmem, + u32 value, + struct i915_request **out); + +void intel_migrate_fini(struct intel_migrate *m); + +#endif /* __INTEL_MIGRATE__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_migrate_types.h b/drivers/gpu/drm/i915/gt/intel_migrate_types.h new file mode 100644 index 000000000000..d98230597f42 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_migrate_types.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __INTEL_MIGRATE_TYPES__ +#define __INTEL_MIGRATE_TYPES__ + +struct intel_context; + +struct intel_migrate { + struct intel_context *context; +}; + +#endif /* __INTEL_MIGRATE_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 17848807f111..582c4423b95d 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -352,7 +352,7 @@ static unsigned int get_mocs_settings(const struct drm_i915_private *i915, table->size = ARRAY_SIZE(icl_mocs_table); table->table = icl_mocs_table; table->n_entries = GEN9_NUM_MOCS_ENTRIES; - } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) { + } else if (IS_GEN9_BC(i915)) { table->size = ARRAY_SIZE(skl_mocs_table); table->n_entries = GEN9_NUM_MOCS_ENTRIES; table->table = skl_mocs_table; diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index 259d7eb4e165..799d382eea79 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -62,20 +62,25 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) u32 pg_enable; int i; - /* 2b: Program RC6 thresholds.*/ - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); - set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); + /* + * With GuCRC, these parameters are set by GuC + */ + if (!intel_uc_uses_guc_rc(>->uc)) { + /* 2b: Program RC6 thresholds.*/ + set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); + set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ + set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ + for_each_engine(engine, rc6_to_gt(rc6), id) + set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GUC_MAX_IDLE_COUNT, 0xA); + set(uncore, GUC_MAX_IDLE_COUNT, 0xA); - set(uncore, GEN6_RC_SLEEP, 0); + set(uncore, GEN6_RC_SLEEP, 0); - set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ + set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ + } /* * 2c: Program Coarse Power Gating Policies. @@ -98,11 +103,19 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60); set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60); - /* 3a: Enable RC6 */ - rc6->ctl_enable = - GEN6_RC_CTL_HW_ENABLE | - GEN6_RC_CTL_RC6_ENABLE | - GEN6_RC_CTL_EI_MODE(1); + /* 3a: Enable RC6 + * + * With GuCRC, we do not enable bit 31 of RC_CTL, + * thus allowing GuC to control RC6 entry/exit fully instead. + * We will not set the HW ENABLE and EI bits + */ + if (!intel_guc_rc_enable(>->uc.guc)) + rc6->ctl_enable = GEN6_RC_CTL_RC6_ENABLE; + else + rc6->ctl_enable = + GEN6_RC_CTL_HW_ENABLE | + GEN6_RC_CTL_RC6_ENABLE | + GEN6_RC_CTL_EI_MODE(1); pg_enable = GEN9_RENDER_PG_ENABLE | @@ -126,7 +139,7 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) enum intel_engine_id id; /* 2b: Program RC6 thresholds.*/ - if (GRAPHICS_VER(rc6_to_i915(rc6)) >= 10) { + if (GRAPHICS_VER(rc6_to_i915(rc6)) >= 11) { set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); } else if (IS_SKYLAKE(rc6_to_i915(rc6))) { @@ -513,6 +526,10 @@ static void __intel_rc6_disable(struct intel_rc6 *rc6) { struct drm_i915_private *i915 = rc6_to_i915(rc6); struct intel_uncore *uncore = rc6_to_uncore(rc6); + struct intel_gt *gt = rc6_to_gt(rc6); + + /* Take control of RC6 back from GuC */ + intel_guc_rc_disable(>->uc.guc); intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); if (GRAPHICS_VER(i915) >= 9) diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c index 4ae1f717a94c..a74b72f50cc9 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c @@ -10,7 +10,7 @@ #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "gem/i915_gem_ttm.h" -#include "intel_region_lmem.h" +#include "gt/intel_gt.h" static int init_fake_lmem_bar(struct intel_memory_region *mem) { @@ -158,7 +158,7 @@ intel_gt_setup_fake_lmem(struct intel_gt *gt) static bool get_legacy_lowmem_region(struct intel_uncore *uncore, u64 *start, u32 *size) { - if (!IS_DG1_REVID(uncore->i915, DG1_REVID_A0, DG1_REVID_B0)) + if (!IS_DG1_GT_STEP(uncore->i915, STEP_A0, STEP_C0)) return false; *start = 0; diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.h b/drivers/gpu/drm/i915/gt/intel_renderstate.h index 48f009203917..4da4c5234ef0 100644 --- a/drivers/gpu/drm/i915/gt/intel_renderstate.h +++ b/drivers/gpu/drm/i915/gt/intel_renderstate.h @@ -8,6 +8,7 @@ #include <linux/types.h> #include "i915_gem.h" +#include "i915_gem_ww.h" struct i915_request; struct intel_context; diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 72251638d4ea..91200c43951f 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -22,7 +22,6 @@ #include "intel_reset.h" #include "uc/intel_guc.h" -#include "uc/intel_guc_submission.h" #define RESET_MAX_RETRIES 3 @@ -39,21 +38,6 @@ static void rmw_clear_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) intel_uncore_rmw_fw(uncore, reg, clr, 0); } -static void skip_context(struct i915_request *rq) -{ - struct intel_context *hung_ctx = rq->context; - - list_for_each_entry_from_rcu(rq, &hung_ctx->timeline->requests, link) { - if (!i915_request_is_active(rq)) - return; - - if (rq->context == hung_ctx) { - i915_request_set_error_once(rq, -EIO); - __i915_request_skip(rq); - } - } -} - static void client_mark_guilty(struct i915_gem_context *ctx, bool banned) { struct drm_i915_file_private *file_priv = ctx->file_priv; @@ -88,10 +72,8 @@ static bool mark_guilty(struct i915_request *rq) bool banned; int i; - if (intel_context_is_closed(rq->context)) { - intel_context_set_banned(rq->context); + if (intel_context_is_closed(rq->context)) return true; - } rcu_read_lock(); ctx = rcu_dereference(rq->context->gem_context); @@ -123,11 +105,9 @@ static bool mark_guilty(struct i915_request *rq) banned = !i915_gem_context_is_recoverable(ctx); if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES)) banned = true; - if (banned) { + if (banned) drm_dbg(&ctx->i915->drm, "context %s: guilty %d, banned\n", ctx->name, atomic_read(&ctx->guilty_count)); - intel_context_set_banned(rq->context); - } client_mark_guilty(ctx, banned); @@ -149,6 +129,8 @@ static void mark_innocent(struct i915_request *rq) void __i915_request_reset(struct i915_request *rq, bool guilty) { + bool banned = false; + RQ_TRACE(rq, "guilty? %s\n", yesno(guilty)); GEM_BUG_ON(__i915_request_is_complete(rq)); @@ -156,13 +138,15 @@ void __i915_request_reset(struct i915_request *rq, bool guilty) if (guilty) { i915_request_set_error_once(rq, -EIO); __i915_request_skip(rq); - if (mark_guilty(rq)) - skip_context(rq); + banned = mark_guilty(rq); } else { i915_request_set_error_once(rq, -EAGAIN); mark_innocent(rq); } rcu_read_unlock(); + + if (banned) + intel_context_ban(rq->context, rq); } static bool i915_in_reset(struct pci_dev *pdev) @@ -515,8 +499,14 @@ static int gen11_reset_engines(struct intel_gt *gt, [VCS1] = GEN11_GRDOM_MEDIA2, [VCS2] = GEN11_GRDOM_MEDIA3, [VCS3] = GEN11_GRDOM_MEDIA4, + [VCS4] = GEN11_GRDOM_MEDIA5, + [VCS5] = GEN11_GRDOM_MEDIA6, + [VCS6] = GEN11_GRDOM_MEDIA7, + [VCS7] = GEN11_GRDOM_MEDIA8, [VECS0] = GEN11_GRDOM_VECS, [VECS1] = GEN11_GRDOM_VECS2, + [VECS2] = GEN11_GRDOM_VECS3, + [VECS3] = GEN11_GRDOM_VECS4, }; struct intel_engine_cs *engine; intel_engine_mask_t tmp; @@ -826,6 +816,8 @@ static int gt_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask) __intel_engine_reset(engine, stalled_mask & engine->mask); local_bh_enable(); + intel_uc_reset(>->uc, true); + intel_ggtt_restore_fences(gt->ggtt); return err; @@ -850,6 +842,8 @@ static void reset_finish(struct intel_gt *gt, intel_engine_mask_t awake) if (awake & engine->mask) intel_engine_pm_put(engine); } + + intel_uc_reset_finish(>->uc); } static void nop_submit_request(struct i915_request *request) @@ -903,6 +897,7 @@ static void __intel_gt_set_wedged(struct intel_gt *gt) for_each_engine(engine, gt, id) if (engine->reset.cancel) engine->reset.cancel(engine); + intel_uc_cancel_requests(>->uc); local_bh_enable(); reset_finish(gt, awake); @@ -1191,6 +1186,9 @@ int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) ENGINE_TRACE(engine, "flags=%lx\n", gt->reset.flags); GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, >->reset.flags)); + if (intel_engine_uses_guc(engine)) + return -ENODEV; + if (!intel_engine_pm_get_if_awake(engine)) return 0; @@ -1201,13 +1199,10 @@ int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) "Resetting %s for %s\n", engine->name, msg); atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]); - if (intel_engine_uses_guc(engine)) - ret = intel_guc_reset_engine(&engine->gt->uc.guc, engine); - else - ret = intel_gt_reset_engine(engine); + ret = intel_gt_reset_engine(engine); if (ret) { /* If we fail here, we expect to fallback to a global reset */ - ENGINE_TRACE(engine, "Failed to reset, err: %d\n", ret); + ENGINE_TRACE(engine, "Failed to reset %s, err: %d\n", engine->name, ret); goto out; } @@ -1341,7 +1336,8 @@ void intel_gt_handle_error(struct intel_gt *gt, * Try engine reset when available. We fall back to full reset if * single reset fails. */ - if (intel_has_reset_engine(gt) && !intel_gt_is_wedged(gt)) { + if (!intel_uc_uses_guc_submission(>->uc) && + intel_has_reset_engine(gt) && !intel_gt_is_wedged(gt)) { local_bh_disable(); for_each_engine_masked(engine, gt, engine_mask, tmp) { BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE); diff --git a/drivers/gpu/drm/i915/gt/intel_ring.h b/drivers/gpu/drm/i915/gt/intel_ring.h index dbf5f14a136f..1b32dadfb8c3 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.h +++ b/drivers/gpu/drm/i915/gt/intel_ring.h @@ -49,6 +49,7 @@ static inline void intel_ring_advance(struct i915_request *rq, u32 *cs) * intel_ring_begin()). */ GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs); + GEM_BUG_ON(!IS_ALIGNED(rq->ring->emit, 8)); /* RING_TAIL qword align */ } static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos) diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 37d74d4ed59b..2958e2fae380 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -16,6 +16,7 @@ #include "intel_reset.h" #include "intel_ring.h" #include "shmem_utils.h" +#include "intel_engine_heartbeat.h" /* Rough estimate of the typical request size, performing a flush, * set-context and then emitting the batch. @@ -342,9 +343,9 @@ static void reset_rewind(struct intel_engine_cs *engine, bool stalled) u32 head; rq = NULL; - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); rcu_read_lock(); - list_for_each_entry(pos, &engine->active.requests, sched.link) { + list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) { if (!__i915_request_is_complete(pos)) { rq = pos; break; @@ -399,7 +400,7 @@ static void reset_rewind(struct intel_engine_cs *engine, bool stalled) } engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } static void reset_finish(struct intel_engine_cs *engine) @@ -411,16 +412,16 @@ static void reset_cancel(struct intel_engine_cs *engine) struct i915_request *request; unsigned long flags; - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); /* Mark all submitted requests as skipped. */ - list_for_each_entry(request, &engine->active.requests, sched.link) + list_for_each_entry(request, &engine->sched_engine->requests, sched.link) i915_request_put(i915_request_mark_eio(request)); intel_engine_signal_breadcrumbs(engine); /* Remaining _unready_ requests will be nop'ed when submitted */ - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } static void i9xx_submit_request(struct i915_request *request) @@ -586,9 +587,44 @@ static void ring_context_reset(struct intel_context *ce) clear_bit(CONTEXT_VALID_BIT, &ce->flags); } +static void ring_context_ban(struct intel_context *ce, + struct i915_request *rq) +{ + struct intel_engine_cs *engine; + + if (!rq || !i915_request_is_active(rq)) + return; + + engine = rq->engine; + lockdep_assert_held(&engine->sched_engine->lock); + list_for_each_entry_continue(rq, &engine->sched_engine->requests, + sched.link) + if (rq->context == ce) { + i915_request_set_error_once(rq, -EIO); + __i915_request_skip(rq); + } +} + +static void ring_context_cancel_request(struct intel_context *ce, + struct i915_request *rq) +{ + struct intel_engine_cs *engine = NULL; + + i915_request_active_engine(rq, &engine); + + if (engine && intel_engine_pulse(engine)) + intel_gt_handle_error(engine->gt, engine->mask, 0, + "request cancellation by %s", + current->comm); +} + static const struct intel_context_ops ring_context_ops = { .alloc = ring_context_alloc, + .cancel_request = ring_context_cancel_request, + + .ban = ring_context_ban, + .pre_pin = ring_context_pre_pin, .pin = ring_context_pin, .unpin = ring_context_unpin, @@ -1047,6 +1083,25 @@ static void setup_irq(struct intel_engine_cs *engine) } } +static void add_to_engine(struct i915_request *rq) +{ + lockdep_assert_held(&rq->engine->sched_engine->lock); + list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); +} + +static void remove_from_engine(struct i915_request *rq) +{ + spin_lock_irq(&rq->engine->sched_engine->lock); + list_del_init(&rq->sched.link); + + /* Prevent further __await_execution() registering a cb, then flush */ + set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); + + spin_unlock_irq(&rq->engine->sched_engine->lock); + + i915_request_notify_execute_cb_imm(rq); +} + static void setup_common(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; @@ -1064,6 +1119,9 @@ static void setup_common(struct intel_engine_cs *engine) engine->reset.cancel = reset_cancel; engine->reset.finish = reset_finish; + engine->add_active_request = add_to_engine; + engine->remove_active_request = remove_from_engine; + engine->cops = &ring_context_ops; engine->request_alloc = ring_request_alloc; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 06e9a8ed4e03..d812b27835f8 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -37,6 +37,20 @@ static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) return rps_to_gt(rps)->uncore; } +static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + return >->uc.guc.slpc; +} + +static bool rps_uses_slpc(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + return intel_uc_uses_guc_slpc(>->uc); +} + static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) { return mask & ~rps->pm_intrmsk_mbz; @@ -167,6 +181,8 @@ static void rps_enable_interrupts(struct intel_rps *rps) { struct intel_gt *gt = rps_to_gt(rps); + GEM_BUG_ON(rps_uses_slpc(rps)); + GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", rps->pm_events, rps_pm_mask(rps, rps->last_freq)); @@ -771,6 +787,8 @@ static int gen6_rps_set(struct intel_rps *rps, u8 val) struct drm_i915_private *i915 = rps_to_i915(rps); u32 swreq; + GEM_BUG_ON(rps_uses_slpc(rps)); + if (GRAPHICS_VER(i915) >= 9) swreq = GEN9_FREQUENCY(val); else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) @@ -861,6 +879,9 @@ void intel_rps_park(struct intel_rps *rps) { int adj; + if (!intel_rps_is_enabled(rps)) + return; + GEM_BUG_ON(atomic_read(&rps->num_waiters)); if (!intel_rps_clear_active(rps)) @@ -999,7 +1020,7 @@ static void gen6_rps_init(struct intel_rps *rps) rps->efficient_freq = rps->rp1_freq; if (IS_HASWELL(i915) || IS_BROADWELL(i915) || - IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 10) { + IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { u32 ddcc_status = 0; if (sandybridge_pcode_read(i915, @@ -1012,7 +1033,7 @@ static void gen6_rps_init(struct intel_rps *rps) rps->max_freq); } - if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 10) { + if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { /* Store the frequency values in 16.66 MHZ units, which is * the natural hardware unit for SKL */ @@ -1356,6 +1377,9 @@ void intel_rps_enable(struct intel_rps *rps) if (!HAS_RPS(i915)) return; + if (rps_uses_slpc(rps)) + return; + intel_gt_check_clock_frequency(rps_to_gt(rps)); intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); @@ -1829,6 +1853,9 @@ void intel_rps_init(struct intel_rps *rps) { struct drm_i915_private *i915 = rps_to_i915(rps); + if (rps_uses_slpc(rps)) + return; + if (IS_CHERRYVIEW(i915)) chv_rps_init(rps); else if (IS_VALLEYVIEW(i915)) @@ -1877,10 +1904,17 @@ void intel_rps_init(struct intel_rps *rps) if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; + + /* GuC needs ARAT expired interrupt unmasked */ + if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) + rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; } void intel_rps_sanitize(struct intel_rps *rps) { + if (rps_uses_slpc(rps)) + return; + if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) rps_disable_interrupts(rps); } @@ -1936,6 +1970,176 @@ u32 intel_rps_read_actual_frequency(struct intel_rps *rps) return freq; } +u32 intel_rps_read_punit_req(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + + return intel_uncore_read(uncore, GEN6_RPNSWREQ); +} + +static u32 intel_rps_get_req(u32 pureq) +{ + u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; + + return req; +} + +u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) +{ + u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); + + return intel_gpu_freq(rps, freq); +} + +u32 intel_rps_get_requested_frequency(struct intel_rps *rps) +{ + if (rps_uses_slpc(rps)) + return intel_rps_read_punit_req_frequency(rps); + else + return intel_gpu_freq(rps, rps->cur_freq); +} + +u32 intel_rps_get_max_frequency(struct intel_rps *rps) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return slpc->max_freq_softlimit; + else + return intel_gpu_freq(rps, rps->max_freq_softlimit); +} + +u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return slpc->rp0_freq; + else + return intel_gpu_freq(rps, rps->rp0_freq); +} + +u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return slpc->rp1_freq; + else + return intel_gpu_freq(rps, rps->rp1_freq); +} + +u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return slpc->min_freq; + else + return intel_gpu_freq(rps, rps->min_freq); +} + +static int set_max_freq(struct intel_rps *rps, u32 val) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + int ret = 0; + + mutex_lock(&rps->lock); + + val = intel_freq_opcode(rps, val); + if (val < rps->min_freq || + val > rps->max_freq || + val < rps->min_freq_softlimit) { + ret = -EINVAL; + goto unlock; + } + + if (val > rps->rp0_freq) + drm_dbg(&i915->drm, "User requested overclocking to %d\n", + intel_gpu_freq(rps, val)); + + rps->max_freq_softlimit = val; + + val = clamp_t(int, rps->cur_freq, + rps->min_freq_softlimit, + rps->max_freq_softlimit); + + /* + * We still need *_set_rps to process the new max_delay and + * update the interrupt limits and PMINTRMSK even though + * frequency request may be unchanged. + */ + intel_rps_set(rps, val); + +unlock: + mutex_unlock(&rps->lock); + + return ret; +} + +int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return intel_guc_slpc_set_max_freq(slpc, val); + else + return set_max_freq(rps, val); +} + +u32 intel_rps_get_min_frequency(struct intel_rps *rps) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return slpc->min_freq_softlimit; + else + return intel_gpu_freq(rps, rps->min_freq_softlimit); +} + +static int set_min_freq(struct intel_rps *rps, u32 val) +{ + int ret = 0; + + mutex_lock(&rps->lock); + + val = intel_freq_opcode(rps, val); + if (val < rps->min_freq || + val > rps->max_freq || + val > rps->max_freq_softlimit) { + ret = -EINVAL; + goto unlock; + } + + rps->min_freq_softlimit = val; + + val = clamp_t(int, rps->cur_freq, + rps->min_freq_softlimit, + rps->max_freq_softlimit); + + /* + * We still need *_set_rps to process the new min_delay and + * update the interrupt limits and PMINTRMSK even though + * frequency request may be unchanged. + */ + intel_rps_set(rps, val); + +unlock: + mutex_unlock(&rps->lock); + + return ret; +} + +int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) +{ + struct intel_guc_slpc *slpc = rps_to_slpc(rps); + + if (rps_uses_slpc(rps)) + return intel_guc_slpc_set_min_freq(slpc, val); + else + return set_min_freq(rps, val); +} + /* External interface for intel_ips.ko */ static struct drm_i915_private __rcu *ips_mchdev; @@ -2129,4 +2333,5 @@ EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_rps.c" +#include "selftest_slpc.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h index 1d2cfc98b510..4213bcce1667 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.h +++ b/drivers/gpu/drm/i915/gt/intel_rps.h @@ -31,6 +31,16 @@ int intel_gpu_freq(struct intel_rps *rps, int val); int intel_freq_opcode(struct intel_rps *rps, int val); u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat1); u32 intel_rps_read_actual_frequency(struct intel_rps *rps); +u32 intel_rps_get_requested_frequency(struct intel_rps *rps); +u32 intel_rps_get_min_frequency(struct intel_rps *rps); +int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val); +u32 intel_rps_get_max_frequency(struct intel_rps *rps); +int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val); +u32 intel_rps_get_rp0_frequency(struct intel_rps *rps); +u32 intel_rps_get_rp1_frequency(struct intel_rps *rps); +u32 intel_rps_get_rpn_frequency(struct intel_rps *rps); +u32 intel_rps_read_punit_req(struct intel_rps *rps); +u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps); void gen5_rps_irq_handler(struct intel_rps *rps); void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c index 367fd44b81c8..bbd272943c3f 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu.c @@ -139,17 +139,36 @@ static void gen12_sseu_info_init(struct intel_gt *gt) * Gen12 has Dual-Subslices, which behave similarly to 2 gen11 SS. * Instead of splitting these, provide userspace with an array * of DSS to more closely represent the hardware resource. + * + * In addition, the concept of slice has been removed in Xe_HP. + * To be compatible with prior generations, assume a single slice + * across the entire device. Then calculate out the DSS for each + * workload type within that software slice. */ - intel_sseu_set_info(sseu, 1, 6, 16); + if (IS_DG2(gt->i915) || IS_XEHPSDV(gt->i915)) + intel_sseu_set_info(sseu, 1, 32, 16); + else + intel_sseu_set_info(sseu, 1, 6, 16); - s_en = intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE) & - GEN11_GT_S_ENA_MASK; + /* + * As mentioned above, Xe_HP does not have the concept of a slice. + * Enable one for software backwards compatibility. + */ + if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) + s_en = 0x1; + else + s_en = intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE) & + GEN11_GT_S_ENA_MASK; dss_en = intel_uncore_read(uncore, GEN12_GT_DSS_ENABLE); /* one bit per pair of EUs */ - eu_en_fuse = ~(intel_uncore_read(uncore, GEN11_EU_DISABLE) & - GEN11_EU_DIS_MASK); + if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) + eu_en_fuse = intel_uncore_read(uncore, XEHP_EU_ENABLE) & XEHP_EU_ENA_MASK; + else + eu_en_fuse = ~(intel_uncore_read(uncore, GEN11_EU_DISABLE) & + GEN11_EU_DIS_MASK); + for (eu = 0; eu < sseu->max_eus_per_subslice / 2; eu++) if (eu_en_fuse & BIT(eu)) eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1); @@ -188,83 +207,6 @@ static void gen11_sseu_info_init(struct intel_gt *gt) sseu->has_eu_pg = 1; } -static void gen10_sseu_info_init(struct intel_gt *gt) -{ - struct intel_uncore *uncore = gt->uncore; - struct sseu_dev_info *sseu = >->info.sseu; - const u32 fuse2 = intel_uncore_read(uncore, GEN8_FUSE2); - const int eu_mask = 0xff; - u32 subslice_mask, eu_en; - int s, ss; - - intel_sseu_set_info(sseu, 6, 4, 8); - - sseu->slice_mask = (fuse2 & GEN10_F2_S_ENA_MASK) >> - GEN10_F2_S_ENA_SHIFT; - - /* Slice0 */ - eu_en = ~intel_uncore_read(uncore, GEN8_EU_DISABLE0); - for (ss = 0; ss < sseu->max_subslices; ss++) - sseu_set_eus(sseu, 0, ss, (eu_en >> (8 * ss)) & eu_mask); - /* Slice1 */ - sseu_set_eus(sseu, 1, 0, (eu_en >> 24) & eu_mask); - eu_en = ~intel_uncore_read(uncore, GEN8_EU_DISABLE1); - sseu_set_eus(sseu, 1, 1, eu_en & eu_mask); - /* Slice2 */ - sseu_set_eus(sseu, 2, 0, (eu_en >> 8) & eu_mask); - sseu_set_eus(sseu, 2, 1, (eu_en >> 16) & eu_mask); - /* Slice3 */ - sseu_set_eus(sseu, 3, 0, (eu_en >> 24) & eu_mask); - eu_en = ~intel_uncore_read(uncore, GEN8_EU_DISABLE2); - sseu_set_eus(sseu, 3, 1, eu_en & eu_mask); - /* Slice4 */ - sseu_set_eus(sseu, 4, 0, (eu_en >> 8) & eu_mask); - sseu_set_eus(sseu, 4, 1, (eu_en >> 16) & eu_mask); - /* Slice5 */ - sseu_set_eus(sseu, 5, 0, (eu_en >> 24) & eu_mask); - eu_en = ~intel_uncore_read(uncore, GEN10_EU_DISABLE3); - sseu_set_eus(sseu, 5, 1, eu_en & eu_mask); - - subslice_mask = (1 << 4) - 1; - subslice_mask &= ~((fuse2 & GEN10_F2_SS_DIS_MASK) >> - GEN10_F2_SS_DIS_SHIFT); - - for (s = 0; s < sseu->max_slices; s++) { - u32 subslice_mask_with_eus = subslice_mask; - - for (ss = 0; ss < sseu->max_subslices; ss++) { - if (sseu_get_eus(sseu, s, ss) == 0) - subslice_mask_with_eus &= ~BIT(ss); - } - - /* - * Slice0 can have up to 3 subslices, but there are only 2 in - * slice1/2. - */ - intel_sseu_set_subslices(sseu, s, s == 0 ? - subslice_mask_with_eus : - subslice_mask_with_eus & 0x3); - } - - sseu->eu_total = compute_eu_total(sseu); - - /* - * CNL is expected to always have a uniform distribution - * of EU across subslices with the exception that any one - * EU in any one subslice may be fused off for die - * recovery. - */ - sseu->eu_per_subslice = - intel_sseu_subslice_total(sseu) ? - DIV_ROUND_UP(sseu->eu_total, intel_sseu_subslice_total(sseu)) : - 0; - - /* No restrictions on Power Gating */ - sseu->has_slice_pg = 1; - sseu->has_subslice_pg = 1; - sseu->has_eu_pg = 1; -} - static void cherryview_sseu_info_init(struct intel_gt *gt) { struct sseu_dev_info *sseu = >->info.sseu; @@ -592,8 +534,6 @@ void intel_sseu_info_init(struct intel_gt *gt) bdw_sseu_info_init(gt); else if (GRAPHICS_VER(i915) == 9) gen9_sseu_info_init(gt); - else if (GRAPHICS_VER(i915) == 10) - gen10_sseu_info_init(gt); else if (GRAPHICS_VER(i915) == 11) gen11_sseu_info_init(gt); else if (GRAPHICS_VER(i915) >= 12) @@ -759,3 +699,21 @@ void intel_sseu_print_topology(const struct sseu_dev_info *sseu, } } } + +u16 intel_slicemask_from_dssmask(u64 dss_mask, int dss_per_slice) +{ + u16 slice_mask = 0; + int i; + + WARN_ON(sizeof(dss_mask) * 8 / dss_per_slice > 8 * sizeof(slice_mask)); + + for (i = 0; dss_mask; i++) { + if (dss_mask & GENMASK(dss_per_slice - 1, 0)) + slice_mask |= BIT(i); + + dss_mask >>= dss_per_slice; + } + + return slice_mask; +} + diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.h b/drivers/gpu/drm/i915/gt/intel_sseu.h index 4cd1a8a7298a..22fef98887c0 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.h +++ b/drivers/gpu/drm/i915/gt/intel_sseu.h @@ -15,13 +15,17 @@ struct drm_i915_private; struct intel_gt; struct drm_printer; -#define GEN_MAX_SLICES (6) /* CNL upper bound */ -#define GEN_MAX_SUBSLICES (8) /* ICL upper bound */ +#define GEN_MAX_SLICES (3) /* SKL upper bound */ +#define GEN_MAX_SUBSLICES (32) /* XEHPSDV upper bound */ #define GEN_SSEU_STRIDE(max_entries) DIV_ROUND_UP(max_entries, BITS_PER_BYTE) #define GEN_MAX_SUBSLICE_STRIDE GEN_SSEU_STRIDE(GEN_MAX_SUBSLICES) #define GEN_MAX_EUS (16) /* TGL upper bound */ #define GEN_MAX_EU_STRIDE GEN_SSEU_STRIDE(GEN_MAX_EUS) +#define GEN_DSS_PER_GSLICE 4 +#define GEN_DSS_PER_CSLICE 8 +#define GEN_DSS_PER_MSLICE 8 + struct sseu_dev_info { u8 slice_mask; u8 subslice_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICE_STRIDE]; @@ -104,4 +108,6 @@ void intel_sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p); void intel_sseu_print_topology(const struct sseu_dev_info *sseu, struct drm_printer *p); +u16 intel_slicemask_from_dssmask(u64 dss_mask, int dss_per_slice); + #endif /* __INTEL_SSEU_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c index 714fe8495775..1ba8b7da9d37 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c @@ -50,10 +50,10 @@ static void cherryview_sseu_device_status(struct intel_gt *gt, #undef SS_MAX } -static void gen10_sseu_device_status(struct intel_gt *gt, +static void gen11_sseu_device_status(struct intel_gt *gt, struct sseu_dev_info *sseu) { -#define SS_MAX 6 +#define SS_MAX 8 struct intel_uncore *uncore = gt->uncore; const struct intel_gt_info *info = >->info; u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2]; @@ -267,8 +267,8 @@ int intel_sseu_status(struct seq_file *m, struct intel_gt *gt) bdw_sseu_device_status(gt, &sseu); else if (GRAPHICS_VER(i915) == 9) gen9_sseu_device_status(gt, &sseu); - else if (GRAPHICS_VER(i915) >= 10) - gen10_sseu_device_status(gt, &sseu); + else if (GRAPHICS_VER(i915) >= 11) + gen11_sseu_device_status(gt, &sseu); } i915_print_sseu_info(m, false, HAS_POOLED_EU(i915), &sseu); diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index c4a126c8caef..1257f4f11e66 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -127,6 +127,15 @@ static void intel_timeline_fini(struct rcu_head *rcu) i915_vma_put(timeline->hwsp_ggtt); i915_active_fini(&timeline->active); + + /* + * A small race exists between intel_gt_retire_requests_timeout and + * intel_timeline_exit which could result in the syncmap not getting + * free'd. Rather than work to hard to seal this race, simply cleanup + * the syncmap on fini. + */ + i915_syncmap_free(&timeline->sync); + kfree(timeline); } diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index b62d1e31a645..aae609d7d85d 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -150,13 +150,14 @@ static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) } static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, - u32 clear, u32 set, u32 read_mask) + u32 clear, u32 set, u32 read_mask, bool masked_reg) { struct i915_wa wa = { .reg = reg, .clr = clear, .set = set, .read = read_mask, + .masked_reg = masked_reg, }; _wa_add(wal, &wa); @@ -165,7 +166,7 @@ static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, static void wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) { - wa_add(wal, reg, clear, set, clear); + wa_add(wal, reg, clear, set, clear, false); } static void @@ -200,20 +201,20 @@ wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) static void wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) { - wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val); + wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true); } static void wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) { - wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val); + wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true); } static void wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, u32 val) { - wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask); + wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true); } static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, @@ -514,53 +515,15 @@ static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); } -static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine, - struct i915_wa_list *wal) -{ - /* WaForceContextSaveRestoreNonCoherent:cnl */ - wa_masked_en(wal, CNL_HDC_CHICKEN0, - HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT); - - /* WaDisableReplayBufferBankArbitrationOptimization:cnl */ - wa_masked_en(wal, COMMON_SLICE_CHICKEN2, - GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); - - /* WaPushConstantDereferenceHoldDisable:cnl */ - wa_masked_en(wal, GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE); - - /* FtrEnableFastAnisoL1BankingFix:cnl */ - wa_masked_en(wal, HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX); - - /* WaDisable3DMidCmdPreemption:cnl */ - wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); - - /* WaDisableGPGPUMidCmdPreemption:cnl */ - wa_masked_field_set(wal, GEN8_CS_CHICKEN1, - GEN9_PREEMPT_GPGPU_LEVEL_MASK, - GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); - - /* WaDisableEarlyEOT:cnl */ - wa_masked_en(wal, GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT); -} - static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { - struct drm_i915_private *i915 = engine->i915; - - /* WaDisableBankHangMode:icl */ + /* Wa_1406697149 (WaDisableBankHangMode:icl) */ wa_write(wal, GEN8_L3CNTLREG, intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) | GEN8_ERRDETBCTRL); - /* Wa_1604370585:icl (pre-prod) - * Formerly known as WaPushConstantDereferenceHoldDisable - */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) - wa_masked_en(wal, GEN7_ROW_CHICKEN2, - PUSH_CONSTANT_DEREF_DISABLE); - /* WaForceEnableNonCoherent:icl * This is not the same workaround as in early Gen9 platforms, where * lacking this could cause system hangs, but coherency performance @@ -570,23 +533,11 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, */ wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT); - /* Wa_2006611047:icl (pre-prod) - * Formerly known as WaDisableImprovedTdlClkGating - */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) - wa_masked_en(wal, GEN7_ROW_CHICKEN2, - GEN11_TDL_CLOCK_GATING_FIX_DISABLE); - - /* Wa_2006665173:icl (pre-prod) */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) - wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, - GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC); - /* WaEnableFloatBlendOptimization:icl */ - wa_write_clr_set(wal, - GEN10_CACHE_MODE_SS, - 0, /* write-only, so skip validation */ - _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE)); + wa_add(wal, GEN10_CACHE_MODE_SS, 0, + _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE), + 0 /* write-only, so skip validation */, + true); /* WaDisableGPGPUMidThreadPreemption:icl */ wa_masked_field_set(wal, GEN8_CS_CHICKEN1, @@ -631,7 +582,7 @@ static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine, FF_MODE2, FF_MODE2_TDS_TIMER_MASK, FF_MODE2_TDS_TIMER_128, - 0); + 0, false); } static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, @@ -640,15 +591,16 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, gen12_ctx_gt_tuning_init(engine, wal); /* - * Wa_1409142259:tgl - * Wa_1409347922:tgl - * Wa_1409252684:tgl - * Wa_1409217633:tgl - * Wa_1409207793:tgl - * Wa_1409178076:tgl - * Wa_1408979724:tgl - * Wa_14010443199:rkl - * Wa_14010698770:rkl + * Wa_1409142259:tgl,dg1,adl-p + * Wa_1409347922:tgl,dg1,adl-p + * Wa_1409252684:tgl,dg1,adl-p + * Wa_1409217633:tgl,dg1,adl-p + * Wa_1409207793:tgl,dg1,adl-p + * Wa_1409178076:tgl,dg1,adl-p + * Wa_1408979724:tgl,dg1,adl-p + * Wa_14010443199:tgl,rkl,dg1,adl-p + * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p + * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p */ wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, GEN12_DISABLE_CPS_AWARE_COLOR_PIPE); @@ -668,7 +620,14 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, FF_MODE2, FF_MODE2_GS_TIMER_MASK, FF_MODE2_GS_TIMER_224, - 0); + 0, false); + + /* + * Wa_14012131227:dg1 + * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p + */ + wa_masked_en(wal, GEN7_COMMON_SLICE_CHICKEN1, + GEN9_RHWO_OPTIMIZATION_DISABLE); } static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, @@ -703,8 +662,6 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, gen12_ctx_workarounds_init(engine, wal); else if (GRAPHICS_VER(i915) == 11) icl_ctx_workarounds_init(engine, wal); - else if (IS_CANNONLAKE(i915)) - cnl_ctx_workarounds_init(engine, wal); else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) cfl_ctx_workarounds_init(engine, wal); else if (IS_GEMINILAKE(i915)) @@ -839,7 +796,7 @@ hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) wa_add(wal, HSW_ROW_CHICKEN3, 0, _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), - 0 /* XXX does this reg exist? */); + 0 /* XXX does this reg exist? */, true); /* WaVSRefCountFullforceMissDisable:hsw */ wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); @@ -882,30 +839,19 @@ skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); /* WaInPlaceDecompressionHang:skl */ - if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER)) + if (IS_SKL_GT_STEP(i915, STEP_A0, STEP_H0)) wa_write_or(wal, GEN9_GAMT_ECO_REG_RW_IA, GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } static void -bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) -{ - gen9_gt_workarounds_init(i915, wal); - - /* WaInPlaceDecompressionHang:bxt */ - wa_write_or(wal, - GEN9_GAMT_ECO_REG_RW_IA, - GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); -} - -static void kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { gen9_gt_workarounds_init(i915, wal); /* WaDisableDynamicCreditSharing:kbl */ - if (IS_KBL_GT_STEP(i915, 0, STEP_B0)) + if (IS_KBL_GT_STEP(i915, 0, STEP_C0)) wa_write_or(wal, GAMT_CHKN_BIT_REG, GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); @@ -943,98 +889,144 @@ cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } +static void __set_mcr_steering(struct i915_wa_list *wal, + i915_reg_t steering_reg, + unsigned int slice, unsigned int subslice) +{ + u32 mcr, mcr_mask; + + mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); + mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; + + wa_write_clr_set(wal, steering_reg, mcr_mask, mcr); +} + +static void __add_mcr_wa(struct drm_i915_private *i915, struct i915_wa_list *wal, + unsigned int slice, unsigned int subslice) +{ + drm_dbg(&i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice); + + __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice); +} + static void -wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) +icl_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) { const struct sseu_dev_info *sseu = &i915->gt.info.sseu; unsigned int slice, subslice; - u32 l3_en, mcr, mcr_mask; - GEM_BUG_ON(GRAPHICS_VER(i915) < 10); + GEM_BUG_ON(GRAPHICS_VER(i915) < 11); + GEM_BUG_ON(hweight8(sseu->slice_mask) > 1); + slice = 0; /* - * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl - * L3Banks could be fused off in single slice scenario. If that is - * the case, we might need to program MCR select to a valid L3Bank - * by default, to make sure we correctly read certain registers - * later on (in the range 0xB100 - 0xB3FF). + * Although a platform may have subslices, we need to always steer + * reads to the lowest instance that isn't fused off. When Render + * Power Gating is enabled, grabbing forcewake will only power up a + * single subslice (the "minconfig") if there isn't a real workload + * that needs to be run; this means that if we steer register reads to + * one of the higher subslices, we run the risk of reading back 0's or + * random garbage. + */ + subslice = __ffs(intel_sseu_get_subslices(sseu, slice)); + + /* + * If the subslice we picked above also steers us to a valid L3 bank, + * then we can just rely on the default steering and won't need to + * worry about explicitly re-steering L3BANK reads later. + */ + if (i915->gt.info.l3bank_mask & BIT(subslice)) + i915->gt.steering_table[L3BANK] = NULL; + + __add_mcr_wa(i915, wal, slice, subslice); +} + +static void +xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) +{ + struct drm_i915_private *i915 = gt->i915; + const struct sseu_dev_info *sseu = >->info.sseu; + unsigned long slice, subslice = 0, slice_mask = 0; + u64 dss_mask = 0; + u32 lncf_mask = 0; + int i; + + /* + * On Xe_HP the steering increases in complexity. There are now several + * more units that require steering and we're not guaranteed to be able + * to find a common setting for all of them. These are: + * - GSLICE (fusable) + * - DSS (sub-unit within gslice; fusable) + * - L3 Bank (fusable) + * - MSLICE (fusable) + * - LNCF (sub-unit within mslice; always present if mslice is present) * - * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl - * Before any MMIO read into slice/subslice specific registers, MCR - * packet control register needs to be programmed to point to any - * enabled s/ss pair. Otherwise, incorrect values will be returned. - * This means each subsequent MMIO read will be forwarded to an - * specific s/ss combination, but this is OK since these registers - * are consistent across s/ss in almost all cases. In the rare - * occasions, such as INSTDONE, where this value is dependent - * on s/ss combo, the read should be done with read_subslice_reg. + * We'll do our default/implicit steering based on GSLICE (in the + * sliceid field) and DSS (in the subsliceid field). If we can + * find overlap between the valid MSLICE and/or LNCF values with + * a suitable GSLICE, then we can just re-use the default value and + * skip and explicit steering at runtime. * - * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both - * to which subslice, or to which L3 bank, the respective mmio reads - * will go, we have to find a common index which works for both - * accesses. + * We only need to look for overlap between GSLICE/MSLICE/LNCF to find + * a valid sliceid value. DSS steering is the only type of steering + * that utilizes the 'subsliceid' bits. * - * Case where we cannot find a common index fortunately should not - * happen in production hardware, so we only emit a warning instead of - * implementing something more complex that requires checking the range - * of every MMIO read. + * Also note that, even though the steering domain is called "GSlice" + * and it is encoded in the register using the gslice format, the spec + * says that the combined (geometry | compute) fuse should be used to + * select the steering. */ - if (GRAPHICS_VER(i915) >= 10 && is_power_of_2(sseu->slice_mask)) { - u32 l3_fuse = - intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) & - GEN10_L3BANK_MASK; + /* Find the potential gslice candidates */ + dss_mask = intel_sseu_get_subslices(sseu, 0); + slice_mask = intel_slicemask_from_dssmask(dss_mask, GEN_DSS_PER_GSLICE); - drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse); - l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse); - } else { - l3_en = ~0; - } + /* + * Find the potential LNCF candidates. Either LNCF within a valid + * mslice is fine. + */ + for_each_set_bit(i, >->info.mslice_mask, GEN12_MAX_MSLICES) + lncf_mask |= (0x3 << (i * 2)); - slice = fls(sseu->slice_mask) - 1; - subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice)); - if (!subslice) { - drm_warn(&i915->drm, - "No common index found between subslice mask %x and L3 bank mask %x!\n", - intel_sseu_get_subslices(sseu, slice), l3_en); - subslice = fls(l3_en); - drm_WARN_ON(&i915->drm, !subslice); - } - subslice--; - - if (GRAPHICS_VER(i915) >= 11) { - mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); - mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; - } else { - mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); - mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; + /* + * Are there any sliceid values that work for both GSLICE and LNCF + * steering? + */ + if (slice_mask & lncf_mask) { + slice_mask &= lncf_mask; + gt->steering_table[LNCF] = NULL; } - drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr); + /* How about sliceid values that also work for MSLICE steering? */ + if (slice_mask & gt->info.mslice_mask) { + slice_mask &= gt->info.mslice_mask; + gt->steering_table[MSLICE] = NULL; + } - wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr); -} + slice = __ffs(slice_mask); + subslice = __ffs(dss_mask >> (slice * GEN_DSS_PER_GSLICE)); + WARN_ON(subslice > GEN_DSS_PER_GSLICE); + WARN_ON(dss_mask >> (slice * GEN_DSS_PER_GSLICE) == 0); -static void -cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) -{ - wa_init_mcr(i915, wal); + __add_mcr_wa(i915, wal, slice, subslice); - /* WaInPlaceDecompressionHang:cnl */ - wa_write_or(wal, - GEN9_GAMT_ECO_REG_RW_IA, - GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); + /* + * SQIDI ranges are special because they use different steering + * registers than everything else we work with. On XeHP SDV and + * DG2-G10, any value in the steering registers will work fine since + * all instances are present, but DG2-G11 only has SQIDI instances at + * ID's 2 and 3, so we need to steer to one of those. For simplicity + * we'll just steer to a hardcoded "2" since that value will work + * everywhere. + */ + __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2); + __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2); } static void icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - wa_init_mcr(i915, wal); - - /* WaInPlaceDecompressionHang:icl */ - wa_write_or(wal, - GEN9_GAMT_ECO_REG_RW_IA, - GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); + icl_wa_init_mcr(i915, wal); /* WaModifyGamTlbPartitioning:icl */ wa_write_clr_set(wal, @@ -1057,18 +1049,6 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) GEN8_GAMW_ECO_DEV_RW_IA, GAMW_ECO_DEV_CTX_RELOAD_DISABLE); - /* Wa_1405779004:icl (pre-prod) */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) - wa_write_or(wal, - SLICE_UNIT_LEVEL_CLKGATE, - MSCUNIT_CLKGATE_DIS); - - /* Wa_1406838659:icl (pre-prod) */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) - wa_write_or(wal, - INF_UNIT_LEVEL_CLKGATE, - CGPSF_CLKGATE_DIS); - /* Wa_1406463099:icl * Formerly known as WaGamTlbPendError */ @@ -1078,10 +1058,16 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) /* Wa_1607087056:icl,ehl,jsl */ if (IS_ICELAKE(i915) || - IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) + IS_JSL_EHL_GT_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); + + /* + * This is not a documented workaround, but rather an optimization + * to reduce sampler power. + */ + wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); } /* @@ -1111,10 +1097,13 @@ static void gen12_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - wa_init_mcr(i915, wal); + icl_wa_init_mcr(i915, wal); - /* Wa_14011060649:tgl,rkl,dg1,adls */ + /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */ wa_14011060649(i915, wal); + + /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */ + wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); } static void @@ -1123,19 +1112,19 @@ tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) gen12_gt_workarounds_init(i915, wal); /* Wa_1409420604:tgl */ - if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) + if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE2, CPSSUNIT_CLKGATE_DIS); /* Wa_1607087056:tgl also know as BUG:1409180338 */ - if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) + if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); /* Wa_1408615072:tgl[a0] */ - if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) + if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE_DIS_TGL); } @@ -1146,7 +1135,7 @@ dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) gen12_gt_workarounds_init(i915, wal); /* Wa_1607087056:dg1 */ - if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0)) + if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); @@ -1165,9 +1154,17 @@ dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void +xehpsdv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + xehp_init_mcr(&i915->gt, wal); +} + +static void gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) { - if (IS_DG1(i915)) + if (IS_XEHPSDV(i915)) + xehpsdv_gt_workarounds_init(i915, wal); + else if (IS_DG1(i915)) dg1_gt_workarounds_init(i915, wal); else if (IS_TIGERLAKE(i915)) tgl_gt_workarounds_init(i915, wal); @@ -1175,8 +1172,6 @@ gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) gen12_gt_workarounds_init(i915, wal); else if (GRAPHICS_VER(i915) == 11) icl_gt_workarounds_init(i915, wal); - else if (IS_CANNONLAKE(i915)) - cnl_gt_workarounds_init(i915, wal); else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) cfl_gt_workarounds_init(i915, wal); else if (IS_GEMINILAKE(i915)) @@ -1184,7 +1179,7 @@ gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) else if (IS_KABYLAKE(i915)) kbl_gt_workarounds_init(i915, wal); else if (IS_BROXTON(i915)) - bxt_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(i915, wal); else if (IS_SKYLAKE(i915)) skl_gt_workarounds_init(i915, wal); else if (IS_HASWELL(i915)) @@ -1247,8 +1242,9 @@ wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from) } static void -wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal) +wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) { + struct intel_uncore *uncore = gt->uncore; enum forcewake_domains fw; unsigned long flags; struct i915_wa *wa; @@ -1263,13 +1259,16 @@ wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal) intel_uncore_forcewake_get__locked(uncore, fw); for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { - if (wa->clr) - intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set); - else - intel_uncore_write_fw(uncore, wa->reg, wa->set); + u32 val, old = 0; + + /* open-coded rmw due to steering */ + old = wa->clr ? intel_gt_read_register_fw(gt, wa->reg) : 0; + val = (old & ~wa->clr) | wa->set; + if (val != old || !wa->clr) + intel_uncore_write_fw(uncore, wa->reg, val); + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) - wa_verify(wa, - intel_uncore_read_fw(uncore, wa->reg), + wa_verify(wa, intel_gt_read_register_fw(gt, wa->reg), wal->name, "application"); } @@ -1279,28 +1278,39 @@ wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal) void intel_gt_apply_workarounds(struct intel_gt *gt) { - wa_list_apply(gt->uncore, >->i915->gt_wa_list); + wa_list_apply(gt, >->i915->gt_wa_list); } -static bool wa_list_verify(struct intel_uncore *uncore, +static bool wa_list_verify(struct intel_gt *gt, const struct i915_wa_list *wal, const char *from) { + struct intel_uncore *uncore = gt->uncore; struct i915_wa *wa; + enum forcewake_domains fw; + unsigned long flags; unsigned int i; bool ok = true; + fw = wal_get_fw_for_rmw(uncore, wal); + + spin_lock_irqsave(&uncore->lock, flags); + intel_uncore_forcewake_get__locked(uncore, fw); + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) ok &= wa_verify(wa, - intel_uncore_read(uncore, wa->reg), + intel_gt_read_register_fw(gt, wa->reg), wal->name, from); + intel_uncore_forcewake_put__locked(uncore, fw); + spin_unlock_irqrestore(&uncore->lock, flags); + return ok; } bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from) { - return wa_list_verify(gt->uncore, >->i915->gt_wa_list, from); + return wa_list_verify(gt, >->i915->gt_wa_list, from); } __maybe_unused @@ -1438,17 +1448,6 @@ static void cml_whitelist_build(struct intel_engine_cs *engine) cfl_whitelist_build(engine); } -static void cnl_whitelist_build(struct intel_engine_cs *engine) -{ - struct i915_wa_list *w = &engine->whitelist; - - if (engine->class != RENDER_CLASS) - return; - - /* WaEnablePreemptionGranularityControlByUMD:cnl */ - whitelist_reg(w, GEN8_CS_CHICKEN1); -} - static void icl_whitelist_build(struct intel_engine_cs *engine) { struct i915_wa_list *w = &engine->whitelist; @@ -1542,7 +1541,7 @@ static void dg1_whitelist_build(struct intel_engine_cs *engine) tgl_whitelist_build(engine); /* GEN:BUG:1409280441:dg1 */ - if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) && + if (IS_DG1_GT_STEP(engine->i915, STEP_A0, STEP_B0) && (engine->class == RENDER_CLASS || engine->class == COPY_ENGINE_CLASS)) whitelist_reg_ext(w, RING_ID(engine->mmio_base), @@ -1562,8 +1561,6 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine) tgl_whitelist_build(engine); else if (GRAPHICS_VER(i915) == 11) icl_whitelist_build(engine); - else if (IS_CANNONLAKE(i915)) - cnl_whitelist_build(engine); else if (IS_COMETLAKE(i915)) cml_whitelist_build(engine); else if (IS_COFFEELAKE(i915)) @@ -1612,8 +1609,8 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct drm_i915_private *i915 = engine->i915; - if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) || - IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) { + if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || + IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) { /* * Wa_1607138336:tgl[a0],dg1[a0] * Wa_1607063988:tgl[a0],dg1[a0] @@ -1623,7 +1620,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN12_DISABLE_POSH_BUSY_FF_DOP_CG); } - if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) { + if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) { /* * Wa_1606679103:tgl * (see also Wa_1606682166:icl) @@ -1633,44 +1630,46 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN7_DISABLE_SAMPLER_PREFETCH); } - if (IS_ALDERLAKE_S(i915) || IS_DG1(i915) || + if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { - /* Wa_1606931601:tgl,rkl,dg1,adl-s */ + /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */ wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); /* * Wa_1407928979:tgl A* * Wa_18011464164:tgl[B0+],dg1[B0+] * Wa_22010931296:tgl[B0+],dg1[B0+] - * Wa_14010919138:rkl,dg1,adl-s + * Wa_14010919138:rkl,dg1,adl-s,adl-p */ wa_write_or(wal, GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE); /* - * Wa_1606700617:tgl,dg1 - * Wa_22010271021:tgl,rkl,dg1, adl-s + * Wa_1606700617:tgl,dg1,adl-p + * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p + * Wa_14010826681:tgl,dg1,rkl,adl-p */ wa_masked_en(wal, GEN9_CS_DEBUG_MODE1, FF_DOP_CLOCK_GATE_DISABLE); } - if (IS_ALDERLAKE_S(i915) || IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) || + if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || + IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { - /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s */ + /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */ wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_PUSH_CONST_DEREF_HOLD_DIS); /* * Wa_1409085225:tgl - * Wa_14010229206:tgl,rkl,dg1[a0],adl-s + * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p */ wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); } - if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) || + if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { /* * Wa_1607030317:tgl @@ -1688,8 +1687,9 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN8_RC_SEMA_IDLE_MSG_DISABLE); } - if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { - /* Wa_1406941453:tgl,rkl,dg1 */ + if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || + IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) { + /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */ wa_masked_en(wal, GEN10_SAMPLER_MODE, ENABLE_SMALLPL); @@ -1701,11 +1701,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) _3D_CHICKEN3, _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE); - /* WaPipelineFlushCoherentLines:icl */ - wa_write_or(wal, - GEN8_L3SQCREG4, - GEN8_LQSC_FLUSH_COHERENT_LINES); - /* * Wa_1405543622:icl * Formerly known as WaGAPZPriorityScheme @@ -1735,19 +1730,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN8_L3SQCREG4, GEN11_LQSC_CLEAN_EVICT_DISABLE); - /* WaForwardProgressSoftReset:icl */ - wa_write_or(wal, - GEN10_SCRATCH_LNCF2, - PMFLUSHDONE_LNICRSDROP | - PMFLUSH_GAPL3UNBLOCK | - PMFLUSHDONE_LNEBLK); - - /* Wa_1406609255:icl (pre-prod) */ - if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) - wa_write_or(wal, - GEN7_SARCHKMD, - GEN7_DISABLE_DEMAND_PREFETCH); - /* Wa_1606682166:icl */ wa_write_or(wal, GEN7_SARCHKMD, @@ -1947,10 +1929,10 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * disable bit, which we don't touch here, but it's good * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). */ - wa_add(wal, GEN7_GT_MODE, 0, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, - GEN6_WIZ_HASHING_16x4), - GEN6_WIZ_HASHING_16x4); + wa_masked_field_set(wal, + GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK, + GEN6_WIZ_HASHING_16x4); } if (IS_GRAPHICS_VER(i915, 6, 7)) @@ -2000,10 +1982,10 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * disable bit, which we don't touch here, but it's good * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). */ - wa_add(wal, - GEN6_GT_MODE, 0, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), - GEN6_WIZ_HASHING_16x4); + wa_masked_field_set(wal, + GEN6_GT_MODE, + GEN6_WIZ_HASHING_MASK, + GEN6_WIZ_HASHING_16x4); /* WaDisable_RenderCache_OperationalFlush:snb */ wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); @@ -2024,7 +2006,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) wa_add(wal, MI_MODE, 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH), /* XXX bit doesn't stick on Broadwater */ - IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH); + IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true); if (GRAPHICS_VER(i915) == 4) /* @@ -2039,7 +2021,8 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) */ wa_add(wal, ECOSKPD, 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE), - 0 /* XXX bit doesn't stick on Broadwater */); + 0 /* XXX bit doesn't stick on Broadwater */, + true); } static void @@ -2048,7 +2031,7 @@ xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) struct drm_i915_private *i915 = engine->i915; /* WaKBLVECSSemaphoreWaitPoll:kbl */ - if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_E0)) { + if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_F0)) { wa_write(wal, RING_SEMA_WAIT_POLL(engine->mmio_base), 1); @@ -2081,7 +2064,7 @@ void intel_engine_init_workarounds(struct intel_engine_cs *engine) void intel_engine_apply_workarounds(struct intel_engine_cs *engine) { - wa_list_apply(engine->uncore, &engine->wa_list); + wa_list_apply(engine->gt, &engine->wa_list); } struct mcr_range { @@ -2107,12 +2090,31 @@ static const struct mcr_range mcr_ranges_gen12[] = { {}, }; +static const struct mcr_range mcr_ranges_xehp[] = { + { .start = 0x4000, .end = 0x4aff }, + { .start = 0x5200, .end = 0x52ff }, + { .start = 0x5400, .end = 0x7fff }, + { .start = 0x8140, .end = 0x815f }, + { .start = 0x8c80, .end = 0x8dff }, + { .start = 0x94d0, .end = 0x955f }, + { .start = 0x9680, .end = 0x96ff }, + { .start = 0xb000, .end = 0xb3ff }, + { .start = 0xc800, .end = 0xcfff }, + { .start = 0xd800, .end = 0xd8ff }, + { .start = 0xdc00, .end = 0xffff }, + { .start = 0x17000, .end = 0x17fff }, + { .start = 0x24a00, .end = 0x24a7f }, + {}, +}; + static bool mcr_range(struct drm_i915_private *i915, u32 offset) { const struct mcr_range *mcr_ranges; int i; - if (GRAPHICS_VER(i915) >= 12) + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) + mcr_ranges = mcr_ranges_xehp; + else if (GRAPHICS_VER(i915) >= 12) mcr_ranges = mcr_ranges_gen12; else if (GRAPHICS_VER(i915) >= 8) mcr_ranges = mcr_ranges_gen8; diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h index c214111ea367..1e873681795d 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h +++ b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h @@ -15,6 +15,7 @@ struct i915_wa { u32 clr; u32 set; u32 read; + bool masked_reg; }; struct i915_wa_list { diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index 32589c6625e1..2c1af030310c 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -235,6 +235,34 @@ static void mock_submit_request(struct i915_request *request) spin_unlock_irqrestore(&engine->hw_lock, flags); } +static void mock_add_to_engine(struct i915_request *rq) +{ + lockdep_assert_held(&rq->engine->sched_engine->lock); + list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); +} + +static void mock_remove_from_engine(struct i915_request *rq) +{ + struct intel_engine_cs *engine, *locked; + + /* + * Virtual engines complicate acquiring the engine timeline lock, + * as their rq->engine pointer is not stable until under that + * engine lock. The simple ploy we use is to take the lock then + * check that the rq still belongs to the newly locked engine. + */ + + locked = READ_ONCE(rq->engine); + spin_lock_irq(&locked->sched_engine->lock); + while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { + spin_unlock(&locked->sched_engine->lock); + spin_lock(&engine->sched_engine->lock); + locked = engine; + } + list_del_init(&rq->sched.link); + spin_unlock_irq(&locked->sched_engine->lock); +} + static void mock_reset_prepare(struct intel_engine_cs *engine) { } @@ -253,10 +281,10 @@ static void mock_reset_cancel(struct intel_engine_cs *engine) del_timer_sync(&mock->hw_delay); - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); /* Mark all submitted requests as skipped. */ - list_for_each_entry(rq, &engine->active.requests, sched.link) + list_for_each_entry(rq, &engine->sched_engine->requests, sched.link) i915_request_put(i915_request_mark_eio(rq)); intel_engine_signal_breadcrumbs(engine); @@ -269,7 +297,7 @@ static void mock_reset_cancel(struct intel_engine_cs *engine) } INIT_LIST_HEAD(&mock->hw_queue); - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } static void mock_reset_finish(struct intel_engine_cs *engine) @@ -283,7 +311,8 @@ static void mock_engine_release(struct intel_engine_cs *engine) GEM_BUG_ON(timer_pending(&mock->hw_delay)); - intel_breadcrumbs_free(engine->breadcrumbs); + i915_sched_engine_put(engine->sched_engine); + intel_breadcrumbs_put(engine->breadcrumbs); intel_context_unpin(engine->kernel_context); intel_context_put(engine->kernel_context); @@ -320,6 +349,8 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, engine->base.emit_flush = mock_emit_flush; engine->base.emit_fini_breadcrumb = mock_emit_breadcrumb; engine->base.submit_request = mock_submit_request; + engine->base.add_active_request = mock_add_to_engine; + engine->base.remove_active_request = mock_remove_from_engine; engine->base.reset.prepare = mock_reset_prepare; engine->base.reset.rewind = mock_reset_rewind; @@ -345,14 +376,18 @@ int mock_engine_init(struct intel_engine_cs *engine) { struct intel_context *ce; - intel_engine_init_active(engine, ENGINE_MOCK); + engine->sched_engine = i915_sched_engine_create(ENGINE_MOCK); + if (!engine->sched_engine) + return -ENOMEM; + engine->sched_engine->private_data = engine; + intel_engine_init_execlists(engine); intel_engine_init__pm(engine); intel_engine_init_retire(engine); engine->breadcrumbs = intel_breadcrumbs_create(NULL); if (!engine->breadcrumbs) - return -ENOMEM; + goto err_schedule; ce = create_kernel_context(engine); if (IS_ERR(ce)) @@ -365,7 +400,9 @@ int mock_engine_init(struct intel_engine_cs *engine) return 0; err_breadcrumbs: - intel_breadcrumbs_free(engine->breadcrumbs); + intel_breadcrumbs_put(engine->breadcrumbs); +err_schedule: + i915_sched_engine_put(engine->sched_engine); return -ENOMEM; } diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c index 26685b927169..fa7b99a671dd 100644 --- a/drivers/gpu/drm/i915/gt/selftest_context.c +++ b/drivers/gpu/drm/i915/gt/selftest_context.c @@ -209,7 +209,13 @@ static int __live_active_context(struct intel_engine_cs *engine) * This test makes sure that the context is kept alive until a * subsequent idle-barrier (emitted when the engine wakeref hits 0 * with no more outstanding requests). + * + * In GuC submission mode we don't use idle barriers and we instead + * get a message from the GuC to signal that it is safe to unpin the + * context from memory. */ + if (intel_engine_uses_guc(engine)) + return 0; if (intel_engine_pm_is_awake(engine)) { pr_err("%s is awake before starting %s!\n", @@ -357,7 +363,11 @@ static int __live_remote_context(struct intel_engine_cs *engine) * on the context image remotely (intel_context_prepare_remote_request), * which inserts foreign fences into intel_context.active, does not * clobber the idle-barrier. + * + * In GuC submission mode we don't use idle barriers. */ + if (intel_engine_uses_guc(engine)) + return 0; if (intel_engine_pm_is_awake(engine)) { pr_err("%s is awake before starting %s!\n", diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index 4896e4ccad50..317eebf086c3 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -405,3 +405,25 @@ void st_engine_heartbeat_enable(struct intel_engine_cs *engine) engine->props.heartbeat_interval_ms = engine->defaults.heartbeat_interval_ms; } + +void st_engine_heartbeat_disable_no_pm(struct intel_engine_cs *engine) +{ + engine->props.heartbeat_interval_ms = 0; + + /* + * Park the heartbeat but without holding the PM lock as that + * makes the engines appear not-idle. Note that if/when unpark + * is called due to the PM lock being acquired later the + * heartbeat still won't be enabled because of the above = 0. + */ + if (intel_engine_pm_get_if_awake(engine)) { + intel_engine_park_heartbeat(engine); + intel_engine_pm_put(engine); + } +} + +void st_engine_heartbeat_enable_no_pm(struct intel_engine_cs *engine) +{ + engine->props.heartbeat_interval_ms = + engine->defaults.heartbeat_interval_ms; +} diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.h b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.h index cd27113d5400..81da2cd8e406 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.h +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.h @@ -9,6 +9,8 @@ struct intel_engine_cs; void st_engine_heartbeat_disable(struct intel_engine_cs *engine); +void st_engine_heartbeat_disable_no_pm(struct intel_engine_cs *engine); void st_engine_heartbeat_enable(struct intel_engine_cs *engine); +void st_engine_heartbeat_enable_no_pm(struct intel_engine_cs *engine); #endif /* SELFTEST_ENGINE_HEARTBEAT_H */ diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c index 72cca3f0da21..75569666105d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c @@ -173,8 +173,8 @@ static int __live_engine_timestamps(struct intel_engine_cs *engine) d_ctx = trifilter(s_ctx); d_ctx *= engine->gt->clock_frequency; - if (IS_ICELAKE(engine->i915)) - d_ring *= 12500000; /* Fixed 80ns for icl ctx timestamp? */ + if (GRAPHICS_VER(engine->i915) == 11) + d_ring *= 12500000; /* Fixed 80ns for GEN11 ctx timestamp? */ else d_ring *= engine->gt->clock_frequency; diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index 1c8108d30b85..f12ffe797639 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -43,7 +43,7 @@ static int wait_for_submit(struct intel_engine_cs *engine, unsigned long timeout) { /* Ignore our own attempts to suppress excess tasklets */ - tasklet_hi_schedule(&engine->execlists.tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); timeout += jiffies; do { @@ -273,7 +273,7 @@ static int live_unlite_restore(struct intel_gt *gt, int prio) }; /* Alternatively preempt the spinner with ce[1] */ - engine->schedule(rq[1], &attr); + engine->sched_engine->schedule(rq[1], &attr); } /* And switch back to ce[0] for good measure */ @@ -553,13 +553,13 @@ static int live_pin_rewind(void *arg) static int engine_lock_reset_tasklet(struct intel_engine_cs *engine) { - tasklet_disable(&engine->execlists.tasklet); + tasklet_disable(&engine->sched_engine->tasklet); local_bh_disable(); if (test_and_set_bit(I915_RESET_ENGINE + engine->id, &engine->gt->reset.flags)) { local_bh_enable(); - tasklet_enable(&engine->execlists.tasklet); + tasklet_enable(&engine->sched_engine->tasklet); intel_gt_set_wedged(engine->gt); return -EBUSY; @@ -574,7 +574,7 @@ static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine) &engine->gt->reset.flags); local_bh_enable(); - tasklet_enable(&engine->execlists.tasklet); + tasklet_enable(&engine->sched_engine->tasklet); } static int live_hold_reset(void *arg) @@ -628,7 +628,7 @@ static int live_hold_reset(void *arg) if (err) goto out; - engine->execlists.tasklet.callback(&engine->execlists.tasklet); + engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); GEM_BUG_ON(execlists_active(&engine->execlists) != rq); i915_request_get(rq); @@ -917,7 +917,7 @@ release_queue(struct intel_engine_cs *engine, i915_request_add(rq); local_bh_disable(); - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); local_bh_enable(); /* kick tasklet */ i915_request_put(rq); @@ -1200,7 +1200,7 @@ static int live_timeslice_rewind(void *arg) while (i915_request_is_active(rq[A2])) { /* semaphore yield! */ /* Wait for the timeslice to kick in */ del_timer(&engine->execlists.timer); - tasklet_hi_schedule(&engine->execlists.tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); intel_engine_flush_submission(engine); } /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */ @@ -1342,7 +1342,7 @@ static int live_timeslice_queue(void *arg) err = PTR_ERR(rq); goto err_heartbeat; } - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); err = wait_for_submit(engine, rq, HZ / 2); if (err) { pr_err("%s: Timed out trying to submit semaphores\n", @@ -1539,12 +1539,12 @@ static int live_busywait_preempt(void *arg) * preempt the busywaits used to synchronise between rings. */ - ctx_hi = kernel_context(gt->i915); + ctx_hi = kernel_context(gt->i915, NULL); if (!ctx_hi) return -ENOMEM; ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; - ctx_lo = kernel_context(gt->i915); + ctx_lo = kernel_context(gt->i915, NULL); if (!ctx_lo) goto err_ctx_hi; ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; @@ -1741,12 +1741,12 @@ static int live_preempt(void *arg) if (igt_spinner_init(&spin_lo, gt)) goto err_spin_hi; - ctx_hi = kernel_context(gt->i915); + ctx_hi = kernel_context(gt->i915, NULL); if (!ctx_hi) goto err_spin_lo; ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; - ctx_lo = kernel_context(gt->i915); + ctx_lo = kernel_context(gt->i915, NULL); if (!ctx_lo) goto err_ctx_hi; ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; @@ -1833,11 +1833,11 @@ static int live_late_preempt(void *arg) if (igt_spinner_init(&spin_lo, gt)) goto err_spin_hi; - ctx_hi = kernel_context(gt->i915); + ctx_hi = kernel_context(gt->i915, NULL); if (!ctx_hi) goto err_spin_lo; - ctx_lo = kernel_context(gt->i915); + ctx_lo = kernel_context(gt->i915, NULL); if (!ctx_lo) goto err_ctx_hi; @@ -1884,7 +1884,7 @@ static int live_late_preempt(void *arg) } attr.priority = I915_PRIORITY_MAX; - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); if (!igt_wait_for_spinner(&spin_hi, rq)) { pr_err("High priority context failed to preempt the low priority context\n"); @@ -1927,7 +1927,7 @@ struct preempt_client { static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c) { - c->ctx = kernel_context(gt->i915); + c->ctx = kernel_context(gt->i915, NULL); if (!c->ctx) return -ENOMEM; @@ -2497,7 +2497,7 @@ static int live_suppress_self_preempt(void *arg) i915_request_add(rq_b); GEM_BUG_ON(i915_request_completed(rq_a)); - engine->schedule(rq_a, &attr); + engine->sched_engine->schedule(rq_a, &attr); igt_spinner_end(&a.spin); if (!igt_wait_for_spinner(&b.spin, rq_b)) { @@ -2629,7 +2629,7 @@ static int live_chain_preempt(void *arg) i915_request_get(rq); i915_request_add(rq); - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); igt_spinner_end(&hi.spin); if (i915_request_wait(rq, 0, HZ / 5) < 0) { @@ -2810,7 +2810,7 @@ static int __live_preempt_ring(struct intel_engine_cs *engine, goto err_ce; } - tmp->ring = __intel_context_ring_size(ring_sz); + tmp->ring_size = ring_sz; err = intel_context_pin(tmp); if (err) { @@ -2988,7 +2988,7 @@ static int live_preempt_gang(void *arg) break; /* Submit each spinner at increasing priority */ - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); } while (prio <= I915_PRIORITY_MAX && !__igt_timeout(end_time, NULL)); pr_debug("%s: Preempt chain of %d requests\n", @@ -3236,7 +3236,7 @@ static int preempt_user(struct intel_engine_cs *engine, i915_request_get(rq); i915_request_add(rq); - engine->schedule(rq, &attr); + engine->sched_engine->schedule(rq, &attr); if (i915_request_wait(rq, 0, HZ / 2) < 0) err = -ETIME; @@ -3384,12 +3384,12 @@ static int live_preempt_timeout(void *arg) if (igt_spinner_init(&spin_lo, gt)) return -ENOMEM; - ctx_hi = kernel_context(gt->i915); + ctx_hi = kernel_context(gt->i915, NULL); if (!ctx_hi) goto err_spin_lo; ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; - ctx_lo = kernel_context(gt->i915); + ctx_lo = kernel_context(gt->i915, NULL); if (!ctx_lo) goto err_ctx_hi; ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; @@ -3561,12 +3561,16 @@ static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) #define BATCH BIT(0) { struct task_struct *tsk[I915_NUM_ENGINES] = {}; - struct preempt_smoke arg[I915_NUM_ENGINES]; + struct preempt_smoke *arg; struct intel_engine_cs *engine; enum intel_engine_id id; unsigned long count; int err = 0; + arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL); + if (!arg) + return -ENOMEM; + for_each_engine(engine, smoke->gt, id) { arg[id] = *smoke; arg[id].engine = engine; @@ -3574,7 +3578,7 @@ static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) arg[id].batch = NULL; arg[id].count = 0; - tsk[id] = kthread_run(smoke_crescendo_thread, &arg, + tsk[id] = kthread_run(smoke_crescendo_thread, arg, "igt/smoke:%d", id); if (IS_ERR(tsk[id])) { err = PTR_ERR(tsk[id]); @@ -3603,6 +3607,8 @@ static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", count, flags, smoke->gt->info.num_engines, smoke->ncontext); + + kfree(arg); return 0; } @@ -3676,7 +3682,7 @@ static int live_preempt_smoke(void *arg) } for (n = 0; n < smoke.ncontext; n++) { - smoke.contexts[n] = kernel_context(smoke.gt->i915); + smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL); if (!smoke.contexts[n]) goto err_ctx; } @@ -3727,7 +3733,7 @@ static int nop_virtual_engine(struct intel_gt *gt, GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve)); for (n = 0; n < nctx; n++) { - ve[n] = intel_execlists_create_virtual(siblings, nsibling); + ve[n] = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ve[n])) { err = PTR_ERR(ve[n]); nctx = n; @@ -3923,7 +3929,7 @@ static int mask_virtual_engine(struct intel_gt *gt, * restrict it to our desired engine within the virtual engine. */ - ve = intel_execlists_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_close; @@ -4054,7 +4060,7 @@ static int slicein_virtual_engine(struct intel_gt *gt, i915_request_add(rq); } - ce = intel_execlists_create_virtual(siblings, nsibling); + ce = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; @@ -4106,7 +4112,7 @@ static int sliceout_virtual_engine(struct intel_gt *gt, /* XXX We do not handle oversubscription and fairness with normal rq */ for (n = 0; n < nsibling; n++) { - ce = intel_execlists_create_virtual(siblings, nsibling); + ce = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; @@ -4208,7 +4214,7 @@ static int preserved_virtual_engine(struct intel_gt *gt, if (err) goto out_scratch; - ve = intel_execlists_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_scratch; @@ -4328,234 +4334,6 @@ static int live_virtual_preserved(void *arg) return 0; } -static int bond_virtual_engine(struct intel_gt *gt, - unsigned int class, - struct intel_engine_cs **siblings, - unsigned int nsibling, - unsigned int flags) -#define BOND_SCHEDULE BIT(0) -{ - struct intel_engine_cs *master; - struct i915_request *rq[16]; - enum intel_engine_id id; - struct igt_spinner spin; - unsigned long n; - int err; - - /* - * A set of bonded requests is intended to be run concurrently - * across a number of engines. We use one request per-engine - * and a magic fence to schedule each of the bonded requests - * at the same time. A consequence of our current scheduler is that - * we only move requests to the HW ready queue when the request - * becomes ready, that is when all of its prerequisite fences have - * been signaled. As one of those fences is the master submit fence, - * there is a delay on all secondary fences as the HW may be - * currently busy. Equally, as all the requests are independent, - * they may have other fences that delay individual request - * submission to HW. Ergo, we do not guarantee that all requests are - * immediately submitted to HW at the same time, just that if the - * rules are abided by, they are ready at the same time as the - * first is submitted. Userspace can embed semaphores in its batch - * to ensure parallel execution of its phases as it requires. - * Though naturally it gets requested that perhaps the scheduler should - * take care of parallel execution, even across preemption events on - * different HW. (The proper answer is of course "lalalala".) - * - * With the submit-fence, we have identified three possible phases - * of synchronisation depending on the master fence: queued (not - * ready), executing, and signaled. The first two are quite simple - * and checked below. However, the signaled master fence handling is - * contentious. Currently we do not distinguish between a signaled - * fence and an expired fence, as once signaled it does not convey - * any information about the previous execution. It may even be freed - * and hence checking later it may not exist at all. Ergo we currently - * do not apply the bonding constraint for an already signaled fence, - * as our expectation is that it should not constrain the secondaries - * and is outside of the scope of the bonded request API (i.e. all - * userspace requests are meant to be running in parallel). As - * it imposes no constraint, and is effectively a no-op, we do not - * check below as normal execution flows are checked extensively above. - * - * XXX Is the degenerate handling of signaled submit fences the - * expected behaviour for userpace? - */ - - GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1); - - if (igt_spinner_init(&spin, gt)) - return -ENOMEM; - - err = 0; - rq[0] = ERR_PTR(-ENOMEM); - for_each_engine(master, gt, id) { - struct i915_sw_fence fence = {}; - struct intel_context *ce; - - if (master->class == class) - continue; - - ce = intel_context_create(master); - if (IS_ERR(ce)) { - err = PTR_ERR(ce); - goto out; - } - - memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq)); - - rq[0] = igt_spinner_create_request(&spin, ce, MI_NOOP); - intel_context_put(ce); - if (IS_ERR(rq[0])) { - err = PTR_ERR(rq[0]); - goto out; - } - i915_request_get(rq[0]); - - if (flags & BOND_SCHEDULE) { - onstack_fence_init(&fence); - err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit, - &fence, - GFP_KERNEL); - } - - i915_request_add(rq[0]); - if (err < 0) - goto out; - - if (!(flags & BOND_SCHEDULE) && - !igt_wait_for_spinner(&spin, rq[0])) { - err = -EIO; - goto out; - } - - for (n = 0; n < nsibling; n++) { - struct intel_context *ve; - - ve = intel_execlists_create_virtual(siblings, nsibling); - if (IS_ERR(ve)) { - err = PTR_ERR(ve); - onstack_fence_fini(&fence); - goto out; - } - - err = intel_virtual_engine_attach_bond(ve->engine, - master, - siblings[n]); - if (err) { - intel_context_put(ve); - onstack_fence_fini(&fence); - goto out; - } - - err = intel_context_pin(ve); - intel_context_put(ve); - if (err) { - onstack_fence_fini(&fence); - goto out; - } - - rq[n + 1] = i915_request_create(ve); - intel_context_unpin(ve); - if (IS_ERR(rq[n + 1])) { - err = PTR_ERR(rq[n + 1]); - onstack_fence_fini(&fence); - goto out; - } - i915_request_get(rq[n + 1]); - - err = i915_request_await_execution(rq[n + 1], - &rq[0]->fence, - ve->engine->bond_execute); - i915_request_add(rq[n + 1]); - if (err < 0) { - onstack_fence_fini(&fence); - goto out; - } - } - onstack_fence_fini(&fence); - intel_engine_flush_submission(master); - igt_spinner_end(&spin); - - if (i915_request_wait(rq[0], 0, HZ / 10) < 0) { - pr_err("Master request did not execute (on %s)!\n", - rq[0]->engine->name); - err = -EIO; - goto out; - } - - for (n = 0; n < nsibling; n++) { - if (i915_request_wait(rq[n + 1], 0, - MAX_SCHEDULE_TIMEOUT) < 0) { - err = -EIO; - goto out; - } - - if (rq[n + 1]->engine != siblings[n]) { - pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n", - siblings[n]->name, - rq[n + 1]->engine->name, - rq[0]->engine->name); - err = -EINVAL; - goto out; - } - } - - for (n = 0; !IS_ERR(rq[n]); n++) - i915_request_put(rq[n]); - rq[0] = ERR_PTR(-ENOMEM); - } - -out: - for (n = 0; !IS_ERR(rq[n]); n++) - i915_request_put(rq[n]); - if (igt_flush_test(gt->i915)) - err = -EIO; - - igt_spinner_fini(&spin); - return err; -} - -static int live_virtual_bond(void *arg) -{ - static const struct phase { - const char *name; - unsigned int flags; - } phases[] = { - { "", 0 }, - { "schedule", BOND_SCHEDULE }, - { }, - }; - struct intel_gt *gt = arg; - struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; - unsigned int class; - int err; - - if (intel_uc_uses_guc_submission(>->uc)) - return 0; - - for (class = 0; class <= MAX_ENGINE_CLASS; class++) { - const struct phase *p; - int nsibling; - - nsibling = select_siblings(gt, class, siblings); - if (nsibling < 2) - continue; - - for (p = phases; p->name; p++) { - err = bond_virtual_engine(gt, - class, siblings, nsibling, - p->flags); - if (err) { - pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n", - __func__, p->name, class, nsibling, err); - return err; - } - } - } - - return 0; -} - static int reset_virtual_engine(struct intel_gt *gt, struct intel_engine_cs **siblings, unsigned int nsibling) @@ -4576,7 +4354,7 @@ static int reset_virtual_engine(struct intel_gt *gt, if (igt_spinner_init(&spin, gt)) return -ENOMEM; - ve = intel_execlists_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_spin; @@ -4606,13 +4384,13 @@ static int reset_virtual_engine(struct intel_gt *gt, if (err) goto out_heartbeat; - engine->execlists.tasklet.callback(&engine->execlists.tasklet); + engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet); GEM_BUG_ON(execlists_active(&engine->execlists) != rq); /* Fake a preemption event; failed of course */ - spin_lock_irq(&engine->active.lock); + spin_lock_irq(&engine->sched_engine->lock); __unwind_incomplete_requests(engine); - spin_unlock_irq(&engine->active.lock); + spin_unlock_irq(&engine->sched_engine->lock); GEM_BUG_ON(rq->engine != engine); /* Reset the engine while keeping our active request on hold */ @@ -4721,7 +4499,6 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_virtual_mask), SUBTEST(live_virtual_preserved), SUBTEST(live_virtual_slice), - SUBTEST(live_virtual_bond), SUBTEST(live_virtual_reset), }; diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 853246fad05f..2c1ed32ca5ac 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -17,6 +17,8 @@ #include "selftests/igt_flush_test.h" #include "selftests/igt_reset.h" #include "selftests/igt_atomic.h" +#include "selftests/igt_spinner.h" +#include "selftests/intel_scheduler_helpers.h" #include "selftests/mock_drm.h" @@ -42,7 +44,7 @@ static int hang_init(struct hang *h, struct intel_gt *gt) memset(h, 0, sizeof(*h)); h->gt = gt; - h->ctx = kernel_context(gt->i915); + h->ctx = kernel_context(gt->i915, NULL); if (IS_ERR(h->ctx)) return PTR_ERR(h->ctx); @@ -378,6 +380,7 @@ static int igt_reset_nop(void *arg) ce = intel_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); + pr_err("[%s] Create context failed: %d!\n", engine->name, err); break; } @@ -387,6 +390,8 @@ static int igt_reset_nop(void *arg) rq = intel_context_create_request(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); + pr_err("[%s] Create request failed: %d!\n", + engine->name, err); break; } @@ -401,24 +406,31 @@ static int igt_reset_nop(void *arg) igt_global_reset_unlock(gt); if (intel_gt_is_wedged(gt)) { + pr_err("[%s] GT is wedged!\n", engine->name); err = -EIO; break; } if (i915_reset_count(global) != reset_count + ++count) { - pr_err("Full GPU reset not recorded!\n"); + pr_err("[%s] Reset not recorded: %d vs %d + %d!\n", + engine->name, i915_reset_count(global), reset_count, count); err = -EINVAL; break; } err = igt_flush_test(gt->i915); - if (err) + if (err) { + pr_err("[%s] Flush failed: %d!\n", engine->name, err); break; + } } while (time_before(jiffies, end_time)); pr_info("%s: %d resets\n", __func__, count); - if (igt_flush_test(gt->i915)) + if (igt_flush_test(gt->i915)) { + pr_err("Post flush failed: %d!\n", err); err = -EIO; + } + return err; } @@ -440,9 +452,19 @@ static int igt_reset_nop_engine(void *arg) IGT_TIMEOUT(end_time); int err; + if (intel_engine_uses_guc(engine)) { + /* Engine level resets are triggered by GuC when a hang + * is detected. They can't be triggered by the KMD any + * more. Thus a nop batch cannot be used as a reset test + */ + continue; + } + ce = intel_context_create(engine); - if (IS_ERR(ce)) + if (IS_ERR(ce)) { + pr_err("[%s] Create context failed: %pe!\n", engine->name, ce); return PTR_ERR(ce); + } reset_count = i915_reset_count(global); reset_engine_count = i915_reset_engine_count(global, engine); @@ -549,9 +571,15 @@ static int igt_reset_fail_engine(void *arg) IGT_TIMEOUT(end_time); int err; + /* Can't manually break the reset if i915 doesn't perform it */ + if (intel_engine_uses_guc(engine)) + continue; + ce = intel_context_create(engine); - if (IS_ERR(ce)) + if (IS_ERR(ce)) { + pr_err("[%s] Create context failed: %pe!\n", engine->name, ce); return PTR_ERR(ce); + } st_engine_heartbeat_disable(engine); set_bit(I915_RESET_ENGINE + id, >->reset.flags); @@ -686,8 +714,12 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) for_each_engine(engine, gt, id) { unsigned int reset_count, reset_engine_count; unsigned long count; + bool using_guc = intel_engine_uses_guc(engine); IGT_TIMEOUT(end_time); + if (using_guc && !active) + continue; + if (active && !intel_engine_can_store_dword(engine)) continue; @@ -705,13 +737,24 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) set_bit(I915_RESET_ENGINE + id, >->reset.flags); count = 0; do { - if (active) { - struct i915_request *rq; + struct i915_request *rq = NULL; + struct intel_selftest_saved_policy saved; + int err2; + + err = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET); + if (err) { + pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); + break; + } + if (active) { rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); - break; + pr_err("[%s] Create hang request failed: %d!\n", + engine->name, err); + goto restore; } i915_request_get(rq); @@ -727,34 +770,59 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) i915_request_put(rq); err = -EIO; - break; + goto restore; } + } - i915_request_put(rq); + if (!using_guc) { + err = intel_engine_reset(engine, NULL); + if (err) { + pr_err("intel_engine_reset(%s) failed, err:%d\n", + engine->name, err); + goto skip; + } } - err = intel_engine_reset(engine, NULL); - if (err) { - pr_err("intel_engine_reset(%s) failed, err:%d\n", - engine->name, err); - break; + if (rq) { + /* Ensure the reset happens and kills the engine */ + err = intel_selftest_wait_for_rq(rq); + if (err) + pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", + engine->name, rq->fence.context, + rq->fence.seqno, rq->context->guc_id, err); } +skip: + if (rq) + i915_request_put(rq); + if (i915_reset_count(global) != reset_count) { pr_err("Full GPU reset recorded! (engine reset expected)\n"); err = -EINVAL; - break; + goto restore; } - if (i915_reset_engine_count(global, engine) != - ++reset_engine_count) { - pr_err("%s engine reset not recorded!\n", - engine->name); - err = -EINVAL; - break; + /* GuC based resets are not logged per engine */ + if (!using_guc) { + if (i915_reset_engine_count(global, engine) != + ++reset_engine_count) { + pr_err("%s engine reset not recorded!\n", + engine->name); + err = -EINVAL; + goto restore; + } } count++; + +restore: + err2 = intel_selftest_restore_policy(engine, &saved); + if (err2) + pr_err("[%s] Restore policy failed: %d!\n", engine->name, err); + if (err == 0) + err = err2; + if (err) + break; } while (time_before(jiffies, end_time)); clear_bit(I915_RESET_ENGINE + id, >->reset.flags); st_engine_heartbeat_enable(engine); @@ -765,12 +833,16 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) break; err = igt_flush_test(gt->i915); - if (err) + if (err) { + pr_err("[%s] Flush failed: %d!\n", engine->name, err); break; + } } - if (intel_gt_is_wedged(gt)) + if (intel_gt_is_wedged(gt)) { + pr_err("GT is wedged!\n"); err = -EIO; + } if (active) hang_fini(&h); @@ -807,7 +879,7 @@ static int active_request_put(struct i915_request *rq) if (!rq) return 0; - if (i915_request_wait(rq, 0, 5 * HZ) < 0) { + if (i915_request_wait(rq, 0, 10 * HZ) < 0) { GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n", rq->engine->name, rq->fence.context, @@ -837,6 +909,7 @@ static int active_engine(void *data) ce[count] = intel_context_create(engine); if (IS_ERR(ce[count])) { err = PTR_ERR(ce[count]); + pr_err("[%s] Create context #%ld failed: %d!\n", engine->name, count, err); while (--count) intel_context_put(ce[count]); return err; @@ -852,23 +925,26 @@ static int active_engine(void *data) new = intel_context_create_request(ce[idx]); if (IS_ERR(new)) { err = PTR_ERR(new); + pr_err("[%s] Create request #%d failed: %d!\n", engine->name, idx, err); break; } rq[idx] = i915_request_get(new); i915_request_add(new); - if (engine->schedule && arg->flags & TEST_PRIORITY) { + if (engine->sched_engine->schedule && arg->flags & TEST_PRIORITY) { struct i915_sched_attr attr = { .priority = i915_prandom_u32_max_state(512, &prng), }; - engine->schedule(rq[idx], &attr); + engine->sched_engine->schedule(rq[idx], &attr); } err = active_request_put(old); - if (err) + if (err) { + pr_err("[%s] Request put failed: %d!\n", engine->name, err); break; + } cond_resched(); } @@ -876,6 +952,9 @@ static int active_engine(void *data) for (count = 0; count < ARRAY_SIZE(rq); count++) { int err__ = active_request_put(rq[count]); + if (err) + pr_err("[%s] Request put #%ld failed: %d!\n", engine->name, count, err); + /* Keep the first error */ if (!err) err = err__; @@ -916,10 +995,13 @@ static int __igt_reset_engines(struct intel_gt *gt, struct active_engine threads[I915_NUM_ENGINES] = {}; unsigned long device = i915_reset_count(global); unsigned long count = 0, reported; + bool using_guc = intel_engine_uses_guc(engine); IGT_TIMEOUT(end_time); - if (flags & TEST_ACTIVE && - !intel_engine_can_store_dword(engine)) + if (flags & TEST_ACTIVE) { + if (!intel_engine_can_store_dword(engine)) + continue; + } else if (using_guc) continue; if (!wait_for_idle(engine)) { @@ -949,6 +1031,7 @@ static int __igt_reset_engines(struct intel_gt *gt, "igt/%s", other->name); if (IS_ERR(tsk)) { err = PTR_ERR(tsk); + pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err); goto unwind; } @@ -958,16 +1041,27 @@ static int __igt_reset_engines(struct intel_gt *gt, yield(); /* start all threads before we begin */ - st_engine_heartbeat_disable(engine); + st_engine_heartbeat_disable_no_pm(engine); set_bit(I915_RESET_ENGINE + id, >->reset.flags); do { struct i915_request *rq = NULL; + struct intel_selftest_saved_policy saved; + int err2; + + err = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET); + if (err) { + pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); + break; + } if (flags & TEST_ACTIVE) { rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); - break; + pr_err("[%s] Create hang request failed: %d!\n", + engine->name, err); + goto restore; } i915_request_get(rq); @@ -983,32 +1077,44 @@ static int __igt_reset_engines(struct intel_gt *gt, i915_request_put(rq); err = -EIO; - break; + goto restore; } + } else { + intel_engine_pm_get(engine); } - err = intel_engine_reset(engine, NULL); - if (err) { - pr_err("i915_reset_engine(%s:%s): failed, err=%d\n", - engine->name, test_name, err); - break; + if (!using_guc) { + err = intel_engine_reset(engine, NULL); + if (err) { + pr_err("i915_reset_engine(%s:%s): failed, err=%d\n", + engine->name, test_name, err); + goto restore; + } + } + + if (rq) { + /* Ensure the reset happens and kills the engine */ + err = intel_selftest_wait_for_rq(rq); + if (err) + pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", + engine->name, rq->fence.context, + rq->fence.seqno, rq->context->guc_id, err); } count++; if (rq) { if (rq->fence.error != -EIO) { - pr_err("i915_reset_engine(%s:%s):" - " failed to reset request %llx:%lld\n", + pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n", engine->name, test_name, rq->fence.context, - rq->fence.seqno); + rq->fence.seqno, rq->context->guc_id); i915_request_put(rq); GEM_TRACE_DUMP(); intel_gt_set_wedged(gt); err = -EIO; - break; + goto restore; } if (i915_request_wait(rq, 0, HZ / 5) < 0) { @@ -1027,12 +1133,15 @@ static int __igt_reset_engines(struct intel_gt *gt, GEM_TRACE_DUMP(); intel_gt_set_wedged(gt); err = -EIO; - break; + goto restore; } i915_request_put(rq); } + if (!(flags & TEST_ACTIVE)) + intel_engine_pm_put(engine); + if (!(flags & TEST_SELF) && !wait_for_idle(engine)) { struct drm_printer p = drm_info_printer(gt->i915->drm.dev); @@ -1044,22 +1153,34 @@ static int __igt_reset_engines(struct intel_gt *gt, "%s\n", engine->name); err = -EIO; - break; + goto restore; } + +restore: + err2 = intel_selftest_restore_policy(engine, &saved); + if (err2) + pr_err("[%s] Restore policy failed: %d!\n", engine->name, err2); + if (err == 0) + err = err2; + if (err) + break; } while (time_before(jiffies, end_time)); clear_bit(I915_RESET_ENGINE + id, >->reset.flags); - st_engine_heartbeat_enable(engine); + st_engine_heartbeat_enable_no_pm(engine); pr_info("i915_reset_engine(%s:%s): %lu resets\n", engine->name, test_name, count); - reported = i915_reset_engine_count(global, engine); - reported -= threads[engine->id].resets; - if (reported != count) { - pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", - engine->name, test_name, count, reported); - if (!err) - err = -EINVAL; + /* GuC based resets are not logged per engine */ + if (!using_guc) { + reported = i915_reset_engine_count(global, engine); + reported -= threads[engine->id].resets; + if (reported != count) { + pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", + engine->name, test_name, count, reported); + if (!err) + err = -EINVAL; + } } unwind: @@ -1078,15 +1199,18 @@ unwind: } put_task_struct(threads[tmp].task); - if (other->uabi_class != engine->uabi_class && - threads[tmp].resets != - i915_reset_engine_count(global, other)) { - pr_err("Innocent engine %s was reset (count=%ld)\n", - other->name, - i915_reset_engine_count(global, other) - - threads[tmp].resets); - if (!err) - err = -EINVAL; + /* GuC based resets are not logged per engine */ + if (!using_guc) { + if (other->uabi_class != engine->uabi_class && + threads[tmp].resets != + i915_reset_engine_count(global, other)) { + pr_err("Innocent engine %s was reset (count=%ld)\n", + other->name, + i915_reset_engine_count(global, other) - + threads[tmp].resets); + if (!err) + err = -EINVAL; + } } } @@ -1101,8 +1225,10 @@ unwind: break; err = igt_flush_test(gt->i915); - if (err) + if (err) { + pr_err("[%s] Flush failed: %d!\n", engine->name, err); break; + } } if (intel_gt_is_wedged(gt)) @@ -1180,12 +1306,15 @@ static int igt_reset_wait(void *arg) igt_global_reset_lock(gt); err = hang_init(&h, gt); - if (err) + if (err) { + pr_err("[%s] Hang init failed: %d!\n", engine->name, err); goto unlock; + } rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); + pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); goto fini; } @@ -1310,12 +1439,15 @@ static int __igt_reset_evict_vma(struct intel_gt *gt, /* Check that we can recover an unbind stuck on a hanging request */ err = hang_init(&h, gt); - if (err) + if (err) { + pr_err("[%s] Hang init failed: %d!\n", engine->name, err); return err; + } obj = i915_gem_object_create_internal(gt->i915, SZ_1M); if (IS_ERR(obj)) { err = PTR_ERR(obj); + pr_err("[%s] Create object failed: %d!\n", engine->name, err); goto fini; } @@ -1330,12 +1462,14 @@ static int __igt_reset_evict_vma(struct intel_gt *gt, arg.vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(arg.vma)) { err = PTR_ERR(arg.vma); + pr_err("[%s] VMA instance failed: %d!\n", engine->name, err); goto out_obj; } rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); + pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); goto out_obj; } @@ -1347,6 +1481,7 @@ static int __igt_reset_evict_vma(struct intel_gt *gt, err = i915_vma_pin(arg.vma, 0, 0, pin_flags); if (err) { i915_request_add(rq); + pr_err("[%s] VMA pin failed: %d!\n", engine->name, err); goto out_obj; } @@ -1363,8 +1498,14 @@ static int __igt_reset_evict_vma(struct intel_gt *gt, i915_vma_lock(arg.vma); err = i915_request_await_object(rq, arg.vma->obj, flags & EXEC_OBJECT_WRITE); - if (err == 0) + if (err == 0) { err = i915_vma_move_to_active(arg.vma, rq, flags); + if (err) + pr_err("[%s] Move to active failed: %d!\n", engine->name, err); + } else { + pr_err("[%s] Request await failed: %d!\n", engine->name, err); + } + i915_vma_unlock(arg.vma); if (flags & EXEC_OBJECT_NEEDS_FENCE) @@ -1392,6 +1533,7 @@ static int __igt_reset_evict_vma(struct intel_gt *gt, tsk = kthread_run(fn, &arg, "igt/evict_vma"); if (IS_ERR(tsk)) { err = PTR_ERR(tsk); + pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err); tsk = NULL; goto out_reset; } @@ -1508,17 +1650,29 @@ static int igt_reset_queue(void *arg) goto unlock; for_each_engine(engine, gt, id) { + struct intel_selftest_saved_policy saved; struct i915_request *prev; IGT_TIMEOUT(end_time); unsigned int count; + bool using_guc = intel_engine_uses_guc(engine); if (!intel_engine_can_store_dword(engine)) continue; + if (using_guc) { + err = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK); + if (err) { + pr_err("[%s] Modify policy failed: %d!\n", engine->name, err); + goto fini; + } + } + prev = hang_create_request(&h, engine); if (IS_ERR(prev)) { err = PTR_ERR(prev); - goto fini; + pr_err("[%s] Create 'prev' hang request failed: %d!\n", engine->name, err); + goto restore; } i915_request_get(prev); @@ -1532,7 +1686,8 @@ static int igt_reset_queue(void *arg) rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); - goto fini; + pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); + goto restore; } i915_request_get(rq); @@ -1557,7 +1712,7 @@ static int igt_reset_queue(void *arg) GEM_TRACE_DUMP(); intel_gt_set_wedged(gt); - goto fini; + goto restore; } if (!wait_until_running(&h, prev)) { @@ -1575,7 +1730,7 @@ static int igt_reset_queue(void *arg) intel_gt_set_wedged(gt); err = -EIO; - goto fini; + goto restore; } reset_count = fake_hangcheck(gt, BIT(id)); @@ -1586,7 +1741,7 @@ static int igt_reset_queue(void *arg) i915_request_put(rq); i915_request_put(prev); err = -EINVAL; - goto fini; + goto restore; } if (rq->fence.error) { @@ -1595,7 +1750,7 @@ static int igt_reset_queue(void *arg) i915_request_put(rq); i915_request_put(prev); err = -EINVAL; - goto fini; + goto restore; } if (i915_reset_count(global) == reset_count) { @@ -1603,7 +1758,7 @@ static int igt_reset_queue(void *arg) i915_request_put(rq); i915_request_put(prev); err = -EINVAL; - goto fini; + goto restore; } i915_request_put(prev); @@ -1618,9 +1773,24 @@ static int igt_reset_queue(void *arg) i915_request_put(prev); - err = igt_flush_test(gt->i915); +restore: + if (using_guc) { + int err2 = intel_selftest_restore_policy(engine, &saved); + + if (err2) + pr_err("%s:%d> [%s] Restore policy failed: %d!\n", + __func__, __LINE__, engine->name, err2); + if (err == 0) + err = err2; + } if (err) + goto fini; + + err = igt_flush_test(gt->i915); + if (err) { + pr_err("[%s] Flush failed: %d!\n", engine->name, err); break; + } } fini: @@ -1653,12 +1823,15 @@ static int igt_handle_error(void *arg) return 0; err = hang_init(&h, gt); - if (err) + if (err) { + pr_err("[%s] Hang init failed: %d!\n", engine->name, err); return err; + } rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); + pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); goto err_fini; } @@ -1702,7 +1875,7 @@ static int __igt_atomic_reset_engine(struct intel_engine_cs *engine, const struct igt_atomic_section *p, const char *mode) { - struct tasklet_struct * const t = &engine->execlists.tasklet; + struct tasklet_struct * const t = &engine->sched_engine->tasklet; int err; GEM_TRACE("i915_reset_engine(%s:%s) under %s\n", @@ -1743,12 +1916,15 @@ static int igt_atomic_reset_engine(struct intel_engine_cs *engine, return err; err = hang_init(&h, engine->gt); - if (err) + if (err) { + pr_err("[%s] Hang init failed: %d!\n", engine->name, err); return err; + } rq = hang_create_request(&h, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); + pr_err("[%s] Create hang request failed: %d!\n", engine->name, err); goto out; } diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 3119016d9910..b0977a3b699b 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -49,7 +49,7 @@ static int wait_for_submit(struct intel_engine_cs *engine, unsigned long timeout) { /* Ignore our own attempts to suppress excess tasklets */ - tasklet_hi_schedule(&engine->execlists.tasklet); + tasklet_hi_schedule(&engine->sched_engine->tasklet); timeout += jiffies; do { @@ -1613,12 +1613,12 @@ static void garbage_reset(struct intel_engine_cs *engine, local_bh_disable(); if (!test_and_set_bit(bit, lock)) { - tasklet_disable(&engine->execlists.tasklet); + tasklet_disable(&engine->sched_engine->tasklet); if (!rq->fence.error) __intel_engine_reset_bh(engine, NULL); - tasklet_enable(&engine->execlists.tasklet); + tasklet_enable(&engine->sched_engine->tasklet); clear_and_wake_up_bit(bit, lock); } local_bh_enable(); diff --git a/drivers/gpu/drm/i915/gt/selftest_migrate.c b/drivers/gpu/drm/i915/gt/selftest_migrate.c new file mode 100644 index 000000000000..12ef2837c89b --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_migrate.c @@ -0,0 +1,669 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/sort.h> + +#include "selftests/i915_random.h" + +static const unsigned int sizes[] = { + SZ_4K, + SZ_64K, + SZ_2M, + CHUNK_SZ - SZ_4K, + CHUNK_SZ, + CHUNK_SZ + SZ_4K, + SZ_64M, +}; + +static struct drm_i915_gem_object * +create_lmem_or_internal(struct drm_i915_private *i915, size_t size) +{ + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_create_lmem(i915, size, 0); + if (!IS_ERR(obj)) + return obj; + + return i915_gem_object_create_internal(i915, size); +} + +static int copy(struct intel_migrate *migrate, + int (*fn)(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *src, + struct drm_i915_gem_object *dst, + struct i915_request **out), + u32 sz, struct rnd_state *prng) +{ + struct drm_i915_private *i915 = migrate->context->engine->i915; + struct drm_i915_gem_object *src, *dst; + struct i915_request *rq; + struct i915_gem_ww_ctx ww; + u32 *vaddr; + int err = 0; + int i; + + src = create_lmem_or_internal(i915, sz); + if (IS_ERR(src)) + return 0; + + dst = i915_gem_object_create_internal(i915, sz); + if (IS_ERR(dst)) + goto err_free_src; + + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(src, &ww); + if (err) + continue; + + err = i915_gem_object_lock(dst, &ww); + if (err) + continue; + + vaddr = i915_gem_object_pin_map(src, I915_MAP_WC); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + continue; + } + + for (i = 0; i < sz / sizeof(u32); i++) + vaddr[i] = i; + i915_gem_object_flush_map(src); + + vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + goto unpin_src; + } + + for (i = 0; i < sz / sizeof(u32); i++) + vaddr[i] = ~i; + i915_gem_object_flush_map(dst); + + err = fn(migrate, &ww, src, dst, &rq); + if (!err) + continue; + + if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) + pr_err("%ps failed, size: %u\n", fn, sz); + if (rq) { + i915_request_wait(rq, 0, HZ); + i915_request_put(rq); + } + i915_gem_object_unpin_map(dst); +unpin_src: + i915_gem_object_unpin_map(src); + } + if (err) + goto err_out; + + if (rq) { + if (i915_request_wait(rq, 0, HZ) < 0) { + pr_err("%ps timed out, size: %u\n", fn, sz); + err = -ETIME; + } + i915_request_put(rq); + } + + for (i = 0; !err && i < sz / PAGE_SIZE; i++) { + int x = i * 1024 + i915_prandom_u32_max_state(1024, prng); + + if (vaddr[x] != x) { + pr_err("%ps failed, size: %u, offset: %zu\n", + fn, sz, x * sizeof(u32)); + igt_hexdump(vaddr + i * 1024, 4096); + err = -EINVAL; + } + } + + i915_gem_object_unpin_map(dst); + i915_gem_object_unpin_map(src); + +err_out: + i915_gem_object_put(dst); +err_free_src: + i915_gem_object_put(src); + + return err; +} + +static int clear(struct intel_migrate *migrate, + int (*fn)(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *obj, + u32 value, + struct i915_request **out), + u32 sz, struct rnd_state *prng) +{ + struct drm_i915_private *i915 = migrate->context->engine->i915; + struct drm_i915_gem_object *obj; + struct i915_request *rq; + struct i915_gem_ww_ctx ww; + u32 *vaddr; + int err = 0; + int i; + + obj = create_lmem_or_internal(i915, sz); + if (IS_ERR(obj)) + return 0; + + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(obj, &ww); + if (err) + continue; + + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + continue; + } + + for (i = 0; i < sz / sizeof(u32); i++) + vaddr[i] = ~i; + i915_gem_object_flush_map(obj); + + err = fn(migrate, &ww, obj, sz, &rq); + if (!err) + continue; + + if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) + pr_err("%ps failed, size: %u\n", fn, sz); + if (rq) { + i915_request_wait(rq, 0, HZ); + i915_request_put(rq); + } + i915_gem_object_unpin_map(obj); + } + if (err) + goto err_out; + + if (rq) { + if (i915_request_wait(rq, 0, HZ) < 0) { + pr_err("%ps timed out, size: %u\n", fn, sz); + err = -ETIME; + } + i915_request_put(rq); + } + + for (i = 0; !err && i < sz / PAGE_SIZE; i++) { + int x = i * 1024 + i915_prandom_u32_max_state(1024, prng); + + if (vaddr[x] != sz) { + pr_err("%ps failed, size: %u, offset: %zu\n", + fn, sz, x * sizeof(u32)); + igt_hexdump(vaddr + i * 1024, 4096); + err = -EINVAL; + } + } + + i915_gem_object_unpin_map(obj); +err_out: + i915_gem_object_put(obj); + + return err; +} + +static int __migrate_copy(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *src, + struct drm_i915_gem_object *dst, + struct i915_request **out) +{ + return intel_migrate_copy(migrate, ww, NULL, + src->mm.pages->sgl, src->cache_level, + i915_gem_object_is_lmem(src), + dst->mm.pages->sgl, dst->cache_level, + i915_gem_object_is_lmem(dst), + out); +} + +static int __global_copy(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *src, + struct drm_i915_gem_object *dst, + struct i915_request **out) +{ + return intel_context_migrate_copy(migrate->context, NULL, + src->mm.pages->sgl, src->cache_level, + i915_gem_object_is_lmem(src), + dst->mm.pages->sgl, dst->cache_level, + i915_gem_object_is_lmem(dst), + out); +} + +static int +migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) +{ + return copy(migrate, __migrate_copy, sz, prng); +} + +static int +global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) +{ + return copy(migrate, __global_copy, sz, prng); +} + +static int __migrate_clear(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *obj, + u32 value, + struct i915_request **out) +{ + return intel_migrate_clear(migrate, ww, NULL, + obj->mm.pages->sgl, + obj->cache_level, + i915_gem_object_is_lmem(obj), + value, out); +} + +static int __global_clear(struct intel_migrate *migrate, + struct i915_gem_ww_ctx *ww, + struct drm_i915_gem_object *obj, + u32 value, + struct i915_request **out) +{ + return intel_context_migrate_clear(migrate->context, NULL, + obj->mm.pages->sgl, + obj->cache_level, + i915_gem_object_is_lmem(obj), + value, out); +} + +static int +migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) +{ + return clear(migrate, __migrate_clear, sz, prng); +} + +static int +global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) +{ + return clear(migrate, __global_clear, sz, prng); +} + +static int live_migrate_copy(void *arg) +{ + struct intel_migrate *migrate = arg; + struct drm_i915_private *i915 = migrate->context->engine->i915; + I915_RND_STATE(prng); + int i; + + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + int err; + + err = migrate_copy(migrate, sizes[i], &prng); + if (err == 0) + err = global_copy(migrate, sizes[i], &prng); + i915_gem_drain_freed_objects(i915); + if (err) + return err; + } + + return 0; +} + +static int live_migrate_clear(void *arg) +{ + struct intel_migrate *migrate = arg; + struct drm_i915_private *i915 = migrate->context->engine->i915; + I915_RND_STATE(prng); + int i; + + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + int err; + + err = migrate_clear(migrate, sizes[i], &prng); + if (err == 0) + err = global_clear(migrate, sizes[i], &prng); + + i915_gem_drain_freed_objects(i915); + if (err) + return err; + } + + return 0; +} + +struct threaded_migrate { + struct intel_migrate *migrate; + struct task_struct *tsk; + struct rnd_state prng; +}; + +static int threaded_migrate(struct intel_migrate *migrate, + int (*fn)(void *arg), + unsigned int flags) +{ + const unsigned int n_cpus = num_online_cpus() + 1; + struct threaded_migrate *thread; + I915_RND_STATE(prng); + unsigned int i; + int err = 0; + + thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL); + if (!thread) + return 0; + + for (i = 0; i < n_cpus; ++i) { + struct task_struct *tsk; + + thread[i].migrate = migrate; + thread[i].prng = + I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng)); + + tsk = kthread_run(fn, &thread[i], "igt-%d", i); + if (IS_ERR(tsk)) { + err = PTR_ERR(tsk); + break; + } + + get_task_struct(tsk); + thread[i].tsk = tsk; + } + + msleep(10); /* start all threads before we kthread_stop() */ + + for (i = 0; i < n_cpus; ++i) { + struct task_struct *tsk = thread[i].tsk; + int status; + + if (IS_ERR_OR_NULL(tsk)) + continue; + + status = kthread_stop(tsk); + if (status && !err) + err = status; + + put_task_struct(tsk); + } + + kfree(thread); + return err; +} + +static int __thread_migrate_copy(void *arg) +{ + struct threaded_migrate *tm = arg; + + return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); +} + +static int thread_migrate_copy(void *arg) +{ + return threaded_migrate(arg, __thread_migrate_copy, 0); +} + +static int __thread_global_copy(void *arg) +{ + struct threaded_migrate *tm = arg; + + return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); +} + +static int thread_global_copy(void *arg) +{ + return threaded_migrate(arg, __thread_global_copy, 0); +} + +static int __thread_migrate_clear(void *arg) +{ + struct threaded_migrate *tm = arg; + + return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); +} + +static int __thread_global_clear(void *arg) +{ + struct threaded_migrate *tm = arg; + + return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); +} + +static int thread_migrate_clear(void *arg) +{ + return threaded_migrate(arg, __thread_migrate_clear, 0); +} + +static int thread_global_clear(void *arg) +{ + return threaded_migrate(arg, __thread_global_clear, 0); +} + +int intel_migrate_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(live_migrate_copy), + SUBTEST(live_migrate_clear), + SUBTEST(thread_migrate_copy), + SUBTEST(thread_migrate_clear), + SUBTEST(thread_global_copy), + SUBTEST(thread_global_clear), + }; + struct intel_gt *gt = &i915->gt; + + if (!gt->migrate.context) + return 0; + + return i915_subtests(tests, >->migrate); +} + +static struct drm_i915_gem_object * +create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem) +{ + struct drm_i915_gem_object *obj = NULL; + int err; + + if (try_lmem) + obj = i915_gem_object_create_lmem(gt->i915, sz, 0); + + if (IS_ERR_OR_NULL(obj)) { + obj = i915_gem_object_create_internal(gt->i915, sz); + if (IS_ERR(obj)) + return obj; + } + + i915_gem_object_trylock(obj); + err = i915_gem_object_pin_pages(obj); + if (err) { + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + return ERR_PTR(err); + } + + return obj; +} + +static int wrap_ktime_compare(const void *A, const void *B) +{ + const ktime_t *a = A, *b = B; + + return ktime_compare(*a, *b); +} + +static int __perf_clear_blt(struct intel_context *ce, + struct scatterlist *sg, + enum i915_cache_level cache_level, + bool is_lmem, + size_t sz) +{ + ktime_t t[5]; + int pass; + int err = 0; + + for (pass = 0; pass < ARRAY_SIZE(t); pass++) { + struct i915_request *rq; + ktime_t t0, t1; + + t0 = ktime_get(); + + err = intel_context_migrate_clear(ce, NULL, sg, cache_level, + is_lmem, 0, &rq); + if (rq) { + if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) + err = -EIO; + i915_request_put(rq); + } + if (err) + break; + + t1 = ktime_get(); + t[pass] = ktime_sub(t1, t0); + } + if (err) + return err; + + sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); + pr_info("%s: %zd KiB fill: %lld MiB/s\n", + ce->engine->name, sz >> 10, + div64_u64(mul_u32_u32(4 * sz, + 1000 * 1000 * 1000), + t[1] + 2 * t[2] + t[3]) >> 20); + return 0; +} + +static int perf_clear_blt(void *arg) +{ + struct intel_gt *gt = arg; + static const unsigned long sizes[] = { + SZ_4K, + SZ_64K, + SZ_2M, + SZ_64M + }; + int i; + + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + struct drm_i915_gem_object *dst; + int err; + + dst = create_init_lmem_internal(gt, sizes[i], true); + if (IS_ERR(dst)) + return PTR_ERR(dst); + + err = __perf_clear_blt(gt->migrate.context, + dst->mm.pages->sgl, + I915_CACHE_NONE, + i915_gem_object_is_lmem(dst), + sizes[i]); + + i915_gem_object_unlock(dst); + i915_gem_object_put(dst); + if (err) + return err; + } + + return 0; +} + +static int __perf_copy_blt(struct intel_context *ce, + struct scatterlist *src, + enum i915_cache_level src_cache_level, + bool src_is_lmem, + struct scatterlist *dst, + enum i915_cache_level dst_cache_level, + bool dst_is_lmem, + size_t sz) +{ + ktime_t t[5]; + int pass; + int err = 0; + + for (pass = 0; pass < ARRAY_SIZE(t); pass++) { + struct i915_request *rq; + ktime_t t0, t1; + + t0 = ktime_get(); + + err = intel_context_migrate_copy(ce, NULL, + src, src_cache_level, + src_is_lmem, + dst, dst_cache_level, + dst_is_lmem, + &rq); + if (rq) { + if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) + err = -EIO; + i915_request_put(rq); + } + if (err) + break; + + t1 = ktime_get(); + t[pass] = ktime_sub(t1, t0); + } + if (err) + return err; + + sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); + pr_info("%s: %zd KiB copy: %lld MiB/s\n", + ce->engine->name, sz >> 10, + div64_u64(mul_u32_u32(4 * sz, + 1000 * 1000 * 1000), + t[1] + 2 * t[2] + t[3]) >> 20); + return 0; +} + +static int perf_copy_blt(void *arg) +{ + struct intel_gt *gt = arg; + static const unsigned long sizes[] = { + SZ_4K, + SZ_64K, + SZ_2M, + SZ_64M + }; + int i; + + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + struct drm_i915_gem_object *src, *dst; + int err; + + src = create_init_lmem_internal(gt, sizes[i], true); + if (IS_ERR(src)) + return PTR_ERR(src); + + dst = create_init_lmem_internal(gt, sizes[i], false); + if (IS_ERR(dst)) { + err = PTR_ERR(dst); + goto err_src; + } + + err = __perf_copy_blt(gt->migrate.context, + src->mm.pages->sgl, + I915_CACHE_NONE, + i915_gem_object_is_lmem(src), + dst->mm.pages->sgl, + I915_CACHE_NONE, + i915_gem_object_is_lmem(dst), + sizes[i]); + + i915_gem_object_unlock(dst); + i915_gem_object_put(dst); +err_src: + i915_gem_object_unlock(src); + i915_gem_object_put(src); + if (err) + return err; + } + + return 0; +} + +int intel_migrate_perf_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(perf_clear_blt), + SUBTEST(perf_copy_blt), + }; + struct intel_gt *gt = &i915->gt; + + if (intel_gt_is_wedged(gt)) + return 0; + + if (!gt->migrate.context) + return 0; + + return intel_gt_live_subtests(tests, gt); +} diff --git a/drivers/gpu/drm/i915/gt/selftest_mocs.c b/drivers/gpu/drm/i915/gt/selftest_mocs.c index b9bb0e6e97f7..13d25bf2a94a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_mocs.c +++ b/drivers/gpu/drm/i915/gt/selftest_mocs.c @@ -10,6 +10,7 @@ #include "gem/selftests/mock_context.h" #include "selftests/igt_reset.h" #include "selftests/igt_spinner.h" +#include "selftests/intel_scheduler_helpers.h" struct live_mocs { struct drm_i915_mocs_table table; @@ -28,7 +29,7 @@ static struct intel_context *mocs_context_create(struct intel_engine_cs *engine) return ce; /* We build large requests to read the registers from the ring */ - ce->ring = __intel_context_ring_size(SZ_16K); + ce->ring_size = SZ_16K; return ce; } @@ -318,7 +319,8 @@ static int live_mocs_clean(void *arg) } static int active_engine_reset(struct intel_context *ce, - const char *reason) + const char *reason, + bool using_guc) { struct igt_spinner spin; struct i915_request *rq; @@ -335,9 +337,13 @@ static int active_engine_reset(struct intel_context *ce, } err = request_add_spin(rq, &spin); - if (err == 0) + if (err == 0 && !using_guc) err = intel_engine_reset(ce->engine, reason); + /* Ensure the reset happens and kills the engine */ + if (err == 0) + err = intel_selftest_wait_for_rq(rq); + igt_spinner_end(&spin); igt_spinner_fini(&spin); @@ -345,21 +351,23 @@ static int active_engine_reset(struct intel_context *ce, } static int __live_mocs_reset(struct live_mocs *mocs, - struct intel_context *ce) + struct intel_context *ce, bool using_guc) { struct intel_gt *gt = ce->engine->gt; int err; if (intel_has_reset_engine(gt)) { - err = intel_engine_reset(ce->engine, "mocs"); - if (err) - return err; - - err = check_mocs_engine(mocs, ce); - if (err) - return err; + if (!using_guc) { + err = intel_engine_reset(ce->engine, "mocs"); + if (err) + return err; + + err = check_mocs_engine(mocs, ce); + if (err) + return err; + } - err = active_engine_reset(ce, "mocs"); + err = active_engine_reset(ce, "mocs", using_guc); if (err) return err; @@ -395,19 +403,33 @@ static int live_mocs_reset(void *arg) igt_global_reset_lock(gt); for_each_engine(engine, gt, id) { + bool using_guc = intel_engine_uses_guc(engine); + struct intel_selftest_saved_policy saved; struct intel_context *ce; + int err2; + + err = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET); + if (err) + break; ce = mocs_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); - break; + goto restore; } intel_engine_pm_get(engine); - err = __live_mocs_reset(&mocs, ce); - intel_engine_pm_put(engine); + err = __live_mocs_reset(&mocs, ce, using_guc); + + intel_engine_pm_put(engine); intel_context_put(ce); + +restore: + err2 = intel_selftest_restore_policy(engine, &saved); + if (err == 0) + err = err2; if (err) break; } diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index 8784257ec808..7a50c9f4071b 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -321,7 +321,7 @@ static int igt_atomic_engine_reset(void *arg) goto out_unlock; for_each_engine(engine, gt, id) { - struct tasklet_struct *t = &engine->execlists.tasklet; + struct tasklet_struct *t = &engine->sched_engine->tasklet; if (t->func) tasklet_disable(t); diff --git a/drivers/gpu/drm/i915/gt/selftest_slpc.c b/drivers/gpu/drm/i915/gt/selftest_slpc.c new file mode 100644 index 000000000000..9334bad131a2 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_slpc.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#define NUM_STEPS 5 +#define H2G_DELAY 50000 +#define delay_for_h2g() usleep_range(H2G_DELAY, H2G_DELAY + 10000) +#define FREQUENCY_REQ_UNIT DIV_ROUND_CLOSEST(GT_FREQUENCY_MULTIPLIER, \ + GEN9_FREQ_SCALER) + +static int slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 freq) +{ + int ret; + + ret = intel_guc_slpc_set_min_freq(slpc, freq); + if (ret) + pr_err("Could not set min frequency to [%u]\n", freq); + else /* Delay to ensure h2g completes */ + delay_for_h2g(); + + return ret; +} + +static int slpc_set_max_freq(struct intel_guc_slpc *slpc, u32 freq) +{ + int ret; + + ret = intel_guc_slpc_set_max_freq(slpc, freq); + if (ret) + pr_err("Could not set maximum frequency [%u]\n", + freq); + else /* Delay to ensure h2g completes */ + delay_for_h2g(); + + return ret; +} + +static int live_slpc_clamp_min(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_gt *gt = &i915->gt; + struct intel_guc_slpc *slpc = >->uc.guc.slpc; + struct intel_rps *rps = >->rps; + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + u32 slpc_min_freq, slpc_max_freq; + int err = 0; + + if (!intel_uc_uses_guc_slpc(>->uc)) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + if (intel_guc_slpc_get_max_freq(slpc, &slpc_max_freq)) { + pr_err("Could not get SLPC max freq\n"); + return -EIO; + } + + if (intel_guc_slpc_get_min_freq(slpc, &slpc_min_freq)) { + pr_err("Could not get SLPC min freq\n"); + return -EIO; + } + + if (slpc_min_freq == slpc_max_freq) { + pr_err("Min/Max are fused to the same value\n"); + return -EINVAL; + } + + intel_gt_pm_wait_for_idle(gt); + intel_gt_pm_get(gt); + for_each_engine(engine, gt, id) { + struct i915_request *rq; + u32 step, min_freq, req_freq; + u32 act_freq, max_act_freq; + + if (!intel_engine_can_store_dword(engine)) + continue; + + /* Go from min to max in 5 steps */ + step = (slpc_max_freq - slpc_min_freq) / NUM_STEPS; + max_act_freq = slpc_min_freq; + for (min_freq = slpc_min_freq; min_freq < slpc_max_freq; + min_freq += step) { + err = slpc_set_min_freq(slpc, min_freq); + if (err) + break; + + st_engine_heartbeat_disable(engine); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + st_engine_heartbeat_enable(engine); + break; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("%s: Spinner did not start\n", + engine->name); + igt_spinner_end(&spin); + st_engine_heartbeat_enable(engine); + intel_gt_set_wedged(engine->gt); + err = -EIO; + break; + } + + /* Wait for GuC to detect business and raise + * requested frequency if necessary. + */ + delay_for_h2g(); + + req_freq = intel_rps_read_punit_req_frequency(rps); + + /* GuC requests freq in multiples of 50/3 MHz */ + if (req_freq < (min_freq - FREQUENCY_REQ_UNIT)) { + pr_err("SWReq is %d, should be at least %d\n", req_freq, + min_freq - FREQUENCY_REQ_UNIT); + igt_spinner_end(&spin); + st_engine_heartbeat_enable(engine); + err = -EINVAL; + break; + } + + act_freq = intel_rps_read_actual_frequency(rps); + if (act_freq > max_act_freq) + max_act_freq = act_freq; + + igt_spinner_end(&spin); + st_engine_heartbeat_enable(engine); + } + + pr_info("Max actual frequency for %s was %d\n", + engine->name, max_act_freq); + + /* Actual frequency should rise above min */ + if (max_act_freq == slpc_min_freq) { + pr_err("Actual freq did not rise above min\n"); + err = -EINVAL; + } + + if (err) + break; + } + + /* Restore min/max frequencies */ + slpc_set_max_freq(slpc, slpc_max_freq); + slpc_set_min_freq(slpc, slpc_min_freq); + + if (igt_flush_test(gt->i915)) + err = -EIO; + + intel_gt_pm_put(gt); + igt_spinner_fini(&spin); + intel_gt_pm_wait_for_idle(gt); + + return err; +} + +static int live_slpc_clamp_max(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_gt *gt = &i915->gt; + struct intel_guc_slpc *slpc; + struct intel_rps *rps; + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + u32 slpc_min_freq, slpc_max_freq; + + slpc = >->uc.guc.slpc; + rps = >->rps; + + if (!intel_uc_uses_guc_slpc(>->uc)) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + if (intel_guc_slpc_get_max_freq(slpc, &slpc_max_freq)) { + pr_err("Could not get SLPC max freq\n"); + return -EIO; + } + + if (intel_guc_slpc_get_min_freq(slpc, &slpc_min_freq)) { + pr_err("Could not get SLPC min freq\n"); + return -EIO; + } + + if (slpc_min_freq == slpc_max_freq) { + pr_err("Min/Max are fused to the same value\n"); + return -EINVAL; + } + + intel_gt_pm_wait_for_idle(gt); + intel_gt_pm_get(gt); + for_each_engine(engine, gt, id) { + struct i915_request *rq; + u32 max_freq, req_freq; + u32 act_freq, max_act_freq; + u32 step; + + if (!intel_engine_can_store_dword(engine)) + continue; + + /* Go from max to min in 5 steps */ + step = (slpc_max_freq - slpc_min_freq) / NUM_STEPS; + max_act_freq = slpc_min_freq; + for (max_freq = slpc_max_freq; max_freq > slpc_min_freq; + max_freq -= step) { + err = slpc_set_max_freq(slpc, max_freq); + if (err) + break; + + st_engine_heartbeat_disable(engine); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + st_engine_heartbeat_enable(engine); + err = PTR_ERR(rq); + break; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("%s: SLPC spinner did not start\n", + engine->name); + igt_spinner_end(&spin); + st_engine_heartbeat_enable(engine); + intel_gt_set_wedged(engine->gt); + err = -EIO; + break; + } + + delay_for_h2g(); + + /* Verify that SWREQ indeed was set to specific value */ + req_freq = intel_rps_read_punit_req_frequency(rps); + + /* GuC requests freq in multiples of 50/3 MHz */ + if (req_freq > (max_freq + FREQUENCY_REQ_UNIT)) { + pr_err("SWReq is %d, should be at most %d\n", req_freq, + max_freq + FREQUENCY_REQ_UNIT); + igt_spinner_end(&spin); + st_engine_heartbeat_enable(engine); + err = -EINVAL; + break; + } + + act_freq = intel_rps_read_actual_frequency(rps); + if (act_freq > max_act_freq) + max_act_freq = act_freq; + + st_engine_heartbeat_enable(engine); + igt_spinner_end(&spin); + + if (err) + break; + } + + pr_info("Max actual frequency for %s was %d\n", + engine->name, max_act_freq); + + /* Actual frequency should rise above min */ + if (max_act_freq == slpc_min_freq) { + pr_err("Actual freq did not rise above min\n"); + err = -EINVAL; + } + + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + + if (err) + break; + } + + /* Restore min/max freq */ + slpc_set_max_freq(slpc, slpc_max_freq); + slpc_set_min_freq(slpc, slpc_min_freq); + + intel_gt_pm_put(gt); + igt_spinner_fini(&spin); + intel_gt_pm_wait_for_idle(gt); + + return err; +} + +int intel_slpc_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(live_slpc_clamp_max), + SUBTEST(live_slpc_clamp_min), + }; + + if (intel_gt_is_wedged(&i915->gt)) + return 0; + + return i915_live_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index 64da0c91dec1..d0b6a3afcf44 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -874,7 +874,7 @@ static int create_watcher(struct hwsp_watcher *w, if (IS_ERR(ce)) return PTR_ERR(ce); - ce->ring = __intel_context_ring_size(ringsz); + ce->ring_size = ringsz; w->rq = intel_context_create_request(ce); intel_context_put(ce); if (IS_ERR(w->rq)) diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index c30754daf4b1..e623ac45f4aa 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -12,6 +12,7 @@ #include "selftests/igt_flush_test.h" #include "selftests/igt_reset.h" #include "selftests/igt_spinner.h" +#include "selftests/intel_scheduler_helpers.h" #include "selftests/mock_drm.h" #include "gem/selftests/igt_gem_utils.h" @@ -261,28 +262,34 @@ static int do_engine_reset(struct intel_engine_cs *engine) return intel_engine_reset(engine, "live_workarounds"); } +static int do_guc_reset(struct intel_engine_cs *engine) +{ + /* Currently a no-op as the reset is handled by GuC */ + return 0; +} + static int switch_to_scratch_context(struct intel_engine_cs *engine, - struct igt_spinner *spin) + struct igt_spinner *spin, + struct i915_request **rq) { struct intel_context *ce; - struct i915_request *rq; int err = 0; ce = intel_context_create(engine); if (IS_ERR(ce)) return PTR_ERR(ce); - rq = igt_spinner_create_request(spin, ce, MI_NOOP); + *rq = igt_spinner_create_request(spin, ce, MI_NOOP); intel_context_put(ce); - if (IS_ERR(rq)) { + if (IS_ERR(*rq)) { spin = NULL; - err = PTR_ERR(rq); + err = PTR_ERR(*rq); goto err; } - err = request_add_spin(rq, spin); + err = request_add_spin(*rq, spin); err: if (err && spin) igt_spinner_end(spin); @@ -296,6 +303,7 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine, { struct intel_context *ce, *tmp; struct igt_spinner spin; + struct i915_request *rq; intel_wakeref_t wakeref; int err; @@ -316,13 +324,24 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine, goto out_spin; } - err = switch_to_scratch_context(engine, &spin); + err = switch_to_scratch_context(engine, &spin, &rq); if (err) goto out_spin; + /* Ensure the spinner hasn't aborted */ + if (i915_request_completed(rq)) { + pr_err("%s spinner failed to start\n", name); + err = -ETIMEDOUT; + goto out_spin; + } + with_intel_runtime_pm(engine->uncore->rpm, wakeref) err = reset(engine); + /* Ensure the reset happens and kills the engine */ + if (err == 0) + err = intel_selftest_wait_for_rq(rq); + igt_spinner_end(&spin); if (err) { @@ -787,9 +806,28 @@ static int live_reset_whitelist(void *arg) continue; if (intel_has_reset_engine(gt)) { - err = check_whitelist_across_reset(engine, - do_engine_reset, - "engine"); + if (intel_engine_uses_guc(engine)) { + struct intel_selftest_saved_policy saved; + int err2; + + err = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET); + if (err) + goto out; + + err = check_whitelist_across_reset(engine, + do_guc_reset, + "guc"); + + err2 = intel_selftest_restore_policy(engine, &saved); + if (err == 0) + err = err2; + } else { + err = check_whitelist_across_reset(engine, + do_engine_reset, + "engine"); + } + if (err) goto out; } @@ -1147,7 +1185,7 @@ verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, enum intel_engine_id id; bool ok = true; - ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str); + ok &= wa_list_verify(gt, &lists->gt_wa_list, str); for_each_engine(engine, gt, id) { struct intel_context *ce; @@ -1175,31 +1213,36 @@ live_gpu_reset_workarounds(void *arg) { struct intel_gt *gt = arg; intel_wakeref_t wakeref; - struct wa_lists lists; + struct wa_lists *lists; bool ok; if (!intel_has_gpu_reset(gt)) return 0; + lists = kzalloc(sizeof(*lists), GFP_KERNEL); + if (!lists) + return -ENOMEM; + pr_info("Verifying after GPU reset...\n"); igt_global_reset_lock(gt); wakeref = intel_runtime_pm_get(gt->uncore->rpm); - reference_lists_init(gt, &lists); + reference_lists_init(gt, lists); - ok = verify_wa_lists(gt, &lists, "before reset"); + ok = verify_wa_lists(gt, lists, "before reset"); if (!ok) goto out; intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); - ok = verify_wa_lists(gt, &lists, "after reset"); + ok = verify_wa_lists(gt, lists, "after reset"); out: - reference_lists_fini(gt, &lists); + reference_lists_fini(gt, lists); intel_runtime_pm_put(gt->uncore->rpm, wakeref); igt_global_reset_unlock(gt); + kfree(lists); return ok ? 0 : -ESRCH; } @@ -1214,43 +1257,57 @@ live_engine_reset_workarounds(void *arg) struct igt_spinner spin; struct i915_request *rq; intel_wakeref_t wakeref; - struct wa_lists lists; + struct wa_lists *lists; int ret = 0; if (!intel_has_reset_engine(gt)) return 0; + lists = kzalloc(sizeof(*lists), GFP_KERNEL); + if (!lists) + return -ENOMEM; + igt_global_reset_lock(gt); wakeref = intel_runtime_pm_get(gt->uncore->rpm); - reference_lists_init(gt, &lists); + reference_lists_init(gt, lists); for_each_engine(engine, gt, id) { + struct intel_selftest_saved_policy saved; + bool using_guc = intel_engine_uses_guc(engine); bool ok; + int ret2; pr_info("Verifying after %s reset...\n", engine->name); + ret = intel_selftest_modify_policy(engine, &saved, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET); + if (ret) + break; + ce = intel_context_create(engine); if (IS_ERR(ce)) { ret = PTR_ERR(ce); - break; + goto restore; } - ok = verify_wa_lists(gt, &lists, "before reset"); - if (!ok) { - ret = -ESRCH; - goto err; - } + if (!using_guc) { + ok = verify_wa_lists(gt, lists, "before reset"); + if (!ok) { + ret = -ESRCH; + goto err; + } - ret = intel_engine_reset(engine, "live_workarounds:idle"); - if (ret) { - pr_err("%s: Reset failed while idle\n", engine->name); - goto err; - } + ret = intel_engine_reset(engine, "live_workarounds:idle"); + if (ret) { + pr_err("%s: Reset failed while idle\n", engine->name); + goto err; + } - ok = verify_wa_lists(gt, &lists, "after idle reset"); - if (!ok) { - ret = -ESRCH; - goto err; + ok = verify_wa_lists(gt, lists, "after idle reset"); + if (!ok) { + ret = -ESRCH; + goto err; + } } ret = igt_spinner_init(&spin, engine->gt); @@ -1271,32 +1328,49 @@ live_engine_reset_workarounds(void *arg) goto err; } - ret = intel_engine_reset(engine, "live_workarounds:active"); - if (ret) { - pr_err("%s: Reset failed on an active spinner\n", - engine->name); - igt_spinner_fini(&spin); - goto err; + /* Ensure the spinner hasn't aborted */ + if (i915_request_completed(rq)) { + ret = -ETIMEDOUT; + goto skip; } + if (!using_guc) { + ret = intel_engine_reset(engine, "live_workarounds:active"); + if (ret) { + pr_err("%s: Reset failed on an active spinner\n", + engine->name); + igt_spinner_fini(&spin); + goto err; + } + } + + /* Ensure the reset happens and kills the engine */ + if (ret == 0) + ret = intel_selftest_wait_for_rq(rq); + +skip: igt_spinner_end(&spin); igt_spinner_fini(&spin); - ok = verify_wa_lists(gt, &lists, "after busy reset"); - if (!ok) { + ok = verify_wa_lists(gt, lists, "after busy reset"); + if (!ok) ret = -ESRCH; - goto err; - } err: intel_context_put(ce); + +restore: + ret2 = intel_selftest_restore_policy(engine, &saved); + if (ret == 0) + ret = ret2; if (ret) break; } - reference_lists_fini(gt, &lists); + reference_lists_fini(gt, lists); intel_runtime_pm_put(gt->uncore->rpm, wakeref); igt_global_reset_unlock(gt); + kfree(lists); igt_flush_test(gt->i915); diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h index 90efef8a73e4..8ff582222aff 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h @@ -6,6 +6,113 @@ #ifndef _ABI_GUC_ACTIONS_ABI_H #define _ABI_GUC_ACTIONS_ABI_H +/** + * DOC: HOST2GUC_REGISTER_CTB + * + * This message is used as part of the `CTB based communication`_ setup. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_REGISTER_CTB` = 0x4505 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:12 | RESERVED = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | **TYPE** - type for the `CT Buffer`_ | + * | | | | + * | | | - _`GUC_CTB_TYPE_HOST2GUC` = 0 | + * | | | - _`GUC_CTB_TYPE_GUC2HOST` = 1 | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **SIZE** - size of the `CT Buffer`_ in 4K units minus 1 | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **DESC_ADDR** - GGTT address of the `CTB Descriptor`_ | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **BUFF_ADDF** - GGTT address of the `CT Buffer`_ | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_REGISTER_CTB 0x4505 + +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_MBZ (0xfffff << 12) +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_TYPE (0xf << 8) +#define GUC_CTB_TYPE_HOST2GUC 0u +#define GUC_CTB_TYPE_GUC2HOST 1u +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_SIZE (0xff << 0) +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR GUC_HXG_REQUEST_MSG_n_DATAn +#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR GUC_HXG_REQUEST_MSG_n_DATAn + +#define HOST2GUC_REGISTER_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_REGISTER_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: HOST2GUC_DEREGISTER_CTB + * + * This message is used as part of the `CTB based communication`_ teardown. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_DEREGISTER_CTB` = 0x4506 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:12 | RESERVED = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | **TYPE** - type of the `CT Buffer`_ | + * | | | | + * | | | see `GUC_ACTION_HOST2GUC_REGISTER_CTB`_ | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_DEREGISTER_CTB 0x4506 + +#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_MBZ (0xfffff << 12) +#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE (0xf << 8) +#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_MBZ2 (0xff << 0) + +#define HOST2GUC_DEREGISTER_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_DEREGISTER_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/* legacy definitions */ + enum intel_guc_action { INTEL_GUC_ACTION_DEFAULT = 0x0, INTEL_GUC_ACTION_REQUEST_PREEMPTION = 0x2, @@ -17,13 +124,33 @@ enum intel_guc_action { INTEL_GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302, INTEL_GUC_ACTION_ENTER_S_STATE = 0x501, INTEL_GUC_ACTION_EXIT_S_STATE = 0x502, - INTEL_GUC_ACTION_SLPC_REQUEST = 0x3003, + INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE = 0x506, + INTEL_GUC_ACTION_SCHED_CONTEXT = 0x1000, + INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET = 0x1001, + INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE = 0x1002, + INTEL_GUC_ACTION_SCHED_ENGINE_MODE_SET = 0x1003, + INTEL_GUC_ACTION_SCHED_ENGINE_MODE_DONE = 0x1004, + INTEL_GUC_ACTION_SET_CONTEXT_PRIORITY = 0x1005, + INTEL_GUC_ACTION_SET_CONTEXT_EXECUTION_QUANTUM = 0x1006, + INTEL_GUC_ACTION_SET_CONTEXT_PREEMPTION_TIMEOUT = 0x1007, + INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION = 0x1008, + INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION = 0x1009, + INTEL_GUC_ACTION_SETUP_PC_GUCRC = 0x3004, INTEL_GUC_ACTION_AUTHENTICATE_HUC = 0x4000, + INTEL_GUC_ACTION_REGISTER_CONTEXT = 0x4502, + INTEL_GUC_ACTION_DEREGISTER_CONTEXT = 0x4503, INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER = 0x4505, INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506, + INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, + INTEL_GUC_ACTION_RESET_CLIENT = 0x5507, INTEL_GUC_ACTION_LIMIT }; +enum intel_guc_rc_options { + INTEL_GUCRC_HOST_CONTROL, + INTEL_GUCRC_FIRMWARE_CONTROL, +}; + enum intel_guc_preempt_options { INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q = 0x4, INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q = 0x8, diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h new file mode 100644 index 000000000000..7a8d4bfc5f6a --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _GUC_ACTIONS_SLPC_ABI_H_ +#define _GUC_ACTIONS_SLPC_ABI_H_ + +#include <linux/types.h> +#include "i915_reg.h" + +/** + * DOC: SLPC SHARED DATA STRUCTURE + * + * +----+------+--------------------------------------------------------------+ + * | CL | Bytes| Description | + * +====+======+==============================================================+ + * | 1 | 0-3 | SHARED DATA SIZE | + * | +------+--------------------------------------------------------------+ + * | | 4-7 | GLOBAL STATE | + * | +------+--------------------------------------------------------------+ + * | | 8-11 | DISPLAY DATA ADDRESS | + * | +------+--------------------------------------------------------------+ + * | | 12:63| PADDING | + * +----+------+--------------------------------------------------------------+ + * | | 0:63 | PADDING(PLATFORM INFO) | + * +----+------+--------------------------------------------------------------+ + * | 3 | 0-3 | TASK STATE DATA | + * + +------+--------------------------------------------------------------+ + * | | 4:63 | PADDING | + * +----+------+--------------------------------------------------------------+ + * |4-21|0:1087| OVERRIDE PARAMS AND BIT FIELDS | + * +----+------+--------------------------------------------------------------+ + * | | | PADDING + EXTRA RESERVED PAGE | + * +----+------+--------------------------------------------------------------+ + */ + +/* + * SLPC exposes certain parameters for global configuration by the host. + * These are referred to as override parameters, because in most cases + * the host will not need to modify the default values used by SLPC. + * SLPC remembers the default values which allows the host to easily restore + * them by simply unsetting the override. The host can set or unset override + * parameters during SLPC (re-)initialization using the SLPC Reset event. + * The host can also set or unset override parameters on the fly using the + * Parameter Set and Parameter Unset events + */ + +#define SLPC_MAX_OVERRIDE_PARAMETERS 256 +#define SLPC_OVERRIDE_BITFIELD_SIZE \ + (SLPC_MAX_OVERRIDE_PARAMETERS / 32) + +#define SLPC_PAGE_SIZE_BYTES 4096 +#define SLPC_CACHELINE_SIZE_BYTES 64 +#define SLPC_SHARED_DATA_SIZE_BYTE_HEADER SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE SLPC_PAGE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_MAX (2 * SLPC_PAGE_SIZE_BYTES) + +/* + * Cacheline size aligned (Total size needed for + * SLPM_KMD_MAX_OVERRIDE_PARAMETERS=256 is 1088 bytes) + */ +#define SLPC_OVERRIDE_PARAMS_TOTAL_BYTES (((((SLPC_MAX_OVERRIDE_PARAMETERS * 4) \ + + ((SLPC_MAX_OVERRIDE_PARAMETERS / 32) * 4)) \ + + (SLPC_CACHELINE_SIZE_BYTES - 1)) / SLPC_CACHELINE_SIZE_BYTES) * \ + SLPC_CACHELINE_SIZE_BYTES) + +#define SLPC_SHARED_DATA_SIZE_BYTE_OTHER (SLPC_SHARED_DATA_SIZE_BYTE_MAX - \ + (SLPC_SHARED_DATA_SIZE_BYTE_HEADER \ + + SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO \ + + SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE \ + + SLPC_OVERRIDE_PARAMS_TOTAL_BYTES \ + + SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE)) + +enum slpc_task_enable { + SLPC_PARAM_TASK_DEFAULT = 0, + SLPC_PARAM_TASK_ENABLED, + SLPC_PARAM_TASK_DISABLED, + SLPC_PARAM_TASK_UNKNOWN +}; + +enum slpc_global_state { + SLPC_GLOBAL_STATE_NOT_RUNNING = 0, + SLPC_GLOBAL_STATE_INITIALIZING = 1, + SLPC_GLOBAL_STATE_RESETTING = 2, + SLPC_GLOBAL_STATE_RUNNING = 3, + SLPC_GLOBAL_STATE_SHUTTING_DOWN = 4, + SLPC_GLOBAL_STATE_ERROR = 5 +}; + +enum slpc_param_id { + SLPC_PARAM_TASK_ENABLE_GTPERF = 0, + SLPC_PARAM_TASK_DISABLE_GTPERF = 1, + SLPC_PARAM_TASK_ENABLE_BALANCER = 2, + SLPC_PARAM_TASK_DISABLE_BALANCER = 3, + SLPC_PARAM_TASK_ENABLE_DCC = 4, + SLPC_PARAM_TASK_DISABLE_DCC = 5, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ = 6, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ = 7, + SLPC_PARAM_GLOBAL_MIN_GT_SLICE_FREQ_MHZ = 8, + SLPC_PARAM_GLOBAL_MAX_GT_SLICE_FREQ_MHZ = 9, + SLPC_PARAM_GTPERF_THRESHOLD_MAX_FPS = 10, + SLPC_PARAM_GLOBAL_DISABLE_GT_FREQ_MANAGEMENT = 11, + SLPC_PARAM_GTPERF_ENABLE_FRAMERATE_STALLING = 12, + SLPC_PARAM_GLOBAL_DISABLE_RC6_MODE_CHANGE = 13, + SLPC_PARAM_GLOBAL_OC_UNSLICE_FREQ_MHZ = 14, + SLPC_PARAM_GLOBAL_OC_SLICE_FREQ_MHZ = 15, + SLPC_PARAM_GLOBAL_ENABLE_IA_GT_BALANCING = 16, + SLPC_PARAM_GLOBAL_ENABLE_ADAPTIVE_BURST_TURBO = 17, + SLPC_PARAM_GLOBAL_ENABLE_EVAL_MODE = 18, + SLPC_PARAM_GLOBAL_ENABLE_BALANCER_IN_NON_GAMING_MODE = 19, + SLPC_PARAM_GLOBAL_RT_MODE_TURBO_FREQ_DELTA_MHZ = 20, + SLPC_PARAM_PWRGATE_RC_MODE = 21, + SLPC_PARAM_EDR_MODE_COMPUTE_TIMEOUT_MS = 22, + SLPC_PARAM_EDR_QOS_FREQ_MHZ = 23, + SLPC_PARAM_MEDIA_FF_RATIO_MODE = 24, + SLPC_PARAM_ENABLE_IA_FREQ_LIMITING = 25, + SLPC_PARAM_STRATEGIES = 26, + SLPC_PARAM_POWER_PROFILE = 27, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY = 28, + SLPC_MAX_PARAM = 32, +}; + +enum slpc_event_id { + SLPC_EVENT_RESET = 0, + SLPC_EVENT_SHUTDOWN = 1, + SLPC_EVENT_PLATFORM_INFO_CHANGE = 2, + SLPC_EVENT_DISPLAY_MODE_CHANGE = 3, + SLPC_EVENT_FLIP_COMPLETE = 4, + SLPC_EVENT_QUERY_TASK_STATE = 5, + SLPC_EVENT_PARAMETER_SET = 6, + SLPC_EVENT_PARAMETER_UNSET = 7, +}; + +struct slpc_task_state_data { + union { + u32 task_status_padding; + struct { + u32 status; +#define SLPC_GTPERF_TASK_ENABLED REG_BIT(0) +#define SLPC_DCC_TASK_ENABLED REG_BIT(11) +#define SLPC_IN_DCC REG_BIT(12) +#define SLPC_BALANCER_ENABLED REG_BIT(15) +#define SLPC_IBC_TASK_ENABLED REG_BIT(16) +#define SLPC_BALANCER_IA_LMT_ENABLED REG_BIT(17) +#define SLPC_BALANCER_IA_LMT_ACTIVE REG_BIT(18) + }; + }; + union { + u32 freq_padding; + struct { +#define SLPC_MAX_UNSLICE_FREQ_MASK REG_GENMASK(7, 0) +#define SLPC_MIN_UNSLICE_FREQ_MASK REG_GENMASK(15, 8) +#define SLPC_MAX_SLICE_FREQ_MASK REG_GENMASK(23, 16) +#define SLPC_MIN_SLICE_FREQ_MASK REG_GENMASK(31, 24) + u32 freq; + }; + }; +} __packed; + +struct slpc_shared_data_header { + /* Total size in bytes of this shared buffer. */ + u32 size; + u32 global_state; + u32 display_data_addr; +} __packed; + +struct slpc_override_params { + u32 bits[SLPC_OVERRIDE_BITFIELD_SIZE]; + u32 values[SLPC_MAX_OVERRIDE_PARAMETERS]; +} __packed; + +struct slpc_shared_data { + struct slpc_shared_data_header header; + u8 shared_data_header_pad[SLPC_SHARED_DATA_SIZE_BYTE_HEADER - + sizeof(struct slpc_shared_data_header)]; + + u8 platform_info_pad[SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO]; + + struct slpc_task_state_data task_state_data; + u8 task_state_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE - + sizeof(struct slpc_task_state_data)]; + + struct slpc_override_params override_params; + u8 override_params_pad[SLPC_OVERRIDE_PARAMS_TOTAL_BYTES - + sizeof(struct slpc_override_params)]; + + u8 shared_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_OTHER]; + + /* PAGE 2 (4096 bytes), mode based parameter will be removed soon */ + u8 reserved_mode_definition[4096]; +} __packed; + +/** + * DOC: SLPC H2G MESSAGE FORMAT + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_PC_SLPM_REQUEST` = 0x3003 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:8 | **EVENT_ID** | + * + +-------+--------------------------------------------------------------+ + * | | 7:0 | **EVENT_ARGC** - number of data arguments | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **EVENT_DATA1** | + * +---+-------+--------------------------------------------------------------+ + * |...| 31:0 | ... | + * +---+-------+--------------------------------------------------------------+ + * |2+n| 31:0 | **EVENT_DATAn** | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST 0x3003 + +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MIN_LEN \ + (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS 9 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MAX_LEN \ + (HOST2GUC_PC_SLPC_REQUEST_REQUEST_MSG_MIN_LEN + \ + HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID (0xff << 8) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC (0xff << 0) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_N_EVENT_DATA_N GUC_HXG_REQUEST_MSG_n_DATAn + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h index d38935f47ecf..99e1fad5ca20 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h @@ -7,6 +7,111 @@ #define _ABI_GUC_COMMUNICATION_CTB_ABI_H #include <linux/types.h> +#include <linux/build_bug.h> + +#include "guc_messages_abi.h" + +/** + * DOC: CT Buffer + * + * Circular buffer used to send `CTB Message`_ + */ + +/** + * DOC: CTB Descriptor + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | **HEAD** - offset (in dwords) to the last dword that was | + * | | | read from the `CT Buffer`_. | + * | | | It can only be updated by the receiver. | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **TAIL** - offset (in dwords) to the last dword that was | + * | | | written to the `CT Buffer`_. | + * | | | It can only be updated by the sender. | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **STATUS** - status of the CTB | + * | | | | + * | | | - _`GUC_CTB_STATUS_NO_ERROR` = 0 (normal operation) | + * | | | - _`GUC_CTB_STATUS_OVERFLOW` = 1 (head/tail too large) | + * | | | - _`GUC_CTB_STATUS_UNDERFLOW` = 2 (truncated message) | + * | | | - _`GUC_CTB_STATUS_MISMATCH` = 4 (head/tail modified) | + * +---+-------+--------------------------------------------------------------+ + * |...| | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + * | 15| 31:0 | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ + +struct guc_ct_buffer_desc { + u32 head; + u32 tail; + u32 status; +#define GUC_CTB_STATUS_NO_ERROR 0 +#define GUC_CTB_STATUS_OVERFLOW (1 << 0) +#define GUC_CTB_STATUS_UNDERFLOW (1 << 1) +#define GUC_CTB_STATUS_MISMATCH (1 << 2) + u32 reserved[13]; +} __packed; +static_assert(sizeof(struct guc_ct_buffer_desc) == 64); + +/** + * DOC: CTB Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **FENCE** - message identifier | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | **FORMAT** - format of the CTB message | + * | | | - _`GUC_CTB_FORMAT_HXG` = 0 - see `CTB HXG Message`_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | **RESERVED** | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **NUM_DWORDS** - length of the CTB message (w/o header) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | optional (depends on FORMAT) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HDR_LEN 1u +#define GUC_CTB_MSG_MIN_LEN GUC_CTB_HDR_LEN +#define GUC_CTB_MSG_MAX_LEN 256u +#define GUC_CTB_MSG_0_FENCE (0xffff << 16) +#define GUC_CTB_MSG_0_FORMAT (0xf << 12) +#define GUC_CTB_FORMAT_HXG 0u +#define GUC_CTB_MSG_0_RESERVED (0xf << 8) +#define GUC_CTB_MSG_0_NUM_DWORDS (0xff << 0) + +/** + * DOC: CTB HXG Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | FENCE | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | FORMAT = GUC_CTB_FORMAT_HXG_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | RESERVED = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | NUM_DWORDS = length (in dwords) of the embedded HXG message | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | +--------------------------------------------------------+ | + * +---+-------+ | | | + * |...| | | Embedded `HXG Message`_ | | + * +---+-------+ | | | + * | n | 31:0 | +--------------------------------------------------------+ | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HXG_MSG_MIN_LEN (GUC_CTB_MSG_MIN_LEN + GUC_HXG_MSG_MIN_LEN) +#define GUC_CTB_HXG_MSG_MAX_LEN GUC_CTB_MSG_MAX_LEN /** * DOC: CTB based communication @@ -61,28 +166,6 @@ */ /* - * Describes single command transport buffer. - * Used by both guc-master and clients. - */ -struct guc_ct_buffer_desc { - u32 addr; /* gfx address */ - u64 host_private; /* host private data */ - u32 size; /* size in bytes */ - u32 head; /* offset updated by GuC*/ - u32 tail; /* offset updated by owner */ - u32 is_in_error; /* error indicator */ - u32 reserved1; - u32 reserved2; - u32 owner; /* id of the channel owner */ - u32 owner_sub_id; /* owner-defined field for extra tracking */ - u32 reserved[5]; -} __packed; - -/* Type of command transport buffer */ -#define INTEL_GUC_CT_BUFFER_TYPE_SEND 0x0u -#define INTEL_GUC_CT_BUFFER_TYPE_RECV 0x1u - -/* * Definition of the command transport message header (DW0) * * bit[4..0] message len (in dwords) diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_mmio_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_mmio_abi.h index be066a62e9e0..bbf1ddb77434 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_mmio_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_mmio_abi.h @@ -7,46 +7,43 @@ #define _ABI_GUC_COMMUNICATION_MMIO_ABI_H /** - * DOC: MMIO based communication + * DOC: GuC MMIO based communication * - * The MMIO based communication between Host and GuC uses software scratch - * registers, where first register holds data treated as message header, - * and other registers are used to hold message payload. + * The MMIO based communication between Host and GuC relies on special + * hardware registers which format could be defined by the software + * (so called scratch registers). * - * For Gen9+, GuC uses software scratch registers 0xC180-0xC1B8, - * but no H2G command takes more than 8 parameters and the GuC FW - * itself uses an 8-element array to store the H2G message. + * Each MMIO based message, both Host to GuC (H2G) and GuC to Host (G2H) + * messages, which maximum length depends on number of available scratch + * registers, is directly written into those scratch registers. * - * +-----------+---------+---------+---------+ - * | MMIO[0] | MMIO[1] | ... | MMIO[n] | - * +-----------+---------+---------+---------+ - * | header | optional payload | - * +======+====+=========+=========+=========+ - * | 31:28|type| | | | - * +------+----+ | | | - * | 27:16|data| | | | - * +------+----+ | | | - * | 15:0|code| | | | - * +------+----+---------+---------+---------+ + * For Gen9+, there are 16 software scratch registers 0xC180-0xC1B8, + * but no H2G command takes more than 4 parameters and the GuC firmware + * itself uses an 4-element array to store the H2G message. * - * The message header consists of: + * For Gen11+, there are additional 4 registers 0x190240-0x19024C, which + * are, regardless on lower count, preferred over legacy ones. * - * - **type**, indicates message type - * - **code**, indicates message code, is specific for **type** - * - **data**, indicates message data, optional, depends on **code** - * - * The following message **types** are supported: - * - * - **REQUEST**, indicates Host-to-GuC request, requested GuC action code - * must be priovided in **code** field. Optional action specific parameters - * can be provided in remaining payload registers or **data** field. - * - * - **RESPONSE**, indicates GuC-to-Host response from earlier GuC request, - * action response status will be provided in **code** field. Optional - * response data can be returned in remaining payload registers or **data** - * field. + * The MMIO based communication is mainly used during driver initialization + * phase to setup the `CTB based communication`_ that will be used afterwards. */ -#define GUC_MAX_MMIO_MSG_LEN 8 +#define GUC_MAX_MMIO_MSG_LEN 4 + +/** + * DOC: MMIO HXG Message + * + * Format of the MMIO messages follows definitions of `HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | +--------------------------------------------------------+ | + * +---+-------+ | | | + * |...| | | Embedded `HXG Message`_ | | + * +---+-------+ | | | + * | n | 31:0 | +--------------------------------------------------------+ | + * +---+-------+--------------------------------------------------------------+ + */ #endif /* _ABI_GUC_COMMUNICATION_MMIO_ABI_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h index 775e21f3058c..29ac823acd4c 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h @@ -6,6 +6,219 @@ #ifndef _ABI_GUC_MESSAGES_ABI_H #define _ABI_GUC_MESSAGES_ABI_H +/** + * DOC: HXG Message + * + * All messages exchanged with GuC are defined using 32 bit dwords. + * First dword is treated as a message header. Remaining dwords are optional. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | | | | + * | 0 | 31 | **ORIGIN** - originator of the message | + * | | | - _`GUC_HXG_ORIGIN_HOST` = 0 | + * | | | - _`GUC_HXG_ORIGIN_GUC` = 1 | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | **TYPE** - message type | + * | | | - _`GUC_HXG_TYPE_REQUEST` = 0 | + * | | | - _`GUC_HXG_TYPE_EVENT` = 1 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_BUSY` = 3 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_RETRY` = 5 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_FAILURE` = 6 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_SUCCESS` = 7 | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **AUX** - auxiliary data (depends on TYPE) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **PAYLOAD** - optional payload (depends on TYPE) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_MSG_MIN_LEN 1u +#define GUC_HXG_MSG_0_ORIGIN (0x1 << 31) +#define GUC_HXG_ORIGIN_HOST 0u +#define GUC_HXG_ORIGIN_GUC 1u +#define GUC_HXG_MSG_0_TYPE (0x7 << 28) +#define GUC_HXG_TYPE_REQUEST 0u +#define GUC_HXG_TYPE_EVENT 1u +#define GUC_HXG_TYPE_NO_RESPONSE_BUSY 3u +#define GUC_HXG_TYPE_NO_RESPONSE_RETRY 5u +#define GUC_HXG_TYPE_RESPONSE_FAILURE 6u +#define GUC_HXG_TYPE_RESPONSE_SUCCESS 7u +#define GUC_HXG_MSG_0_AUX (0xfffffff << 0) +#define GUC_HXG_MSG_n_PAYLOAD (0xffffffff << 0) + +/** + * DOC: HXG Request + * + * The `HXG Request`_ message should be used to initiate synchronous activity + * for which confirmation or return data is expected. + * + * The recipient of this message shall use `HXG Response`_, `HXG Failure`_ + * or `HXG Retry`_ message as a definite reply, and may use `HXG Busy`_ + * message as a intermediate reply. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - request data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - requested action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_REQUEST_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_REQUEST_MSG_0_DATA0 (0xfff << 16) +#define GUC_HXG_REQUEST_MSG_0_ACTION (0xffff << 0) +#define GUC_HXG_REQUEST_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Event + * + * The `HXG Event`_ message should be used to initiate asynchronous activity + * that does not involves immediate confirmation nor data. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - event data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - event action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional event data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_EVENT_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_EVENT_MSG_0_DATA0 (0xfff << 16) +#define GUC_HXG_EVENT_MSG_0_ACTION (0xffff << 0) +#define GUC_HXG_EVENT_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Busy + * + * The `HXG Busy`_ message may be used to acknowledge reception of the `HXG Request`_ + * message if the recipient expects that it processing will be longer than default + * timeout. + * + * The @COUNTER field may be used as a progress indicator. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_BUSY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **COUNTER** - progress indicator | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_BUSY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_BUSY_MSG_0_COUNTER GUC_HXG_MSG_0_AUX + +/** + * DOC: HXG Retry + * + * The `HXG Retry`_ message should be used by recipient to indicate that the + * `HXG Request`_ message was dropped and it should be resent again. + * + * The @REASON field may be used to provide additional information. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_RETRY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **REASON** - reason for retry | + * | | | - _`GUC_HXG_RETRY_REASON_UNSPECIFIED` = 0 | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RETRY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RETRY_MSG_0_REASON GUC_HXG_MSG_0_AUX +#define GUC_HXG_RETRY_REASON_UNSPECIFIED 0u + +/** + * DOC: HXG Failure + * + * The `HXG Failure`_ message shall be used as a reply to the `HXG Request`_ + * message that could not be processed due to an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_FAILURE_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **HINT** - additional error hint | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ERROR** - error/result code | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_FAILURE_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_FAILURE_MSG_0_HINT (0xfff << 16) +#define GUC_HXG_FAILURE_MSG_0_ERROR (0xffff << 0) + +/** + * DOC: HXG Response + * + * The `HXG Response`_ message shall be used as a reply to the `HXG Request`_ + * message that was successfully processed without an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **DATA0** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RESPONSE_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RESPONSE_MSG_0_DATA0 GUC_HXG_MSG_0_AUX +#define GUC_HXG_RESPONSE_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/* deprecated */ #define INTEL_GUC_MSG_TYPE_SHIFT 28 #define INTEL_GUC_MSG_TYPE_MASK (0xF << INTEL_GUC_MSG_TYPE_SHIFT) #define INTEL_GUC_MSG_DATA_SHIFT 16 diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index f147cb389a20..fbfcae727d7f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -7,6 +7,7 @@ #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" #include "intel_guc.h" +#include "intel_guc_slpc.h" #include "intel_guc_ads.h" #include "intel_guc_submission.h" #include "i915_drv.h" @@ -157,6 +158,8 @@ void intel_guc_init_early(struct intel_guc *guc) intel_guc_ct_init_early(&guc->ct); intel_guc_log_init_early(&guc->log); intel_guc_submission_init_early(guc); + intel_guc_slpc_init_early(&guc->slpc); + intel_guc_rc_init_early(guc); mutex_init(&guc->send_mutex); spin_lock_init(&guc->irq_lock); @@ -180,6 +183,11 @@ void intel_guc_init_early(struct intel_guc *guc) } } +void intel_guc_init_late(struct intel_guc *guc) +{ + intel_guc_ads_init_late(guc); +} + static u32 guc_ctl_debug_flags(struct intel_guc *guc) { u32 level = intel_guc_log_get_level(&guc->log); @@ -201,6 +209,9 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) if (!intel_guc_submission_is_used(guc)) flags |= GUC_CTL_DISABLE_SCHEDULER; + if (intel_guc_slpc_is_used(guc)) + flags |= GUC_CTL_ENABLE_SLPC; + return flags; } @@ -219,24 +230,19 @@ static u32 guc_ctl_log_params_flags(struct intel_guc *guc) BUILD_BUG_ON(!CRASH_BUFFER_SIZE); BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, UNIT)); - BUILD_BUG_ON(!DPC_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(DPC_BUFFER_SIZE, UNIT)); - BUILD_BUG_ON(!ISR_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(ISR_BUFFER_SIZE, UNIT)); + BUILD_BUG_ON(!DEBUG_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, UNIT)); BUILD_BUG_ON((CRASH_BUFFER_SIZE / UNIT - 1) > (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT)); - BUILD_BUG_ON((DPC_BUFFER_SIZE / UNIT - 1) > - (GUC_LOG_DPC_MASK >> GUC_LOG_DPC_SHIFT)); - BUILD_BUG_ON((ISR_BUFFER_SIZE / UNIT - 1) > - (GUC_LOG_ISR_MASK >> GUC_LOG_ISR_SHIFT)); + BUILD_BUG_ON((DEBUG_BUFFER_SIZE / UNIT - 1) > + (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT)); flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL | FLAG | ((CRASH_BUFFER_SIZE / UNIT - 1) << GUC_LOG_CRASH_SHIFT) | - ((DPC_BUFFER_SIZE / UNIT - 1) << GUC_LOG_DPC_SHIFT) | - ((ISR_BUFFER_SIZE / UNIT - 1) << GUC_LOG_ISR_SHIFT) | + ((DEBUG_BUFFER_SIZE / UNIT - 1) << GUC_LOG_DEBUG_SHIFT) | (offset << GUC_LOG_BUF_ADDR_SHIFT); #undef UNIT @@ -331,6 +337,12 @@ int intel_guc_init(struct intel_guc *guc) goto err_ct; } + if (intel_guc_slpc_is_used(guc)) { + ret = intel_guc_slpc_init(&guc->slpc); + if (ret) + goto err_submission; + } + /* now that everything is perma-pinned, initialize the parameters */ guc_init_params(guc); @@ -341,6 +353,8 @@ int intel_guc_init(struct intel_guc *guc) return 0; +err_submission: + intel_guc_submission_fini(guc); err_ct: intel_guc_ct_fini(&guc->ct); err_ads: @@ -363,6 +377,9 @@ void intel_guc_fini(struct intel_guc *guc) i915_ggtt_disable_guc(gt->ggtt); + if (intel_guc_slpc_is_used(guc)) + intel_guc_slpc_fini(&guc->slpc); + if (intel_guc_submission_is_used(guc)) intel_guc_submission_fini(guc); @@ -376,29 +393,27 @@ void intel_guc_fini(struct intel_guc *guc) /* * This function implements the MMIO based host to GuC interface. */ -int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len, +int intel_guc_send_mmio(struct intel_guc *guc, const u32 *request, u32 len, u32 *response_buf, u32 response_buf_size) { + struct drm_i915_private *i915 = guc_to_gt(guc)->i915; struct intel_uncore *uncore = guc_to_gt(guc)->uncore; - u32 status; + u32 header; int i; int ret; GEM_BUG_ON(!len); GEM_BUG_ON(len > guc->send_regs.count); - /* We expect only action code */ - GEM_BUG_ON(*action & ~INTEL_GUC_MSG_CODE_MASK); - - /* If CT is available, we expect to use MMIO only during init/fini */ - GEM_BUG_ON(*action != INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER && - *action != INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER); + GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) != GUC_HXG_ORIGIN_HOST); + GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) != GUC_HXG_TYPE_REQUEST); mutex_lock(&guc->send_mutex); intel_uncore_forcewake_get(uncore, guc->send_regs.fw_domains); +retry: for (i = 0; i < len; i++) - intel_uncore_write(uncore, guc_send_reg(guc, i), action[i]); + intel_uncore_write(uncore, guc_send_reg(guc, i), request[i]); intel_uncore_posting_read(uncore, guc_send_reg(guc, i - 1)); @@ -410,30 +425,74 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len, */ ret = __intel_wait_for_register_fw(uncore, guc_send_reg(guc, 0), - INTEL_GUC_MSG_TYPE_MASK, - INTEL_GUC_MSG_TYPE_RESPONSE << - INTEL_GUC_MSG_TYPE_SHIFT, - 10, 10, &status); - /* If GuC explicitly returned an error, convert it to -EIO */ - if (!ret && !INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(status)) - ret = -EIO; + GUC_HXG_MSG_0_ORIGIN, + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, + GUC_HXG_ORIGIN_GUC), + 10, 10, &header); + if (unlikely(ret)) { +timeout: + drm_err(&i915->drm, "mmio request %#x: no reply %x\n", + request[0], header); + goto out; + } - if (ret) { - DRM_ERROR("MMIO: GuC action %#x failed with error %d %#x\n", - action[0], ret, status); + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { +#define done ({ header = intel_uncore_read(uncore, guc_send_reg(guc, 0)); \ + FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != GUC_HXG_ORIGIN_GUC || \ + FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != GUC_HXG_TYPE_NO_RESPONSE_BUSY; }) + + ret = wait_for(done, 1000); + if (unlikely(ret)) + goto timeout; + if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != + GUC_HXG_ORIGIN_GUC)) + goto proto; +#undef done + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header); + + drm_dbg(&i915->drm, "mmio request %#x: retrying, reason %u\n", + request[0], reason); + goto retry; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == GUC_HXG_TYPE_RESPONSE_FAILURE) { + u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header); + u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header); + + drm_err(&i915->drm, "mmio request %#x: failure %x/%u\n", + request[0], error, hint); + ret = -ENXIO; + goto out; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { +proto: + drm_err(&i915->drm, "mmio request %#x: unexpected reply %#x\n", + request[0], header); + ret = -EPROTO; goto out; } if (response_buf) { - int count = min(response_buf_size, guc->send_regs.count - 1); + int count = min(response_buf_size, guc->send_regs.count); + + GEM_BUG_ON(!count); - for (i = 0; i < count; i++) + response_buf[0] = header; + + for (i = 1; i < count; i++) response_buf[i] = intel_uncore_read(uncore, - guc_send_reg(guc, i + 1)); - } + guc_send_reg(guc, i)); - /* Use data from the GuC response as our return value */ - ret = INTEL_GUC_MSG_TO_DATA(status); + /* Use number of copied dwords as our return value */ + ret = count; + } else { + /* Use data from the GuC response as our return value */ + ret = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header); + } out: intel_uncore_forcewake_put(uncore, guc->send_regs.fw_domains); @@ -487,65 +546,35 @@ int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset) */ int intel_guc_suspend(struct intel_guc *guc) { - struct intel_uncore *uncore = guc_to_gt(guc)->uncore; int ret; - u32 status; u32 action[] = { - INTEL_GUC_ACTION_ENTER_S_STATE, - GUC_POWER_D1, /* any value greater than GUC_POWER_D0 */ + INTEL_GUC_ACTION_RESET_CLIENT, }; - /* - * If GuC communication is enabled but submission is not supported, - * we do not need to suspend the GuC. - */ - if (!intel_guc_submission_is_used(guc) || !intel_guc_is_ready(guc)) + if (!intel_guc_is_ready(guc)) return 0; - /* - * The ENTER_S_STATE action queues the save/restore operation in GuC FW - * and then returns, so waiting on the H2G is not enough to guarantee - * GuC is done. When all the processing is done, GuC writes - * INTEL_GUC_SLEEP_STATE_SUCCESS to scratch register 14, so we can poll - * on that. Note that GuC does not ensure that the value in the register - * is different from INTEL_GUC_SLEEP_STATE_SUCCESS while the action is - * in progress so we need to take care of that ourselves as well. - */ - - intel_uncore_write(uncore, SOFT_SCRATCH(14), - INTEL_GUC_SLEEP_STATE_INVALID_MASK); - - ret = intel_guc_send(guc, action, ARRAY_SIZE(action)); - if (ret) - return ret; - - ret = __intel_wait_for_register(uncore, SOFT_SCRATCH(14), - INTEL_GUC_SLEEP_STATE_INVALID_MASK, - 0, 0, 10, &status); - if (ret) - return ret; - - if (status != INTEL_GUC_SLEEP_STATE_SUCCESS) { - DRM_ERROR("GuC failed to change sleep state. " - "action=0x%x, err=%u\n", - action[0], status); - return -EIO; + if (intel_guc_submission_is_used(guc)) { + /* + * This H2G MMIO command tears down the GuC in two steps. First it will + * generate a G2H CTB for every active context indicating a reset. In + * practice the i915 shouldn't ever get a G2H as suspend should only be + * called when the GPU is idle. Next, it tears down the CTBs and this + * H2G MMIO command completes. + * + * Don't abort on a failure code from the GuC. Keep going and do the + * clean up in santize() and re-initialisation on resume and hopefully + * the error here won't be problematic. + */ + ret = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0); + if (ret) + DRM_ERROR("GuC suspend: RESET_CLIENT action failed with error %d!\n", ret); } - return 0; -} - -/** - * intel_guc_reset_engine() - ask GuC to reset an engine - * @guc: intel_guc structure - * @engine: engine to be reset - */ -int intel_guc_reset_engine(struct intel_guc *guc, - struct intel_engine_cs *engine) -{ - /* XXX: to be implemented with submission interface rework */ + /* Signal that the GuC isn't running. */ + intel_guc_sanitize(guc); - return -ENODEV; + return 0; } /** @@ -554,7 +583,12 @@ int intel_guc_reset_engine(struct intel_guc *guc, */ int intel_guc_resume(struct intel_guc *guc) { - /* XXX: to be implemented with submission interface rework */ + /* + * NB: This function can still be called even if GuC submission is + * disabled, e.g. if GuC is enabled for HuC authentication only. Thus, + * if any code is later added here, it must be support doing nothing + * if submission is disabled (as per intel_guc_suspend). + */ return 0; } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 4abc59f6f3cd..2e27fe59786b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -6,12 +6,16 @@ #ifndef _INTEL_GUC_H_ #define _INTEL_GUC_H_ +#include <linux/xarray.h> +#include <linux/delay.h> + #include "intel_uncore.h" #include "intel_guc_fw.h" #include "intel_guc_fwif.h" #include "intel_guc_ct.h" #include "intel_guc_log.h" #include "intel_guc_reg.h" +#include "intel_guc_slpc_types.h" #include "intel_uc_fw.h" #include "i915_utils.h" #include "i915_vma.h" @@ -27,24 +31,47 @@ struct intel_guc { struct intel_uc_fw fw; struct intel_guc_log log; struct intel_guc_ct ct; + struct intel_guc_slpc slpc; + + /* Global engine used to submit requests to GuC */ + struct i915_sched_engine *sched_engine; + struct i915_request *stalled_request; /* intel_guc_recv interrupt related state */ spinlock_t irq_lock; unsigned int msg_enabled_mask; + atomic_t outstanding_submission_g2h; + struct { void (*reset)(struct intel_guc *guc); void (*enable)(struct intel_guc *guc); void (*disable)(struct intel_guc *guc); } interrupts; + /* + * contexts_lock protects the pool of free guc ids and a linked list of + * guc ids available to be stolen + */ + spinlock_t contexts_lock; + struct ida guc_ids; + struct list_head guc_id_list; + + bool submission_supported; bool submission_selected; + bool rc_supported; + bool rc_selected; struct i915_vma *ads_vma; struct __guc_ads_blob *ads_blob; + u32 ads_regset_size; + u32 ads_golden_ctxt_size; - struct i915_vma *stage_desc_pool; - void *stage_desc_pool_vaddr; + struct i915_vma *lrc_desc_pool; + void *lrc_desc_pool_vaddr; + + /* guc_id to intel_context lookup */ + struct xarray context_lookup; /* Control params for fw initialization */ u32 params[GUC_CTL_MAX_DWORDS]; @@ -74,7 +101,15 @@ static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) static inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len) { - return intel_guc_ct_send(&guc->ct, action, len, NULL, 0); + return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0); +} + +static +inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len, + u32 g2h_len_dw) +{ + return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, + MAKE_SEND_FLAGS(g2h_len_dw)); } static inline int @@ -82,7 +117,43 @@ intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len, u32 *response_buf, u32 response_buf_size) { return intel_guc_ct_send(&guc->ct, action, len, - response_buf, response_buf_size); + response_buf, response_buf_size, 0); +} + +static inline int intel_guc_send_busy_loop(struct intel_guc *guc, + const u32 *action, + u32 len, + u32 g2h_len_dw, + bool loop) +{ + int err; + unsigned int sleep_period_ms = 1; + bool not_atomic = !in_atomic() && !irqs_disabled(); + + /* + * FIXME: Have caller pass in if we are in an atomic context to avoid + * using in_atomic(). It is likely safe here as we check for irqs + * disabled which basically all the spin locks in the i915 do but + * regardless this should be cleaned up. + */ + + /* No sleeping with spin locks, just busy loop */ + might_sleep_if(loop && not_atomic); + +retry: + err = intel_guc_send_nb(guc, action, len, g2h_len_dw); + if (unlikely(err == -EBUSY && loop)) { + if (likely(not_atomic)) { + if (msleep_interruptible(sleep_period_ms)) + return -EINTR; + sleep_period_ms = sleep_period_ms << 1; + } else { + cpu_relax(); + } + goto retry; + } + + return err; } static inline void intel_guc_to_host_event_handler(struct intel_guc *guc) @@ -118,6 +189,7 @@ static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc, } void intel_guc_init_early(struct intel_guc *guc); +void intel_guc_init_late(struct intel_guc *guc); void intel_guc_init_send_regs(struct intel_guc *guc); void intel_guc_write_params(struct intel_guc *guc); int intel_guc_init(struct intel_guc *guc); @@ -160,9 +232,25 @@ static inline bool intel_guc_is_ready(struct intel_guc *guc) return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct); } +static inline void intel_guc_reset_interrupts(struct intel_guc *guc) +{ + guc->interrupts.reset(guc); +} + +static inline void intel_guc_enable_interrupts(struct intel_guc *guc) +{ + guc->interrupts.enable(guc); +} + +static inline void intel_guc_disable_interrupts(struct intel_guc *guc) +{ + guc->interrupts.disable(guc); +} + static inline int intel_guc_sanitize(struct intel_guc *guc) { intel_uc_fw_sanitize(&guc->fw); + intel_guc_disable_interrupts(guc); intel_guc_ct_sanitize(&guc->ct); guc->mmio_msg = 0; @@ -183,8 +271,27 @@ static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask) spin_unlock_irq(&guc->irq_lock); } -int intel_guc_reset_engine(struct intel_guc *guc, - struct intel_engine_cs *engine); +int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout); + +int intel_guc_deregister_done_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len); +int intel_guc_sched_done_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len); +int intel_guc_context_reset_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len); +int intel_guc_engine_failure_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len); + +void intel_guc_find_hung_context(struct intel_engine_cs *engine); + +int intel_guc_global_policies_update(struct intel_guc *guc); + +void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq); + +void intel_guc_submission_reset_prepare(struct intel_guc *guc); +void intel_guc_submission_reset(struct intel_guc *guc, bool stalled); +void intel_guc_submission_reset_finish(struct intel_guc *guc); +void intel_guc_submission_cancel_requests(struct intel_guc *guc); void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 9abfbc6edbd6..6926919bcac6 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -3,8 +3,11 @@ * Copyright © 2014-2019 Intel Corporation */ +#include <linux/bsearch.h> + #include "gt/intel_gt.h" #include "gt/intel_lrc.h" +#include "gt/shmem_utils.h" #include "intel_guc_ads.h" #include "intel_guc_fwif.h" #include "intel_uc.h" @@ -23,10 +26,15 @@ * | guc_policies | * +---------------------------------------+ * | guc_gt_system_info | - * +---------------------------------------+ - * | guc_clients_info | - * +---------------------------------------+ - * | guc_ct_pool_entry[size] | + * +---------------------------------------+ <== static + * | guc_mmio_reg[countA] (engine 0.0) | + * | guc_mmio_reg[countB] (engine 0.1) | + * | guc_mmio_reg[countC] (engine 1.0) | + * | ... | + * +---------------------------------------+ <== dynamic + * | padding | + * +---------------------------------------+ <== 4K aligned + * | golden contexts | * +---------------------------------------+ * | padding | * +---------------------------------------+ <== 4K aligned @@ -39,18 +47,49 @@ struct __guc_ads_blob { struct guc_ads ads; struct guc_policies policies; struct guc_gt_system_info system_info; - struct guc_clients_info clients_info; - struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; + /* From here on, location is dynamic! Refer to above diagram. */ + struct guc_mmio_reg regset[0]; } __packed; +static u32 guc_ads_regset_size(struct intel_guc *guc) +{ + GEM_BUG_ON(!guc->ads_regset_size); + return guc->ads_regset_size; +} + +static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) +{ + return PAGE_ALIGN(guc->ads_golden_ctxt_size); +} + static u32 guc_ads_private_data_size(struct intel_guc *guc) { return PAGE_ALIGN(guc->fw.private_data_size); } +static u32 guc_ads_regset_offset(struct intel_guc *guc) +{ + return offsetof(struct __guc_ads_blob, regset); +} + +static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) +{ + u32 offset; + + offset = guc_ads_regset_offset(guc) + + guc_ads_regset_size(guc); + + return PAGE_ALIGN(offset); +} + static u32 guc_ads_private_data_offset(struct intel_guc *guc) { - return PAGE_ALIGN(sizeof(struct __guc_ads_blob)); + u32 offset; + + offset = guc_ads_golden_ctxt_offset(guc) + + guc_ads_golden_ctxt_size(guc); + + return PAGE_ALIGN(offset); } static u32 guc_ads_blob_size(struct intel_guc *guc) @@ -59,36 +98,66 @@ static u32 guc_ads_blob_size(struct intel_guc *guc) guc_ads_private_data_size(guc); } -static void guc_policy_init(struct guc_policy *policy) +static void guc_policies_init(struct intel_guc *guc, struct guc_policies *policies) { - policy->execution_quantum = POLICY_DEFAULT_EXECUTION_QUANTUM_US; - policy->preemption_time = POLICY_DEFAULT_PREEMPTION_TIME_US; - policy->fault_time = POLICY_DEFAULT_FAULT_TIME_US; - policy->policy_flags = 0; + struct intel_gt *gt = guc_to_gt(guc); + struct drm_i915_private *i915 = gt->i915; + + policies->dpc_promote_time = GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US; + policies->max_num_work_items = GLOBAL_POLICY_MAX_NUM_WI; + + policies->global_flags = 0; + if (i915->params.reset < 2) + policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; + + policies->is_valid = 1; } -static void guc_policies_init(struct guc_policies *policies) +void intel_guc_ads_print_policy_info(struct intel_guc *guc, + struct drm_printer *dp) { - struct guc_policy *policy; - u32 p, i; + struct __guc_ads_blob *blob = guc->ads_blob; - policies->dpc_promote_time = POLICY_DEFAULT_DPC_PROMOTE_TIME_US; - policies->max_num_work_items = POLICY_MAX_NUM_WI; + if (unlikely(!blob)) + return; - for (p = 0; p < GUC_CLIENT_PRIORITY_NUM; p++) { - for (i = 0; i < GUC_MAX_ENGINE_CLASSES; i++) { - policy = &policies->policy[p][i]; + drm_printf(dp, "Global scheduling policies:\n"); + drm_printf(dp, " DPC promote time = %u\n", blob->policies.dpc_promote_time); + drm_printf(dp, " Max num work items = %u\n", blob->policies.max_num_work_items); + drm_printf(dp, " Flags = %u\n", blob->policies.global_flags); +} - guc_policy_init(policy); - } - } +static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) +{ + u32 action[] = { + INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, + policy_offset + }; - policies->is_valid = 1; + return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); } -static void guc_ct_pool_entries_init(struct guc_ct_pool_entry *pool, u32 num) +int intel_guc_global_policies_update(struct intel_guc *guc) { - memset(pool, 0, num * sizeof(*pool)); + struct __guc_ads_blob *blob = guc->ads_blob; + struct intel_gt *gt = guc_to_gt(guc); + intel_wakeref_t wakeref; + int ret; + + if (!blob) + return -EOPNOTSUPP; + + GEM_BUG_ON(!blob->ads.scheduler_policies); + + guc_policies_init(guc, &blob->policies); + + if (!intel_guc_is_ready(guc)) + return 0; + + with_intel_runtime_pm(>->i915->runtime_pm, wakeref) + ret = guc_action_policies_update(guc, blob->ads.scheduler_policies); + + return ret; } static void guc_mapping_table_init(struct intel_gt *gt, @@ -113,53 +182,324 @@ static void guc_mapping_table_init(struct intel_gt *gt, } /* - * The first 80 dwords of the register state context, containing the - * execlists and ppgtt registers. + * The save/restore register list must be pre-calculated to a temporary + * buffer of driver defined size before it can be generated in place + * inside the ADS. */ -#define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) +#define MAX_MMIO_REGS 128 /* Arbitrary size, increase as needed */ +struct temp_regset { + struct guc_mmio_reg *registers; + u32 used; + u32 size; +}; -static void __guc_ads_init(struct intel_guc *guc) +static int guc_mmio_reg_cmp(const void *a, const void *b) +{ + const struct guc_mmio_reg *ra = a; + const struct guc_mmio_reg *rb = b; + + return (int)ra->offset - (int)rb->offset; +} + +static void guc_mmio_reg_add(struct temp_regset *regset, + u32 offset, u32 flags) +{ + u32 count = regset->used; + struct guc_mmio_reg reg = { + .offset = offset, + .flags = flags, + }; + struct guc_mmio_reg *slot; + + GEM_BUG_ON(count >= regset->size); + + /* + * The mmio list is built using separate lists within the driver. + * It's possible that at some point we may attempt to add the same + * register more than once. Do not consider this an error; silently + * move on if the register is already in the list. + */ + if (bsearch(®, regset->registers, count, + sizeof(reg), guc_mmio_reg_cmp)) + return; + + slot = ®set->registers[count]; + regset->used++; + *slot = reg; + + while (slot-- > regset->registers) { + GEM_BUG_ON(slot[0].offset == slot[1].offset); + if (slot[1].offset > slot[0].offset) + break; + + swap(slot[1], slot[0]); + } +} + +#define GUC_MMIO_REG_ADD(regset, reg, masked) \ + guc_mmio_reg_add(regset, \ + i915_mmio_reg_offset((reg)), \ + (masked) ? GUC_REGSET_MASKED : 0) + +static void guc_mmio_regset_init(struct temp_regset *regset, + struct intel_engine_cs *engine) +{ + const u32 base = engine->mmio_base; + struct i915_wa_list *wal = &engine->wa_list; + struct i915_wa *wa; + unsigned int i; + + regset->used = 0; + + GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true); + GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); + GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); + + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) + GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); + + /* Be extra paranoid and include all whitelist registers. */ + for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) + GUC_MMIO_REG_ADD(regset, + RING_FORCE_TO_NONPRIV(base, i), + false); + + /* add in local MOCS registers */ + for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++) + GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false); +} + +static int guc_mmio_reg_state_query(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - struct drm_i915_private *i915 = gt->i915; + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct temp_regset temp_set; + u32 total; + + /* + * Need to actually build the list in order to filter out + * duplicates and other such data dependent constructions. + */ + temp_set.size = MAX_MMIO_REGS; + temp_set.registers = kmalloc_array(temp_set.size, + sizeof(*temp_set.registers), + GFP_KERNEL); + if (!temp_set.registers) + return -ENOMEM; + + total = 0; + for_each_engine(engine, gt, id) { + guc_mmio_regset_init(&temp_set, engine); + total += temp_set.used; + } + + kfree(temp_set.registers); + + return total * sizeof(struct guc_mmio_reg); +} + +static void guc_mmio_reg_state_init(struct intel_guc *guc, + struct __guc_ads_blob *blob) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct temp_regset temp_set; + struct guc_mmio_reg_set *ads_reg_set; + u32 addr_ggtt, offset; + u8 guc_class; + + offset = guc_ads_regset_offset(guc); + addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; + temp_set.registers = (struct guc_mmio_reg *)(((u8 *)blob) + offset); + temp_set.size = guc->ads_regset_size / sizeof(temp_set.registers[0]); + + for_each_engine(engine, gt, id) { + /* Class index is checked in class converter */ + GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); + + guc_class = engine_class_to_guc_class(engine->class); + ads_reg_set = &blob->ads.reg_state_list[guc_class][engine->instance]; + + guc_mmio_regset_init(&temp_set, engine); + if (!temp_set.used) { + ads_reg_set->address = 0; + ads_reg_set->count = 0; + continue; + } + + ads_reg_set->address = addr_ggtt; + ads_reg_set->count = temp_set.used; + + temp_set.size -= temp_set.used; + temp_set.registers += temp_set.used; + addr_ggtt += temp_set.used * sizeof(struct guc_mmio_reg); + } + + GEM_BUG_ON(temp_set.size); +} + +static void fill_engine_enable_masks(struct intel_gt *gt, + struct guc_gt_system_info *info) +{ + info->engine_enabled_masks[GUC_RENDER_CLASS] = 1; + info->engine_enabled_masks[GUC_BLITTER_CLASS] = 1; + info->engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); + info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); +} + +static int guc_prep_golden_context(struct intel_guc *guc, + struct __guc_ads_blob *blob) +{ + struct intel_gt *gt = guc_to_gt(guc); + u32 addr_ggtt, offset; + u32 total_size = 0, alloc_size, real_size; + u8 engine_class, guc_class; + struct guc_gt_system_info *info, local_info; + + /* + * Reserve the memory for the golden contexts and point GuC at it but + * leave it empty for now. The context data will be filled in later + * once there is something available to put there. + * + * Note that the HWSP and ring context are not included. + * + * Note also that the storage must be pinned in the GGTT, so that the + * address won't change after GuC has been told where to find it. The + * GuC will also validate that the LRC base + size fall within the + * allowed GGTT range. + */ + if (blob) { + offset = guc_ads_golden_ctxt_offset(guc); + addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; + info = &blob->system_info; + } else { + memset(&local_info, 0, sizeof(local_info)); + info = &local_info; + fill_engine_enable_masks(gt, info); + } + + for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { + if (engine_class == OTHER_CLASS) + continue; + + guc_class = engine_class_to_guc_class(engine_class); + + if (!info->engine_enabled_masks[guc_class]) + continue; + + real_size = intel_engine_context_size(gt, engine_class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + + if (!blob) + continue; + + blob->ads.eng_state_size[guc_class] = real_size; + blob->ads.golden_context_lrca[guc_class] = addr_ggtt; + addr_ggtt += alloc_size; + } + + if (!blob) + return total_size; + + GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); + return total_size; +} + +static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, gt, id) { + if (engine->class != engine_class) + continue; + + if (!engine->default_state) + continue; + + return engine; + } + + return NULL; +} + +static void guc_init_golden_context(struct intel_guc *guc) +{ struct __guc_ads_blob *blob = guc->ads_blob; - const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE; - u32 base; + struct intel_engine_cs *engine; + struct intel_gt *gt = guc_to_gt(guc); + u32 addr_ggtt, offset; + u32 total_size = 0, alloc_size, real_size; u8 engine_class, guc_class; + u8 *ptr; - /* GuC scheduling policies */ - guc_policies_init(&blob->policies); + /* Skip execlist and PPGTT registers + HWSP */ + const u32 lr_hw_context_size = 80 * sizeof(u32); + const u32 skip_size = LRC_PPHWSP_SZ * PAGE_SIZE + + lr_hw_context_size; + + if (!intel_uc_uses_guc_submission(>->uc)) + return; + + GEM_BUG_ON(!blob); /* - * GuC expects a per-engine-class context image and size - * (minus hwsp and ring context). The context image will be - * used to reinitialize engines after a reset. It must exist - * and be pinned in the GGTT, so that the address won't change after - * we have told GuC where to find it. The context size will be used - * to validate that the LRC base + size fall within allowed GGTT. + * Go back and fill in the golden context data now that it is + * available. */ + offset = guc_ads_golden_ctxt_offset(guc); + addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; + ptr = ((u8 *)blob) + offset; + for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { if (engine_class == OTHER_CLASS) continue; guc_class = engine_class_to_guc_class(engine_class); - /* - * TODO: Set context pointer to default state to allow - * GuC to re-init guilty contexts after internal reset. - */ - blob->ads.golden_context_lrca[guc_class] = 0; - blob->ads.eng_state_size[guc_class] = - intel_engine_context_size(guc_to_gt(guc), - engine_class) - - skipped_size; + if (!blob->system_info.engine_enabled_masks[guc_class]) + continue; + + real_size = intel_engine_context_size(gt, engine_class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + + engine = find_engine_state(gt, engine_class); + if (!engine) { + drm_err(>->i915->drm, "No engine state recorded for class %d!\n", + engine_class); + blob->ads.eng_state_size[guc_class] = 0; + blob->ads.golden_context_lrca[guc_class] = 0; + continue; + } + + GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != real_size); + GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); + addr_ggtt += alloc_size; + + shmem_read(engine->default_state, skip_size, ptr + skip_size, + real_size - skip_size); + ptr += alloc_size; } + GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); +} + +static void __guc_ads_init(struct intel_guc *guc) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct drm_i915_private *i915 = gt->i915; + struct __guc_ads_blob *blob = guc->ads_blob; + u32 base; + + /* GuC scheduling policies */ + guc_policies_init(guc, &blob->policies); + /* System info */ - blob->system_info.engine_enabled_masks[GUC_RENDER_CLASS] = 1; - blob->system_info.engine_enabled_masks[GUC_BLITTER_CLASS] = 1; - blob->system_info.engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); - blob->system_info.engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); + fill_engine_enable_masks(gt, &blob->system_info); blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = hweight8(gt->info.sseu.slice_mask); @@ -174,21 +514,19 @@ static void __guc_ads_init(struct intel_guc *guc) GEN12_DOORBELLS_PER_SQIDI) + 1; } + /* Golden contexts for re-initialising after a watchdog reset */ + guc_prep_golden_context(guc, blob); + guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); base = intel_guc_ggtt_offset(guc, guc->ads_vma); - /* Clients info */ - guc_ct_pool_entries_init(blob->ct_pool, ARRAY_SIZE(blob->ct_pool)); - - blob->clients_info.clients_num = 1; - blob->clients_info.ct_pool_addr = base + ptr_offset(blob, ct_pool); - blob->clients_info.ct_pool_count = ARRAY_SIZE(blob->ct_pool); - /* ADS */ blob->ads.scheduler_policies = base + ptr_offset(blob, policies); blob->ads.gt_system_info = base + ptr_offset(blob, system_info); - blob->ads.clients_info = base + ptr_offset(blob, clients_info); + + /* MMIO save/restore list */ + guc_mmio_reg_state_init(guc, blob); /* Private Data */ blob->ads.private_data = base + guc_ads_private_data_offset(guc); @@ -210,6 +548,19 @@ int intel_guc_ads_create(struct intel_guc *guc) GEM_BUG_ON(guc->ads_vma); + /* Need to calculate the reg state size dynamically: */ + ret = guc_mmio_reg_state_query(guc); + if (ret < 0) + return ret; + guc->ads_regset_size = ret; + + /* Likewise the golden contexts: */ + ret = guc_prep_golden_context(guc, NULL); + if (ret < 0) + return ret; + guc->ads_golden_ctxt_size = ret; + + /* Now the total size can be determined: */ size = guc_ads_blob_size(guc); ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, @@ -222,6 +573,18 @@ int intel_guc_ads_create(struct intel_guc *guc) return 0; } +void intel_guc_ads_init_late(struct intel_guc *guc) +{ + /* + * The golden context setup requires the saved engine state from + * __engines_record_defaults(). However, that requires engines to be + * operational which means the ADS must already have been configured. + * Fortunately, the golden context state is not needed until a hang + * occurs, so it can be filled in during this late init phase. + */ + guc_init_golden_context(guc); +} + void intel_guc_ads_destroy(struct intel_guc *guc) { i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h index b00d3ae1113a..3d85051d57e4 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h @@ -7,9 +7,13 @@ #define _INTEL_GUC_ADS_H_ struct intel_guc; +struct drm_printer; int intel_guc_ads_create(struct intel_guc *guc); void intel_guc_ads_destroy(struct intel_guc *guc); +void intel_guc_ads_init_late(struct intel_guc *guc); void intel_guc_ads_reset(struct intel_guc *guc); +void intel_guc_ads_print_policy_info(struct intel_guc *guc, + struct drm_printer *p); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 8f7b148fef58..22b4733b55e2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -3,6 +3,11 @@ * Copyright © 2016-2019 Intel Corporation */ +#include <linux/circ_buf.h> +#include <linux/ktime.h> +#include <linux/time64.h> +#include <linux/timekeeping.h> + #include "i915_drv.h" #include "intel_guc_ct.h" #include "gt/intel_gt.h" @@ -58,11 +63,17 @@ static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct) * +--------+-----------------------------------------------+------+ * * Size of each `CT Buffer`_ must be multiple of 4K. - * As we don't expect too many messages, for now use minimum sizes. + * We don't expect too many messages in flight at any time, unless we are + * using the GuC submission. In that case each request requires a minimum + * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this + * enough space to avoid backpressure on the driver. We increase the size + * of the receive buffer (relative to the send) to ensure a G2H response + * CTB has a landing spot. */ #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) #define CTB_H2G_BUFFER_SIZE (SZ_4K) -#define CTB_G2H_BUFFER_SIZE (SZ_4K) +#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) +#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) struct ct_request { struct list_head link; @@ -98,66 +109,84 @@ void intel_guc_ct_init_early(struct intel_guc_ct *ct) INIT_LIST_HEAD(&ct->requests.incoming); INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); + init_waitqueue_head(&ct->wq); } static inline const char *guc_ct_buffer_type_to_str(u32 type) { switch (type) { - case INTEL_GUC_CT_BUFFER_TYPE_SEND: + case GUC_CTB_TYPE_HOST2GUC: return "SEND"; - case INTEL_GUC_CT_BUFFER_TYPE_RECV: + case GUC_CTB_TYPE_GUC2HOST: return "RECV"; default: return "<invalid>"; } } -static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc, - u32 cmds_addr, u32 size) +static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) { memset(desc, 0, sizeof(*desc)); - desc->addr = cmds_addr; - desc->size = size; - desc->owner = CTB_OWNER_HOST; } -static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb, u32 cmds_addr) +static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) { - guc_ct_buffer_desc_init(ctb->desc, cmds_addr, ctb->size); + u32 space; + + ctb->broken = false; + ctb->tail = 0; + ctb->head = 0; + space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; + atomic_set(&ctb->space, space); + + guc_ct_buffer_desc_init(ctb->desc); } static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, struct guc_ct_buffer_desc *desc, - u32 *cmds, u32 size) + u32 *cmds, u32 size_in_bytes, u32 resv_space) { - GEM_BUG_ON(size % 4); + GEM_BUG_ON(size_in_bytes % 4); ctb->desc = desc; ctb->cmds = cmds; - ctb->size = size; + ctb->size = size_in_bytes / 4; + ctb->resv_space = resv_space / 4; - guc_ct_buffer_reset(ctb, 0); + guc_ct_buffer_reset(ctb); } -static int guc_action_register_ct_buffer(struct intel_guc *guc, - u32 desc_addr, - u32 type) +static int guc_action_register_ct_buffer(struct intel_guc *guc, u32 type, + u32 desc_addr, u32 buff_addr, u32 size) { - u32 action[] = { - INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER, - desc_addr, - sizeof(struct guc_ct_buffer_desc), - type + u32 request[HOST2GUC_REGISTER_CTB_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_REGISTER_CTB), + FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_SIZE, size / SZ_4K - 1) | + FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_TYPE, type), + FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR, desc_addr), + FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR, buff_addr), }; - /* Can't use generic send(), CT registration must go over MMIO */ - return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0); + GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); + GEM_BUG_ON(size % SZ_4K); + + /* CT registration must go over MMIO */ + return intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); } -static int ct_register_buffer(struct intel_guc_ct *ct, u32 desc_addr, u32 type) +static int ct_register_buffer(struct intel_guc_ct *ct, u32 type, + u32 desc_addr, u32 buff_addr, u32 size) { - int err = guc_action_register_ct_buffer(ct_to_guc(ct), desc_addr, type); + int err; + + err = i915_inject_probe_error(guc_to_gt(ct_to_guc(ct))->i915, -ENXIO); + if (unlikely(err)) + return err; + err = guc_action_register_ct_buffer(ct_to_guc(ct), type, + desc_addr, buff_addr, size); if (unlikely(err)) CT_ERROR(ct, "Failed to register %s buffer (err=%d)\n", guc_ct_buffer_type_to_str(type), err); @@ -166,14 +195,17 @@ static int ct_register_buffer(struct intel_guc_ct *ct, u32 desc_addr, u32 type) static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type) { - u32 action[] = { - INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER, - CTB_OWNER_HOST, - type + u32 request[HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_DEREGISTER_CTB), + FIELD_PREP(HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE, type), }; - /* Can't use generic send(), CT deregistration must go over MMIO */ - return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0); + GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); + + /* CT deregistration must go over MMIO */ + return intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); } static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type) @@ -200,10 +232,15 @@ int intel_guc_ct_init(struct intel_guc_ct *ct) struct guc_ct_buffer_desc *desc; u32 blob_size; u32 cmds_size; + u32 resv_space; void *blob; u32 *cmds; int err; + err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO); + if (err) + return err; + GEM_BUG_ON(ct->vma); blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; @@ -220,19 +257,23 @@ int intel_guc_ct_init(struct intel_guc_ct *ct) desc = blob; cmds = blob + 2 * CTB_DESC_SIZE; cmds_size = CTB_H2G_BUFFER_SIZE; - CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u\n", "send", - ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size); + resv_space = 0; + CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", + ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, + resv_space); - guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size); + guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); /* store pointers to desc and cmds for recv ctb */ desc = blob + CTB_DESC_SIZE; cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; cmds_size = CTB_G2H_BUFFER_SIZE; - CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u\n", "recv", - ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size); + resv_space = G2H_ROOM_BUFFER_SIZE; + CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", + ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, + resv_space); - guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size); + guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); return 0; } @@ -261,7 +302,7 @@ void intel_guc_ct_fini(struct intel_guc_ct *ct) int intel_guc_ct_enable(struct intel_guc_ct *ct) { struct intel_guc *guc = ct_to_guc(ct); - u32 base, cmds; + u32 base, desc, cmds; void *blob; int err; @@ -277,32 +318,36 @@ int intel_guc_ct_enable(struct intel_guc_ct *ct) GEM_BUG_ON(blob != ct->ctbs.send.desc); /* (re)initialize descriptors */ - cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); - guc_ct_buffer_reset(&ct->ctbs.send, cmds); - - cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); - guc_ct_buffer_reset(&ct->ctbs.recv, cmds); + guc_ct_buffer_reset(&ct->ctbs.send); + guc_ct_buffer_reset(&ct->ctbs.recv); /* * Register both CT buffers starting with RECV buffer. * Descriptors are in first half of the blob. */ - err = ct_register_buffer(ct, base + ptrdiff(ct->ctbs.recv.desc, blob), - INTEL_GUC_CT_BUFFER_TYPE_RECV); + desc = base + ptrdiff(ct->ctbs.recv.desc, blob); + cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); + err = ct_register_buffer(ct, GUC_CTB_TYPE_GUC2HOST, + desc, cmds, ct->ctbs.recv.size * 4); + if (unlikely(err)) goto err_out; - err = ct_register_buffer(ct, base + ptrdiff(ct->ctbs.send.desc, blob), - INTEL_GUC_CT_BUFFER_TYPE_SEND); + desc = base + ptrdiff(ct->ctbs.send.desc, blob); + cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); + err = ct_register_buffer(ct, GUC_CTB_TYPE_HOST2GUC, + desc, cmds, ct->ctbs.send.size * 4); + if (unlikely(err)) goto err_deregister; ct->enabled = true; + ct->stall_time = KTIME_MAX; return 0; err_deregister: - ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV); + ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); err_out: CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); return err; @@ -321,8 +366,8 @@ void intel_guc_ct_disable(struct intel_guc_ct *ct) ct->enabled = false; if (intel_guc_is_fw_running(guc)) { - ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_SEND); - ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV); + ct_deregister_buffer(ct, GUC_CTB_TYPE_HOST2GUC); + ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); } } @@ -354,81 +399,63 @@ static void write_barrier(struct intel_guc_ct *ct) } } -/** - * DOC: CTB Host to GuC request - * - * Format of the CTB Host to GuC request message is as follows:: - * - * +------------+---------+---------+---------+---------+ - * | msg[0] | [1] | [2] | ... | [n-1] | - * +------------+---------+---------+---------+---------+ - * | MESSAGE | MESSAGE PAYLOAD | - * + HEADER +---------+---------+---------+---------+ - * | | 0 | 1 | ... | n | - * +============+=========+=========+=========+=========+ - * | len >= 1 | FENCE | request specific data | - * +------+-----+---------+---------+---------+---------+ - * - * ^-----------------len-------------------^ - */ - static int ct_write(struct intel_guc_ct *ct, const u32 *action, u32 len /* in dwords */, - u32 fence) + u32 fence, u32 flags) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; struct guc_ct_buffer_desc *desc = ctb->desc; - u32 head = desc->head; - u32 tail = desc->tail; + u32 tail = ctb->tail; u32 size = ctb->size; - u32 used; u32 header; + u32 hxg; + u32 type; u32 *cmds = ctb->cmds; unsigned int i; - if (unlikely(desc->is_in_error)) - return -EPIPE; - - if (unlikely(!IS_ALIGNED(head | tail, 4) || - (tail | head) >= size)) + if (unlikely(desc->status)) goto corrupted; - /* later calculations will be done in dwords */ - head /= 4; - tail /= 4; - size /= 4; - - /* - * tail == head condition indicates empty. GuC FW does not support - * using up the entire buffer to get tail == head meaning full. - */ - if (tail < head) - used = (size - head) + tail; - else - used = tail - head; + GEM_BUG_ON(tail > size); - /* make sure there is a space including extra dw for the fence */ - if (unlikely(used + len + 1 >= size)) - return -ENOSPC; +#ifdef CONFIG_DRM_I915_DEBUG_GUC + if (unlikely(tail != READ_ONCE(desc->tail))) { + CT_ERROR(ct, "Tail was modified %u != %u\n", + desc->tail, tail); + desc->status |= GUC_CTB_STATUS_MISMATCH; + goto corrupted; + } + if (unlikely(READ_ONCE(desc->head) >= size)) { + CT_ERROR(ct, "Invalid head offset %u >= %u)\n", + desc->head, size); + desc->status |= GUC_CTB_STATUS_OVERFLOW; + goto corrupted; + } +#endif /* - * Write the message. The format is the following: - * DW0: header (including action code) - * DW1: fence - * DW2+: action data + * dw0: CT header (including fence) + * dw1: HXG header (including action code) + * dw2+: action data */ - header = (len << GUC_CT_MSG_LEN_SHIFT) | - GUC_CT_MSG_SEND_STATUS | - (action[0] << GUC_CT_MSG_ACTION_SHIFT); + header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | + FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | + FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); - CT_DEBUG(ct, "writing %*ph %*ph %*ph\n", - 4, &header, 4, &fence, 4 * (len - 1), &action[1]); + type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT : + GUC_HXG_TYPE_REQUEST; + hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | + FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | + GUC_HXG_EVENT_MSG_0_DATA0, action[0]); + + CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", + tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); cmds[tail] = header; tail = (tail + 1) % size; - cmds[tail] = fence; + cmds[tail] = hxg; tail = (tail + 1) % size; for (i = 1; i < len; i++) { @@ -443,14 +470,20 @@ static int ct_write(struct intel_guc_ct *ct, */ write_barrier(ct); - /* now update desc tail (back in bytes) */ - desc->tail = tail * 4; + /* update local copies */ + ctb->tail = tail; + GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); + atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); + + /* now update descriptor */ + WRITE_ONCE(desc->tail, tail); + return 0; corrupted: - CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n", - desc->addr, desc->head, desc->tail, desc->size); - desc->is_in_error = 1; + CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", + desc->head, desc->tail, desc->status); + ctb->broken = true; return -EPIPE; } @@ -459,7 +492,7 @@ corrupted: * @req: pointer to pending request * @status: placeholder for status * - * For each sent request, Guc shall send bac CT response message. + * For each sent request, GuC shall send back CT response message. * Our message handler will update status of tracked request once * response message with given fence is received. Wait here and * check for valid response status value. @@ -475,12 +508,18 @@ static int wait_for_ct_request_update(struct ct_request *req, u32 *status) /* * Fast commands should complete in less than 10us, so sample quickly * up to that length of time, then switch to a slower sleep-wait loop. - * No GuC command should ever take longer than 10ms. + * No GuC command should ever take longer than 10ms but many GuC + * commands can be inflight at time, so use a 1s timeout on the slower + * sleep-wait loop. */ -#define done INTEL_GUC_MSG_IS_RESPONSE(READ_ONCE(req->status)) - err = wait_for_us(done, 10); +#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 +#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 +#define done \ + (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ + GUC_HXG_ORIGIN_GUC) + err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); if (err) - err = wait_for(done, 10); + err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); #undef done if (unlikely(err)) @@ -490,6 +529,131 @@ static int wait_for_ct_request_update(struct ct_request *req, u32 *status) return err; } +#define GUC_CTB_TIMEOUT_MS 1500 +static inline bool ct_deadlocked(struct intel_guc_ct *ct) +{ + long timeout = GUC_CTB_TIMEOUT_MS; + bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; + + if (unlikely(ret)) { + struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; + struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; + + CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", + ktime_ms_delta(ktime_get(), ct->stall_time), + send->status, recv->status); + ct->ctbs.send.broken = true; + } + + return ret; +} + +static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) +{ + struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; + + /* + * We leave a certain amount of space in the G2H CTB buffer for + * unexpected G2H CTBs (e.g. logging, engine hang, etc...) + */ + return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; +} + +static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) +{ + lockdep_assert_held(&ct->ctbs.send.lock); + + GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); + + if (g2h_len_dw) + atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); +} + +static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) +{ + atomic_add(g2h_len_dw, &ct->ctbs.recv.space); +} + +static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) +{ + struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; + struct guc_ct_buffer_desc *desc = ctb->desc; + u32 head; + u32 space; + + if (atomic_read(&ctb->space) >= len_dw) + return true; + + head = READ_ONCE(desc->head); + if (unlikely(head > ctb->size)) { + CT_ERROR(ct, "Invalid head offset %u >= %u)\n", + head, ctb->size); + desc->status |= GUC_CTB_STATUS_OVERFLOW; + ctb->broken = true; + return false; + } + + space = CIRC_SPACE(ctb->tail, head, ctb->size); + atomic_set(&ctb->space, space); + + return space >= len_dw; +} + +static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) +{ + lockdep_assert_held(&ct->ctbs.send.lock); + + if (unlikely(!h2g_has_room(ct, h2g_dw) || !g2h_has_room(ct, g2h_dw))) { + if (ct->stall_time == KTIME_MAX) + ct->stall_time = ktime_get(); + + if (unlikely(ct_deadlocked(ct))) + return -EPIPE; + else + return -EBUSY; + } + + ct->stall_time = KTIME_MAX; + return 0; +} + +#define G2H_LEN_DW(f) ({ \ + typeof(f) f_ = (f); \ + FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ + FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ + GUC_CTB_HXG_MSG_MIN_LEN : 0; \ +}) +static int ct_send_nb(struct intel_guc_ct *ct, + const u32 *action, + u32 len, + u32 flags) +{ + struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; + unsigned long spin_flags; + u32 g2h_len_dw = G2H_LEN_DW(flags); + u32 fence; + int ret; + + spin_lock_irqsave(&ctb->lock, spin_flags); + + ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); + if (unlikely(ret)) + goto out; + + fence = ct_get_next_fence(ct); + ret = ct_write(ct, action, len, fence, flags); + if (unlikely(ret)) + goto out; + + g2h_reserve_space(ct, g2h_len_dw); + intel_guc_notify(ct_to_guc(ct)); + +out: + spin_unlock_irqrestore(&ctb->lock, spin_flags); + + return ret; +} + static int ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, @@ -497,8 +661,10 @@ static int ct_send(struct intel_guc_ct *ct, u32 response_buf_size, u32 *status) { + struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; struct ct_request request; unsigned long flags; + unsigned int sleep_period_ms = 1; u32 fence; int err; @@ -506,8 +672,33 @@ static int ct_send(struct intel_guc_ct *ct, GEM_BUG_ON(!len); GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK); GEM_BUG_ON(!response_buf && response_buf_size); + might_sleep(); + + /* + * We use a lazy spin wait loop here as we believe that if the CT + * buffers are sized correctly the flow control condition should be + * rare. Reserving the maximum size in the G2H credits as we don't know + * how big the response is going to be. + */ +retry: + spin_lock_irqsave(&ctb->lock, flags); + if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || + !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { + if (ct->stall_time == KTIME_MAX) + ct->stall_time = ktime_get(); + spin_unlock_irqrestore(&ctb->lock, flags); + + if (unlikely(ct_deadlocked(ct))) + return -EPIPE; + + if (msleep_interruptible(sleep_period_ms)) + return -EINTR; + sleep_period_ms = sleep_period_ms << 1; + + goto retry; + } - spin_lock_irqsave(&ct->ctbs.send.lock, flags); + ct->stall_time = KTIME_MAX; fence = ct_get_next_fence(ct); request.fence = fence; @@ -519,9 +710,10 @@ static int ct_send(struct intel_guc_ct *ct, list_add_tail(&request.link, &ct->requests.pending); spin_unlock(&ct->requests.lock); - err = ct_write(ct, action, len, fence); + err = ct_write(ct, action, len, fence, 0); + g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); - spin_unlock_irqrestore(&ct->ctbs.send.lock, flags); + spin_unlock_irqrestore(&ctb->lock, flags); if (unlikely(err)) goto unlink; @@ -529,24 +721,25 @@ static int ct_send(struct intel_guc_ct *ct, intel_guc_notify(ct_to_guc(ct)); err = wait_for_ct_request_update(&request, status); + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); if (unlikely(err)) goto unlink; - if (!INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(*status)) { + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { err = -EIO; goto unlink; } if (response_buf) { /* There shall be no data in the status */ - WARN_ON(INTEL_GUC_MSG_TO_DATA(request.status)); + WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); /* Return actual response len */ err = request.response_len; } else { /* There shall be no response payload */ WARN_ON(request.response_len); /* Return data decoded from the status dword */ - err = INTEL_GUC_MSG_TO_DATA(*status); + err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); } unlink: @@ -561,16 +754,25 @@ unlink: * Command Transport (CT) buffer based GuC send function. */ int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, - u32 *response_buf, u32 response_buf_size) + u32 *response_buf, u32 response_buf_size, u32 flags) { u32 status = ~0; /* undefined */ int ret; if (unlikely(!ct->enabled)) { - WARN(1, "Unexpected send: action=%#x\n", *action); + struct intel_guc *guc = ct_to_guc(ct); + struct intel_uc *uc = container_of(guc, struct intel_uc, guc); + + WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); return -ENODEV; } + if (unlikely(ct->ctbs.send.broken)) + return -EPIPE; + + if (flags & INTEL_GUC_CT_SEND_NB) + return ct_send_nb(ct, action, len, flags); + ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); if (unlikely(ret < 0)) { CT_ERROR(ct, "Sending action %#x failed (err=%d status=%#X)\n", @@ -583,21 +785,6 @@ int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, return ret; } -static inline unsigned int ct_header_get_len(u32 header) -{ - return (header >> GUC_CT_MSG_LEN_SHIFT) & GUC_CT_MSG_LEN_MASK; -} - -static inline unsigned int ct_header_get_action(u32 header) -{ - return (header >> GUC_CT_MSG_ACTION_SHIFT) & GUC_CT_MSG_ACTION_MASK; -} - -static inline bool ct_header_is_response(u32 header) -{ - return !!(header & GUC_CT_MSG_IS_RESPONSE); -} - static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) { struct ct_incoming_msg *msg; @@ -621,8 +808,8 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; struct guc_ct_buffer_desc *desc = ctb->desc; - u32 head = desc->head; - u32 tail = desc->tail; + u32 head = ctb->head; + u32 tail = READ_ONCE(desc->tail); u32 size = ctb->size; u32 *cmds = ctb->cmds; s32 available; @@ -630,17 +817,28 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) unsigned int i; u32 header; - if (unlikely(desc->is_in_error)) + if (unlikely(ctb->broken)) return -EPIPE; - if (unlikely(!IS_ALIGNED(head | tail, 4) || - (tail | head) >= size)) + if (unlikely(desc->status)) goto corrupted; - /* later calculations will be done in dwords */ - head /= 4; - tail /= 4; - size /= 4; + GEM_BUG_ON(head > size); + +#ifdef CONFIG_DRM_I915_DEBUG_GUC + if (unlikely(head != READ_ONCE(desc->head))) { + CT_ERROR(ct, "Head was modified %u != %u\n", + desc->head, head); + desc->status |= GUC_CTB_STATUS_MISMATCH; + goto corrupted; + } +#endif + if (unlikely(tail >= size)) { + CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", + tail, size); + desc->status |= GUC_CTB_STATUS_OVERFLOW; + goto corrupted; + } /* tail == head condition indicates empty */ available = tail - head; @@ -652,14 +850,14 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) /* beware of buffer wrap case */ if (unlikely(available < 0)) available += size; - CT_DEBUG(ct, "available %d (%u:%u)\n", available, head, tail); + CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); GEM_BUG_ON(available < 0); header = cmds[head]; head = (head + 1) % size; /* message len with header */ - len = ct_header_get_len(header) + 1; + len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; if (unlikely(len > (u32)available)) { CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 4, &header, @@ -667,6 +865,7 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) size - head : available - 1), &cmds[head], 4 * (head + available - 1 > size ? available - 1 - size + head : 0), &cmds[0]); + desc->status |= GUC_CTB_STATUS_UNDERFLOW; goto corrupted; } @@ -689,65 +888,39 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) } CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); - desc->head = head * 4; + /* update local copies */ + ctb->head = head; + + /* now update descriptor */ + WRITE_ONCE(desc->head, head); + return available - len; corrupted: - CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n", - desc->addr, desc->head, desc->tail, desc->size); - desc->is_in_error = 1; + CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", + desc->head, desc->tail, desc->status); + ctb->broken = true; return -EPIPE; } -/** - * DOC: CTB GuC to Host response - * - * Format of the CTB GuC to Host response message is as follows:: - * - * +------------+---------+---------+---------+---------+---------+ - * | msg[0] | [1] | [2] | [3] | ... | [n-1] | - * +------------+---------+---------+---------+---------+---------+ - * | MESSAGE | MESSAGE PAYLOAD | - * + HEADER +---------+---------+---------+---------+---------+ - * | | 0 | 1 | 2 | ... | n | - * +============+=========+=========+=========+=========+=========+ - * | len >= 2 | FENCE | STATUS | response specific data | - * +------+-----+---------+---------+---------+---------+---------+ - * - * ^-----------------------len-----------------------^ - */ - static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) { - u32 header = response->msg[0]; - u32 len = ct_header_get_len(header); - u32 fence; - u32 status; - u32 datalen; + u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); + u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); + const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; + const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; + u32 datalen = len - GUC_HXG_MSG_MIN_LEN; struct ct_request *req; unsigned long flags; bool found = false; int err = 0; - GEM_BUG_ON(!ct_header_is_response(header)); - - /* Response payload shall at least include fence and status */ - if (unlikely(len < 2)) { - CT_ERROR(ct, "Corrupted response (len %u)\n", len); - return -EPROTO; - } - - fence = response->msg[1]; - status = response->msg[2]; - datalen = len - 2; - - /* Format of the status follows RESPONSE message */ - if (unlikely(!INTEL_GUC_MSG_IS_RESPONSE(status))) { - CT_ERROR(ct, "Corrupted response (status %#x)\n", status); - return -EPROTO; - } + GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); + GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); + GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && + FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); - CT_DEBUG(ct, "response fence %u status %#x\n", fence, status); + CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); spin_lock_irqsave(&ct->requests.lock, flags); list_for_each_entry(req, &ct->requests.pending, link) { @@ -763,18 +936,22 @@ static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *r err = -EMSGSIZE; } if (datalen) - memcpy(req->response_buf, response->msg + 3, 4 * datalen); + memcpy(req->response_buf, data, 4 * datalen); req->response_len = datalen; - WRITE_ONCE(req->status, status); + WRITE_ONCE(req->status, hxg[0]); found = true; break; } - spin_unlock_irqrestore(&ct->requests.lock, flags); - if (!found) { CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence); - return -ENOKEY; + CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence, + ct->requests.last_fence); + list_for_each_entry(req, &ct->requests.pending, link) + CT_ERROR(ct, "request %u awaits response\n", + req->fence); + err = -ENOKEY; } + spin_unlock_irqrestore(&ct->requests.lock, flags); if (unlikely(err)) return err; @@ -786,14 +963,16 @@ static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *r static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) { struct intel_guc *guc = ct_to_guc(ct); - u32 header, action, len; + const u32 *hxg; const u32 *payload; + u32 hxg_len, action, len; int ret; - header = request->msg[0]; - payload = &request->msg[1]; - action = ct_header_get_action(header); - len = ct_header_get_len(header); + hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; + hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; + payload = &hxg[GUC_HXG_MSG_MIN_LEN]; + action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + len = hxg_len - GUC_HXG_MSG_MIN_LEN; CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); @@ -801,6 +980,19 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r case INTEL_GUC_ACTION_DEFAULT: ret = intel_guc_to_host_process_recv_msg(guc, payload, len); break; + case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + ret = intel_guc_deregister_done_process_msg(guc, payload, + len); + break; + case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + ret = intel_guc_sched_done_process_msg(guc, payload, len); + break; + case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: + ret = intel_guc_context_reset_process_msg(guc, payload, len); + break; + case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: + ret = intel_guc_engine_failure_process_msg(guc, payload, len); + break; default: ret = -EOPNOTSUPP; break; @@ -855,29 +1047,24 @@ static void ct_incoming_request_worker_func(struct work_struct *w) queue_work(system_unbound_wq, &ct->requests.worker); } -/** - * DOC: CTB GuC to Host request - * - * Format of the CTB GuC to Host request message is as follows:: - * - * +------------+---------+---------+---------+---------+---------+ - * | msg[0] | [1] | [2] | [3] | ... | [n-1] | - * +------------+---------+---------+---------+---------+---------+ - * | MESSAGE | MESSAGE PAYLOAD | - * + HEADER +---------+---------+---------+---------+---------+ - * | | 0 | 1 | 2 | ... | n | - * +============+=========+=========+=========+=========+=========+ - * | len | request specific data | - * +------+-----+---------+---------+---------+---------+---------+ - * - * ^-----------------------len-----------------------^ - */ - -static int ct_handle_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) +static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) { + const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); unsigned long flags; - GEM_BUG_ON(ct_header_is_response(request->msg[0])); + GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); + + /* + * Adjusting the space must be done in IRQ or deadlock can occur as the + * CTB processing in the below workqueue can send CTBs which creates a + * circular dependency if the space was returned there. + */ + switch (action) { + case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + g2h_release_space(ct, request->size); + } spin_lock_irqsave(&ct->requests.lock, flags); list_add_tail(&request->link, &ct->requests.incoming); @@ -887,15 +1074,53 @@ static int ct_handle_request(struct intel_guc_ct *ct, struct ct_incoming_msg *re return 0; } -static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) +static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) { - u32 header = msg->msg[0]; + u32 origin, type; + u32 *hxg; int err; - if (ct_header_is_response(header)) + if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) + return -EBADMSG; + + hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; + + origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); + if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { + err = -EPROTO; + goto failed; + } + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); + switch (type) { + case GUC_HXG_TYPE_EVENT: + err = ct_handle_event(ct, msg); + break; + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + case GUC_HXG_TYPE_RESPONSE_FAILURE: err = ct_handle_response(ct, msg); + break; + default: + err = -EOPNOTSUPP; + } + + if (unlikely(err)) { +failed: + CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", + ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); + } + return err; +} + +static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) +{ + u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); + int err; + + if (format == GUC_CTB_FORMAT_HXG) + err = ct_handle_hxg(ct, msg); else - err = ct_handle_request(ct, msg); + err = -EOPNOTSUPP; if (unlikely(err)) { CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", @@ -958,3 +1183,25 @@ void intel_guc_ct_event_handler(struct intel_guc_ct *ct) ct_try_receive_message(ct); } + +void intel_guc_ct_print_info(struct intel_guc_ct *ct, + struct drm_printer *p) +{ + drm_printf(p, "CT %s\n", enableddisabled(ct->enabled)); + + if (!ct->enabled) + return; + + drm_printf(p, "H2G Space: %u\n", + atomic_read(&ct->ctbs.send.space) * 4); + drm_printf(p, "Head: %u\n", + ct->ctbs.send.desc->head); + drm_printf(p, "Tail: %u\n", + ct->ctbs.send.desc->tail); + drm_printf(p, "G2H Space: %u\n", + atomic_read(&ct->ctbs.recv.space) * 4); + drm_printf(p, "Head: %u\n", + ct->ctbs.recv.desc->head); + drm_printf(p, "Tail: %u\n", + ct->ctbs.recv.desc->tail); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h index cb222f202301..f709a19c7e21 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h @@ -9,11 +9,14 @@ #include <linux/interrupt.h> #include <linux/spinlock.h> #include <linux/workqueue.h> +#include <linux/ktime.h> +#include <linux/wait.h> #include "intel_guc_fwif.h" struct i915_vma; struct intel_guc; +struct drm_printer; /** * DOC: Command Transport (CT). @@ -31,16 +34,25 @@ struct intel_guc; * @lock: protects access to the commands buffer and buffer descriptor * @desc: pointer to the buffer descriptor * @cmds: pointer to the commands buffer - * @size: size of the commands buffer + * @size: size of the commands buffer in dwords + * @resv_space: reserved space in buffer in dwords + * @head: local shadow copy of head in dwords + * @tail: local shadow copy of tail in dwords + * @space: local shadow copy of space in dwords + * @broken: flag to indicate if descriptor data is broken */ struct intel_guc_ct_buffer { spinlock_t lock; struct guc_ct_buffer_desc *desc; u32 *cmds; u32 size; + u32 resv_space; + u32 tail; + u32 head; + atomic_t space; + bool broken; }; - /** Top-level structure for Command Transport related data * * Includes a pair of CT buffers for bi-directional communication and tracking @@ -58,8 +70,11 @@ struct intel_guc_ct { struct tasklet_struct receive_tasklet; + /** @wq: wait queue for g2h chanenl */ + wait_queue_head_t wq; + struct { - u32 last_fence; /* last fence used to send request */ + u16 last_fence; /* last fence used to send request */ spinlock_t lock; /* protects pending requests list */ struct list_head pending; /* requests waiting for response */ @@ -67,6 +82,9 @@ struct intel_guc_ct { struct list_head incoming; /* incoming requests */ struct work_struct worker; /* handler for incoming requests */ } requests; + + /** @stall_time: time of first time a CTB submission is stalled */ + ktime_t stall_time; }; void intel_guc_ct_init_early(struct intel_guc_ct *ct); @@ -85,8 +103,18 @@ static inline bool intel_guc_ct_enabled(struct intel_guc_ct *ct) return ct->enabled; } +#define INTEL_GUC_CT_SEND_NB BIT(31) +#define INTEL_GUC_CT_SEND_G2H_DW_SHIFT 0 +#define INTEL_GUC_CT_SEND_G2H_DW_MASK (0xff << INTEL_GUC_CT_SEND_G2H_DW_SHIFT) +#define MAKE_SEND_FLAGS(len) ({ \ + typeof(len) len_ = (len); \ + GEM_BUG_ON(!FIELD_FIT(INTEL_GUC_CT_SEND_G2H_DW_MASK, len_)); \ + (FIELD_PREP(INTEL_GUC_CT_SEND_G2H_DW_MASK, len_) | INTEL_GUC_CT_SEND_NB); \ +}) int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, - u32 *response_buf, u32 response_buf_size); + u32 *response_buf, u32 response_buf_size, u32 flags); void intel_guc_ct_event_handler(struct intel_guc_ct *ct); +void intel_guc_ct_print_info(struct intel_guc_ct *ct, struct drm_printer *p); + #endif /* _INTEL_GUC_CT_H_ */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c index fe7cb7b29a1e..887c8c8f35db 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c @@ -9,6 +9,10 @@ #include "intel_guc.h" #include "intel_guc_debugfs.h" #include "intel_guc_log_debugfs.h" +#include "gt/uc/intel_guc_ct.h" +#include "gt/uc/intel_guc_ads.h" +#include "gt/uc/intel_guc_submission.h" +#include "gt/uc/intel_guc_slpc.h" static int guc_info_show(struct seq_file *m, void *data) { @@ -22,16 +26,57 @@ static int guc_info_show(struct seq_file *m, void *data) drm_puts(&p, "\n"); intel_guc_log_info(&guc->log, &p); - /* Add more as required ... */ + if (!intel_guc_submission_is_used(guc)) + return 0; + + intel_guc_ct_print_info(&guc->ct, &p); + intel_guc_submission_print_info(guc, &p); + intel_guc_ads_print_policy_info(guc, &p); return 0; } DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_info); +static int guc_registered_contexts_show(struct seq_file *m, void *data) +{ + struct intel_guc *guc = m->private; + struct drm_printer p = drm_seq_file_printer(m); + + if (!intel_guc_submission_is_used(guc)) + return -ENODEV; + + intel_guc_submission_print_context_info(guc, &p); + + return 0; +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_registered_contexts); + +static int guc_slpc_info_show(struct seq_file *m, void *unused) +{ + struct intel_guc *guc = m->private; + struct intel_guc_slpc *slpc = &guc->slpc; + struct drm_printer p = drm_seq_file_printer(m); + + if (!intel_guc_slpc_is_used(guc)) + return -ENODEV; + + return intel_guc_slpc_print_info(slpc, &p); +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_slpc_info); + +static bool intel_eval_slpc_support(void *data) +{ + struct intel_guc *guc = (struct intel_guc *)data; + + return intel_guc_slpc_is_used(guc); +} + void intel_guc_debugfs_register(struct intel_guc *guc, struct dentry *root) { static const struct debugfs_gt_file files[] = { { "guc_info", &guc_info_fops, NULL }, + { "guc_registered_contexts", &guc_registered_contexts_fops, NULL }, + { "guc_slpc_info", &guc_slpc_info_fops, &intel_eval_slpc_support}, }; if (!intel_guc_is_supported(guc)) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index e9a9d85e2aa3..fa4be13c8854 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -12,19 +12,27 @@ #include "gt/intel_engine_types.h" #include "abi/guc_actions_abi.h" +#include "abi/guc_actions_slpc_abi.h" #include "abi/guc_errors_abi.h" #include "abi/guc_communication_mmio_abi.h" #include "abi/guc_communication_ctb_abi.h" #include "abi/guc_messages_abi.h" +/* Payload length only i.e. don't include G2H header length */ +#define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 2 +#define G2H_LEN_DW_DEREGISTER_CONTEXT 1 + +#define GUC_CONTEXT_DISABLE 0 +#define GUC_CONTEXT_ENABLE 1 + #define GUC_CLIENT_PRIORITY_KMD_HIGH 0 #define GUC_CLIENT_PRIORITY_HIGH 1 #define GUC_CLIENT_PRIORITY_KMD_NORMAL 2 #define GUC_CLIENT_PRIORITY_NORMAL 3 #define GUC_CLIENT_PRIORITY_NUM 4 -#define GUC_MAX_STAGE_DESCRIPTORS 1024 -#define GUC_INVALID_STAGE_ID GUC_MAX_STAGE_DESCRIPTORS +#define GUC_MAX_LRC_DESCRIPTORS 65535 +#define GUC_INVALID_LRC_ID GUC_MAX_LRC_DESCRIPTORS #define GUC_RENDER_ENGINE 0 #define GUC_VIDEO_ENGINE 1 @@ -81,15 +89,14 @@ #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) #define GUC_LOG_CRASH_SHIFT 4 #define GUC_LOG_CRASH_MASK (0x3 << GUC_LOG_CRASH_SHIFT) -#define GUC_LOG_DPC_SHIFT 6 -#define GUC_LOG_DPC_MASK (0x7 << GUC_LOG_DPC_SHIFT) -#define GUC_LOG_ISR_SHIFT 9 -#define GUC_LOG_ISR_MASK (0x7 << GUC_LOG_ISR_SHIFT) +#define GUC_LOG_DEBUG_SHIFT 6 +#define GUC_LOG_DEBUG_MASK (0xF << GUC_LOG_DEBUG_SHIFT) #define GUC_LOG_BUF_ADDR_SHIFT 12 #define GUC_CTL_WA 1 #define GUC_CTL_FEATURE 2 #define GUC_CTL_DISABLE_SCHEDULER (1 << 14) +#define GUC_CTL_ENABLE_SLPC BIT(2) #define GUC_CTL_DEBUG 3 #define GUC_LOG_VERBOSITY_SHIFT 0 @@ -136,6 +143,11 @@ #define GUC_ID_TO_ENGINE_INSTANCE(guc_id) \ (((guc_id) & GUC_ENGINE_INSTANCE_MASK) >> GUC_ENGINE_INSTANCE_SHIFT) +#define SLPC_EVENT(id, c) (\ +FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \ +FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, c) \ +) + static inline u8 engine_class_to_guc_class(u8 class) { BUILD_BUG_ON(GUC_RENDER_CLASS != RENDER_CLASS); @@ -177,66 +189,40 @@ struct guc_process_desc { u32 reserved[30]; } __packed; -/* engine id and context id is packed into guc_execlist_context.context_id*/ -#define GUC_ELC_CTXID_OFFSET 0 -#define GUC_ELC_ENGINE_OFFSET 29 +#define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) -/* The execlist context including software and HW information */ -struct guc_execlist_context { - u32 context_desc; - u32 context_id; - u32 ring_status; - u32 ring_lrca; - u32 ring_begin; - u32 ring_end; - u32 ring_next_free_location; - u32 ring_current_tail_pointer_value; - u8 engine_state_submit_value; - u8 engine_state_wait_value; - u16 pagefault_count; - u16 engine_submit_queue_count; -} __packed; +#define CONTEXT_POLICY_DEFAULT_EXECUTION_QUANTUM_US 1000000 +#define CONTEXT_POLICY_DEFAULT_PREEMPTION_TIME_US 500000 + +/* Preempt to idle on quantum expiry */ +#define CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE BIT(0) /* - * This structure describes a stage set arranged for a particular communication - * between uKernel (GuC) and Driver (KMD). Technically, this is known as a - * "GuC Context descriptor" in the specs, but we use the term "stage descriptor" - * to avoid confusion with all the other things already named "context" in the - * driver. A static pool of these descriptors are stored inside a GEM object - * (stage_desc_pool) which is held for the entire lifetime of our interaction - * with the GuC, being allocated before the GuC is loaded with its firmware. + * GuC Context registration descriptor. + * FIXME: This is only required to exist during context registration. + * The current 1:1 between guc_lrc_desc and LRCs for the lifetime of the LRC + * is not required. */ -struct guc_stage_desc { - u32 sched_common_area; - u32 stage_id; - u32 pas_id; - u8 engines_used; - u64 db_trigger_cpu; - u32 db_trigger_uk; - u64 db_trigger_phy; - u16 db_id; - - struct guc_execlist_context lrc[GUC_MAX_ENGINES_NUM]; - - u8 attribute; - +struct guc_lrc_desc { + u32 hw_context_desc; + u32 slpm_perf_mode_hint; /* SPLC v1 only */ + u32 slpm_freq_hint; + u32 engine_submit_mask; /* In logical space */ + u8 engine_class; + u8 reserved0[3]; u32 priority; - - u32 wq_sampled_tail_offset; - u32 wq_total_submit_enqueues; - u32 process_desc; u32 wq_addr; u32 wq_size; - - u32 engine_presence; - - u8 engine_suspended; - - u8 reserved0[3]; - u64 reserved1[1]; - - u64 desc_private; + u32 context_flags; /* CONTEXT_REGISTRATION_* */ + /* Time for one workload to execute. (in micro seconds) */ + u32 execution_quantum; + /* Time to wait for a preemption request to complete before issuing a + * reset. (in micro seconds). + */ + u32 preemption_timeout; + u32 policy_flags; /* CONTEXT_POLICY_* */ + u32 reserved1[19]; } __packed; #define GUC_POWER_UNSPECIFIED 0 @@ -247,32 +233,14 @@ struct guc_stage_desc { /* Scheduling policy settings */ -/* Reset engine upon preempt failure */ -#define POLICY_RESET_ENGINE (1<<0) -/* Preempt to idle on quantum expiry */ -#define POLICY_PREEMPT_TO_IDLE (1<<1) +#define GLOBAL_POLICY_MAX_NUM_WI 15 -#define POLICY_MAX_NUM_WI 15 -#define POLICY_DEFAULT_DPC_PROMOTE_TIME_US 500000 -#define POLICY_DEFAULT_EXECUTION_QUANTUM_US 1000000 -#define POLICY_DEFAULT_PREEMPTION_TIME_US 500000 -#define POLICY_DEFAULT_FAULT_TIME_US 250000 +/* Don't reset an engine upon preemption failure */ +#define GLOBAL_POLICY_DISABLE_ENGINE_RESET BIT(0) -struct guc_policy { - /* Time for one workload to execute. (in micro seconds) */ - u32 execution_quantum; - /* Time to wait for a preemption request to completed before issuing a - * reset. (in micro seconds). */ - u32 preemption_time; - /* How much time to allow to run after the first fault is observed. - * Then preempt afterwards. (in micro seconds) */ - u32 fault_time; - u32 policy_flags; - u32 reserved[8]; -} __packed; +#define GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US 500000 struct guc_policies { - struct guc_policy policy[GUC_CLIENT_PRIORITY_NUM][GUC_MAX_ENGINE_CLASSES]; u32 submission_queue_depth[GUC_MAX_ENGINE_CLASSES]; /* In micro seconds. How much time to allow before DPC processing is * called back via interrupt (to prevent DPC queue drain starving). @@ -286,6 +254,7 @@ struct guc_policies { * idle. */ u32 max_num_work_items; + u32 global_flags; u32 reserved[4]; } __packed; @@ -311,29 +280,13 @@ struct guc_gt_system_info { u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; } __packed; -/* Clients info */ -struct guc_ct_pool_entry { - struct guc_ct_buffer_desc desc; - u32 reserved[7]; -} __packed; - -#define GUC_CT_POOL_SIZE 2 - -struct guc_clients_info { - u32 clients_num; - u32 reserved0[13]; - u32 ct_pool_addr; - u32 ct_pool_count; - u32 reserved[4]; -} __packed; - /* GuC Additional Data Struct */ struct guc_ads { struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; u32 reserved0; u32 scheduler_policies; u32 gt_system_info; - u32 clients_info; + u32 reserved1; u32 control_data; u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; @@ -344,8 +297,7 @@ struct guc_ads { /* GuC logging structures */ enum guc_log_buffer_type { - GUC_ISR_LOG_BUFFER, - GUC_DPC_LOG_BUFFER, + GUC_DEBUG_LOG_BUFFER, GUC_CRASH_DUMP_LOG_BUFFER, GUC_MAX_LOG_BUFFER }; @@ -414,23 +366,6 @@ struct guc_shared_ctx_data { struct guc_ctx_report preempt_ctx_report[GUC_MAX_ENGINES_NUM]; } __packed; -#define __INTEL_GUC_MSG_GET(T, m) \ - (((m) & INTEL_GUC_MSG_ ## T ## _MASK) >> INTEL_GUC_MSG_ ## T ## _SHIFT) -#define INTEL_GUC_MSG_TO_TYPE(m) __INTEL_GUC_MSG_GET(TYPE, m) -#define INTEL_GUC_MSG_TO_DATA(m) __INTEL_GUC_MSG_GET(DATA, m) -#define INTEL_GUC_MSG_TO_CODE(m) __INTEL_GUC_MSG_GET(CODE, m) - -#define __INTEL_GUC_MSG_TYPE_IS(T, m) \ - (INTEL_GUC_MSG_TO_TYPE(m) == INTEL_GUC_MSG_TYPE_ ## T) -#define INTEL_GUC_MSG_IS_REQUEST(m) __INTEL_GUC_MSG_TYPE_IS(REQUEST, m) -#define INTEL_GUC_MSG_IS_RESPONSE(m) __INTEL_GUC_MSG_TYPE_IS(RESPONSE, m) - -#define INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(m) \ - (typecheck(u32, (m)) && \ - ((m) & (INTEL_GUC_MSG_TYPE_MASK | INTEL_GUC_MSG_CODE_MASK)) == \ - ((INTEL_GUC_MSG_TYPE_RESPONSE << INTEL_GUC_MSG_TYPE_SHIFT) | \ - (INTEL_GUC_RESPONSE_STATUS_SUCCESS << INTEL_GUC_MSG_CODE_SHIFT))) - /* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ enum intel_guc_recv_message { INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1), diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c index c36d5eb5bbb9..ac0931f0374b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c @@ -197,10 +197,8 @@ static bool guc_check_log_buf_overflow(struct intel_guc_log *log, static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type) { switch (type) { - case GUC_ISR_LOG_BUFFER: - return ISR_BUFFER_SIZE; - case GUC_DPC_LOG_BUFFER: - return DPC_BUFFER_SIZE; + case GUC_DEBUG_LOG_BUFFER: + return DEBUG_BUFFER_SIZE; case GUC_CRASH_DUMP_LOG_BUFFER: return CRASH_BUFFER_SIZE; default: @@ -245,7 +243,7 @@ static void guc_read_update_log_buffer(struct intel_guc_log *log) src_data += PAGE_SIZE; dst_data += PAGE_SIZE; - for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { + for (type = GUC_DEBUG_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { /* * Make a copy of the state structure, inside GuC log buffer * (which is uncached mapped), on the stack to avoid reading @@ -463,21 +461,16 @@ int intel_guc_log_create(struct intel_guc_log *log) * +===============================+ 00B * | Crash dump state header | * +-------------------------------+ 32B - * | DPC state header | + * | Debug state header | * +-------------------------------+ 64B - * | ISR state header | - * +-------------------------------+ 96B * | | * +===============================+ PAGE_SIZE (4KB) * | Crash Dump logs | * +===============================+ + CRASH_SIZE - * | DPC logs | - * +===============================+ + DPC_SIZE - * | ISR logs | - * +===============================+ + ISR_SIZE + * | Debug logs | + * +===============================+ + DEBUG_SIZE */ - guc_log_size = PAGE_SIZE + CRASH_BUFFER_SIZE + DPC_BUFFER_SIZE + - ISR_BUFFER_SIZE; + guc_log_size = PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE; vma = intel_guc_allocate_vma(guc, guc_log_size); if (IS_ERR(vma)) { @@ -675,10 +668,8 @@ static const char * stringify_guc_log_type(enum guc_log_buffer_type type) { switch (type) { - case GUC_ISR_LOG_BUFFER: - return "ISR"; - case GUC_DPC_LOG_BUFFER: - return "DPC"; + case GUC_DEBUG_LOG_BUFFER: + return "DEBUG"; case GUC_CRASH_DUMP_LOG_BUFFER: return "CRASH"; default: @@ -708,7 +699,7 @@ void intel_guc_log_info(struct intel_guc_log *log, struct drm_printer *p) drm_printf(p, "\tRelay full count: %u\n", log->relay.full_count); - for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { + for (type = GUC_DEBUG_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { drm_printf(p, "\t%s:\tflush count %10u, overflow count %10u\n", stringify_guc_log_type(type), log->stats[type].flush, diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h index 11fccd0b2294..ac1ee1d5ce10 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h @@ -17,12 +17,10 @@ struct intel_guc; #ifdef CONFIG_DRM_I915_DEBUG_GUC #define CRASH_BUFFER_SIZE SZ_2M -#define DPC_BUFFER_SIZE SZ_8M -#define ISR_BUFFER_SIZE SZ_8M +#define DEBUG_BUFFER_SIZE SZ_16M #else #define CRASH_BUFFER_SIZE SZ_8K -#define DPC_BUFFER_SIZE SZ_32K -#define ISR_BUFFER_SIZE SZ_32K +#define DEBUG_BUFFER_SIZE SZ_64K #endif /* diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c new file mode 100644 index 000000000000..fc805d466d99 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "intel_guc_rc.h" +#include "gt/intel_gt.h" +#include "i915_drv.h" + +static bool __guc_rc_supported(struct intel_guc *guc) +{ + /* GuC RC is unavailable for pre-Gen12 */ + return guc->submission_supported && + GRAPHICS_VER(guc_to_gt(guc)->i915) >= 12; +} + +static bool __guc_rc_selected(struct intel_guc *guc) +{ + if (!intel_guc_rc_is_supported(guc)) + return false; + + return guc->submission_selected; +} + +void intel_guc_rc_init_early(struct intel_guc *guc) +{ + guc->rc_supported = __guc_rc_supported(guc); + guc->rc_selected = __guc_rc_selected(guc); +} + +static int guc_action_control_gucrc(struct intel_guc *guc, bool enable) +{ + u32 rc_mode = enable ? INTEL_GUCRC_FIRMWARE_CONTROL : + INTEL_GUCRC_HOST_CONTROL; + u32 action[] = { + INTEL_GUC_ACTION_SETUP_PC_GUCRC, + rc_mode + }; + int ret; + + ret = intel_guc_send(guc, action, ARRAY_SIZE(action)); + ret = ret > 0 ? -EPROTO : ret; + + return ret; +} + +static int __guc_rc_control(struct intel_guc *guc, bool enable) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct drm_device *drm = &guc_to_gt(guc)->i915->drm; + int ret; + + if (!intel_uc_uses_guc_rc(>->uc)) + return -EOPNOTSUPP; + + if (!intel_guc_is_ready(guc)) + return -EINVAL; + + ret = guc_action_control_gucrc(guc, enable); + if (ret) { + drm_err(drm, "Failed to %s GuC RC (%pe)\n", + enabledisable(enable), ERR_PTR(ret)); + return ret; + } + + drm_info(>->i915->drm, "GuC RC: %s\n", + enableddisabled(enable)); + + return 0; +} + +int intel_guc_rc_enable(struct intel_guc *guc) +{ + return __guc_rc_control(guc, true); +} + +int intel_guc_rc_disable(struct intel_guc *guc) +{ + return __guc_rc_control(guc, false); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.h new file mode 100644 index 000000000000..57e86c337838 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_GUC_RC_H_ +#define _INTEL_GUC_RC_H_ + +#include "intel_guc_submission.h" + +void intel_guc_rc_init_early(struct intel_guc *guc); + +static inline bool intel_guc_rc_is_supported(struct intel_guc *guc) +{ + return guc->rc_supported; +} + +static inline bool intel_guc_rc_is_wanted(struct intel_guc *guc) +{ + return guc->submission_selected && intel_guc_rc_is_supported(guc); +} + +static inline bool intel_guc_rc_is_used(struct intel_guc *guc) +{ + return intel_guc_submission_is_used(guc) && intel_guc_rc_is_wanted(guc); +} + +int intel_guc_rc_enable(struct intel_guc *guc); +int intel_guc_rc_disable(struct intel_guc *guc); + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c new file mode 100644 index 000000000000..65a3e7fdb2b2 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_guc_slpc.h" +#include "gt/intel_gt.h" + +static inline struct intel_guc *slpc_to_guc(struct intel_guc_slpc *slpc) +{ + return container_of(slpc, struct intel_guc, slpc); +} + +static inline struct intel_gt *slpc_to_gt(struct intel_guc_slpc *slpc) +{ + return guc_to_gt(slpc_to_guc(slpc)); +} + +static inline struct drm_i915_private *slpc_to_i915(struct intel_guc_slpc *slpc) +{ + return slpc_to_gt(slpc)->i915; +} + +static bool __detect_slpc_supported(struct intel_guc *guc) +{ + /* GuC SLPC is unavailable for pre-Gen12 */ + return guc->submission_supported && + GRAPHICS_VER(guc_to_gt(guc)->i915) >= 12; +} + +static bool __guc_slpc_selected(struct intel_guc *guc) +{ + if (!intel_guc_slpc_is_supported(guc)) + return false; + + return guc->submission_selected; +} + +void intel_guc_slpc_init_early(struct intel_guc_slpc *slpc) +{ + struct intel_guc *guc = slpc_to_guc(slpc); + + slpc->supported = __detect_slpc_supported(guc); + slpc->selected = __guc_slpc_selected(guc); +} + +static void slpc_mem_set_param(struct slpc_shared_data *data, + u32 id, u32 value) +{ + GEM_BUG_ON(id >= SLPC_MAX_OVERRIDE_PARAMETERS); + /* + * When the flag bit is set, corresponding value will be read + * and applied by SLPC. + */ + data->override_params.bits[id >> 5] |= (1 << (id % 32)); + data->override_params.values[id] = value; +} + +static void slpc_mem_set_enabled(struct slpc_shared_data *data, + u8 enable_id, u8 disable_id) +{ + /* + * Enabling a param involves setting the enable_id + * to 1 and disable_id to 0. + */ + slpc_mem_set_param(data, enable_id, 1); + slpc_mem_set_param(data, disable_id, 0); +} + +static void slpc_mem_set_disabled(struct slpc_shared_data *data, + u8 enable_id, u8 disable_id) +{ + /* + * Disabling a param involves setting the enable_id + * to 0 and disable_id to 1. + */ + slpc_mem_set_param(data, disable_id, 1); + slpc_mem_set_param(data, enable_id, 0); +} + +int intel_guc_slpc_init(struct intel_guc_slpc *slpc) +{ + struct intel_guc *guc = slpc_to_guc(slpc); + struct drm_i915_private *i915 = slpc_to_i915(slpc); + u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); + int err; + + GEM_BUG_ON(slpc->vma); + + err = intel_guc_allocate_and_map_vma(guc, size, &slpc->vma, (void **)&slpc->vaddr); + if (unlikely(err)) { + drm_err(&i915->drm, + "Failed to allocate SLPC struct (err=%pe)\n", + ERR_PTR(err)); + return err; + } + + slpc->max_freq_softlimit = 0; + slpc->min_freq_softlimit = 0; + + return err; +} + +static u32 slpc_get_state(struct intel_guc_slpc *slpc) +{ + struct slpc_shared_data *data; + + GEM_BUG_ON(!slpc->vma); + + drm_clflush_virt_range(slpc->vaddr, sizeof(u32)); + data = slpc->vaddr; + + return data->header.global_state; +} + +static int guc_action_slpc_set_param(struct intel_guc *guc, u8 id, u32 value) +{ + u32 request[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_PARAMETER_SET, 2), + id, + value, + }; + int ret; + + ret = intel_guc_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int guc_action_slpc_unset_param(struct intel_guc *guc, u8 id) +{ + u32 request[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 2), + id, + }; + + return intel_guc_send(guc, request, ARRAY_SIZE(request)); +} + +static bool slpc_is_running(struct intel_guc_slpc *slpc) +{ + return slpc_get_state(slpc) == SLPC_GLOBAL_STATE_RUNNING; +} + +static int guc_action_slpc_query(struct intel_guc *guc, u32 offset) +{ + u32 request[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_QUERY_TASK_STATE, 2), + offset, + 0, + }; + int ret; + + ret = intel_guc_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int slpc_query_task_state(struct intel_guc_slpc *slpc) +{ + struct intel_guc *guc = slpc_to_guc(slpc); + struct drm_i915_private *i915 = slpc_to_i915(slpc); + u32 offset = intel_guc_ggtt_offset(guc, slpc->vma); + int ret; + + ret = guc_action_slpc_query(guc, offset); + if (unlikely(ret)) + drm_err(&i915->drm, "Failed to query task state (%pe)\n", + ERR_PTR(ret)); + + drm_clflush_virt_range(slpc->vaddr, SLPC_PAGE_SIZE_BYTES); + + return ret; +} + +static int slpc_set_param(struct intel_guc_slpc *slpc, u8 id, u32 value) +{ + struct intel_guc *guc = slpc_to_guc(slpc); + struct drm_i915_private *i915 = slpc_to_i915(slpc); + int ret; + + GEM_BUG_ON(id >= SLPC_MAX_PARAM); + + ret = guc_action_slpc_set_param(guc, id, value); + if (ret) + drm_err(&i915->drm, "Failed to set param %d to %u (%pe)\n", + id, value, ERR_PTR(ret)); + + return ret; +} + +static int slpc_unset_param(struct intel_guc_slpc *slpc, + u8 id) +{ + struct intel_guc *guc = slpc_to_guc(slpc); + + GEM_BUG_ON(id >= SLPC_MAX_PARAM); + + return guc_action_slpc_unset_param(guc, id); +} + +static const char *slpc_global_state_to_string(enum slpc_global_state state) +{ + switch (state) { + case SLPC_GLOBAL_STATE_NOT_RUNNING: + return "not running"; + case SLPC_GLOBAL_STATE_INITIALIZING: + return "initializing"; + case SLPC_GLOBAL_STATE_RESETTING: + return "resetting"; + case SLPC_GLOBAL_STATE_RUNNING: + return "running"; + case SLPC_GLOBAL_STATE_SHUTTING_DOWN: + return "shutting down"; + case SLPC_GLOBAL_STATE_ERROR: + return "error"; + default: + return "unknown"; + } +} + +static const char *slpc_get_state_string(struct intel_guc_slpc *slpc) +{ + return slpc_global_state_to_string(slpc_get_state(slpc)); +} + +static int guc_action_slpc_reset(struct intel_guc *guc, u32 offset) +{ + u32 request[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_RESET, 2), + offset, + 0, + }; + int ret; + + ret = intel_guc_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int slpc_reset(struct intel_guc_slpc *slpc) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + struct intel_guc *guc = slpc_to_guc(slpc); + u32 offset = intel_guc_ggtt_offset(guc, slpc->vma); + int ret; + + ret = guc_action_slpc_reset(guc, offset); + + if (unlikely(ret < 0)) { + drm_err(&i915->drm, "SLPC reset action failed (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + if (!ret) { + if (wait_for(slpc_is_running(slpc), SLPC_RESET_TIMEOUT_MS)) { + drm_err(&i915->drm, "SLPC not enabled! State = %s\n", + slpc_get_state_string(slpc)); + return -EIO; + } + } + + return 0; +} + +static u32 slpc_decode_min_freq(struct intel_guc_slpc *slpc) +{ + struct slpc_shared_data *data = slpc->vaddr; + + GEM_BUG_ON(!slpc->vma); + + return DIV_ROUND_CLOSEST(REG_FIELD_GET(SLPC_MIN_UNSLICE_FREQ_MASK, + data->task_state_data.freq) * + GT_FREQUENCY_MULTIPLIER, GEN9_FREQ_SCALER); +} + +static u32 slpc_decode_max_freq(struct intel_guc_slpc *slpc) +{ + struct slpc_shared_data *data = slpc->vaddr; + + GEM_BUG_ON(!slpc->vma); + + return DIV_ROUND_CLOSEST(REG_FIELD_GET(SLPC_MAX_UNSLICE_FREQ_MASK, + data->task_state_data.freq) * + GT_FREQUENCY_MULTIPLIER, GEN9_FREQ_SCALER); +} + +static void slpc_shared_data_reset(struct slpc_shared_data *data) +{ + memset(data, 0, sizeof(struct slpc_shared_data)); + + data->header.size = sizeof(struct slpc_shared_data); + + /* Enable only GTPERF task, disable others */ + slpc_mem_set_enabled(data, SLPC_PARAM_TASK_ENABLE_GTPERF, + SLPC_PARAM_TASK_DISABLE_GTPERF); + + slpc_mem_set_disabled(data, SLPC_PARAM_TASK_ENABLE_BALANCER, + SLPC_PARAM_TASK_DISABLE_BALANCER); + + slpc_mem_set_disabled(data, SLPC_PARAM_TASK_ENABLE_DCC, + SLPC_PARAM_TASK_DISABLE_DCC); +} + +/** + * intel_guc_slpc_set_max_freq() - Set max frequency limit for SLPC. + * @slpc: pointer to intel_guc_slpc. + * @val: frequency (MHz) + * + * This function will invoke GuC SLPC action to update the max frequency + * limit for unslice. + * + * Return: 0 on success, non-zero error code on failure. + */ +int intel_guc_slpc_set_max_freq(struct intel_guc_slpc *slpc, u32 val) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + intel_wakeref_t wakeref; + int ret; + + if (val < slpc->min_freq || + val > slpc->rp0_freq || + val < slpc->min_freq_softlimit) + return -EINVAL; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + ret = slpc_set_param(slpc, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ, + val); + + /* Return standardized err code for sysfs calls */ + if (ret) + ret = -EIO; + } + + if (!ret) + slpc->max_freq_softlimit = val; + + return ret; +} + +/** + * intel_guc_slpc_get_max_freq() - Get max frequency limit for SLPC. + * @slpc: pointer to intel_guc_slpc. + * @val: pointer to val which will hold max frequency (MHz) + * + * This function will invoke GuC SLPC action to read the max frequency + * limit for unslice. + * + * Return: 0 on success, non-zero error code on failure. + */ +int intel_guc_slpc_get_max_freq(struct intel_guc_slpc *slpc, u32 *val) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + intel_wakeref_t wakeref; + int ret = 0; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + /* Force GuC to update task data */ + ret = slpc_query_task_state(slpc); + + if (!ret) + *val = slpc_decode_max_freq(slpc); + } + + return ret; +} + +/** + * intel_guc_slpc_set_min_freq() - Set min frequency limit for SLPC. + * @slpc: pointer to intel_guc_slpc. + * @val: frequency (MHz) + * + * This function will invoke GuC SLPC action to update the min unslice + * frequency. + * + * Return: 0 on success, non-zero error code on failure. + */ +int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + intel_wakeref_t wakeref; + int ret; + + if (val < slpc->min_freq || + val > slpc->rp0_freq || + val > slpc->max_freq_softlimit) + return -EINVAL; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + ret = slpc_set_param(slpc, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ, + val); + + /* Return standardized err code for sysfs calls */ + if (ret) + ret = -EIO; + } + + if (!ret) + slpc->min_freq_softlimit = val; + + return ret; +} + +/** + * intel_guc_slpc_get_min_freq() - Get min frequency limit for SLPC. + * @slpc: pointer to intel_guc_slpc. + * @val: pointer to val which will hold min frequency (MHz) + * + * This function will invoke GuC SLPC action to read the min frequency + * limit for unslice. + * + * Return: 0 on success, non-zero error code on failure. + */ +int intel_guc_slpc_get_min_freq(struct intel_guc_slpc *slpc, u32 *val) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + intel_wakeref_t wakeref; + int ret = 0; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + /* Force GuC to update task data */ + ret = slpc_query_task_state(slpc); + + if (!ret) + *val = slpc_decode_min_freq(slpc); + } + + return ret; +} + +void intel_guc_pm_intrmsk_enable(struct intel_gt *gt) +{ + u32 pm_intrmsk_mbz = 0; + + /* + * Allow GuC to receive ARAT timer expiry event. + * This interrupt register is setup by RPS code + * when host based Turbo is enabled. + */ + pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; + + intel_uncore_rmw(gt->uncore, + GEN6_PMINTRMSK, pm_intrmsk_mbz, 0); +} + +static int slpc_set_softlimits(struct intel_guc_slpc *slpc) +{ + int ret = 0; + + /* + * Softlimits are initially equivalent to platform limits + * unless they have deviated from defaults, in which case, + * we retain the values and set min/max accordingly. + */ + if (!slpc->max_freq_softlimit) + slpc->max_freq_softlimit = slpc->rp0_freq; + else if (slpc->max_freq_softlimit != slpc->rp0_freq) + ret = intel_guc_slpc_set_max_freq(slpc, + slpc->max_freq_softlimit); + + if (unlikely(ret)) + return ret; + + if (!slpc->min_freq_softlimit) + slpc->min_freq_softlimit = slpc->min_freq; + else if (slpc->min_freq_softlimit != slpc->min_freq) + return intel_guc_slpc_set_min_freq(slpc, + slpc->min_freq_softlimit); + + return 0; +} + +static int slpc_ignore_eff_freq(struct intel_guc_slpc *slpc, bool ignore) +{ + int ret = 0; + + if (ignore) { + ret = slpc_set_param(slpc, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY, + ignore); + if (!ret) + return slpc_set_param(slpc, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ, + slpc->min_freq); + } else { + ret = slpc_unset_param(slpc, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY); + if (!ret) + return slpc_unset_param(slpc, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ); + } + + return ret; +} + +static int slpc_use_fused_rp0(struct intel_guc_slpc *slpc) +{ + /* Force SLPC to used platform rp0 */ + return slpc_set_param(slpc, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ, + slpc->rp0_freq); +} + +static void slpc_get_rp_values(struct intel_guc_slpc *slpc) +{ + u32 rp_state_cap; + + rp_state_cap = intel_uncore_read(slpc_to_gt(slpc)->uncore, + GEN6_RP_STATE_CAP); + + slpc->rp0_freq = REG_FIELD_GET(RP0_CAP_MASK, rp_state_cap) * + GT_FREQUENCY_MULTIPLIER; + slpc->rp1_freq = REG_FIELD_GET(RP1_CAP_MASK, rp_state_cap) * + GT_FREQUENCY_MULTIPLIER; + slpc->min_freq = REG_FIELD_GET(RPN_CAP_MASK, rp_state_cap) * + GT_FREQUENCY_MULTIPLIER; +} + +/* + * intel_guc_slpc_enable() - Start SLPC + * @slpc: pointer to intel_guc_slpc. + * + * SLPC is enabled by setting up the shared data structure and + * sending reset event to GuC SLPC. Initial data is setup in + * intel_guc_slpc_init. Here we send the reset event. We do + * not currently need a slpc_disable since this is taken care + * of automatically when a reset/suspend occurs and the GuC + * CTB is destroyed. + * + * Return: 0 on success, non-zero error code on failure. + */ +int intel_guc_slpc_enable(struct intel_guc_slpc *slpc) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + int ret; + + GEM_BUG_ON(!slpc->vma); + + slpc_shared_data_reset(slpc->vaddr); + + ret = slpc_reset(slpc); + if (unlikely(ret < 0)) { + drm_err(&i915->drm, "SLPC Reset event returned (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + ret = slpc_query_task_state(slpc); + if (unlikely(ret < 0)) + return ret; + + intel_guc_pm_intrmsk_enable(&i915->gt); + + slpc_get_rp_values(slpc); + + /* Ignore efficient freq and set min to platform min */ + ret = slpc_ignore_eff_freq(slpc, true); + if (unlikely(ret)) { + drm_err(&i915->drm, "Failed to set SLPC min to RPn (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + /* Set SLPC max limit to RP0 */ + ret = slpc_use_fused_rp0(slpc); + if (unlikely(ret)) { + drm_err(&i915->drm, "Failed to set SLPC max to RP0 (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + /* Revert SLPC min/max to softlimits if necessary */ + ret = slpc_set_softlimits(slpc); + if (unlikely(ret)) { + drm_err(&i915->drm, "Failed to set SLPC softlimits (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + return 0; +} + +int intel_guc_slpc_print_info(struct intel_guc_slpc *slpc, struct drm_printer *p) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + struct slpc_shared_data *data = slpc->vaddr; + struct slpc_task_state_data *slpc_tasks; + intel_wakeref_t wakeref; + int ret = 0; + + GEM_BUG_ON(!slpc->vma); + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + ret = slpc_query_task_state(slpc); + + if (!ret) { + slpc_tasks = &data->task_state_data; + + drm_printf(p, "\tSLPC state: %s\n", slpc_get_state_string(slpc)); + drm_printf(p, "\tGTPERF task active: %s\n", + yesno(slpc_tasks->status & SLPC_GTPERF_TASK_ENABLED)); + drm_printf(p, "\tMax freq: %u MHz\n", + slpc_decode_max_freq(slpc)); + drm_printf(p, "\tMin freq: %u MHz\n", + slpc_decode_min_freq(slpc)); + } + } + + return ret; +} + +void intel_guc_slpc_fini(struct intel_guc_slpc *slpc) +{ + if (!slpc->vma) + return; + + i915_vma_unpin_and_release(&slpc->vma, I915_VMA_RELEASE_MAP); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h new file mode 100644 index 000000000000..e45054d5b9b4 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_GUC_SLPC_H_ +#define _INTEL_GUC_SLPC_H_ + +#include "intel_guc_submission.h" +#include "intel_guc_slpc_types.h" + +struct intel_gt; +struct drm_printer; + +static inline bool intel_guc_slpc_is_supported(struct intel_guc *guc) +{ + return guc->slpc.supported; +} + +static inline bool intel_guc_slpc_is_wanted(struct intel_guc *guc) +{ + return guc->slpc.selected; +} + +static inline bool intel_guc_slpc_is_used(struct intel_guc *guc) +{ + return intel_guc_submission_is_used(guc) && intel_guc_slpc_is_wanted(guc); +} + +void intel_guc_slpc_init_early(struct intel_guc_slpc *slpc); + +int intel_guc_slpc_init(struct intel_guc_slpc *slpc); +int intel_guc_slpc_enable(struct intel_guc_slpc *slpc); +void intel_guc_slpc_fini(struct intel_guc_slpc *slpc); +int intel_guc_slpc_set_max_freq(struct intel_guc_slpc *slpc, u32 val); +int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val); +int intel_guc_slpc_get_max_freq(struct intel_guc_slpc *slpc, u32 *val); +int intel_guc_slpc_get_min_freq(struct intel_guc_slpc *slpc, u32 *val); +int intel_guc_slpc_print_info(struct intel_guc_slpc *slpc, struct drm_printer *p); +void intel_guc_pm_intrmsk_enable(struct intel_gt *gt); + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h new file mode 100644 index 000000000000..41d13527666f --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_GUC_SLPC_TYPES_H_ +#define _INTEL_GUC_SLPC_TYPES_H_ + +#include <linux/types.h> + +#define SLPC_RESET_TIMEOUT_MS 5 + +struct intel_guc_slpc { + struct i915_vma *vma; + struct slpc_shared_data *vaddr; + bool supported; + bool selected; + + /* platform frequency limits */ + u32 min_freq; + u32 rp0_freq; + u32 rp1_freq; + + /* frequency softlimits */ + u32 min_freq_softlimit; + u32 max_freq_softlimit; +}; + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 7c8ff9792f7b..87d8dc8f51b9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -10,10 +10,13 @@ #include "gt/intel_breadcrumbs.h" #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" +#include "gt/intel_engine_heartbeat.h" #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_requests.h" #include "gt/intel_lrc.h" +#include "gt/intel_lrc_reg.h" #include "gt/intel_mocs.h" #include "gt/intel_ring.h" @@ -58,244 +61,705 @@ * */ +/* GuC Virtual Engine */ +struct guc_virtual_engine { + struct intel_engine_cs base; + struct intel_context context; +}; + +static struct intel_context * +guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count); + #define GUC_REQUEST_SIZE 64 /* bytes */ -static inline struct i915_priolist *to_priolist(struct rb_node *rb) +/* + * Below is a set of functions which control the GuC scheduling state which do + * not require a lock as all state transitions are mutually exclusive. i.e. It + * is not possible for the context pinning code and submission, for the same + * context, to be executing simultaneously. We still need an atomic as it is + * possible for some of the bits to changing at the same time though. + */ +#define SCHED_STATE_NO_LOCK_ENABLED BIT(0) +#define SCHED_STATE_NO_LOCK_PENDING_ENABLE BIT(1) +#define SCHED_STATE_NO_LOCK_REGISTERED BIT(2) +static inline bool context_enabled(struct intel_context *ce) { - return rb_entry(rb, struct i915_priolist, node); + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_ENABLED); } -static struct guc_stage_desc *__get_stage_desc(struct intel_guc *guc, u32 id) +static inline void set_context_enabled(struct intel_context *ce) { - struct guc_stage_desc *base = guc->stage_desc_pool_vaddr; + atomic_or(SCHED_STATE_NO_LOCK_ENABLED, &ce->guc_sched_state_no_lock); +} - return &base[id]; +static inline void clr_context_enabled(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_ENABLED, + &ce->guc_sched_state_no_lock); } -static int guc_stage_desc_pool_create(struct intel_guc *guc) +static inline bool context_pending_enable(struct intel_context *ce) { - u32 size = PAGE_ALIGN(sizeof(struct guc_stage_desc) * - GUC_MAX_STAGE_DESCRIPTORS); + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_PENDING_ENABLE); +} - return intel_guc_allocate_and_map_vma(guc, size, &guc->stage_desc_pool, - &guc->stage_desc_pool_vaddr); +static inline void set_context_pending_enable(struct intel_context *ce) +{ + atomic_or(SCHED_STATE_NO_LOCK_PENDING_ENABLE, + &ce->guc_sched_state_no_lock); +} + +static inline void clr_context_pending_enable(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_PENDING_ENABLE, + &ce->guc_sched_state_no_lock); } -static void guc_stage_desc_pool_destroy(struct intel_guc *guc) +static inline bool context_registered(struct intel_context *ce) { - i915_vma_unpin_and_release(&guc->stage_desc_pool, I915_VMA_RELEASE_MAP); + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_REGISTERED); +} + +static inline void set_context_registered(struct intel_context *ce) +{ + atomic_or(SCHED_STATE_NO_LOCK_REGISTERED, + &ce->guc_sched_state_no_lock); +} + +static inline void clr_context_registered(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_REGISTERED, + &ce->guc_sched_state_no_lock); } /* - * Initialise/clear the stage descriptor shared with the GuC firmware. - * - * This descriptor tells the GuC where (in GGTT space) to find the important - * data structures related to work submission (process descriptor, write queue, - * etc). + * Below is a set of functions which control the GuC scheduling state which + * require a lock, aside from the special case where the functions are called + * from guc_lrc_desc_pin(). In that case it isn't possible for any other code + * path to be executing on the context. */ -static void guc_stage_desc_init(struct intel_guc *guc) +#define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0) +#define SCHED_STATE_DESTROYED BIT(1) +#define SCHED_STATE_PENDING_DISABLE BIT(2) +#define SCHED_STATE_BANNED BIT(3) +#define SCHED_STATE_BLOCKED_SHIFT 4 +#define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT) +#define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT) +static inline void init_sched_state(struct intel_context *ce) +{ + /* Only should be called from guc_lrc_desc_pin() */ + atomic_set(&ce->guc_sched_state_no_lock, 0); + ce->guc_state.sched_state = 0; +} + +static inline bool +context_wait_for_deregister_to_register(struct intel_context *ce) +{ + return ce->guc_state.sched_state & + SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER; +} + +static inline void +set_context_wait_for_deregister_to_register(struct intel_context *ce) { - struct guc_stage_desc *desc; + /* Only should be called from guc_lrc_desc_pin() without lock */ + ce->guc_state.sched_state |= + SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER; +} - /* we only use 1 stage desc, so hardcode it to 0 */ - desc = __get_stage_desc(guc, 0); - memset(desc, 0, sizeof(*desc)); +static inline void +clr_context_wait_for_deregister_to_register(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= + ~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER; +} - desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE | - GUC_STAGE_DESC_ATTR_KERNEL; +static inline bool +context_destroyed(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_DESTROYED; +} - desc->stage_id = 0; - desc->priority = GUC_CLIENT_PRIORITY_KMD_NORMAL; +static inline void +set_context_destroyed(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_DESTROYED; +} - desc->wq_size = GUC_WQ_SIZE; +static inline bool context_pending_disable(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE; } -static void guc_stage_desc_fini(struct intel_guc *guc) +static inline void set_context_pending_disable(struct intel_context *ce) { - struct guc_stage_desc *desc; + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE; +} - desc = __get_stage_desc(guc, 0); - memset(desc, 0, sizeof(*desc)); +static inline void clr_context_pending_disable(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE; } -static void guc_add_request(struct intel_guc *guc, struct i915_request *rq) +static inline bool context_banned(struct intel_context *ce) { - /* Leaving stub as this function will be used in future patches */ + return ce->guc_state.sched_state & SCHED_STATE_BANNED; } -/* - * When we're doing submissions using regular execlists backend, writing to - * ELSP from CPU side is enough to make sure that writes to ringbuffer pages - * pinned in mappable aperture portion of GGTT are visible to command streamer. - * Writes done by GuC on our behalf are not guaranteeing such ordering, - * therefore, to ensure the flush, we're issuing a POSTING READ. - */ -static void flush_ggtt_writes(struct i915_vma *vma) +static inline void set_context_banned(struct intel_context *ce) { - if (i915_vma_is_map_and_fenceable(vma)) - intel_uncore_posting_read_fw(vma->vm->gt->uncore, - GUC_STATUS); + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_BANNED; } -static void guc_submit(struct intel_engine_cs *engine, - struct i915_request **out, - struct i915_request **end) +static inline void clr_context_banned(struct intel_context *ce) { - struct intel_guc *guc = &engine->gt->uc.guc; + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_BANNED; +} - do { - struct i915_request *rq = *out++; +static inline u32 context_blocked(struct intel_context *ce) +{ + return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >> + SCHED_STATE_BLOCKED_SHIFT; +} - flush_ggtt_writes(rq->ring->vma); - guc_add_request(guc, rq); - } while (out != end); +static inline void incr_context_blocked(struct intel_context *ce) +{ + lockdep_assert_held(&ce->engine->sched_engine->lock); + lockdep_assert_held(&ce->guc_state.lock); + + ce->guc_state.sched_state += SCHED_STATE_BLOCKED; + + GEM_BUG_ON(!context_blocked(ce)); /* Overflow check */ } -static inline int rq_prio(const struct i915_request *rq) +static inline void decr_context_blocked(struct intel_context *ce) { - return rq->sched.attr.priority; + lockdep_assert_held(&ce->engine->sched_engine->lock); + lockdep_assert_held(&ce->guc_state.lock); + + GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */ + + ce->guc_state.sched_state -= SCHED_STATE_BLOCKED; } -static struct i915_request *schedule_in(struct i915_request *rq, int idx) +static inline bool context_guc_id_invalid(struct intel_context *ce) { - trace_i915_request_in(rq, idx); + return ce->guc_id == GUC_INVALID_LRC_ID; +} + +static inline void set_context_guc_id_invalid(struct intel_context *ce) +{ + ce->guc_id = GUC_INVALID_LRC_ID; +} + +static inline struct intel_guc *ce_to_guc(struct intel_context *ce) +{ + return &ce->engine->gt->uc.guc; +} + +static inline struct i915_priolist *to_priolist(struct rb_node *rb) +{ + return rb_entry(rb, struct i915_priolist, node); +} + +static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, u32 index) +{ + struct guc_lrc_desc *base = guc->lrc_desc_pool_vaddr; + + GEM_BUG_ON(index >= GUC_MAX_LRC_DESCRIPTORS); + + return &base[index]; +} + +static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id) +{ + struct intel_context *ce = xa_load(&guc->context_lookup, id); + + GEM_BUG_ON(id >= GUC_MAX_LRC_DESCRIPTORS); + + return ce; +} + +static int guc_lrc_desc_pool_create(struct intel_guc *guc) +{ + u32 size; + int ret; + + size = PAGE_ALIGN(sizeof(struct guc_lrc_desc) * + GUC_MAX_LRC_DESCRIPTORS); + ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool, + (void **)&guc->lrc_desc_pool_vaddr); + if (ret) + return ret; + + return 0; +} + +static void guc_lrc_desc_pool_destroy(struct intel_guc *guc) +{ + guc->lrc_desc_pool_vaddr = NULL; + i915_vma_unpin_and_release(&guc->lrc_desc_pool, I915_VMA_RELEASE_MAP); +} + +static inline bool guc_submission_initialized(struct intel_guc *guc) +{ + return !!guc->lrc_desc_pool_vaddr; +} + +static inline void reset_lrc_desc(struct intel_guc *guc, u32 id) +{ + if (likely(guc_submission_initialized(guc))) { + struct guc_lrc_desc *desc = __get_lrc_desc(guc, id); + unsigned long flags; + + memset(desc, 0, sizeof(*desc)); + + /* + * xarray API doesn't have xa_erase_irqsave wrapper, so calling + * the lower level functions directly. + */ + xa_lock_irqsave(&guc->context_lookup, flags); + __xa_erase(&guc->context_lookup, id); + xa_unlock_irqrestore(&guc->context_lookup, flags); + } +} + +static inline bool lrc_desc_registered(struct intel_guc *guc, u32 id) +{ + return __get_context(guc, id); +} + +static inline void set_lrc_desc_registered(struct intel_guc *guc, u32 id, + struct intel_context *ce) +{ + unsigned long flags; /* - * Currently we are not tracking the rq->context being inflight - * (ce->inflight = rq->engine). It is only used by the execlists - * backend at the moment, a similar counting strategy would be - * required if we generalise the inflight tracking. + * xarray API doesn't have xa_save_irqsave wrapper, so calling the + * lower level functions directly. */ + xa_lock_irqsave(&guc->context_lookup, flags); + __xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC); + xa_unlock_irqrestore(&guc->context_lookup, flags); +} + +static int guc_submission_send_busy_loop(struct intel_guc *guc, + const u32 *action, + u32 len, + u32 g2h_len_dw, + bool loop) +{ + int err; + + err = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop); + + if (!err && g2h_len_dw) + atomic_inc(&guc->outstanding_submission_g2h); + + return err; +} + +int intel_guc_wait_for_pending_msg(struct intel_guc *guc, + atomic_t *wait_var, + bool interruptible, + long timeout) +{ + const int state = interruptible ? + TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; + DEFINE_WAIT(wait); - __intel_gt_pm_get(rq->engine->gt); - return i915_request_get(rq); + might_sleep(); + GEM_BUG_ON(timeout < 0); + + if (!atomic_read(wait_var)) + return 0; + + if (!timeout) + return -ETIME; + + for (;;) { + prepare_to_wait(&guc->ct.wq, &wait, state); + + if (!atomic_read(wait_var)) + break; + + if (signal_pending_state(state, current)) { + timeout = -EINTR; + break; + } + + if (!timeout) { + timeout = -ETIME; + break; + } + + timeout = io_schedule_timeout(timeout); + } + finish_wait(&guc->ct.wq, &wait); + + return (timeout < 0) ? timeout : 0; } -static void schedule_out(struct i915_request *rq) +int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout) { - trace_i915_request_out(rq); + if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc)) + return 0; - intel_gt_pm_put_async(rq->engine->gt); - i915_request_put(rq); + return intel_guc_wait_for_pending_msg(guc, + &guc->outstanding_submission_g2h, + true, timeout); } -static void __guc_dequeue(struct intel_engine_cs *engine) +static int guc_lrc_desc_pin(struct intel_context *ce, bool loop); + +static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) { - struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_request **first = execlists->inflight; - struct i915_request ** const last_port = first + execlists->port_mask; - struct i915_request *last = first[0]; - struct i915_request **port; - bool submit = false; - struct rb_node *rb; + int err = 0; + struct intel_context *ce = rq->context; + u32 action[3]; + int len = 0; + u32 g2h_len_dw = 0; + bool enabled; - lockdep_assert_held(&engine->active.lock); + /* + * Corner case where requests were sitting in the priority list or a + * request resubmitted after the context was banned. + */ + if (unlikely(intel_context_is_banned(ce))) { + i915_request_put(i915_request_mark_eio(rq)); + intel_engine_signal_breadcrumbs(ce->engine); + goto out; + } - if (last) { - if (*++first) - return; + GEM_BUG_ON(!atomic_read(&ce->guc_id_ref)); + GEM_BUG_ON(context_guc_id_invalid(ce)); - last = NULL; + /* + * Corner case where the GuC firmware was blown away and reloaded while + * this context was pinned. + */ + if (unlikely(!lrc_desc_registered(guc, ce->guc_id))) { + err = guc_lrc_desc_pin(ce, false); + if (unlikely(err)) + goto out; } /* - * We write directly into the execlists->inflight queue and don't use - * the execlists->pending queue, as we don't have a distinct switch - * event. + * The request / context will be run on the hardware when scheduling + * gets enabled in the unblock. */ - port = first; - while ((rb = rb_first_cached(&execlists->queue))) { + if (unlikely(context_blocked(ce))) + goto out; + + enabled = context_enabled(ce); + + if (!enabled) { + action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET; + action[len++] = ce->guc_id; + action[len++] = GUC_CONTEXT_ENABLE; + set_context_pending_enable(ce); + intel_context_get(ce); + g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; + } else { + action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT; + action[len++] = ce->guc_id; + } + + err = intel_guc_send_nb(guc, action, len, g2h_len_dw); + if (!enabled && !err) { + trace_intel_context_sched_enable(ce); + atomic_inc(&guc->outstanding_submission_g2h); + set_context_enabled(ce); + } else if (!enabled) { + clr_context_pending_enable(ce); + intel_context_put(ce); + } + if (likely(!err)) + trace_i915_request_guc_submit(rq); + +out: + return err; +} + +static inline void guc_set_lrc_tail(struct i915_request *rq) +{ + rq->context->lrc_reg_state[CTX_RING_TAIL] = + intel_ring_set_tail(rq->ring, rq->tail); +} + +static inline int rq_prio(const struct i915_request *rq) +{ + return rq->sched.attr.priority; +} + +static int guc_dequeue_one_context(struct intel_guc *guc) +{ + struct i915_sched_engine * const sched_engine = guc->sched_engine; + struct i915_request *last = NULL; + bool submit = false; + struct rb_node *rb; + int ret; + + lockdep_assert_held(&sched_engine->lock); + + if (guc->stalled_request) { + submit = true; + last = guc->stalled_request; + goto resubmit; + } + + while ((rb = rb_first_cached(&sched_engine->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; priolist_for_each_request_consume(rq, rn, p) { - if (last && rq->context != last->context) { - if (port == last_port) - goto done; - - *port = schedule_in(last, - port - execlists->inflight); - port++; - } + if (last && rq->context != last->context) + goto done; list_del_init(&rq->sched.link); + __i915_request_submit(rq); - submit = true; + + trace_i915_request_in(rq, 0); last = rq; + submit = true; } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &sched_engine->queue); i915_priolist_free(p); } done: - execlists->queue_priority_hint = - rb ? to_priolist(rb)->priority : INT_MIN; if (submit) { - *port = schedule_in(last, port - execlists->inflight); - *++port = NULL; - guc_submit(engine, first, port); + guc_set_lrc_tail(last); +resubmit: + ret = guc_add_request(guc, last); + if (unlikely(ret == -EPIPE)) + goto deadlk; + else if (ret == -EBUSY) { + tasklet_schedule(&sched_engine->tasklet); + guc->stalled_request = last; + return false; + } } - execlists->active = execlists->inflight; + + guc->stalled_request = NULL; + return submit; + +deadlk: + sched_engine->tasklet.callback = NULL; + tasklet_disable_nosync(&sched_engine->tasklet); + return false; } static void guc_submission_tasklet(struct tasklet_struct *t) { - struct intel_engine_cs * const engine = - from_tasklet(engine, t, execlists.tasklet); - struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_request **port, *rq; + struct i915_sched_engine *sched_engine = + from_tasklet(sched_engine, t, tasklet); unsigned long flags; + bool loop; - spin_lock_irqsave(&engine->active.lock, flags); - - for (port = execlists->inflight; (rq = *port); port++) { - if (!i915_request_completed(rq)) - break; + spin_lock_irqsave(&sched_engine->lock, flags); - schedule_out(rq); - } - if (port != execlists->inflight) { - int idx = port - execlists->inflight; - int rem = ARRAY_SIZE(execlists->inflight) - idx; - memmove(execlists->inflight, port, rem * sizeof(*port)); - } + do { + loop = guc_dequeue_one_context(sched_engine->private_data); + } while (loop); - __guc_dequeue(engine); + i915_sched_engine_reset_on_empty(sched_engine); - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&sched_engine->lock, flags); } static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir) { - if (iir & GT_RENDER_USER_INTERRUPT) { + if (iir & GT_RENDER_USER_INTERRUPT) intel_engine_signal_breadcrumbs(engine); - tasklet_hi_schedule(&engine->execlists.tasklet); +} + +static void __guc_context_destroy(struct intel_context *ce); +static void release_guc_id(struct intel_guc *guc, struct intel_context *ce); +static void guc_signal_context_fence(struct intel_context *ce); +static void guc_cancel_context_requests(struct intel_context *ce); +static void guc_blocked_fence_complete(struct intel_context *ce); + +static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc) +{ + struct intel_context *ce; + unsigned long index, flags; + bool pending_disable, pending_enable, deregister, destroyed, banned; + + xa_for_each(&guc->context_lookup, index, ce) { + /* Flush context */ + spin_lock_irqsave(&ce->guc_state.lock, flags); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + /* + * Once we are at this point submission_disabled() is guaranteed + * to be visible to all callers who set the below flags (see above + * flush and flushes in reset_prepare). If submission_disabled() + * is set, the caller shouldn't set these flags. + */ + + destroyed = context_destroyed(ce); + pending_enable = context_pending_enable(ce); + pending_disable = context_pending_disable(ce); + deregister = context_wait_for_deregister_to_register(ce); + banned = context_banned(ce); + init_sched_state(ce); + + if (pending_enable || destroyed || deregister) { + atomic_dec(&guc->outstanding_submission_g2h); + if (deregister) + guc_signal_context_fence(ce); + if (destroyed) { + release_guc_id(guc, ce); + __guc_context_destroy(ce); + } + if (pending_enable || deregister) + intel_context_put(ce); + } + + /* Not mutualy exclusive with above if statement. */ + if (pending_disable) { + guc_signal_context_fence(ce); + if (banned) { + guc_cancel_context_requests(ce); + intel_engine_signal_breadcrumbs(ce->engine); + } + intel_context_sched_disable_unpin(ce); + atomic_dec(&guc->outstanding_submission_g2h); + spin_lock_irqsave(&ce->guc_state.lock, flags); + guc_blocked_fence_complete(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + intel_context_put(ce); + } } } -static void guc_reset_prepare(struct intel_engine_cs *engine) +static inline bool +submission_disabled(struct intel_guc *guc) +{ + struct i915_sched_engine * const sched_engine = guc->sched_engine; + + return unlikely(!sched_engine || + !__tasklet_is_enabled(&sched_engine->tasklet)); +} + +static void disable_submission(struct intel_guc *guc) { - struct intel_engine_execlists * const execlists = &engine->execlists; + struct i915_sched_engine * const sched_engine = guc->sched_engine; + + if (__tasklet_is_enabled(&sched_engine->tasklet)) { + GEM_BUG_ON(!guc->ct.enabled); + __tasklet_disable_sync_once(&sched_engine->tasklet); + sched_engine->tasklet.callback = NULL; + } +} + +static void enable_submission(struct intel_guc *guc) +{ + struct i915_sched_engine * const sched_engine = guc->sched_engine; + unsigned long flags; - ENGINE_TRACE(engine, "\n"); + spin_lock_irqsave(&guc->sched_engine->lock, flags); + sched_engine->tasklet.callback = guc_submission_tasklet; + wmb(); /* Make sure callback visible */ + if (!__tasklet_is_enabled(&sched_engine->tasklet) && + __tasklet_enable(&sched_engine->tasklet)) { + GEM_BUG_ON(!guc->ct.enabled); + + /* And kick in case we missed a new request submission. */ + tasklet_hi_schedule(&sched_engine->tasklet); + } + spin_unlock_irqrestore(&guc->sched_engine->lock, flags); +} + +static void guc_flush_submissions(struct intel_guc *guc) +{ + struct i915_sched_engine * const sched_engine = guc->sched_engine; + unsigned long flags; + + spin_lock_irqsave(&sched_engine->lock, flags); + spin_unlock_irqrestore(&sched_engine->lock, flags); +} + +void intel_guc_submission_reset_prepare(struct intel_guc *guc) +{ + int i; + + if (unlikely(!guc_submission_initialized(guc))) { + /* Reset called during driver load? GuC not yet initialised! */ + return; + } + + intel_gt_park_heartbeats(guc_to_gt(guc)); + disable_submission(guc); + guc->interrupts.disable(guc); + + /* Flush IRQ handler */ + spin_lock_irq(&guc_to_gt(guc)->irq_lock); + spin_unlock_irq(&guc_to_gt(guc)->irq_lock); + + guc_flush_submissions(guc); /* - * Prevent request submission to the hardware until we have - * completed the reset in i915_gem_reset_finish(). If a request - * is completed by one engine, it may then queue a request - * to a second via its execlists->tasklet *just* as we are - * calling engine->init_hw() and also writing the ELSP. - * Turning off the execlists->tasklet until the reset is over - * prevents the race. + * Handle any outstanding G2Hs before reset. Call IRQ handler directly + * each pass as interrupt have been disabled. We always scrub for + * outstanding G2H as it is possible for outstanding_submission_g2h to + * be incremented after the context state update. */ - __tasklet_disable_sync_once(&execlists->tasklet); + for (i = 0; i < 4 && atomic_read(&guc->outstanding_submission_g2h); ++i) { + intel_guc_to_host_event_handler(guc); +#define wait_for_reset(guc, wait_var) \ + intel_guc_wait_for_pending_msg(guc, wait_var, false, (HZ / 20)) + do { + wait_for_reset(guc, &guc->outstanding_submission_g2h); + } while (!list_empty(&guc->ct.requests.incoming)); + } + scrub_guc_desc_for_outstanding_g2h(guc); +} + +static struct intel_engine_cs * +guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling) +{ + struct intel_engine_cs *engine; + intel_engine_mask_t tmp, mask = ve->mask; + unsigned int num_siblings = 0; + + for_each_engine_masked(engine, ve->gt, mask, tmp) + if (num_siblings++ == sibling) + return engine; + + return NULL; +} + +static inline struct intel_engine_cs * +__context_to_physical_engine(struct intel_context *ce) +{ + struct intel_engine_cs *engine = ce->engine; + + if (intel_engine_is_virtual(engine)) + engine = guc_virtual_get_sibling(engine, 0); + + return engine; } -static void guc_reset_state(struct intel_context *ce, - struct intel_engine_cs *engine, - u32 head, - bool scrub) +static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub) { + struct intel_engine_cs *engine = __context_to_physical_engine(ce); + + if (intel_context_is_banned(ce)) + return; + GEM_BUG_ON(!intel_context_is_pinned(ce)); /* @@ -313,37 +777,132 @@ static void guc_reset_state(struct intel_context *ce, lrc_update_regs(ce, engine, head); } -static void guc_reset_rewind(struct intel_engine_cs *engine, bool stalled) +static void guc_reset_nop(struct intel_engine_cs *engine) { - struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_request *rq; +} + +static void guc_rewind_nop(struct intel_engine_cs *engine, bool stalled) +{ +} + +static void +__unwind_incomplete_requests(struct intel_context *ce) +{ + struct i915_request *rq, *rn; + struct list_head *pl; + int prio = I915_PRIORITY_INVALID; + struct i915_sched_engine * const sched_engine = + ce->engine->sched_engine; unsigned long flags; - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&sched_engine->lock, flags); + spin_lock(&ce->guc_active.lock); + list_for_each_entry_safe(rq, rn, + &ce->guc_active.requests, + sched.link) { + if (i915_request_completed(rq)) + continue; - /* Push back any incomplete requests for replay after the reset. */ - rq = execlists_unwind_incomplete_requests(execlists); - if (!rq) - goto out_unlock; + list_del_init(&rq->sched.link); + spin_unlock(&ce->guc_active.lock); + + __i915_request_unsubmit(rq); + + /* Push the request back into the queue for later resubmission. */ + GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); + if (rq_prio(rq) != prio) { + prio = rq_prio(rq); + pl = i915_sched_lookup_priolist(sched_engine, prio); + } + GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine)); + + list_add_tail(&rq->sched.link, pl); + set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + + spin_lock(&ce->guc_active.lock); + } + spin_unlock(&ce->guc_active.lock); + spin_unlock_irqrestore(&sched_engine->lock, flags); +} + +static void __guc_reset_context(struct intel_context *ce, bool stalled) +{ + struct i915_request *rq; + u32 head; + + intel_context_get(ce); + + /* + * GuC will implicitly mark the context as non-schedulable + * when it sends the reset notification. Make sure our state + * reflects this change. The context will be marked enabled + * on resubmission. + */ + clr_context_enabled(ce); + + rq = intel_context_find_active_request(ce); + if (!rq) { + head = ce->ring->tail; + stalled = false; + goto out_replay; + } if (!i915_request_started(rq)) stalled = false; + GEM_BUG_ON(i915_active_is_idle(&ce->active)); + head = intel_ring_wrap(ce->ring, rq->head); __i915_request_reset(rq, stalled); - guc_reset_state(rq->context, engine, rq->head, stalled); -out_unlock: - spin_unlock_irqrestore(&engine->active.lock, flags); +out_replay: + guc_reset_state(ce, head, stalled); + __unwind_incomplete_requests(ce); + intel_context_put(ce); +} + +void intel_guc_submission_reset(struct intel_guc *guc, bool stalled) +{ + struct intel_context *ce; + unsigned long index; + + if (unlikely(!guc_submission_initialized(guc))) { + /* Reset called during driver load? GuC not yet initialised! */ + return; + } + + xa_for_each(&guc->context_lookup, index, ce) + if (intel_context_is_pinned(ce)) + __guc_reset_context(ce, stalled); + + /* GuC is blown away, drop all references to contexts */ + xa_destroy(&guc->context_lookup); } -static void guc_reset_cancel(struct intel_engine_cs *engine) +static void guc_cancel_context_requests(struct intel_context *ce) +{ + struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine; + struct i915_request *rq; + unsigned long flags; + + /* Mark all executing requests as skipped. */ + spin_lock_irqsave(&sched_engine->lock, flags); + spin_lock(&ce->guc_active.lock); + list_for_each_entry(rq, &ce->guc_active.requests, sched.link) + i915_request_put(i915_request_mark_eio(rq)); + spin_unlock(&ce->guc_active.lock); + spin_unlock_irqrestore(&sched_engine->lock, flags); +} + +static void +guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine) { - struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_request *rq, *rn; struct rb_node *rb; unsigned long flags; - ENGINE_TRACE(engine, "\n"); + /* Can be called during boot if GuC fails to load */ + if (!sched_engine) + return; /* * Before we call engine->cancel_requests(), we should have exclusive @@ -359,47 +918,67 @@ static void guc_reset_cancel(struct intel_engine_cs *engine) * submission's irq state, we also wish to remind ourselves that * it is irq state.) */ - spin_lock_irqsave(&engine->active.lock, flags); - - /* Mark all executing requests as skipped. */ - list_for_each_entry(rq, &engine->active.requests, sched.link) { - i915_request_set_error_once(rq, -EIO); - i915_request_mark_complete(rq); - } + spin_lock_irqsave(&sched_engine->lock, flags); /* Flush the queued requests to the timeline list (for retiring). */ - while ((rb = rb_first_cached(&execlists->queue))) { + while ((rb = rb_first_cached(&sched_engine->queue))) { struct i915_priolist *p = to_priolist(rb); priolist_for_each_request_consume(rq, rn, p) { list_del_init(&rq->sched.link); + __i915_request_submit(rq); - dma_fence_set_error(&rq->fence, -EIO); - i915_request_mark_complete(rq); + + i915_request_put(i915_request_mark_eio(rq)); } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &sched_engine->queue); i915_priolist_free(p); } /* Remaining _unready_ requests will be nop'ed when submitted */ - execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; + sched_engine->queue_priority_hint = INT_MIN; + sched_engine->queue = RB_ROOT_CACHED; - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&sched_engine->lock, flags); } -static void guc_reset_finish(struct intel_engine_cs *engine) +void intel_guc_submission_cancel_requests(struct intel_guc *guc) { - struct intel_engine_execlists * const execlists = &engine->execlists; + struct intel_context *ce; + unsigned long index; - if (__tasklet_enable(&execlists->tasklet)) - /* And kick in case we missed a new request submission. */ - tasklet_hi_schedule(&execlists->tasklet); + xa_for_each(&guc->context_lookup, index, ce) + if (intel_context_is_pinned(ce)) + guc_cancel_context_requests(ce); + + guc_cancel_sched_engine_requests(guc->sched_engine); - ENGINE_TRACE(engine, "depth->%d\n", - atomic_read(&execlists->tasklet.count)); + /* GuC is blown away, drop all references to contexts */ + xa_destroy(&guc->context_lookup); +} + +void intel_guc_submission_reset_finish(struct intel_guc *guc) +{ + /* Reset called during driver load or during wedge? */ + if (unlikely(!guc_submission_initialized(guc) || + test_bit(I915_WEDGED, &guc_to_gt(guc)->reset.flags))) { + return; + } + + /* + * Technically possible for either of these values to be non-zero here, + * but very unlikely + harmless. Regardless let's add a warn so we can + * see in CI if this happens frequently / a precursor to taking down the + * machine. + */ + GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h)); + atomic_set(&guc->outstanding_submission_g2h, 0); + + intel_guc_global_policies_update(guc); + enable_submission(guc); + intel_gt_unpark_heartbeats(guc_to_gt(guc)); } /* @@ -410,43 +989,986 @@ int intel_guc_submission_init(struct intel_guc *guc) { int ret; - if (guc->stage_desc_pool) + if (guc->lrc_desc_pool) return 0; - ret = guc_stage_desc_pool_create(guc); + ret = guc_lrc_desc_pool_create(guc); if (ret) return ret; /* * Keep static analysers happy, let them know that we allocated the * vma after testing that it didn't exist earlier. */ - GEM_BUG_ON(!guc->stage_desc_pool); + GEM_BUG_ON(!guc->lrc_desc_pool); + + xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); + + spin_lock_init(&guc->contexts_lock); + INIT_LIST_HEAD(&guc->guc_id_list); + ida_init(&guc->guc_ids); return 0; } void intel_guc_submission_fini(struct intel_guc *guc) { - if (guc->stage_desc_pool) { - guc_stage_desc_pool_destroy(guc); + if (!guc->lrc_desc_pool) + return; + + guc_lrc_desc_pool_destroy(guc); + i915_sched_engine_put(guc->sched_engine); +} + +static inline void queue_request(struct i915_sched_engine *sched_engine, + struct i915_request *rq, + int prio) +{ + GEM_BUG_ON(!list_empty(&rq->sched.link)); + list_add_tail(&rq->sched.link, + i915_sched_lookup_priolist(sched_engine, prio)); + set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); +} + +static int guc_bypass_tasklet_submit(struct intel_guc *guc, + struct i915_request *rq) +{ + int ret; + + __i915_request_submit(rq); + + trace_i915_request_in(rq, 0); + + guc_set_lrc_tail(rq); + ret = guc_add_request(guc, rq); + if (ret == -EBUSY) + guc->stalled_request = rq; + + if (unlikely(ret == -EPIPE)) + disable_submission(guc); + + return ret; +} + +static void guc_submit_request(struct i915_request *rq) +{ + struct i915_sched_engine *sched_engine = rq->engine->sched_engine; + struct intel_guc *guc = &rq->engine->gt->uc.guc; + unsigned long flags; + + /* Will be called from irq-context when using foreign fences. */ + spin_lock_irqsave(&sched_engine->lock, flags); + + if (submission_disabled(guc) || guc->stalled_request || + !i915_sched_engine_is_empty(sched_engine)) + queue_request(sched_engine, rq, rq_prio(rq)); + else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY) + tasklet_hi_schedule(&sched_engine->tasklet); + + spin_unlock_irqrestore(&sched_engine->lock, flags); +} + +static int new_guc_id(struct intel_guc *guc) +{ + return ida_simple_get(&guc->guc_ids, 0, + GUC_MAX_LRC_DESCRIPTORS, GFP_KERNEL | + __GFP_RETRY_MAYFAIL | __GFP_NOWARN); +} + +static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce) +{ + if (!context_guc_id_invalid(ce)) { + ida_simple_remove(&guc->guc_ids, ce->guc_id); + reset_lrc_desc(guc, ce->guc_id); + set_context_guc_id_invalid(ce); } + if (!list_empty(&ce->guc_id_link)) + list_del_init(&ce->guc_id_link); } -static int guc_context_alloc(struct intel_context *ce) +static void release_guc_id(struct intel_guc *guc, struct intel_context *ce) { - return lrc_alloc(ce, ce->engine); + unsigned long flags; + + spin_lock_irqsave(&guc->contexts_lock, flags); + __release_guc_id(guc, ce); + spin_unlock_irqrestore(&guc->contexts_lock, flags); +} + +static int steal_guc_id(struct intel_guc *guc) +{ + struct intel_context *ce; + int guc_id; + + lockdep_assert_held(&guc->contexts_lock); + + if (!list_empty(&guc->guc_id_list)) { + ce = list_first_entry(&guc->guc_id_list, + struct intel_context, + guc_id_link); + + GEM_BUG_ON(atomic_read(&ce->guc_id_ref)); + GEM_BUG_ON(context_guc_id_invalid(ce)); + + list_del_init(&ce->guc_id_link); + guc_id = ce->guc_id; + clr_context_registered(ce); + set_context_guc_id_invalid(ce); + return guc_id; + } else { + return -EAGAIN; + } +} + +static int assign_guc_id(struct intel_guc *guc, u16 *out) +{ + int ret; + + lockdep_assert_held(&guc->contexts_lock); + + ret = new_guc_id(guc); + if (unlikely(ret < 0)) { + ret = steal_guc_id(guc); + if (ret < 0) + return ret; + } + + *out = ret; + return 0; +} + +#define PIN_GUC_ID_TRIES 4 +static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) +{ + int ret = 0; + unsigned long flags, tries = PIN_GUC_ID_TRIES; + + GEM_BUG_ON(atomic_read(&ce->guc_id_ref)); + +try_again: + spin_lock_irqsave(&guc->contexts_lock, flags); + + if (context_guc_id_invalid(ce)) { + ret = assign_guc_id(guc, &ce->guc_id); + if (ret) + goto out_unlock; + ret = 1; /* Indidcates newly assigned guc_id */ + } + if (!list_empty(&ce->guc_id_link)) + list_del_init(&ce->guc_id_link); + atomic_inc(&ce->guc_id_ref); + +out_unlock: + spin_unlock_irqrestore(&guc->contexts_lock, flags); + + /* + * -EAGAIN indicates no guc_ids are available, let's retire any + * outstanding requests to see if that frees up a guc_id. If the first + * retire didn't help, insert a sleep with the timeslice duration before + * attempting to retire more requests. Double the sleep period each + * subsequent pass before finally giving up. The sleep period has max of + * 100ms and minimum of 1ms. + */ + if (ret == -EAGAIN && --tries) { + if (PIN_GUC_ID_TRIES - tries > 1) { + unsigned int timeslice_shifted = + ce->engine->props.timeslice_duration_ms << + (PIN_GUC_ID_TRIES - tries - 2); + unsigned int max = min_t(unsigned int, 100, + timeslice_shifted); + + msleep(max_t(unsigned int, max, 1)); + } + intel_gt_retire_requests(guc_to_gt(guc)); + goto try_again; + } + + return ret; +} + +static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce) +{ + unsigned long flags; + + GEM_BUG_ON(atomic_read(&ce->guc_id_ref) < 0); + + if (unlikely(context_guc_id_invalid(ce))) + return; + + spin_lock_irqsave(&guc->contexts_lock, flags); + if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id_link) && + !atomic_read(&ce->guc_id_ref)) + list_add_tail(&ce->guc_id_link, &guc->guc_id_list); + spin_unlock_irqrestore(&guc->contexts_lock, flags); +} + +static int __guc_action_register_context(struct intel_guc *guc, + u32 guc_id, + u32 offset, + bool loop) +{ + u32 action[] = { + INTEL_GUC_ACTION_REGISTER_CONTEXT, + guc_id, + offset, + }; + + return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), + 0, loop); +} + +static int register_context(struct intel_context *ce, bool loop) +{ + struct intel_guc *guc = ce_to_guc(ce); + u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool) + + ce->guc_id * sizeof(struct guc_lrc_desc); + int ret; + + trace_intel_context_register(ce); + + ret = __guc_action_register_context(guc, ce->guc_id, offset, loop); + if (likely(!ret)) + set_context_registered(ce); + + return ret; +} + +static int __guc_action_deregister_context(struct intel_guc *guc, + u32 guc_id, + bool loop) +{ + u32 action[] = { + INTEL_GUC_ACTION_DEREGISTER_CONTEXT, + guc_id, + }; + + return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), + G2H_LEN_DW_DEREGISTER_CONTEXT, + loop); +} + +static int deregister_context(struct intel_context *ce, u32 guc_id, bool loop) +{ + struct intel_guc *guc = ce_to_guc(ce); + + trace_intel_context_deregister(ce); + + return __guc_action_deregister_context(guc, guc_id, loop); +} + +static intel_engine_mask_t adjust_engine_mask(u8 class, intel_engine_mask_t mask) +{ + switch (class) { + case RENDER_CLASS: + return mask >> RCS0; + case VIDEO_ENHANCEMENT_CLASS: + return mask >> VECS0; + case VIDEO_DECODE_CLASS: + return mask >> VCS0; + case COPY_ENGINE_CLASS: + return mask >> BCS0; + default: + MISSING_CASE(class); + return 0; + } +} + +static void guc_context_policy_init(struct intel_engine_cs *engine, + struct guc_lrc_desc *desc) +{ + desc->policy_flags = 0; + + if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION) + desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE; + + /* NB: For both of these, zero means disabled. */ + desc->execution_quantum = engine->props.timeslice_duration_ms * 1000; + desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000; +} + +static inline u8 map_i915_prio_to_guc_prio(int prio); + +static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) +{ + struct intel_engine_cs *engine = ce->engine; + struct intel_runtime_pm *runtime_pm = engine->uncore->rpm; + struct intel_guc *guc = &engine->gt->uc.guc; + u32 desc_idx = ce->guc_id; + struct guc_lrc_desc *desc; + const struct i915_gem_context *ctx; + int prio = I915_CONTEXT_DEFAULT_PRIORITY; + bool context_registered; + intel_wakeref_t wakeref; + int ret = 0; + + GEM_BUG_ON(!engine->mask); + + /* + * Ensure LRC + CT vmas are is same region as write barrier is done + * based on CT vma region. + */ + GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) != + i915_gem_object_is_lmem(ce->ring->vma->obj)); + + context_registered = lrc_desc_registered(guc, desc_idx); + + rcu_read_lock(); + ctx = rcu_dereference(ce->gem_context); + if (ctx) + prio = ctx->sched.priority; + rcu_read_unlock(); + + reset_lrc_desc(guc, desc_idx); + set_lrc_desc_registered(guc, desc_idx, ce); + + desc = __get_lrc_desc(guc, desc_idx); + desc->engine_class = engine_class_to_guc_class(engine->class); + desc->engine_submit_mask = adjust_engine_mask(engine->class, + engine->mask); + desc->hw_context_desc = ce->lrc.lrca; + ce->guc_prio = map_i915_prio_to_guc_prio(prio); + desc->priority = ce->guc_prio; + desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD; + guc_context_policy_init(engine, desc); + init_sched_state(ce); + + /* + * The context_lookup xarray is used to determine if the hardware + * context is currently registered. There are two cases in which it + * could be registered either the guc_id has been stolen from another + * context or the lrc descriptor address of this context has changed. In + * either case the context needs to be deregistered with the GuC before + * registering this context. + */ + if (context_registered) { + trace_intel_context_steal_guc_id(ce); + if (!loop) { + set_context_wait_for_deregister_to_register(ce); + intel_context_get(ce); + } else { + bool disabled; + unsigned long flags; + + /* Seal race with Reset */ + spin_lock_irqsave(&ce->guc_state.lock, flags); + disabled = submission_disabled(guc); + if (likely(!disabled)) { + set_context_wait_for_deregister_to_register(ce); + intel_context_get(ce); + } + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + if (unlikely(disabled)) { + reset_lrc_desc(guc, desc_idx); + return 0; /* Will get registered later */ + } + } + + /* + * If stealing the guc_id, this ce has the same guc_id as the + * context whose guc_id was stolen. + */ + with_intel_runtime_pm(runtime_pm, wakeref) + ret = deregister_context(ce, ce->guc_id, loop); + if (unlikely(ret == -EBUSY)) { + clr_context_wait_for_deregister_to_register(ce); + intel_context_put(ce); + } else if (unlikely(ret == -ENODEV)) { + ret = 0; /* Will get registered later */ + } + } else { + with_intel_runtime_pm(runtime_pm, wakeref) + ret = register_context(ce, loop); + if (unlikely(ret == -EBUSY)) + reset_lrc_desc(guc, desc_idx); + else if (unlikely(ret == -ENODEV)) + ret = 0; /* Will get registered later */ + } + + return ret; +} + +static int __guc_context_pre_pin(struct intel_context *ce, + struct intel_engine_cs *engine, + struct i915_gem_ww_ctx *ww, + void **vaddr) +{ + return lrc_pre_pin(ce, engine, ww, vaddr); +} + +static int __guc_context_pin(struct intel_context *ce, + struct intel_engine_cs *engine, + void *vaddr) +{ + if (i915_ggtt_offset(ce->state) != + (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK)) + set_bit(CONTEXT_LRCA_DIRTY, &ce->flags); + + /* + * GuC context gets pinned in guc_request_alloc. See that function for + * explaination of why. + */ + + return lrc_pin(ce, engine, vaddr); } static int guc_context_pre_pin(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void **vaddr) { - return lrc_pre_pin(ce, ce->engine, ww, vaddr); + return __guc_context_pre_pin(ce, ce->engine, ww, vaddr); } static int guc_context_pin(struct intel_context *ce, void *vaddr) { - return lrc_pin(ce, ce->engine, vaddr); + return __guc_context_pin(ce, ce->engine, vaddr); +} + +static void guc_context_unpin(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + + unpin_guc_id(guc, ce); + lrc_unpin(ce); +} + +static void guc_context_post_unpin(struct intel_context *ce) +{ + lrc_post_unpin(ce); +} + +static void __guc_context_sched_enable(struct intel_guc *guc, + struct intel_context *ce) +{ + u32 action[] = { + INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET, + ce->guc_id, + GUC_CONTEXT_ENABLE + }; + + trace_intel_context_sched_enable(ce); + + guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true); +} + +static void __guc_context_sched_disable(struct intel_guc *guc, + struct intel_context *ce, + u16 guc_id) +{ + u32 action[] = { + INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET, + guc_id, /* ce->guc_id not stable */ + GUC_CONTEXT_DISABLE + }; + + GEM_BUG_ON(guc_id == GUC_INVALID_LRC_ID); + + trace_intel_context_sched_disable(ce); + + guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true); +} + +static void guc_blocked_fence_complete(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + + if (!i915_sw_fence_done(&ce->guc_blocked)) + i915_sw_fence_complete(&ce->guc_blocked); +} + +static void guc_blocked_fence_reinit(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_blocked)); + + /* + * This fence is always complete unless a pending schedule disable is + * outstanding. We arm the fence here and complete it when we receive + * the pending schedule disable complete message. + */ + i915_sw_fence_fini(&ce->guc_blocked); + i915_sw_fence_reinit(&ce->guc_blocked); + i915_sw_fence_await(&ce->guc_blocked); + i915_sw_fence_commit(&ce->guc_blocked); +} + +static u16 prep_context_pending_disable(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + + set_context_pending_disable(ce); + clr_context_enabled(ce); + guc_blocked_fence_reinit(ce); + intel_context_get(ce); + + return ce->guc_id; +} + +static struct i915_sw_fence *guc_context_block(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + struct i915_sched_engine *sched_engine = ce->engine->sched_engine; + unsigned long flags; + struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; + intel_wakeref_t wakeref; + u16 guc_id; + bool enabled; + + spin_lock_irqsave(&ce->guc_state.lock, flags); + + /* + * Sync with submission path, increment before below changes to context + * state. + */ + spin_lock(&sched_engine->lock); + incr_context_blocked(ce); + spin_unlock(&sched_engine->lock); + + enabled = context_enabled(ce); + if (unlikely(!enabled || submission_disabled(guc))) { + if (enabled) + clr_context_enabled(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + return &ce->guc_blocked; + } + + /* + * We add +2 here as the schedule disable complete CTB handler calls + * intel_context_sched_disable_unpin (-2 to pin_count). + */ + atomic_add(2, &ce->pin_count); + + guc_id = prep_context_pending_disable(ce); + + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + with_intel_runtime_pm(runtime_pm, wakeref) + __guc_context_sched_disable(guc, ce, guc_id); + + return &ce->guc_blocked; +} + +static void guc_context_unblock(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + struct i915_sched_engine *sched_engine = ce->engine->sched_engine; + unsigned long flags; + struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; + intel_wakeref_t wakeref; + bool enable; + + GEM_BUG_ON(context_enabled(ce)); + + spin_lock_irqsave(&ce->guc_state.lock, flags); + + if (unlikely(submission_disabled(guc) || + !intel_context_is_pinned(ce) || + context_pending_disable(ce) || + context_blocked(ce) > 1)) { + enable = false; + } else { + enable = true; + set_context_pending_enable(ce); + set_context_enabled(ce); + intel_context_get(ce); + } + + /* + * Sync with submission path, decrement after above changes to context + * state. + */ + spin_lock(&sched_engine->lock); + decr_context_blocked(ce); + spin_unlock(&sched_engine->lock); + + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + if (enable) { + with_intel_runtime_pm(runtime_pm, wakeref) + __guc_context_sched_enable(guc, ce); + } +} + +static void guc_context_cancel_request(struct intel_context *ce, + struct i915_request *rq) +{ + if (i915_sw_fence_signaled(&rq->submit)) { + struct i915_sw_fence *fence = guc_context_block(ce); + + i915_sw_fence_wait(fence); + if (!i915_request_completed(rq)) { + __i915_request_skip(rq); + guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head), + true); + } + guc_context_unblock(ce); + } +} + +static void __guc_context_set_preemption_timeout(struct intel_guc *guc, + u16 guc_id, + u32 preemption_timeout) +{ + u32 action[] = { + INTEL_GUC_ACTION_SET_CONTEXT_PREEMPTION_TIMEOUT, + guc_id, + preemption_timeout + }; + + intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); +} + +static void guc_context_ban(struct intel_context *ce, struct i915_request *rq) +{ + struct intel_guc *guc = ce_to_guc(ce); + struct intel_runtime_pm *runtime_pm = + &ce->engine->gt->i915->runtime_pm; + intel_wakeref_t wakeref; + unsigned long flags; + + guc_flush_submissions(guc); + + spin_lock_irqsave(&ce->guc_state.lock, flags); + set_context_banned(ce); + + if (submission_disabled(guc) || + (!context_enabled(ce) && !context_pending_disable(ce))) { + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + guc_cancel_context_requests(ce); + intel_engine_signal_breadcrumbs(ce->engine); + } else if (!context_pending_disable(ce)) { + u16 guc_id; + + /* + * We add +2 here as the schedule disable complete CTB handler + * calls intel_context_sched_disable_unpin (-2 to pin_count). + */ + atomic_add(2, &ce->pin_count); + + guc_id = prep_context_pending_disable(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + /* + * In addition to disabling scheduling, set the preemption + * timeout to the minimum value (1 us) so the banned context + * gets kicked off the HW ASAP. + */ + with_intel_runtime_pm(runtime_pm, wakeref) { + __guc_context_set_preemption_timeout(guc, guc_id, 1); + __guc_context_sched_disable(guc, ce, guc_id); + } + } else { + if (!context_guc_id_invalid(ce)) + with_intel_runtime_pm(runtime_pm, wakeref) + __guc_context_set_preemption_timeout(guc, + ce->guc_id, + 1); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + } +} + +static void guc_context_sched_disable(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + unsigned long flags; + struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm; + intel_wakeref_t wakeref; + u16 guc_id; + bool enabled; + + if (submission_disabled(guc) || context_guc_id_invalid(ce) || + !lrc_desc_registered(guc, ce->guc_id)) { + clr_context_enabled(ce); + goto unpin; + } + + if (!context_enabled(ce)) + goto unpin; + + spin_lock_irqsave(&ce->guc_state.lock, flags); + + /* + * We have to check if the context has been disabled by another thread. + * We also have to check if the context has been pinned again as another + * pin operation is allowed to pass this function. Checking the pin + * count, within ce->guc_state.lock, synchronizes this function with + * guc_request_alloc ensuring a request doesn't slip through the + * 'context_pending_disable' fence. Checking within the spin lock (can't + * sleep) ensures another process doesn't pin this context and generate + * a request before we set the 'context_pending_disable' flag here. + */ + enabled = context_enabled(ce); + if (unlikely(!enabled || submission_disabled(guc))) { + if (enabled) + clr_context_enabled(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + goto unpin; + } + if (unlikely(atomic_add_unless(&ce->pin_count, -2, 2))) { + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + return; + } + guc_id = prep_context_pending_disable(ce); + + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + with_intel_runtime_pm(runtime_pm, wakeref) + __guc_context_sched_disable(guc, ce, guc_id); + + return; +unpin: + intel_context_sched_disable_unpin(ce); +} + +static inline void guc_lrc_desc_unpin(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + + GEM_BUG_ON(!lrc_desc_registered(guc, ce->guc_id)); + GEM_BUG_ON(ce != __get_context(guc, ce->guc_id)); + GEM_BUG_ON(context_enabled(ce)); + + clr_context_registered(ce); + deregister_context(ce, ce->guc_id, true); +} + +static void __guc_context_destroy(struct intel_context *ce) +{ + GEM_BUG_ON(ce->guc_prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] || + ce->guc_prio_count[GUC_CLIENT_PRIORITY_HIGH] || + ce->guc_prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] || + ce->guc_prio_count[GUC_CLIENT_PRIORITY_NORMAL]); + + lrc_fini(ce); + intel_context_fini(ce); + + if (intel_engine_is_virtual(ce->engine)) { + struct guc_virtual_engine *ve = + container_of(ce, typeof(*ve), context); + + if (ve->base.breadcrumbs) + intel_breadcrumbs_put(ve->base.breadcrumbs); + + kfree(ve); + } else { + intel_context_free(ce); + } +} + +static void guc_context_destroy(struct kref *kref) +{ + struct intel_context *ce = container_of(kref, typeof(*ce), ref); + struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; + struct intel_guc *guc = ce_to_guc(ce); + intel_wakeref_t wakeref; + unsigned long flags; + bool disabled; + + /* + * If the guc_id is invalid this context has been stolen and we can free + * it immediately. Also can be freed immediately if the context is not + * registered with the GuC or the GuC is in the middle of a reset. + */ + if (context_guc_id_invalid(ce)) { + __guc_context_destroy(ce); + return; + } else if (submission_disabled(guc) || + !lrc_desc_registered(guc, ce->guc_id)) { + release_guc_id(guc, ce); + __guc_context_destroy(ce); + return; + } + + /* + * We have to acquire the context spinlock and check guc_id again, if it + * is valid it hasn't been stolen and needs to be deregistered. We + * delete this context from the list of unpinned guc_ids available to + * steal to seal a race with guc_lrc_desc_pin(). When the G2H CTB + * returns indicating this context has been deregistered the guc_id is + * returned to the pool of available guc_ids. + */ + spin_lock_irqsave(&guc->contexts_lock, flags); + if (context_guc_id_invalid(ce)) { + spin_unlock_irqrestore(&guc->contexts_lock, flags); + __guc_context_destroy(ce); + return; + } + + if (!list_empty(&ce->guc_id_link)) + list_del_init(&ce->guc_id_link); + spin_unlock_irqrestore(&guc->contexts_lock, flags); + + /* Seal race with Reset */ + spin_lock_irqsave(&ce->guc_state.lock, flags); + disabled = submission_disabled(guc); + if (likely(!disabled)) + set_context_destroyed(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + if (unlikely(disabled)) { + release_guc_id(guc, ce); + __guc_context_destroy(ce); + return; + } + + /* + * We defer GuC context deregistration until the context is destroyed + * in order to save on CTBs. With this optimization ideally we only need + * 1 CTB to register the context during the first pin and 1 CTB to + * deregister the context when the context is destroyed. Without this + * optimization, a CTB would be needed every pin & unpin. + * + * XXX: Need to acqiure the runtime wakeref as this can be triggered + * from context_free_worker when runtime wakeref is not held. + * guc_lrc_desc_unpin requires the runtime as a GuC register is written + * in H2G CTB to deregister the context. A future patch may defer this + * H2G CTB if the runtime wakeref is zero. + */ + with_intel_runtime_pm(runtime_pm, wakeref) + guc_lrc_desc_unpin(ce); +} + +static int guc_context_alloc(struct intel_context *ce) +{ + return lrc_alloc(ce, ce->engine); +} + +static void guc_context_set_prio(struct intel_guc *guc, + struct intel_context *ce, + u8 prio) +{ + u32 action[] = { + INTEL_GUC_ACTION_SET_CONTEXT_PRIORITY, + ce->guc_id, + prio, + }; + + GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH || + prio > GUC_CLIENT_PRIORITY_NORMAL); + + if (ce->guc_prio == prio || submission_disabled(guc) || + !context_registered(ce)) + return; + + guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); + + ce->guc_prio = prio; + trace_intel_context_set_prio(ce); +} + +static inline u8 map_i915_prio_to_guc_prio(int prio) +{ + if (prio == I915_PRIORITY_NORMAL) + return GUC_CLIENT_PRIORITY_KMD_NORMAL; + else if (prio < I915_PRIORITY_NORMAL) + return GUC_CLIENT_PRIORITY_NORMAL; + else if (prio < I915_PRIORITY_DISPLAY) + return GUC_CLIENT_PRIORITY_HIGH; + else + return GUC_CLIENT_PRIORITY_KMD_HIGH; +} + +static inline void add_context_inflight_prio(struct intel_context *ce, + u8 guc_prio) +{ + lockdep_assert_held(&ce->guc_active.lock); + GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_prio_count)); + + ++ce->guc_prio_count[guc_prio]; + + /* Overflow protection */ + GEM_WARN_ON(!ce->guc_prio_count[guc_prio]); +} + +static inline void sub_context_inflight_prio(struct intel_context *ce, + u8 guc_prio) +{ + lockdep_assert_held(&ce->guc_active.lock); + GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_prio_count)); + + /* Underflow protection */ + GEM_WARN_ON(!ce->guc_prio_count[guc_prio]); + + --ce->guc_prio_count[guc_prio]; +} + +static inline void update_context_prio(struct intel_context *ce) +{ + struct intel_guc *guc = &ce->engine->gt->uc.guc; + int i; + + BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0); + BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL); + + lockdep_assert_held(&ce->guc_active.lock); + + for (i = 0; i < ARRAY_SIZE(ce->guc_prio_count); ++i) { + if (ce->guc_prio_count[i]) { + guc_context_set_prio(guc, ce, i); + break; + } + } +} + +static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio) +{ + /* Lower value is higher priority */ + return new_guc_prio < old_guc_prio; +} + +static void add_to_context(struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq)); + + GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI); + + spin_lock(&ce->guc_active.lock); + list_move_tail(&rq->sched.link, &ce->guc_active.requests); + + if (rq->guc_prio == GUC_PRIO_INIT) { + rq->guc_prio = new_guc_prio; + add_context_inflight_prio(ce, rq->guc_prio); + } else if (new_guc_prio_higher(rq->guc_prio, new_guc_prio)) { + sub_context_inflight_prio(ce, rq->guc_prio); + rq->guc_prio = new_guc_prio; + add_context_inflight_prio(ce, rq->guc_prio); + } + update_context_prio(ce); + + spin_unlock(&ce->guc_active.lock); +} + +static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_active.lock); + + if (rq->guc_prio != GUC_PRIO_INIT && + rq->guc_prio != GUC_PRIO_FINI) { + sub_context_inflight_prio(ce, rq->guc_prio); + update_context_prio(ce); + } + rq->guc_prio = GUC_PRIO_FINI; +} + +static void remove_from_context(struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + + spin_lock_irq(&ce->guc_active.lock); + + list_del_init(&rq->sched.link); + clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + + /* Prevent further __await_execution() registering a cb, then flush */ + set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); + + guc_prio_fini(rq, ce); + + spin_unlock_irq(&ce->guc_active.lock); + + atomic_dec(&ce->guc_id_ref); + i915_request_notify_execute_cb_imm(rq); } static const struct intel_context_ops guc_context_ops = { @@ -454,28 +1976,71 @@ static const struct intel_context_ops guc_context_ops = { .pre_pin = guc_context_pre_pin, .pin = guc_context_pin, - .unpin = lrc_unpin, - .post_unpin = lrc_post_unpin, + .unpin = guc_context_unpin, + .post_unpin = guc_context_post_unpin, + + .ban = guc_context_ban, + + .cancel_request = guc_context_cancel_request, .enter = intel_context_enter_engine, .exit = intel_context_exit_engine, + .sched_disable = guc_context_sched_disable, + .reset = lrc_reset, - .destroy = lrc_destroy, + .destroy = guc_context_destroy, + + .create_virtual = guc_create_virtual, }; -static int guc_request_alloc(struct i915_request *request) +static void __guc_signal_context_fence(struct intel_context *ce) { + struct i915_request *rq; + + lockdep_assert_held(&ce->guc_state.lock); + + if (!list_empty(&ce->guc_state.fences)) + trace_intel_context_fence_release(ce); + + list_for_each_entry(rq, &ce->guc_state.fences, guc_fence_link) + i915_sw_fence_complete(&rq->submit); + + INIT_LIST_HEAD(&ce->guc_state.fences); +} + +static void guc_signal_context_fence(struct intel_context *ce) +{ + unsigned long flags; + + spin_lock_irqsave(&ce->guc_state.lock, flags); + clr_context_wait_for_deregister_to_register(ce); + __guc_signal_context_fence(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); +} + +static bool context_needs_register(struct intel_context *ce, bool new_guc_id) +{ + return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) || + !lrc_desc_registered(ce_to_guc(ce), ce->guc_id)) && + !submission_disabled(ce_to_guc(ce)); +} + +static int guc_request_alloc(struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + struct intel_guc *guc = ce_to_guc(ce); + unsigned long flags; int ret; - GEM_BUG_ON(!intel_context_is_pinned(request->context)); + GEM_BUG_ON(!intel_context_is_pinned(rq->context)); /* * Flush enough space to reduce the likelihood of waiting after * we start building the request - in which case we will just * have to repeat work. */ - request->reserved_space += GUC_REQUEST_SIZE; + rq->reserved_space += GUC_REQUEST_SIZE; /* * Note that after this point, we have committed to using @@ -486,40 +2051,232 @@ static int guc_request_alloc(struct i915_request *request) */ /* Unconditionally invalidate GPU caches and TLBs. */ - ret = request->engine->emit_flush(request, EMIT_INVALIDATE); + ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE); if (ret) return ret; - request->reserved_space -= GUC_REQUEST_SIZE; + rq->reserved_space -= GUC_REQUEST_SIZE; + + /* + * Call pin_guc_id here rather than in the pinning step as with + * dma_resv, contexts can be repeatedly pinned / unpinned trashing the + * guc_ids and creating horrible race conditions. This is especially bad + * when guc_ids are being stolen due to over subscription. By the time + * this function is reached, it is guaranteed that the guc_id will be + * persistent until the generated request is retired. Thus, sealing these + * race conditions. It is still safe to fail here if guc_ids are + * exhausted and return -EAGAIN to the user indicating that they can try + * again in the future. + * + * There is no need for a lock here as the timeline mutex ensures at + * most one context can be executing this code path at once. The + * guc_id_ref is incremented once for every request in flight and + * decremented on each retire. When it is zero, a lock around the + * increment (in pin_guc_id) is needed to seal a race with unpin_guc_id. + */ + if (atomic_add_unless(&ce->guc_id_ref, 1, 0)) + goto out; + + ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */ + if (unlikely(ret < 0)) + return ret; + if (context_needs_register(ce, !!ret)) { + ret = guc_lrc_desc_pin(ce, true); + if (unlikely(ret)) { /* unwind */ + if (ret == -EPIPE) { + disable_submission(guc); + goto out; /* GPU will be reset */ + } + atomic_dec(&ce->guc_id_ref); + unpin_guc_id(guc, ce); + return ret; + } + } + + clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags); + +out: + /* + * We block all requests on this context if a G2H is pending for a + * schedule disable or context deregistration as the GuC will fail a + * schedule enable or context registration if either G2H is pending + * respectfully. Once a G2H returns, the fence is released that is + * blocking these requests (see guc_signal_context_fence). + * + * We can safely check the below fields outside of the lock as it isn't + * possible for these fields to transition from being clear to set but + * converse is possible, hence the need for the check within the lock. + */ + if (likely(!context_wait_for_deregister_to_register(ce) && + !context_pending_disable(ce))) + return 0; + + spin_lock_irqsave(&ce->guc_state.lock, flags); + if (context_wait_for_deregister_to_register(ce) || + context_pending_disable(ce)) { + i915_sw_fence_await(&rq->submit); + + list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences); + } + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + return 0; } -static inline void queue_request(struct intel_engine_cs *engine, - struct i915_request *rq, - int prio) +static int guc_virtual_context_pre_pin(struct intel_context *ce, + struct i915_gem_ww_ctx *ww, + void **vaddr) { - GEM_BUG_ON(!list_empty(&rq->sched.link)); - list_add_tail(&rq->sched.link, - i915_sched_lookup_priolist(engine, prio)); - set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); + + return __guc_context_pre_pin(ce, engine, ww, vaddr); } -static void guc_submit_request(struct i915_request *rq) +static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr) { - struct intel_engine_cs *engine = rq->engine; - unsigned long flags; + struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); - /* Will be called from irq-context when using foreign fences. */ - spin_lock_irqsave(&engine->active.lock, flags); + return __guc_context_pin(ce, engine, vaddr); +} - queue_request(engine, rq, rq_prio(rq)); +static void guc_virtual_context_enter(struct intel_context *ce) +{ + intel_engine_mask_t tmp, mask = ce->engine->mask; + struct intel_engine_cs *engine; - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); - GEM_BUG_ON(list_empty(&rq->sched.link)); + for_each_engine_masked(engine, ce->engine->gt, mask, tmp) + intel_engine_pm_get(engine); - tasklet_hi_schedule(&engine->execlists.tasklet); + intel_timeline_enter(ce->timeline); +} + +static void guc_virtual_context_exit(struct intel_context *ce) +{ + intel_engine_mask_t tmp, mask = ce->engine->mask; + struct intel_engine_cs *engine; - spin_unlock_irqrestore(&engine->active.lock, flags); + for_each_engine_masked(engine, ce->engine->gt, mask, tmp) + intel_engine_pm_put(engine); + + intel_timeline_exit(ce->timeline); +} + +static int guc_virtual_context_alloc(struct intel_context *ce) +{ + struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); + + return lrc_alloc(ce, engine); +} + +static const struct intel_context_ops virtual_guc_context_ops = { + .alloc = guc_virtual_context_alloc, + + .pre_pin = guc_virtual_context_pre_pin, + .pin = guc_virtual_context_pin, + .unpin = guc_context_unpin, + .post_unpin = guc_context_post_unpin, + + .ban = guc_context_ban, + + .cancel_request = guc_context_cancel_request, + + .enter = guc_virtual_context_enter, + .exit = guc_virtual_context_exit, + + .sched_disable = guc_context_sched_disable, + + .destroy = guc_context_destroy, + + .get_sibling = guc_virtual_get_sibling, +}; + +static bool +guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b) +{ + struct intel_engine_cs *sibling; + intel_engine_mask_t tmp, mask = b->engine_mask; + bool result = false; + + for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp) + result |= intel_engine_irq_enable(sibling); + + return result; +} + +static void +guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b) +{ + struct intel_engine_cs *sibling; + intel_engine_mask_t tmp, mask = b->engine_mask; + + for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp) + intel_engine_irq_disable(sibling); +} + +static void guc_init_breadcrumbs(struct intel_engine_cs *engine) +{ + int i; + + /* + * In GuC submission mode we do not know which physical engine a request + * will be scheduled on, this creates a problem because the breadcrumb + * interrupt is per physical engine. To work around this we attach + * requests and direct all breadcrumb interrupts to the first instance + * of an engine per class. In addition all breadcrumb interrupts are + * enabled / disabled across an engine class in unison. + */ + for (i = 0; i < MAX_ENGINE_INSTANCE; ++i) { + struct intel_engine_cs *sibling = + engine->gt->engine_class[engine->class][i]; + + if (sibling) { + if (engine->breadcrumbs != sibling->breadcrumbs) { + intel_breadcrumbs_put(engine->breadcrumbs); + engine->breadcrumbs = + intel_breadcrumbs_get(sibling->breadcrumbs); + } + break; + } + } + + if (engine->breadcrumbs) { + engine->breadcrumbs->engine_mask |= engine->mask; + engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs; + engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs; + } +} + +static void guc_bump_inflight_request_prio(struct i915_request *rq, + int prio) +{ + struct intel_context *ce = rq->context; + u8 new_guc_prio = map_i915_prio_to_guc_prio(prio); + + /* Short circuit function */ + if (prio < I915_PRIORITY_NORMAL || + rq->guc_prio == GUC_PRIO_FINI || + (rq->guc_prio != GUC_PRIO_INIT && + !new_guc_prio_higher(rq->guc_prio, new_guc_prio))) + return; + + spin_lock(&ce->guc_active.lock); + if (rq->guc_prio != GUC_PRIO_FINI) { + if (rq->guc_prio != GUC_PRIO_INIT) + sub_context_inflight_prio(ce, rq->guc_prio); + rq->guc_prio = new_guc_prio; + add_context_inflight_prio(ce, rq->guc_prio); + update_context_prio(ce); + } + spin_unlock(&ce->guc_active.lock); +} + +static void guc_retire_inflight_request_prio(struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + + spin_lock(&ce->guc_active.lock); + guc_prio_fini(rq, ce); + spin_unlock(&ce->guc_active.lock); } static void sanitize_hwsp(struct intel_engine_cs *engine) @@ -588,21 +2345,68 @@ static int guc_resume(struct intel_engine_cs *engine) return 0; } +static bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine) +{ + return !sched_engine->tasklet.callback; +} + static void guc_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = guc_submit_request; } +static inline void guc_kernel_context_pin(struct intel_guc *guc, + struct intel_context *ce) +{ + if (context_guc_id_invalid(ce)) + pin_guc_id(guc, ce); + guc_lrc_desc_pin(ce, true); +} + +static inline void guc_init_lrc_mapping(struct intel_guc *guc) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* make sure all descriptors are clean... */ + xa_destroy(&guc->context_lookup); + + /* + * Some contexts might have been pinned before we enabled GuC + * submission, so we need to add them to the GuC bookeeping. + * Also, after a reset the of the GuC we want to make sure that the + * information shared with GuC is properly reset. The kernel LRCs are + * not attached to the gem_context, so they need to be added separately. + * + * Note: we purposefully do not check the return of guc_lrc_desc_pin, + * because that function can only fail if a reset is just starting. This + * is at the end of reset so presumably another reset isn't happening + * and even it did this code would be run again. + */ + + for_each_engine(engine, gt, id) + if (engine->kernel_context) + guc_kernel_context_pin(guc, engine->kernel_context); +} + static void guc_release(struct intel_engine_cs *engine) { engine->sanitize = NULL; /* no longer in control, nothing to sanitize */ - tasklet_kill(&engine->execlists.tasklet); - intel_engine_cleanup_common(engine); lrc_fini_wa_ctx(engine); } +static void virtual_guc_bump_serial(struct intel_engine_cs *engine) +{ + struct intel_engine_cs *e; + intel_engine_mask_t tmp, mask = engine->mask; + + for_each_engine_masked(e, engine->gt, mask, tmp) + e->serial++; +} + static void guc_default_vfuncs(struct intel_engine_cs *engine) { /* Default vfuncs which can be overridden by each engine. */ @@ -611,13 +2415,15 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &guc_context_ops; engine->request_alloc = guc_request_alloc; + engine->add_active_request = add_to_context; + engine->remove_active_request = remove_from_context; - engine->schedule = i915_schedule; + engine->sched_engine->schedule = i915_schedule; - engine->reset.prepare = guc_reset_prepare; - engine->reset.rewind = guc_reset_rewind; - engine->reset.cancel = guc_reset_cancel; - engine->reset.finish = guc_reset_finish; + engine->reset.prepare = guc_reset_nop; + engine->reset.rewind = guc_rewind_nop; + engine->reset.cancel = guc_reset_nop; + engine->reset.finish = guc_reset_nop; engine->emit_flush = gen8_emit_flush_xcs; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; @@ -629,13 +2435,13 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->set_default_submission = guc_set_default_submission; engine->flags |= I915_ENGINE_HAS_PREEMPTION; + engine->flags |= I915_ENGINE_HAS_TIMESLICES; /* * TODO: GuC supports timeslicing and semaphores as well, but they're * handled by the firmware so some minor tweaks are required before * enabling. * - * engine->flags |= I915_ENGINE_HAS_TIMESLICES; * engine->flags |= I915_ENGINE_HAS_SEMAPHORES; */ @@ -666,9 +2472,21 @@ static inline void guc_default_irqs(struct intel_engine_cs *engine) intel_engine_set_irq_handler(engine, cs_irq_handler); } +static void guc_sched_engine_destroy(struct kref *kref) +{ + struct i915_sched_engine *sched_engine = + container_of(kref, typeof(*sched_engine), ref); + struct intel_guc *guc = sched_engine->private_data; + + guc->sched_engine = NULL; + tasklet_kill(&sched_engine->tasklet); /* flush the callback */ + kfree(sched_engine); +} + int intel_guc_submission_setup(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; + struct intel_guc *guc = &engine->gt->uc.guc; /* * The setup relies on several assumptions (e.g. irqs always enabled) @@ -676,10 +2494,28 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine) */ GEM_BUG_ON(GRAPHICS_VER(i915) < 11); - tasklet_setup(&engine->execlists.tasklet, guc_submission_tasklet); + if (!guc->sched_engine) { + guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL); + if (!guc->sched_engine) + return -ENOMEM; + + guc->sched_engine->schedule = i915_schedule; + guc->sched_engine->disabled = guc_sched_engine_disabled; + guc->sched_engine->private_data = guc; + guc->sched_engine->destroy = guc_sched_engine_destroy; + guc->sched_engine->bump_inflight_request_prio = + guc_bump_inflight_request_prio; + guc->sched_engine->retire_inflight_request_prio = + guc_retire_inflight_request_prio; + tasklet_setup(&guc->sched_engine->tasklet, + guc_submission_tasklet); + } + i915_sched_engine_put(engine->sched_engine); + engine->sched_engine = i915_sched_engine_get(guc->sched_engine); guc_default_vfuncs(engine); guc_default_irqs(engine); + guc_init_breadcrumbs(engine); if (engine->class == RENDER_CLASS) rcs_submission_override(engine); @@ -695,18 +2531,19 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine) void intel_guc_submission_enable(struct intel_guc *guc) { - guc_stage_desc_init(guc); + guc_init_lrc_mapping(guc); } void intel_guc_submission_disable(struct intel_guc *guc) { - struct intel_gt *gt = guc_to_gt(guc); - - GEM_BUG_ON(gt->awake); /* GT should be parked first */ - /* Note: By the time we're here, GuC may have already been reset */ +} - guc_stage_desc_fini(guc); +static bool __guc_submission_supported(struct intel_guc *guc) +{ + /* GuC submission is unavailable for pre-Gen11 */ + return intel_guc_is_supported(guc) && + GRAPHICS_VER(guc_to_gt(guc)->i915) >= 11; } static bool __guc_submission_selected(struct intel_guc *guc) @@ -721,5 +2558,481 @@ static bool __guc_submission_selected(struct intel_guc *guc) void intel_guc_submission_init_early(struct intel_guc *guc) { + guc->submission_supported = __guc_submission_supported(guc); guc->submission_selected = __guc_submission_selected(guc); } + +static inline struct intel_context * +g2h_context_lookup(struct intel_guc *guc, u32 desc_idx) +{ + struct intel_context *ce; + + if (unlikely(desc_idx >= GUC_MAX_LRC_DESCRIPTORS)) { + drm_err(&guc_to_gt(guc)->i915->drm, + "Invalid desc_idx %u", desc_idx); + return NULL; + } + + ce = __get_context(guc, desc_idx); + if (unlikely(!ce)) { + drm_err(&guc_to_gt(guc)->i915->drm, + "Context is NULL, desc_idx %u", desc_idx); + return NULL; + } + + return ce; +} + +static void decr_outstanding_submission_g2h(struct intel_guc *guc) +{ + if (atomic_dec_and_test(&guc->outstanding_submission_g2h)) + wake_up_all(&guc->ct.wq); +} + +int intel_guc_deregister_done_process_msg(struct intel_guc *guc, + const u32 *msg, + u32 len) +{ + struct intel_context *ce; + u32 desc_idx = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + return -EPROTO; + } + + ce = g2h_context_lookup(guc, desc_idx); + if (unlikely(!ce)) + return -EPROTO; + + trace_intel_context_deregister_done(ce); + + if (context_wait_for_deregister_to_register(ce)) { + struct intel_runtime_pm *runtime_pm = + &ce->engine->gt->i915->runtime_pm; + intel_wakeref_t wakeref; + + /* + * Previous owner of this guc_id has been deregistered, now safe + * register this context. + */ + with_intel_runtime_pm(runtime_pm, wakeref) + register_context(ce, true); + guc_signal_context_fence(ce); + intel_context_put(ce); + } else if (context_destroyed(ce)) { + /* Context has been destroyed */ + release_guc_id(guc, ce); + __guc_context_destroy(ce); + } + + decr_outstanding_submission_g2h(guc); + + return 0; +} + +int intel_guc_sched_done_process_msg(struct intel_guc *guc, + const u32 *msg, + u32 len) +{ + struct intel_context *ce; + unsigned long flags; + u32 desc_idx = msg[0]; + + if (unlikely(len < 2)) { + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + return -EPROTO; + } + + ce = g2h_context_lookup(guc, desc_idx); + if (unlikely(!ce)) + return -EPROTO; + + if (unlikely(context_destroyed(ce) || + (!context_pending_enable(ce) && + !context_pending_disable(ce)))) { + drm_err(&guc_to_gt(guc)->i915->drm, + "Bad context sched_state 0x%x, 0x%x, desc_idx %u", + atomic_read(&ce->guc_sched_state_no_lock), + ce->guc_state.sched_state, desc_idx); + return -EPROTO; + } + + trace_intel_context_sched_done(ce); + + if (context_pending_enable(ce)) { + clr_context_pending_enable(ce); + } else if (context_pending_disable(ce)) { + bool banned; + + /* + * Unpin must be done before __guc_signal_context_fence, + * otherwise a race exists between the requests getting + * submitted + retired before this unpin completes resulting in + * the pin_count going to zero and the context still being + * enabled. + */ + intel_context_sched_disable_unpin(ce); + + spin_lock_irqsave(&ce->guc_state.lock, flags); + banned = context_banned(ce); + clr_context_banned(ce); + clr_context_pending_disable(ce); + __guc_signal_context_fence(ce); + guc_blocked_fence_complete(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + if (banned) { + guc_cancel_context_requests(ce); + intel_engine_signal_breadcrumbs(ce->engine); + } + } + + decr_outstanding_submission_g2h(guc); + intel_context_put(ce); + + return 0; +} + +static void capture_error_state(struct intel_guc *guc, + struct intel_context *ce) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct drm_i915_private *i915 = gt->i915; + struct intel_engine_cs *engine = __context_to_physical_engine(ce); + intel_wakeref_t wakeref; + + intel_engine_set_hung_context(engine, ce); + with_intel_runtime_pm(&i915->runtime_pm, wakeref) + i915_capture_error_state(gt, engine->mask); + atomic_inc(&i915->gpu_error.reset_engine_count[engine->uabi_class]); +} + +static void guc_context_replay(struct intel_context *ce) +{ + struct i915_sched_engine *sched_engine = ce->engine->sched_engine; + + __guc_reset_context(ce, true); + tasklet_hi_schedule(&sched_engine->tasklet); +} + +static void guc_handle_context_reset(struct intel_guc *guc, + struct intel_context *ce) +{ + trace_intel_context_reset(ce); + + if (likely(!intel_context_is_banned(ce))) { + capture_error_state(guc, ce); + guc_context_replay(ce); + } +} + +int intel_guc_context_reset_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len) +{ + struct intel_context *ce; + int desc_idx; + + if (unlikely(len != 1)) { + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + return -EPROTO; + } + + desc_idx = msg[0]; + ce = g2h_context_lookup(guc, desc_idx); + if (unlikely(!ce)) + return -EPROTO; + + guc_handle_context_reset(guc, ce); + + return 0; +} + +static struct intel_engine_cs * +guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance) +{ + struct intel_gt *gt = guc_to_gt(guc); + u8 engine_class = guc_class_to_engine_class(guc_class); + + /* Class index is checked in class converter */ + GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE); + + return gt->engine_class[engine_class][instance]; +} + +int intel_guc_engine_failure_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len) +{ + struct intel_engine_cs *engine; + u8 guc_class, instance; + u32 reason; + + if (unlikely(len != 3)) { + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + return -EPROTO; + } + + guc_class = msg[0]; + instance = msg[1]; + reason = msg[2]; + + engine = guc_lookup_engine(guc, guc_class, instance); + if (unlikely(!engine)) { + drm_err(&guc_to_gt(guc)->i915->drm, + "Invalid engine %d:%d", guc_class, instance); + return -EPROTO; + } + + intel_gt_handle_error(guc_to_gt(guc), engine->mask, + I915_ERROR_CAPTURE, + "GuC failed to reset %s (reason=0x%08x)\n", + engine->name, reason); + + return 0; +} + +void intel_guc_find_hung_context(struct intel_engine_cs *engine) +{ + struct intel_guc *guc = &engine->gt->uc.guc; + struct intel_context *ce; + struct i915_request *rq; + unsigned long index; + + /* Reset called during driver load? GuC not yet initialised! */ + if (unlikely(!guc_submission_initialized(guc))) + return; + + xa_for_each(&guc->context_lookup, index, ce) { + if (!intel_context_is_pinned(ce)) + continue; + + if (intel_engine_is_virtual(ce->engine)) { + if (!(ce->engine->mask & engine->mask)) + continue; + } else { + if (ce->engine != engine) + continue; + } + + list_for_each_entry(rq, &ce->guc_active.requests, sched.link) { + if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE) + continue; + + intel_engine_set_hung_context(engine, ce); + + /* Can only cope with one hang at a time... */ + return; + } + } +} + +void intel_guc_dump_active_requests(struct intel_engine_cs *engine, + struct i915_request *hung_rq, + struct drm_printer *m) +{ + struct intel_guc *guc = &engine->gt->uc.guc; + struct intel_context *ce; + unsigned long index; + unsigned long flags; + + /* Reset called during driver load? GuC not yet initialised! */ + if (unlikely(!guc_submission_initialized(guc))) + return; + + xa_for_each(&guc->context_lookup, index, ce) { + if (!intel_context_is_pinned(ce)) + continue; + + if (intel_engine_is_virtual(ce->engine)) { + if (!(ce->engine->mask & engine->mask)) + continue; + } else { + if (ce->engine != engine) + continue; + } + + spin_lock_irqsave(&ce->guc_active.lock, flags); + intel_engine_dump_active_requests(&ce->guc_active.requests, + hung_rq, m); + spin_unlock_irqrestore(&ce->guc_active.lock, flags); + } +} + +void intel_guc_submission_print_info(struct intel_guc *guc, + struct drm_printer *p) +{ + struct i915_sched_engine *sched_engine = guc->sched_engine; + struct rb_node *rb; + unsigned long flags; + + if (!sched_engine) + return; + + drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n", + atomic_read(&guc->outstanding_submission_g2h)); + drm_printf(p, "GuC tasklet count: %u\n\n", + atomic_read(&sched_engine->tasklet.count)); + + spin_lock_irqsave(&sched_engine->lock, flags); + drm_printf(p, "Requests in GuC submit tasklet:\n"); + for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) { + struct i915_priolist *pl = to_priolist(rb); + struct i915_request *rq; + + priolist_for_each_request(rq, pl) + drm_printf(p, "guc_id=%u, seqno=%llu\n", + rq->context->guc_id, + rq->fence.seqno); + } + spin_unlock_irqrestore(&sched_engine->lock, flags); + drm_printf(p, "\n"); +} + +static inline void guc_log_context_priority(struct drm_printer *p, + struct intel_context *ce) +{ + int i; + + drm_printf(p, "\t\tPriority: %d\n", + ce->guc_prio); + drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n"); + for (i = GUC_CLIENT_PRIORITY_KMD_HIGH; + i < GUC_CLIENT_PRIORITY_NUM; ++i) { + drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n", + i, ce->guc_prio_count[i]); + } + drm_printf(p, "\n"); +} + +void intel_guc_submission_print_context_info(struct intel_guc *guc, + struct drm_printer *p) +{ + struct intel_context *ce; + unsigned long index; + + xa_for_each(&guc->context_lookup, index, ce) { + drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id); + drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca); + drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n", + ce->ring->head, + ce->lrc_reg_state[CTX_RING_HEAD]); + drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n", + ce->ring->tail, + ce->lrc_reg_state[CTX_RING_TAIL]); + drm_printf(p, "\t\tContext Pin Count: %u\n", + atomic_read(&ce->pin_count)); + drm_printf(p, "\t\tGuC ID Ref Count: %u\n", + atomic_read(&ce->guc_id_ref)); + drm_printf(p, "\t\tSchedule State: 0x%x, 0x%x\n\n", + ce->guc_state.sched_state, + atomic_read(&ce->guc_sched_state_no_lock)); + + guc_log_context_priority(p, ce); + } +} + +static struct intel_context * +guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) +{ + struct guc_virtual_engine *ve; + struct intel_guc *guc; + unsigned int n; + int err; + + ve = kzalloc(sizeof(*ve), GFP_KERNEL); + if (!ve) + return ERR_PTR(-ENOMEM); + + guc = &siblings[0]->gt->uc.guc; + + ve->base.i915 = siblings[0]->i915; + ve->base.gt = siblings[0]->gt; + ve->base.uncore = siblings[0]->uncore; + ve->base.id = -1; + + ve->base.uabi_class = I915_ENGINE_CLASS_INVALID; + ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; + ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; + ve->base.saturated = ALL_ENGINES; + + snprintf(ve->base.name, sizeof(ve->base.name), "virtual"); + + ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine); + + ve->base.cops = &virtual_guc_context_ops; + ve->base.request_alloc = guc_request_alloc; + ve->base.bump_serial = virtual_guc_bump_serial; + + ve->base.submit_request = guc_submit_request; + + ve->base.flags = I915_ENGINE_IS_VIRTUAL; + + intel_context_init(&ve->context, &ve->base); + + for (n = 0; n < count; n++) { + struct intel_engine_cs *sibling = siblings[n]; + + GEM_BUG_ON(!is_power_of_2(sibling->mask)); + if (sibling->mask & ve->base.mask) { + DRM_DEBUG("duplicate %s entry in load balancer\n", + sibling->name); + err = -EINVAL; + goto err_put; + } + + ve->base.mask |= sibling->mask; + + if (n != 0 && ve->base.class != sibling->class) { + DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n", + sibling->class, ve->base.class); + err = -EINVAL; + goto err_put; + } else if (n == 0) { + ve->base.class = sibling->class; + ve->base.uabi_class = sibling->uabi_class; + snprintf(ve->base.name, sizeof(ve->base.name), + "v%dx%d", ve->base.class, count); + ve->base.context_size = sibling->context_size; + + ve->base.add_active_request = + sibling->add_active_request; + ve->base.remove_active_request = + sibling->remove_active_request; + ve->base.emit_bb_start = sibling->emit_bb_start; + ve->base.emit_flush = sibling->emit_flush; + ve->base.emit_init_breadcrumb = + sibling->emit_init_breadcrumb; + ve->base.emit_fini_breadcrumb = + sibling->emit_fini_breadcrumb; + ve->base.emit_fini_breadcrumb_dw = + sibling->emit_fini_breadcrumb_dw; + ve->base.breadcrumbs = + intel_breadcrumbs_get(sibling->breadcrumbs); + + ve->base.flags |= sibling->flags; + + ve->base.props.timeslice_duration_ms = + sibling->props.timeslice_duration_ms; + ve->base.props.preempt_timeout_ms = + sibling->props.preempt_timeout_ms; + } + } + + return &ve->context; + +err_put: + intel_context_put(&ve->context); + return ERR_PTR(err); +} + +bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve) +{ + struct intel_engine_cs *engine; + intel_engine_mask_t tmp, mask = ve->mask; + + for_each_engine_masked(engine, ve->gt, mask, tmp) + if (READ_ONCE(engine->props.heartbeat_interval_ms)) + return true; + + return false; +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h index 3f7005018939..c7ef44fa0c36 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h @@ -10,6 +10,7 @@ #include "intel_guc.h" +struct drm_printer; struct intel_engine_cs; void intel_guc_submission_init_early(struct intel_guc *guc); @@ -20,11 +21,24 @@ void intel_guc_submission_fini(struct intel_guc *guc); int intel_guc_preempt_work_create(struct intel_guc *guc); void intel_guc_preempt_work_destroy(struct intel_guc *guc); int intel_guc_submission_setup(struct intel_engine_cs *engine); +void intel_guc_submission_print_info(struct intel_guc *guc, + struct drm_printer *p); +void intel_guc_submission_print_context_info(struct intel_guc *guc, + struct drm_printer *p); +void intel_guc_dump_active_requests(struct intel_engine_cs *engine, + struct i915_request *hung_rq, + struct drm_printer *m); + +bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve); + +int intel_guc_wait_for_pending_msg(struct intel_guc *guc, + atomic_t *wait_var, + bool interruptible, + long timeout); static inline bool intel_guc_submission_is_supported(struct intel_guc *guc) { - /* XXX: GuC submission is unavailable for now */ - return false; + return guc->submission_supported; } static inline bool intel_guc_submission_is_wanted(struct intel_guc *guc) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index 6d8b9233214e..b104fb7607eb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -34,8 +34,14 @@ static void uc_expand_default_options(struct intel_uc *uc) return; } - /* Default: enable HuC authentication only */ - i915->params.enable_guc = ENABLE_GUC_LOAD_HUC; + /* Intermediate platforms are HuC authentication only */ + if (IS_DG1(i915) || IS_ALDERLAKE_S(i915)) { + i915->params.enable_guc = ENABLE_GUC_LOAD_HUC; + return; + } + + /* Default: enable HuC authentication and GuC submission */ + i915->params.enable_guc = ENABLE_GUC_LOAD_HUC | ENABLE_GUC_SUBMISSION; } /* Reset GuC providing us with fresh state for both GuC and HuC. @@ -69,16 +75,18 @@ static void __confirm_options(struct intel_uc *uc) struct drm_i915_private *i915 = uc_to_gt(uc)->i915; drm_dbg(&i915->drm, - "enable_guc=%d (guc:%s submission:%s huc:%s)\n", + "enable_guc=%d (guc:%s submission:%s huc:%s slpc:%s)\n", i915->params.enable_guc, yesno(intel_uc_wants_guc(uc)), yesno(intel_uc_wants_guc_submission(uc)), - yesno(intel_uc_wants_huc(uc))); + yesno(intel_uc_wants_huc(uc)), + yesno(intel_uc_wants_guc_slpc(uc))); if (i915->params.enable_guc == 0) { GEM_BUG_ON(intel_uc_wants_guc(uc)); GEM_BUG_ON(intel_uc_wants_guc_submission(uc)); GEM_BUG_ON(intel_uc_wants_huc(uc)); + GEM_BUG_ON(intel_uc_wants_guc_slpc(uc)); return; } @@ -120,6 +128,11 @@ void intel_uc_init_early(struct intel_uc *uc) uc->ops = &uc_ops_off; } +void intel_uc_init_late(struct intel_uc *uc) +{ + intel_guc_init_late(&uc->guc); +} + void intel_uc_driver_late_release(struct intel_uc *uc) { } @@ -207,21 +220,6 @@ static void guc_handle_mmio_msg(struct intel_guc *guc) spin_unlock_irq(&guc->irq_lock); } -static void guc_reset_interrupts(struct intel_guc *guc) -{ - guc->interrupts.reset(guc); -} - -static void guc_enable_interrupts(struct intel_guc *guc) -{ - guc->interrupts.enable(guc); -} - -static void guc_disable_interrupts(struct intel_guc *guc) -{ - guc->interrupts.disable(guc); -} - static int guc_enable_communication(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); @@ -242,7 +240,7 @@ static int guc_enable_communication(struct intel_guc *guc) guc_get_mmio_msg(guc); guc_handle_mmio_msg(guc); - guc_enable_interrupts(guc); + intel_guc_enable_interrupts(guc); /* check for CT messages received before we enabled interrupts */ spin_lock_irq(>->irq_lock); @@ -265,7 +263,7 @@ static void guc_disable_communication(struct intel_guc *guc) */ guc_clear_mmio_msg(guc); - guc_disable_interrupts(guc); + intel_guc_disable_interrupts(guc); intel_guc_ct_disable(&guc->ct); @@ -323,9 +321,6 @@ static int __uc_init(struct intel_uc *uc) if (i915_inject_probe_failure(uc_to_gt(uc)->i915)) return -ENOMEM; - /* XXX: GuC submission is unavailable for now */ - GEM_BUG_ON(intel_uc_uses_guc_submission(uc)); - ret = intel_guc_init(guc); if (ret) return ret; @@ -463,7 +458,7 @@ static int __uc_init_hw(struct intel_uc *uc) if (ret) goto err_out; - guc_reset_interrupts(guc); + intel_guc_reset_interrupts(guc); /* WaEnableuKernelHeaderValidFix:skl */ /* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */ @@ -505,12 +500,21 @@ static int __uc_init_hw(struct intel_uc *uc) if (intel_uc_uses_guc_submission(uc)) intel_guc_submission_enable(guc); + if (intel_uc_uses_guc_slpc(uc)) { + ret = intel_guc_slpc_enable(&guc->slpc); + if (ret) + goto err_submission; + } + drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_GUC), guc->fw.path, guc->fw.major_ver_found, guc->fw.minor_ver_found, "submission", enableddisabled(intel_uc_uses_guc_submission(uc))); + drm_info(&i915->drm, "GuC SLPC: %s\n", + enableddisabled(intel_uc_uses_guc_slpc(uc))); + if (intel_uc_uses_huc(uc)) { drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC), @@ -525,6 +529,8 @@ static int __uc_init_hw(struct intel_uc *uc) /* * We've failed to load the firmware :( */ +err_submission: + intel_guc_submission_disable(guc); err_log_capture: __uc_capture_load_err_log(uc); err_out: @@ -565,23 +571,67 @@ void intel_uc_reset_prepare(struct intel_uc *uc) { struct intel_guc *guc = &uc->guc; - if (!intel_guc_is_ready(guc)) + uc->reset_in_progress = true; + + /* Nothing to do if GuC isn't supported */ + if (!intel_uc_supports_guc(uc)) return; + /* Firmware expected to be running when this function is called */ + if (!intel_guc_is_ready(guc)) + goto sanitize; + + if (intel_uc_uses_guc_submission(uc)) + intel_guc_submission_reset_prepare(guc); + +sanitize: __uc_sanitize(uc); } +void intel_uc_reset(struct intel_uc *uc, bool stalled) +{ + struct intel_guc *guc = &uc->guc; + + /* Firmware can not be running when this function is called */ + if (intel_uc_uses_guc_submission(uc)) + intel_guc_submission_reset(guc, stalled); +} + +void intel_uc_reset_finish(struct intel_uc *uc) +{ + struct intel_guc *guc = &uc->guc; + + uc->reset_in_progress = false; + + /* Firmware expected to be running when this function is called */ + if (intel_guc_is_fw_running(guc) && intel_uc_uses_guc_submission(uc)) + intel_guc_submission_reset_finish(guc); +} + +void intel_uc_cancel_requests(struct intel_uc *uc) +{ + struct intel_guc *guc = &uc->guc; + + /* Firmware can not be running when this function is called */ + if (intel_uc_uses_guc_submission(uc)) + intel_guc_submission_cancel_requests(guc); +} + void intel_uc_runtime_suspend(struct intel_uc *uc) { struct intel_guc *guc = &uc->guc; - int err; if (!intel_guc_is_ready(guc)) return; - err = intel_guc_suspend(guc); - if (err) - DRM_DEBUG_DRIVER("Failed to suspend GuC, err=%d", err); + /* + * Wait for any outstanding CTB before tearing down communication /w the + * GuC. + */ +#define OUTSTANDING_CTB_TIMEOUT_PERIOD (HZ / 5) + intel_guc_wait_for_pending_msg(guc, &guc->outstanding_submission_g2h, + false, OUTSTANDING_CTB_TIMEOUT_PERIOD); + GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h)); guc_disable_communication(guc); } @@ -590,17 +640,22 @@ void intel_uc_suspend(struct intel_uc *uc) { struct intel_guc *guc = &uc->guc; intel_wakeref_t wakeref; + int err; if (!intel_guc_is_ready(guc)) return; - with_intel_runtime_pm(uc_to_gt(uc)->uncore->rpm, wakeref) - intel_uc_runtime_suspend(uc); + with_intel_runtime_pm(&uc_to_gt(uc)->i915->runtime_pm, wakeref) { + err = intel_guc_suspend(guc); + if (err) + DRM_DEBUG_DRIVER("Failed to suspend GuC, err=%d", err); + } } static int __uc_resume(struct intel_uc *uc, bool enable_communication) { struct intel_guc *guc = &uc->guc; + struct intel_gt *gt = guc_to_gt(guc); int err; if (!intel_guc_is_fw_running(guc)) @@ -612,6 +667,13 @@ static int __uc_resume(struct intel_uc *uc, bool enable_communication) if (enable_communication) guc_enable_communication(guc); + /* If we are only resuming GuC communication but not reloading + * GuC, we need to ensure the ARAT timer interrupt is enabled + * again. In case of GuC reload, it is enabled during SLPC enable. + */ + if (enable_communication && intel_uc_uses_guc_slpc(uc)) + intel_guc_pm_intrmsk_enable(gt); + err = intel_guc_resume(guc); if (err) { DRM_DEBUG_DRIVER("Failed to resume GuC, err=%d", err); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.h b/drivers/gpu/drm/i915/gt/uc/intel_uc.h index 9c954c589edf..866b462821c0 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.h @@ -7,7 +7,9 @@ #define _INTEL_UC_H_ #include "intel_guc.h" +#include "intel_guc_rc.h" #include "intel_guc_submission.h" +#include "intel_guc_slpc.h" #include "intel_huc.h" #include "i915_params.h" @@ -30,13 +32,19 @@ struct intel_uc { /* Snapshot of GuC log from last failed load */ struct drm_i915_gem_object *load_err_log; + + bool reset_in_progress; }; void intel_uc_init_early(struct intel_uc *uc); +void intel_uc_init_late(struct intel_uc *uc); void intel_uc_driver_late_release(struct intel_uc *uc); void intel_uc_driver_remove(struct intel_uc *uc); void intel_uc_init_mmio(struct intel_uc *uc); void intel_uc_reset_prepare(struct intel_uc *uc); +void intel_uc_reset(struct intel_uc *uc, bool stalled); +void intel_uc_reset_finish(struct intel_uc *uc); +void intel_uc_cancel_requests(struct intel_uc *uc); void intel_uc_suspend(struct intel_uc *uc); void intel_uc_runtime_suspend(struct intel_uc *uc); int intel_uc_resume(struct intel_uc *uc); @@ -77,10 +85,17 @@ __uc_state_checker(x, func, uses, used) uc_state_checkers(guc, guc); uc_state_checkers(huc, huc); uc_state_checkers(guc, guc_submission); +uc_state_checkers(guc, guc_slpc); +uc_state_checkers(guc, guc_rc); #undef uc_state_checkers #undef __uc_state_checker +static inline int intel_uc_wait_for_idle(struct intel_uc *uc, long timeout) +{ + return intel_guc_wait_for_idle(&uc->guc, timeout); +} + #define intel_uc_ops_function(_NAME, _OPS, _TYPE, _RET) \ static inline _TYPE intel_uc_##_NAME(struct intel_uc *uc) \ { \ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index df647c9a8d56..3a16d08608a5 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -48,19 +48,20 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * firmware as TGL. */ #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ - fw_def(ALDERLAKE_S, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ - fw_def(ROCKETLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ - fw_def(JASPERLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ - fw_def(ELKHARTLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ - fw_def(ICELAKE, 0, guc_def(icl, 49, 0, 1), huc_def(icl, 9, 0, 0)) \ - fw_def(COMETLAKE, 5, guc_def(cml, 49, 0, 1), huc_def(cml, 4, 0, 0)) \ - fw_def(COMETLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ - fw_def(COFFEELAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ - fw_def(GEMINILAKE, 0, guc_def(glk, 49, 0, 1), huc_def(glk, 4, 0, 0)) \ - fw_def(KABYLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ - fw_def(BROXTON, 0, guc_def(bxt, 49, 0, 1), huc_def(bxt, 2, 0, 0)) \ - fw_def(SKYLAKE, 0, guc_def(skl, 49, 0, 1), huc_def(skl, 2, 0, 0)) + fw_def(ALDERLAKE_P, 0, guc_def(adlp, 62, 0, 3), huc_def(tgl, 7, 9, 3)) \ + fw_def(ALDERLAKE_S, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ + fw_def(ROCKETLAKE, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ + fw_def(JASPERLAKE, 0, guc_def(ehl, 62, 0, 0), huc_def(ehl, 9, 0, 0)) \ + fw_def(ELKHARTLAKE, 0, guc_def(ehl, 62, 0, 0), huc_def(ehl, 9, 0, 0)) \ + fw_def(ICELAKE, 0, guc_def(icl, 62, 0, 0), huc_def(icl, 9, 0, 0)) \ + fw_def(COMETLAKE, 5, guc_def(cml, 62, 0, 0), huc_def(cml, 4, 0, 0)) \ + fw_def(COMETLAKE, 0, guc_def(kbl, 62, 0, 0), huc_def(kbl, 4, 0, 0)) \ + fw_def(COFFEELAKE, 0, guc_def(kbl, 62, 0, 0), huc_def(kbl, 4, 0, 0)) \ + fw_def(GEMINILAKE, 0, guc_def(glk, 62, 0, 0), huc_def(glk, 4, 0, 0)) \ + fw_def(KABYLAKE, 0, guc_def(kbl, 62, 0, 0), huc_def(kbl, 4, 0, 0)) \ + fw_def(BROXTON, 0, guc_def(bxt, 62, 0, 0), huc_def(bxt, 2, 0, 0)) \ + fw_def(SKYLAKE, 0, guc_def(skl, 62, 0, 0), huc_def(skl, 2, 0, 0)) #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ "i915/" \ diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index cc2c05e18206..e5c2fdfc20e3 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -1055,7 +1055,7 @@ static bool vgpu_ips_enabled(struct intel_vgpu *vgpu) { struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; - if (GRAPHICS_VER(dev_priv) == 9 || GRAPHICS_VER(dev_priv) == 10) { + if (GRAPHICS_VER(dev_priv) == 9) { u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) & GAMW_ECO_ENABLE_64K_IPS_FIELD; diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 06024d321a1a..cde0a477fb49 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -3149,6 +3149,7 @@ static int init_bdw_mmio_info(struct intel_gvt *gvt) MMIO_DFH(_MMIO(0xb100), D_BDW, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0xb10c), D_BDW, F_CMD_ACCESS, NULL, NULL); MMIO_D(_MMIO(0xb110), D_BDW); + MMIO_D(GEN9_SCRATCH_LNCF1, D_BDW_PLUS); MMIO_F(_MMIO(0x24d0), 48, F_CMD_ACCESS | F_CMD_WRITE_PATCH, 0, 0, D_BDW_PLUS, NULL, force_nonpriv_write); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 1ac98f8aba31..7efa386449d1 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -885,7 +885,7 @@ static int intel_vgpu_group_notifier(struct notifier_block *nb, return NOTIFY_OK; } -static int intel_vgpu_open(struct mdev_device *mdev) +static int intel_vgpu_open_device(struct mdev_device *mdev) { struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu); @@ -1004,7 +1004,7 @@ static void __intel_vgpu_release(struct intel_vgpu *vgpu) vgpu->handle = 0; } -static void intel_vgpu_release(struct mdev_device *mdev) +static void intel_vgpu_close_device(struct mdev_device *mdev) { struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); @@ -1753,8 +1753,8 @@ static struct mdev_parent_ops intel_vgpu_ops = { .create = intel_vgpu_create, .remove = intel_vgpu_remove, - .open = intel_vgpu_open, - .release = intel_vgpu_release, + .open_device = intel_vgpu_open_device, + .close_device = intel_vgpu_close_device, .read = intel_vgpu_read, .write = intel_vgpu_write, diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index b8ac80765461..f776c470914d 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -105,6 +105,8 @@ static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { {RCS0, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */ {RCS0, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */ {RCS0, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */ + {RCS0, GEN9_SCRATCH1, 0, false}, /* 0xb11c */ + {RCS0, GEN9_SCRATCH_LNCF1, 0, false}, /* 0xb008 */ {RCS0, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */ {RCS0, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */ {RCS0, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */ diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 734c37c5e347..b56a8e37a3cd 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -1409,11 +1409,8 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) intel_context_set_single_submission(ce); /* Max ring buffer size */ - if (!intel_uc_wants_guc_submission(&engine->gt->uc)) { - const unsigned int ring_size = 512 * SZ_4K; - - ce->ring = __intel_context_ring_size(ring_size); - } + if (!intel_uc_wants_guc_submission(&engine->gt->uc)) + ce->ring_size = SZ_2M; s->shadow[i] = ce; } diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index b1aa1c482c32..3103c1e1fd14 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -13,7 +13,6 @@ #include "i915_drv.h" #include "i915_active.h" -#include "i915_globals.h" /* * Active refs memory management @@ -22,10 +21,7 @@ * they idle (when we know the active requests are inactive) and allocate the * nodes from a local slab cache to hopefully reduce the fragmentation. */ -static struct i915_global_active { - struct i915_global base; - struct kmem_cache *slab_cache; -} global; +static struct kmem_cache *slab_cache; struct active_node { struct rb_node node; @@ -174,7 +170,7 @@ __active_retire(struct i915_active *ref) /* Finally free the discarded timeline tree */ rbtree_postorder_for_each_entry_safe(it, n, &root, node) { GEM_BUG_ON(i915_active_fence_isset(&it->base)); - kmem_cache_free(global.slab_cache, it); + kmem_cache_free(slab_cache, it); } } @@ -322,7 +318,7 @@ active_instance(struct i915_active *ref, u64 idx) * XXX: We should preallocate this before i915_active_ref() is ever * called, but we cannot call into fs_reclaim() anyway, so use GFP_ATOMIC. */ - node = kmem_cache_alloc(global.slab_cache, GFP_ATOMIC); + node = kmem_cache_alloc(slab_cache, GFP_ATOMIC); if (!node) goto out; @@ -788,7 +784,7 @@ void i915_active_fini(struct i915_active *ref) mutex_destroy(&ref->mutex); if (ref->cache) - kmem_cache_free(global.slab_cache, ref->cache); + kmem_cache_free(slab_cache, ref->cache); } static inline bool is_idle_barrier(struct active_node *node, u64 idx) @@ -908,7 +904,7 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, node = reuse_idle_barrier(ref, idx); rcu_read_unlock(); if (!node) { - node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL); + node = kmem_cache_alloc(slab_cache, GFP_KERNEL); if (!node) goto unwind; @@ -956,7 +952,7 @@ unwind: atomic_dec(&ref->count); intel_engine_pm_put(barrier_to_engine(node)); - kmem_cache_free(global.slab_cache, node); + kmem_cache_free(slab_cache, node); } return -ENOMEM; } @@ -1176,27 +1172,16 @@ struct i915_active *i915_active_create(void) #include "selftests/i915_active.c" #endif -static void i915_global_active_shrink(void) +void i915_active_module_exit(void) { - kmem_cache_shrink(global.slab_cache); + kmem_cache_destroy(slab_cache); } -static void i915_global_active_exit(void) +int __init i915_active_module_init(void) { - kmem_cache_destroy(global.slab_cache); -} - -static struct i915_global_active global = { { - .shrink = i915_global_active_shrink, - .exit = i915_global_active_exit, -} }; - -int __init i915_global_active_init(void) -{ - global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN); - if (!global.slab_cache) + slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN); + if (!slab_cache) return -ENOMEM; - i915_global_register(&global.base); return 0; } diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h index d0feda68b874..5fcdb0e2bc9e 100644 --- a/drivers/gpu/drm/i915/i915_active.h +++ b/drivers/gpu/drm/i915/i915_active.h @@ -247,4 +247,7 @@ static inline int __i915_request_await_exclusive(struct i915_request *rq, return err; } +void i915_active_module_exit(void); +int i915_active_module_init(void); + #endif /* _I915_ACTIVE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c new file mode 100644 index 000000000000..7b274c51cac0 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_buddy.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/kmemleak.h> + +#include "i915_buddy.h" + +#include "i915_gem.h" +#include "i915_utils.h" + +static struct kmem_cache *slab_blocks; + +static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm, + struct i915_buddy_block *parent, + unsigned int order, + u64 offset) +{ + struct i915_buddy_block *block; + + GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER); + + block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL); + if (!block) + return NULL; + + block->header = offset; + block->header |= order; + block->parent = parent; + + GEM_BUG_ON(block->header & I915_BUDDY_HEADER_UNUSED); + return block; +} + +static void i915_block_free(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + kmem_cache_free(slab_blocks, block); +} + +static void mark_allocated(struct i915_buddy_block *block) +{ + block->header &= ~I915_BUDDY_HEADER_STATE; + block->header |= I915_BUDDY_ALLOCATED; + + list_del(&block->link); +} + +static void mark_free(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + block->header &= ~I915_BUDDY_HEADER_STATE; + block->header |= I915_BUDDY_FREE; + + list_add(&block->link, + &mm->free_list[i915_buddy_block_order(block)]); +} + +static void mark_split(struct i915_buddy_block *block) +{ + block->header &= ~I915_BUDDY_HEADER_STATE; + block->header |= I915_BUDDY_SPLIT; + + list_del(&block->link); +} + +int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size) +{ + unsigned int i; + u64 offset; + + if (size < chunk_size) + return -EINVAL; + + if (chunk_size < PAGE_SIZE) + return -EINVAL; + + if (!is_power_of_2(chunk_size)) + return -EINVAL; + + size = round_down(size, chunk_size); + + mm->size = size; + mm->chunk_size = chunk_size; + mm->max_order = ilog2(size) - ilog2(chunk_size); + + GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER); + + mm->free_list = kmalloc_array(mm->max_order + 1, + sizeof(struct list_head), + GFP_KERNEL); + if (!mm->free_list) + return -ENOMEM; + + for (i = 0; i <= mm->max_order; ++i) + INIT_LIST_HEAD(&mm->free_list[i]); + + mm->n_roots = hweight64(size); + + mm->roots = kmalloc_array(mm->n_roots, + sizeof(struct i915_buddy_block *), + GFP_KERNEL); + if (!mm->roots) + goto out_free_list; + + offset = 0; + i = 0; + + /* + * Split into power-of-two blocks, in case we are given a size that is + * not itself a power-of-two. + */ + do { + struct i915_buddy_block *root; + unsigned int order; + u64 root_size; + + root_size = rounddown_pow_of_two(size); + order = ilog2(root_size) - ilog2(chunk_size); + + root = i915_block_alloc(mm, NULL, order, offset); + if (!root) + goto out_free_roots; + + mark_free(mm, root); + + GEM_BUG_ON(i > mm->max_order); + GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size); + + mm->roots[i] = root; + + offset += root_size; + size -= root_size; + i++; + } while (size); + + return 0; + +out_free_roots: + while (i--) + i915_block_free(mm, mm->roots[i]); + kfree(mm->roots); +out_free_list: + kfree(mm->free_list); + return -ENOMEM; +} + +void i915_buddy_fini(struct i915_buddy_mm *mm) +{ + int i; + + for (i = 0; i < mm->n_roots; ++i) { + GEM_WARN_ON(!i915_buddy_block_is_free(mm->roots[i])); + i915_block_free(mm, mm->roots[i]); + } + + kfree(mm->roots); + kfree(mm->free_list); +} + +static int split_block(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + unsigned int block_order = i915_buddy_block_order(block) - 1; + u64 offset = i915_buddy_block_offset(block); + + GEM_BUG_ON(!i915_buddy_block_is_free(block)); + GEM_BUG_ON(!i915_buddy_block_order(block)); + + block->left = i915_block_alloc(mm, block, block_order, offset); + if (!block->left) + return -ENOMEM; + + block->right = i915_block_alloc(mm, block, block_order, + offset + (mm->chunk_size << block_order)); + if (!block->right) { + i915_block_free(mm, block->left); + return -ENOMEM; + } + + mark_free(mm, block->left); + mark_free(mm, block->right); + + mark_split(block); + + return 0; +} + +static struct i915_buddy_block * +get_buddy(struct i915_buddy_block *block) +{ + struct i915_buddy_block *parent; + + parent = block->parent; + if (!parent) + return NULL; + + if (parent->left == block) + return parent->right; + + return parent->left; +} + +static void __i915_buddy_free(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + struct i915_buddy_block *parent; + + while ((parent = block->parent)) { + struct i915_buddy_block *buddy; + + buddy = get_buddy(block); + + if (!i915_buddy_block_is_free(buddy)) + break; + + list_del(&buddy->link); + + i915_block_free(mm, block); + i915_block_free(mm, buddy); + + block = parent; + } + + mark_free(mm, block); +} + +void i915_buddy_free(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + GEM_BUG_ON(!i915_buddy_block_is_allocated(block)); + __i915_buddy_free(mm, block); +} + +void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects) +{ + struct i915_buddy_block *block, *on; + + list_for_each_entry_safe(block, on, objects, link) { + i915_buddy_free(mm, block); + cond_resched(); + } + INIT_LIST_HEAD(objects); +} + +/* + * Allocate power-of-two block. The order value here translates to: + * + * 0 = 2^0 * mm->chunk_size + * 1 = 2^1 * mm->chunk_size + * 2 = 2^2 * mm->chunk_size + * ... + */ +struct i915_buddy_block * +i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order) +{ + struct i915_buddy_block *block = NULL; + unsigned int i; + int err; + + for (i = order; i <= mm->max_order; ++i) { + block = list_first_entry_or_null(&mm->free_list[i], + struct i915_buddy_block, + link); + if (block) + break; + } + + if (!block) + return ERR_PTR(-ENOSPC); + + GEM_BUG_ON(!i915_buddy_block_is_free(block)); + + while (i != order) { + err = split_block(mm, block); + if (unlikely(err)) + goto out_free; + + /* Go low */ + block = block->left; + i--; + } + + mark_allocated(block); + kmemleak_update_trace(block); + return block; + +out_free: + if (i != order) + __i915_buddy_free(mm, block); + return ERR_PTR(err); +} + +static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= e2 && e1 >= s2; +} + +static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= s2 && e1 >= e2; +} + +/* + * Allocate range. Note that it's safe to chain together multiple alloc_ranges + * with the same blocks list. + * + * Intended for pre-allocating portions of the address space, for example to + * reserve a block for the initial framebuffer or similar, hence the expectation + * here is that i915_buddy_alloc() is still the main vehicle for + * allocations, so if that's not the case then the drm_mm range allocator is + * probably a much better fit, and so you should probably go use that instead. + */ +int i915_buddy_alloc_range(struct i915_buddy_mm *mm, + struct list_head *blocks, + u64 start, u64 size) +{ + struct i915_buddy_block *block; + struct i915_buddy_block *buddy; + LIST_HEAD(allocated); + LIST_HEAD(dfs); + u64 end; + int err; + int i; + + if (size < mm->chunk_size) + return -EINVAL; + + if (!IS_ALIGNED(size | start, mm->chunk_size)) + return -EINVAL; + + if (range_overflows(start, size, mm->size)) + return -EINVAL; + + for (i = 0; i < mm->n_roots; ++i) + list_add_tail(&mm->roots[i]->tmp_link, &dfs); + + end = start + size - 1; + + do { + u64 block_start; + u64 block_end; + + block = list_first_entry_or_null(&dfs, + struct i915_buddy_block, + tmp_link); + if (!block) + break; + + list_del(&block->tmp_link); + + block_start = i915_buddy_block_offset(block); + block_end = block_start + i915_buddy_block_size(mm, block) - 1; + + if (!overlaps(start, end, block_start, block_end)) + continue; + + if (i915_buddy_block_is_allocated(block)) { + err = -ENOSPC; + goto err_free; + } + + if (contains(start, end, block_start, block_end)) { + if (!i915_buddy_block_is_free(block)) { + err = -ENOSPC; + goto err_free; + } + + mark_allocated(block); + list_add_tail(&block->link, &allocated); + continue; + } + + if (!i915_buddy_block_is_split(block)) { + err = split_block(mm, block); + if (unlikely(err)) + goto err_undo; + } + + list_add(&block->right->tmp_link, &dfs); + list_add(&block->left->tmp_link, &dfs); + } while (1); + + list_splice_tail(&allocated, blocks); + return 0; + +err_undo: + /* + * We really don't want to leave around a bunch of split blocks, since + * bigger is better, so make sure we merge everything back before we + * free the allocated blocks. + */ + buddy = get_buddy(block); + if (buddy && + (i915_buddy_block_is_free(block) && + i915_buddy_block_is_free(buddy))) + __i915_buddy_free(mm, block); + +err_free: + i915_buddy_free_list(mm, &allocated); + return err; +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftests/i915_buddy.c" +#endif + +void i915_buddy_module_exit(void) +{ + kmem_cache_destroy(slab_blocks); +} + +int __init i915_buddy_module_init(void) +{ + slab_blocks = KMEM_CACHE(i915_buddy_block, 0); + if (!slab_blocks) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h new file mode 100644 index 000000000000..3940d632f208 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_buddy.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_BUDDY_H__ +#define __I915_BUDDY_H__ + +#include <linux/bitops.h> +#include <linux/list.h> +#include <linux/slab.h> + +struct i915_buddy_block { +#define I915_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) +#define I915_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) +#define I915_BUDDY_ALLOCATED (1 << 10) +#define I915_BUDDY_FREE (2 << 10) +#define I915_BUDDY_SPLIT (3 << 10) +/* Free to be used, if needed in the future */ +#define I915_BUDDY_HEADER_UNUSED GENMASK_ULL(9, 6) +#define I915_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0) + u64 header; + + struct i915_buddy_block *left; + struct i915_buddy_block *right; + struct i915_buddy_block *parent; + + void *private; /* owned by creator */ + + /* + * While the block is allocated by the user through i915_buddy_alloc*, + * the user has ownership of the link, for example to maintain within + * a list, if so desired. As soon as the block is freed with + * i915_buddy_free* ownership is given back to the mm. + */ + struct list_head link; + struct list_head tmp_link; +}; + +/* Order-zero must be at least PAGE_SIZE */ +#define I915_BUDDY_MAX_ORDER (63 - PAGE_SHIFT) + +/* + * Binary Buddy System. + * + * Locking should be handled by the user, a simple mutex around + * i915_buddy_alloc* and i915_buddy_free* should suffice. + */ +struct i915_buddy_mm { + /* Maintain a free list for each order. */ + struct list_head *free_list; + + /* + * Maintain explicit binary tree(s) to track the allocation of the + * address space. This gives us a simple way of finding a buddy block + * and performing the potentially recursive merge step when freeing a + * block. Nodes are either allocated or free, in which case they will + * also exist on the respective free list. + */ + struct i915_buddy_block **roots; + + /* + * Anything from here is public, and remains static for the lifetime of + * the mm. Everything above is considered do-not-touch. + */ + unsigned int n_roots; + unsigned int max_order; + + /* Must be at least PAGE_SIZE */ + u64 chunk_size; + u64 size; +}; + +static inline u64 +i915_buddy_block_offset(struct i915_buddy_block *block) +{ + return block->header & I915_BUDDY_HEADER_OFFSET; +} + +static inline unsigned int +i915_buddy_block_order(struct i915_buddy_block *block) +{ + return block->header & I915_BUDDY_HEADER_ORDER; +} + +static inline unsigned int +i915_buddy_block_state(struct i915_buddy_block *block) +{ + return block->header & I915_BUDDY_HEADER_STATE; +} + +static inline bool +i915_buddy_block_is_allocated(struct i915_buddy_block *block) +{ + return i915_buddy_block_state(block) == I915_BUDDY_ALLOCATED; +} + +static inline bool +i915_buddy_block_is_free(struct i915_buddy_block *block) +{ + return i915_buddy_block_state(block) == I915_BUDDY_FREE; +} + +static inline bool +i915_buddy_block_is_split(struct i915_buddy_block *block) +{ + return i915_buddy_block_state(block) == I915_BUDDY_SPLIT; +} + +static inline u64 +i915_buddy_block_size(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + return mm->chunk_size << i915_buddy_block_order(block); +} + +int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size); + +void i915_buddy_fini(struct i915_buddy_mm *mm); + +struct i915_buddy_block * +i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order); + +int i915_buddy_alloc_range(struct i915_buddy_mm *mm, + struct list_head *blocks, + u64 start, u64 size); + +void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block); + +void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects); + +void i915_buddy_module_exit(void); +int i915_buddy_module_init(void); + +#endif diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index a3b4d99d64b9..e0403ce9ce69 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1468,42 +1468,43 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, * space. Parsing should be faster in some cases this way. */ batch_end = cmd + batch_length / sizeof(*batch_end); - while (*cmd != MI_BATCH_BUFFER_END) { - u32 length = 1; - - if (*cmd != MI_NOOP) { /* MI_NOOP == 0 */ - desc = find_cmd(engine, *cmd, desc, &default_desc); - if (!desc) { - DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd); - ret = -EINVAL; - break; - } + do { + u32 length; - if (desc->flags & CMD_DESC_FIXED) - length = desc->length.fixed; - else - length = (*cmd & desc->length.mask) + LENGTH_BIAS; + if (*cmd == MI_BATCH_BUFFER_END) + break; - if ((batch_end - cmd) < length) { - DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", - *cmd, - length, - batch_end - cmd); - ret = -EINVAL; - break; - } + desc = find_cmd(engine, *cmd, desc, &default_desc); + if (!desc) { + DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd); + ret = -EINVAL; + break; + } - if (!check_cmd(engine, desc, cmd, length)) { - ret = -EACCES; - break; - } + if (desc->flags & CMD_DESC_FIXED) + length = desc->length.fixed; + else + length = (*cmd & desc->length.mask) + LENGTH_BIAS; - if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { - ret = check_bbstart(cmd, offset, length, batch_length, - batch_addr, shadow_addr, - jump_whitelist); - break; - } + if ((batch_end - cmd) < length) { + DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", + *cmd, + length, + batch_end - cmd); + ret = -EINVAL; + break; + } + + if (!check_cmd(engine, desc, cmd, length)) { + ret = -EACCES; + break; + } + + if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { + ret = check_bbstart(cmd, offset, length, batch_length, + batch_addr, shadow_addr, + jump_whitelist); + break; } if (!IS_ERR_OR_NULL(jump_whitelist)) @@ -1516,7 +1517,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, ret = -EINVAL; break; } - } + } while (1); if (trampoline) { /* diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 0529576f069c..44969f5dde50 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -538,20 +538,20 @@ static int i915_frequency_info(struct seq_file *m, void *unused) max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 : rp_state_cap >> 16) & 0xff; max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : rp_state_cap >> 0) & 0xff; max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); seq_printf(m, "Max overclocked frequency: %dMHz\n", diff --git a/drivers/gpu/drm/i915/i915_debugfs_params.c b/drivers/gpu/drm/i915/i915_debugfs_params.c index 4e2b077692cb..20424275d41e 100644 --- a/drivers/gpu/drm/i915/i915_debugfs_params.c +++ b/drivers/gpu/drm/i915/i915_debugfs_params.c @@ -6,9 +6,21 @@ #include <linux/kernel.h> #include "i915_debugfs_params.h" +#include "gt/intel_gt.h" +#include "gt/uc/intel_guc.h" #include "i915_drv.h" #include "i915_params.h" +#define MATCH_DEBUGFS_NODE_NAME(_file, _name) \ + (strcmp((_file)->f_path.dentry->d_name.name, (_name)) == 0) + +#define GET_I915(i915, name, ptr) \ + do { \ + struct i915_params *params; \ + params = container_of(((void *)(ptr)), typeof(*params), name); \ + (i915) = container_of(params, typeof(*(i915)), params); \ + } while (0) + /* int param */ static int i915_param_int_show(struct seq_file *m, void *data) { @@ -24,6 +36,16 @@ static int i915_param_int_open(struct inode *inode, struct file *file) return single_open(file, i915_param_int_show, inode->i_private); } +static int notify_guc(struct drm_i915_private *i915) +{ + int ret = 0; + + if (intel_uc_uses_guc_submission(&i915->gt.uc)) + ret = intel_guc_global_policies_update(&i915->gt.uc.guc); + + return ret; +} + static ssize_t i915_param_int_write(struct file *file, const char __user *ubuf, size_t len, loff_t *offp) @@ -81,8 +103,10 @@ static ssize_t i915_param_uint_write(struct file *file, const char __user *ubuf, size_t len, loff_t *offp) { + struct drm_i915_private *i915; struct seq_file *m = file->private_data; unsigned int *value = m->private; + unsigned int old = *value; int ret; ret = kstrtouint_from_user(ubuf, len, 0, value); @@ -95,6 +119,14 @@ static ssize_t i915_param_uint_write(struct file *file, *value = b; } + if (!ret && MATCH_DEBUGFS_NODE_NAME(file, "reset")) { + GET_I915(i915, reset, value); + + ret = notify_guc(i915); + if (ret) + *value = old; + } + return ret ?: len; } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 30d8cd8c69b1..59fb4c710c8c 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -271,10 +271,11 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) bool pre = false; pre |= IS_HSW_EARLY_SDV(dev_priv); - pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); - pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); - pre |= IS_KBL_GT_STEP(dev_priv, 0, STEP_A0); - pre |= IS_GLK_REVID(dev_priv, 0, GLK_REVID_A2); + pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6; + pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA; + pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1; + pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3; + pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7; if (pre) { drm_err(&dev_priv->drm, "This is a pre-production stepping. " @@ -619,7 +620,9 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) intel_opregion_setup(dev_priv); - intel_pcode_init(dev_priv); + ret = intel_pcode_init(dev_priv); + if (ret) + goto err_msi; /* * Fill the dram structure to get the system dram info. This will be @@ -1230,6 +1233,10 @@ static int i915_drm_resume(struct drm_device *dev) disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); + ret = intel_pcode_init(dev_priv); + if (ret) + return ret; + sanitize_gpu(dev_priv); ret = i915_ggtt_enable_hw(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 997fbe9532c1..005b1cec7007 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -202,6 +202,68 @@ struct drm_i915_file_private { struct rcu_head rcu; }; + /** @proto_context_lock: Guards all struct i915_gem_proto_context + * operations + * + * This not only guards @proto_context_xa, but is always held + * whenever we manipulate any struct i915_gem_proto_context, + * including finalizing it on first actual use of the GEM context. + * + * See i915_gem_proto_context. + */ + struct mutex proto_context_lock; + + /** @proto_context_xa: xarray of struct i915_gem_proto_context + * + * Historically, the context uAPI allowed for two methods of + * setting context parameters: SET_CONTEXT_PARAM and + * CONTEXT_CREATE_EXT_SETPARAM. The former is allowed to be called + * at any time while the later happens as part of + * GEM_CONTEXT_CREATE. Everything settable via one was settable + * via the other. While some params are fairly simple and setting + * them on a live context is harmless such as the context priority, + * others are far trickier such as the VM or the set of engines. + * In order to swap out the VM, for instance, we have to delay + * until all current in-flight work is complete, swap in the new + * VM, and then continue. This leads to a plethora of potential + * race conditions we'd really rather avoid. + * + * We have since disallowed setting these more complex parameters + * on active contexts. This works by delaying the creation of the + * actual context until after the client is done configuring it + * with SET_CONTEXT_PARAM. From the perspective of the client, it + * has the same u32 context ID the whole time. From the + * perspective of i915, however, it's a struct i915_gem_proto_context + * right up until the point where we attempt to do something which + * the proto-context can't handle. Then the struct i915_gem_context + * gets created. + * + * This is accomplished via a little xarray dance. When + * GEM_CONTEXT_CREATE is called, we create a struct + * i915_gem_proto_context, reserve a slot in @context_xa but leave + * it NULL, and place the proto-context in the corresponding slot + * in @proto_context_xa. Then, in i915_gem_context_lookup(), we + * first check @context_xa. If it's there, we return the struct + * i915_gem_context and we're done. If it's not, we look in + * @proto_context_xa and, if we find it there, we create the actual + * context and kill the proto-context. + * + * In order for this dance to work properly, everything which ever + * touches a struct i915_gem_proto_context is guarded by + * @proto_context_lock, including context creation. Yes, this + * means context creation now takes a giant global lock but it + * can't really be helped and that should never be on any driver's + * fast-path anyway. + */ + struct xarray proto_context_xa; + + /** @context_xa: xarray of fully created i915_gem_context + * + * Write access to this xarray is guarded by @proto_context_lock. + * Otherwise, writers may race with finalize_create_context_locked(). + * + * See @proto_context_xa. + */ struct xarray context_xa; struct xarray vm_xa; @@ -332,15 +394,6 @@ struct drm_i915_display_funcs { void (*read_luts)(struct intel_crtc_state *crtc_state); }; -enum i915_cache_level { - I915_CACHE_NONE = 0, - I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */ - I915_CACHE_L3_LLC, /* gen7+, L3 sits between the domain specifc - caches, eg sampler/render caches, and the - large Last-Level-Cache. LLC is coherent with - the CPU, but L3 is only visible to the GPU. */ - I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */ -}; #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ @@ -556,7 +609,7 @@ struct i915_gem_mm { * notifier_lock for mmu notifiers, memory may not be allocated * while holding this lock. */ - spinlock_t notifier_lock; + rwlock_t notifier_lock; #endif /* shrinker accounting, also useful for userland debugging */ @@ -580,6 +633,9 @@ i915_fence_timeout(const struct drm_i915_private *i915) #define HAS_HW_SAGV_WM(i915) (DISPLAY_VER(i915) >= 13 && !IS_DGFX(i915)) +/* Amount of PSF GV points, BSpec precisely defines this */ +#define I915_NUM_PSF_GV_POINTS 3 + struct ddi_vbt_port_info { /* Non-NULL if port present. */ struct intel_bios_encoder_data *devdata; @@ -1093,12 +1149,16 @@ struct drm_i915_private { INTEL_DRAM_LPDDR5, } type; u8 num_qgv_points; + u8 num_psf_gv_points; } dram_info; struct intel_bw_info { /* for each QGV point */ unsigned int deratedbw[I915_NUM_QGV_POINTS]; + /* for each PSF GV point */ + unsigned int psf_bw[I915_NUM_PSF_GV_POINTS]; u8 num_qgv_points; + u8 num_psf_gv_points; u8 num_planes; } max_bw[6]; @@ -1243,11 +1303,17 @@ static inline struct drm_i915_private *pdev_to_i915(struct pci_dev *pdev) #define INTEL_DEVID(dev_priv) (RUNTIME_INFO(dev_priv)->device_id) +#define IP_VER(ver, rel) ((ver) << 8 | (rel)) + #define GRAPHICS_VER(i915) (INTEL_INFO(i915)->graphics_ver) +#define GRAPHICS_VER_FULL(i915) IP_VER(INTEL_INFO(i915)->graphics_ver, \ + INTEL_INFO(i915)->graphics_rel) #define IS_GRAPHICS_VER(i915, from, until) \ (GRAPHICS_VER(i915) >= (from) && GRAPHICS_VER(i915) <= (until)) #define MEDIA_VER(i915) (INTEL_INFO(i915)->media_ver) +#define MEDIA_VER_FULL(i915) IP_VER(INTEL_INFO(i915)->media_ver, \ + INTEL_INFO(i915)->media_rel) #define IS_MEDIA_VER(i915, from, until) \ (MEDIA_VER(i915) >= (from) && MEDIA_VER(i915) <= (until)) @@ -1255,29 +1321,20 @@ static inline struct drm_i915_private *pdev_to_i915(struct pci_dev *pdev) #define IS_DISPLAY_VER(i915, from, until) \ (DISPLAY_VER(i915) >= (from) && DISPLAY_VER(i915) <= (until)) -#define REVID_FOREVER 0xff #define INTEL_REVID(dev_priv) (to_pci_dev((dev_priv)->drm.dev)->revision) #define HAS_DSB(dev_priv) (INTEL_INFO(dev_priv)->display.has_dsb) -/* - * Return true if revision is in range [since,until] inclusive. - * - * Use 0 for open-ended since, and REVID_FOREVER for open-ended until. - */ -#define IS_REVID(p, since, until) \ - (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) - #define INTEL_DISPLAY_STEP(__i915) (RUNTIME_INFO(__i915)->step.display_step) #define INTEL_GT_STEP(__i915) (RUNTIME_INFO(__i915)->step.gt_step) #define IS_DISPLAY_STEP(__i915, since, until) \ (drm_WARN_ON(&(__i915)->drm, INTEL_DISPLAY_STEP(__i915) == STEP_NONE), \ - INTEL_DISPLAY_STEP(__i915) >= (since) && INTEL_DISPLAY_STEP(__i915) <= (until)) + INTEL_DISPLAY_STEP(__i915) >= (since) && INTEL_DISPLAY_STEP(__i915) < (until)) #define IS_GT_STEP(__i915, since, until) \ (drm_WARN_ON(&(__i915)->drm, INTEL_GT_STEP(__i915) == STEP_NONE), \ - INTEL_GT_STEP(__i915) >= (since) && INTEL_GT_STEP(__i915) <= (until)) + INTEL_GT_STEP(__i915) >= (since) && INTEL_GT_STEP(__i915) < (until)) static __always_inline unsigned int __platform_mask_index(const struct intel_runtime_info *info, @@ -1376,7 +1433,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_GEMINILAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_GEMINILAKE) #define IS_COFFEELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COFFEELAKE) #define IS_COMETLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COMETLAKE) -#define IS_CANNONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_CANNONLAKE) +#define IS_CANNONLAKE(dev_priv) 0 #define IS_ICELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_ICELAKE) #define IS_JSL_EHL(dev_priv) (IS_PLATFORM(dev_priv, INTEL_JASPERLAKE) || \ IS_PLATFORM(dev_priv, INTEL_ELKHARTLAKE)) @@ -1385,6 +1442,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_DG1(dev_priv) IS_PLATFORM(dev_priv, INTEL_DG1) #define IS_ALDERLAKE_S(dev_priv) IS_PLATFORM(dev_priv, INTEL_ALDERLAKE_S) #define IS_ALDERLAKE_P(dev_priv) IS_PLATFORM(dev_priv, INTEL_ALDERLAKE_P) +#define IS_XEHPSDV(dev_priv) IS_PLATFORM(dev_priv, INTEL_XEHPSDV) +#define IS_DG2(dev_priv) IS_PLATFORM(dev_priv, INTEL_DG2) +#define IS_DG2_G10(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_DG2, INTEL_SUBPLATFORM_G10) +#define IS_DG2_G11(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_DG2, INTEL_SUBPLATFORM_G11) #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev_priv) \ @@ -1436,8 +1499,6 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_CML_GT2(dev_priv) (IS_COMETLAKE(dev_priv) && \ INTEL_INFO(dev_priv)->gt == 2) -#define IS_CNL_WITH_PORT_F(dev_priv) \ - IS_SUBPLATFORM(dev_priv, INTEL_CANNONLAKE, INTEL_SUBPLATFORM_PORTF) #define IS_ICL_WITH_PORT_F(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF) @@ -1447,60 +1508,17 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_TGL_Y(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULX) -#define SKL_REVID_A0 0x0 -#define SKL_REVID_B0 0x1 -#define SKL_REVID_C0 0x2 -#define SKL_REVID_D0 0x3 -#define SKL_REVID_E0 0x4 -#define SKL_REVID_F0 0x5 -#define SKL_REVID_G0 0x6 -#define SKL_REVID_H0 0x7 - -#define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until)) - -#define BXT_REVID_A0 0x0 -#define BXT_REVID_A1 0x1 -#define BXT_REVID_B0 0x3 -#define BXT_REVID_B_LAST 0x8 -#define BXT_REVID_C0 0x9 - -#define IS_BXT_REVID(dev_priv, since, until) \ - (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) +#define IS_SKL_GT_STEP(p, since, until) (IS_SKYLAKE(p) && IS_GT_STEP(p, since, until)) #define IS_KBL_GT_STEP(dev_priv, since, until) \ (IS_KABYLAKE(dev_priv) && IS_GT_STEP(dev_priv, since, until)) #define IS_KBL_DISPLAY_STEP(dev_priv, since, until) \ (IS_KABYLAKE(dev_priv) && IS_DISPLAY_STEP(dev_priv, since, until)) -#define GLK_REVID_A0 0x0 -#define GLK_REVID_A1 0x1 -#define GLK_REVID_A2 0x2 -#define GLK_REVID_B0 0x3 - -#define IS_GLK_REVID(dev_priv, since, until) \ - (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until)) - -#define CNL_REVID_A0 0x0 -#define CNL_REVID_B0 0x1 -#define CNL_REVID_C0 0x2 - -#define IS_CNL_REVID(p, since, until) \ - (IS_CANNONLAKE(p) && IS_REVID(p, since, until)) - -#define ICL_REVID_A0 0x0 -#define ICL_REVID_A2 0x1 -#define ICL_REVID_B0 0x3 -#define ICL_REVID_B2 0x4 -#define ICL_REVID_C0 0x5 - -#define IS_ICL_REVID(p, since, until) \ - (IS_ICELAKE(p) && IS_REVID(p, since, until)) - -#define EHL_REVID_A0 0x0 -#define EHL_REVID_B0 0x1 - -#define IS_JSL_EHL_REVID(p, since, until) \ - (IS_JSL_EHL(p) && IS_REVID(p, since, until)) +#define IS_JSL_EHL_GT_STEP(p, since, until) \ + (IS_JSL_EHL(p) && IS_GT_STEP(p, since, until)) +#define IS_JSL_EHL_DISPLAY_STEP(p, since, until) \ + (IS_JSL_EHL(p) && IS_DISPLAY_STEP(p, since, until)) #define IS_TGL_DISPLAY_STEP(__i915, since, until) \ (IS_TIGERLAKE(__i915) && \ @@ -1514,18 +1532,13 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, (IS_TIGERLAKE(__i915) && !(IS_TGL_U(__i915) || IS_TGL_Y(__i915)) && \ IS_GT_STEP(__i915, since, until)) -#define RKL_REVID_A0 0x0 -#define RKL_REVID_B0 0x1 -#define RKL_REVID_C0 0x4 - -#define IS_RKL_REVID(p, since, until) \ - (IS_ROCKETLAKE(p) && IS_REVID(p, since, until)) - -#define DG1_REVID_A0 0x0 -#define DG1_REVID_B0 0x1 +#define IS_RKL_DISPLAY_STEP(p, since, until) \ + (IS_ROCKETLAKE(p) && IS_DISPLAY_STEP(p, since, until)) -#define IS_DG1_REVID(p, since, until) \ - (IS_DG1(p) && IS_REVID(p, since, until)) +#define IS_DG1_GT_STEP(p, since, until) \ + (IS_DG1(p) && IS_GT_STEP(p, since, until)) +#define IS_DG1_DISPLAY_STEP(p, since, until) \ + (IS_DG1(p) && IS_DISPLAY_STEP(p, since, until)) #define IS_ADLS_DISPLAY_STEP(__i915, since, until) \ (IS_ALDERLAKE_S(__i915) && \ @@ -1543,6 +1556,31 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, (IS_ALDERLAKE_P(__i915) && \ IS_GT_STEP(__i915, since, until)) +#define IS_XEHPSDV_GT_STEP(__i915, since, until) \ + (IS_XEHPSDV(__i915) && IS_GT_STEP(__i915, since, until)) + +/* + * DG2 hardware steppings are a bit unusual. The hardware design was forked + * to create two variants (G10 and G11) which have distinct workaround sets. + * The G11 fork of the DG2 design resets the GT stepping back to "A0" for its + * first iteration, even though it's more similar to a G10 B0 stepping in terms + * of functionality and workarounds. However the display stepping does not + * reset in the same manner --- a specific stepping like "B0" has a consistent + * meaning regardless of whether it belongs to a G10 or G11 DG2. + * + * TLDR: All GT workarounds and stepping-specific logic must be applied in + * relation to a specific subplatform (G10 or G11), whereas display workarounds + * and stepping-specific logic will be applied with a general DG2-wide stepping + * number. + */ +#define IS_DG2_GT_STEP(__i915, variant, since, until) \ + (IS_SUBPLATFORM(__i915, INTEL_DG2, INTEL_SUBPLATFORM_##variant) && \ + IS_GT_STEP(__i915, since, until)) + +#define IS_DG2_DISP_STEP(__i915, since, until) \ + (IS_DG2(__i915) && \ + IS_DISPLAY_STEP(__i915, since, until)) + #define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) #define IS_GEN9_LP(dev_priv) (GRAPHICS_VER(dev_priv) == 9 && IS_LP(dev_priv)) #define IS_GEN9_BC(dev_priv) (GRAPHICS_VER(dev_priv) == 9 && !IS_LP(dev_priv)) @@ -1580,8 +1618,6 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_LOGICAL_RING_ELSQ(dev_priv) \ (INTEL_INFO(dev_priv)->has_logical_ring_elsq) -#define HAS_MASTER_UNIT_IRQ(dev_priv) (INTEL_INFO(dev_priv)->has_master_unit_irq) - #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv) #define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type) @@ -1607,12 +1643,10 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, /* WaRsDisableCoarsePowerGating:skl,cnl */ #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ - (IS_CANNONLAKE(dev_priv) || \ - IS_SKL_GT3(dev_priv) || \ - IS_SKL_GT4(dev_priv)) + (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) #define HAS_GMBUS_IRQ(dev_priv) (GRAPHICS_VER(dev_priv) >= 4) -#define HAS_GMBUS_BURST_READ(dev_priv) (GRAPHICS_VER(dev_priv) >= 10 || \ +#define HAS_GMBUS_BURST_READ(dev_priv) (GRAPHICS_VER(dev_priv) >= 11 || \ IS_GEMINILAKE(dev_priv) || \ IS_KABYLAKE(dev_priv)) @@ -1632,6 +1666,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_DP_MST(dev_priv) (INTEL_INFO(dev_priv)->display.has_dp_mst) +#define HAS_CDCLK_CRAWL(dev_priv) (INTEL_INFO(dev_priv)->display.has_cdclk_crawl) #define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->display.has_fpga_dbg) #define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) @@ -1653,6 +1688,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_RUNTIME_PM(dev_priv) (INTEL_INFO(dev_priv)->has_runtime_pm) #define HAS_64BIT_RELOC(dev_priv) (INTEL_INFO(dev_priv)->has_64bit_reloc) +#define HAS_MSLICES(dev_priv) \ + (INTEL_INFO(dev_priv)->has_mslices) + #define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) #define HAS_REGION(i915, i) (INTEL_INFO(i915)->memory_regions & (i)) @@ -1742,9 +1780,6 @@ void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); void i915_gem_init_early(struct drm_i915_private *dev_priv); void i915_gem_cleanup_early(struct drm_i915_private *dev_priv); -struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, - u16 type, u16 instance); - static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915) { /* @@ -1841,24 +1876,18 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, struct dma_buf *i915_gem_prime_export(struct drm_gem_object *gem_obj, int flags); -static inline struct i915_gem_context * -__i915_gem_context_lookup_rcu(struct drm_i915_file_private *file_priv, u32 id) -{ - return xa_load(&file_priv->context_xa, id); -} - -static inline struct i915_gem_context * -i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) +static inline struct i915_address_space * +i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) { - struct i915_gem_context *ctx; + struct i915_address_space *vm; rcu_read_lock(); - ctx = __i915_gem_context_lookup_rcu(file_priv, id); - if (ctx && !kref_get_unless_zero(&ctx->ref)) - ctx = NULL; + vm = xa_load(&file_priv->vm_xa, id); + if (vm && !kref_get_unless_zero(&vm->ref)) + vm = NULL; rcu_read_unlock(); - return ctx; + return vm; } /* i915_gem_evict.c */ @@ -1925,8 +1954,8 @@ int remap_io_sg(struct vm_area_struct *vma, static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) { - if (GRAPHICS_VER(i915) >= 10) - return CNL_HWS_CSB_WRITE_INDEX; + if (GRAPHICS_VER(i915) >= 11) + return ICL_HWS_CSB_WRITE_INDEX; else return I915_HWS_CSB_WRITE_INDEX; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 6a0a3f0e36e1..590efc8b0265 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -469,12 +469,6 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (ret != -ENODEV) goto out; - ret = -ENODEV; - if (obj->ops->pread) - ret = obj->ops->pread(obj, args); - if (ret != -ENODEV) - goto out; - ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); @@ -1207,58 +1201,6 @@ int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) return ret; } -void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr) -{ - ww_acquire_init(&ww->ctx, &reservation_ww_class); - INIT_LIST_HEAD(&ww->obj_list); - ww->intr = intr; - ww->contended = NULL; -} - -static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww) -{ - struct drm_i915_gem_object *obj; - - while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) { - list_del(&obj->obj_link); - i915_gem_object_unlock(obj); - } -} - -void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj) -{ - list_del(&obj->obj_link); - i915_gem_object_unlock(obj); -} - -void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww) -{ - i915_gem_ww_ctx_unlock_all(ww); - WARN_ON(ww->contended); - ww_acquire_fini(&ww->ctx); -} - -int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww) -{ - int ret = 0; - - if (WARN_ON(!ww->contended)) - return -EINVAL; - - i915_gem_ww_ctx_unlock_all(ww); - if (ww->intr) - ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx); - else - dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx); - - if (!ret) - list_add_tail(&ww->contended->obj_link, &ww->obj_list); - - ww->contended = NULL; - - return ret; -} - #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/mock_gem_device.c" #include "selftests/i915_gem.c" diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 440c35f1abc9..d0752e5553db 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -123,16 +123,4 @@ static inline bool __tasklet_is_scheduled(struct tasklet_struct *t) return test_bit(TASKLET_STATE_SCHED, &t->state); } -struct i915_gem_ww_ctx { - struct ww_acquire_ctx ctx; - struct list_head obj_list; - bool intr; - struct drm_i915_gem_object *contended; -}; - -void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ctx, bool intr); -void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ctx); -int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ctx); -void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj); - #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 4d2d59a9942b..2b73ddb11c66 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -27,6 +27,7 @@ */ #include "gem/i915_gem_context.h" +#include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/i915_gem_ww.c b/drivers/gpu/drm/i915/i915_gem_ww.c new file mode 100644 index 000000000000..3f6ff139478e --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_ww.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ +#include <linux/dma-resv.h> +#include "i915_gem_ww.h" +#include "gem/i915_gem_object.h" + +void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr) +{ + ww_acquire_init(&ww->ctx, &reservation_ww_class); + INIT_LIST_HEAD(&ww->obj_list); + ww->intr = intr; + ww->contended = NULL; +} + +static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww) +{ + struct drm_i915_gem_object *obj; + + while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) { + list_del(&obj->obj_link); + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + } +} + +void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj) +{ + list_del(&obj->obj_link); + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); +} + +void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww) +{ + i915_gem_ww_ctx_unlock_all(ww); + WARN_ON(ww->contended); + ww_acquire_fini(&ww->ctx); +} + +int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww) +{ + int ret = 0; + + if (WARN_ON(!ww->contended)) + return -EINVAL; + + i915_gem_ww_ctx_unlock_all(ww); + if (ww->intr) + ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx); + else + dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx); + + if (!ret) + list_add_tail(&ww->contended->obj_link, &ww->obj_list); + else + i915_gem_object_put(ww->contended); + + ww->contended = NULL; + + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_gem_ww.h b/drivers/gpu/drm/i915/i915_gem_ww.h new file mode 100644 index 000000000000..f6b1a796667b --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_ww.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ +#ifndef __I915_GEM_WW_H__ +#define __I915_GEM_WW_H__ + +#include <drm/drm_drv.h> + +struct i915_gem_ww_ctx { + struct ww_acquire_ctx ctx; + struct list_head obj_list; + struct drm_i915_gem_object *contended; + unsigned short intr; + unsigned short loop; +}; + +void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ctx, bool intr); +void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ctx); +int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ctx); +void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj); + +/* Internal functions used by the inlines! Don't use. */ +static inline int __i915_gem_ww_fini(struct i915_gem_ww_ctx *ww, int err) +{ + ww->loop = 0; + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(ww); + if (!err) + ww->loop = 1; + } + + if (!ww->loop) + i915_gem_ww_ctx_fini(ww); + + return err; +} + +static inline void +__i915_gem_ww_init(struct i915_gem_ww_ctx *ww, bool intr) +{ + i915_gem_ww_ctx_init(ww, intr); + ww->loop = 1; +} + +#define for_i915_gem_ww(_ww, _err, _intr) \ + for (__i915_gem_ww_init(_ww, _intr); (_ww)->loop; \ + _err = __i915_gem_ww_fini(_ww, _err)) + +#endif diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 24e18219eb50..77490cb5ff9c 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -15,7 +15,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, struct pci_dev *pdev = to_pci_dev(dev->dev); const struct sseu_dev_info *sseu = &i915->gt.info.sseu; drm_i915_getparam_t *param = data; - int value; + int value = 0; switch (param->param) { case I915_PARAM_IRQ_ACTIVE: @@ -134,6 +134,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, case I915_PARAM_HAS_EXEC_FENCE_ARRAY: case I915_PARAM_HAS_EXEC_SUBMIT_FENCE: case I915_PARAM_HAS_EXEC_TIMELINE_FENCES: + case I915_PARAM_HAS_USERPTR_PROBE: /* For the time being all of these are always true; * if some supported hardware does not have one of these * features this value needs to be provided from @@ -150,7 +151,9 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, return -ENODEV; break; case I915_PARAM_SUBSLICE_MASK: - value = sseu->subslice_mask[0]; + /* Only copy bits from the first slice */ + memcpy(&value, sseu->subslice_mask, + min(sseu->ss_stride, (u8)sizeof(value))); if (!value) return -ENODEV; break; diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c deleted file mode 100644 index 77f1911c463b..000000000000 --- a/drivers/gpu/drm/i915/i915_globals.c +++ /dev/null @@ -1,160 +0,0 @@ -/* - * SPDX-License-Identifier: MIT - * - * Copyright © 2019 Intel Corporation - */ - -#include <linux/slab.h> -#include <linux/workqueue.h> - -#include "i915_active.h" -#include "gem/i915_gem_context.h" -#include "gem/i915_gem_object.h" -#include "i915_globals.h" -#include "i915_request.h" -#include "i915_scheduler.h" -#include "i915_vma.h" - -static LIST_HEAD(globals); - -static atomic_t active; -static atomic_t epoch; -static struct park_work { - struct delayed_work work; - struct rcu_head rcu; - unsigned long flags; -#define PENDING 0 - int epoch; -} park; - -static void i915_globals_shrink(void) -{ - struct i915_global *global; - - /* - * kmem_cache_shrink() discards empty slabs and reorders partially - * filled slabs to prioritise allocating from the mostly full slabs, - * with the aim of reducing fragmentation. - */ - list_for_each_entry(global, &globals, link) - global->shrink(); -} - -static void __i915_globals_grace(struct rcu_head *rcu) -{ - /* Ratelimit parking as shrinking is quite slow */ - schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ)); -} - -static void __i915_globals_queue_rcu(void) -{ - park.epoch = atomic_inc_return(&epoch); - if (!atomic_read(&active)) { - init_rcu_head(&park.rcu); - call_rcu(&park.rcu, __i915_globals_grace); - } -} - -static void __i915_globals_park(struct work_struct *work) -{ - destroy_rcu_head(&park.rcu); - - /* Confirm nothing woke up in the last grace period */ - if (park.epoch != atomic_read(&epoch)) { - __i915_globals_queue_rcu(); - return; - } - - clear_bit(PENDING, &park.flags); - i915_globals_shrink(); -} - -void __init i915_global_register(struct i915_global *global) -{ - GEM_BUG_ON(!global->shrink); - GEM_BUG_ON(!global->exit); - - list_add_tail(&global->link, &globals); -} - -static void __i915_globals_cleanup(void) -{ - struct i915_global *global, *next; - - list_for_each_entry_safe_reverse(global, next, &globals, link) - global->exit(); -} - -static __initconst int (* const initfn[])(void) = { - i915_global_active_init, - i915_global_context_init, - i915_global_gem_context_init, - i915_global_objects_init, - i915_global_request_init, - i915_global_scheduler_init, - i915_global_vma_init, -}; - -int __init i915_globals_init(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(initfn); i++) { - int err; - - err = initfn[i](); - if (err) { - __i915_globals_cleanup(); - return err; - } - } - - INIT_DELAYED_WORK(&park.work, __i915_globals_park); - return 0; -} - -void i915_globals_park(void) -{ - /* - * Defer shrinking the global slab caches (and other work) until - * after a RCU grace period has completed with no activity. This - * is to try and reduce the latency impact on the consumers caused - * by us shrinking the caches the same time as they are trying to - * allocate, with the assumption being that if we idle long enough - * for an RCU grace period to elapse since the last use, it is likely - * to be longer until we need the caches again. - */ - if (!atomic_dec_and_test(&active)) - return; - - /* Queue cleanup after the next RCU grace period has freed slabs */ - if (!test_and_set_bit(PENDING, &park.flags)) - __i915_globals_queue_rcu(); -} - -void i915_globals_unpark(void) -{ - atomic_inc(&epoch); - atomic_inc(&active); -} - -static void __exit __i915_globals_flush(void) -{ - atomic_inc(&active); /* skip shrinking */ - - rcu_barrier(); /* wait for the work to be queued */ - flush_delayed_work(&park.work); - - atomic_dec(&active); -} - -void __exit i915_globals_exit(void) -{ - GEM_BUG_ON(atomic_read(&active)); - - __i915_globals_flush(); - __i915_globals_cleanup(); - - /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */ - rcu_barrier(); -} diff --git a/drivers/gpu/drm/i915/i915_globals.h b/drivers/gpu/drm/i915/i915_globals.h deleted file mode 100644 index 2d199f411a4a..000000000000 --- a/drivers/gpu/drm/i915/i915_globals.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: MIT - * - * Copyright © 2019 Intel Corporation - */ - -#ifndef _I915_GLOBALS_H_ -#define _I915_GLOBALS_H_ - -#include <linux/types.h> - -typedef void (*i915_global_func_t)(void); - -struct i915_global { - struct list_head link; - - i915_global_func_t shrink; - i915_global_func_t exit; -}; - -void i915_global_register(struct i915_global *global); - -int i915_globals_init(void); -void i915_globals_park(void); -void i915_globals_unpark(void); -void i915_globals_exit(void); - -/* constructors */ -int i915_global_active_init(void); -int i915_global_context_init(void); -int i915_global_gem_context_init(void); -int i915_global_objects_init(void); -int i915_global_request_init(void); -int i915_global_scheduler_init(void); -int i915_global_vma_init(void); - -#endif /* _I915_GLOBALS_H_ */ diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 35c97c39f125..9cf6ac575de1 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -727,9 +727,18 @@ static void err_print_gt(struct drm_i915_error_state_buf *m, if (GRAPHICS_VER(m->i915) >= 12) { int i; - for (i = 0; i < GEN12_SFC_DONE_MAX; i++) + for (i = 0; i < GEN12_SFC_DONE_MAX; i++) { + /* + * SFC_DONE resides in the VD forcewake domain, so it + * only exists if the corresponding VCS engine is + * present. + */ + if (!HAS_ENGINE(gt->_gt, _VCS(i * 2))) + continue; + err_printf(m, " SFC_DONE[%d]: 0x%08x\n", i, gt->sfc_done[i]); + } err_printf(m, " GAM_DONE: 0x%08x\n", gt->gam_done); } @@ -1039,7 +1048,7 @@ i915_vma_coredump_create(const struct intel_gt *gt, if (ret) break; } - } else if (i915_gem_object_is_lmem(vma->obj)) { + } else if (__i915_gem_object_is_lmem(vma->obj)) { struct intel_memory_region *mem = vma->obj->mm.region; dma_addr_t dma; @@ -1429,20 +1438,37 @@ capture_engine(struct intel_engine_cs *engine, { struct intel_engine_capture_vma *capture = NULL; struct intel_engine_coredump *ee; - struct i915_request *rq; + struct intel_context *ce; + struct i915_request *rq = NULL; unsigned long flags; ee = intel_engine_coredump_alloc(engine, GFP_KERNEL); if (!ee) return NULL; - spin_lock_irqsave(&engine->active.lock, flags); - rq = intel_engine_find_active_request(engine); + ce = intel_engine_get_hung_context(engine); + if (ce) { + intel_engine_clear_hung_context(engine); + rq = intel_context_find_active_request(ce); + if (!rq || !i915_request_started(rq)) + goto no_request_capture; + } else { + /* + * Getting here with GuC enabled means it is a forced error capture + * with no actual hang. So, no need to attempt the execlist search. + */ + if (!intel_uc_uses_guc_submission(&engine->gt->uc)) { + spin_lock_irqsave(&engine->sched_engine->lock, flags); + rq = intel_engine_execlist_find_hung_request(engine); + spin_unlock_irqrestore(&engine->sched_engine->lock, + flags); + } + } if (rq) capture = intel_engine_coredump_add_request(ee, rq, ATOMIC_MAYFAIL); - spin_unlock_irqrestore(&engine->active.lock, flags); if (!capture) { +no_request_capture: kfree(ee); return NULL; } @@ -1581,6 +1607,14 @@ static void gt_record_regs(struct intel_gt_coredump *gt) if (GRAPHICS_VER(i915) >= 12) { for (i = 0; i < GEN12_SFC_DONE_MAX; i++) { + /* + * SFC_DONE resides in the VD forcewake domain, so it + * only exists if the corresponding VCS engine is + * present. + */ + if (!HAS_ENGINE(gt->_gt, _VCS(i * 2))) + continue; + gt->sfc_done[i] = intel_uncore_read(uncore, GEN12_SFC_DONE(i)); } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 1d4c683c9de9..9bc4f4a8e12e 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -207,7 +207,7 @@ static void intel_hpd_init_pins(struct drm_i915_private *dev_priv) (!HAS_PCH_SPLIT(dev_priv) || HAS_PCH_NOP(dev_priv))) return; - if (HAS_PCH_DG1(dev_priv)) + if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) hpd->pch_hpd = hpd_sde_dg1; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) hpd->pch_hpd = hpd_icp; @@ -2297,11 +2297,10 @@ static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv) GEN9_AUX_CHANNEL_C | GEN9_AUX_CHANNEL_D; - if (IS_CNL_WITH_PORT_F(dev_priv) || DISPLAY_VER(dev_priv) == 11) - mask |= CNL_AUX_CHANNEL_F; - - if (DISPLAY_VER(dev_priv) == 11) + if (DISPLAY_VER(dev_priv) == 11) { + mask |= ICL_AUX_CHANNEL_F; mask |= ICL_AUX_CHANNEL_E; + } return mask; } @@ -2698,11 +2697,9 @@ gen11_display_irq_handler(struct drm_i915_private *i915) enable_rpm_wakeref_asserts(&i915->runtime_pm); } -static __always_inline irqreturn_t -__gen11_irq_handler(struct drm_i915_private * const i915, - u32 (*intr_disable)(void __iomem * const regs), - void (*intr_enable)(void __iomem * const regs)) +static irqreturn_t gen11_irq_handler(int irq, void *arg) { + struct drm_i915_private *i915 = arg; void __iomem * const regs = i915->uncore.regs; struct intel_gt *gt = &i915->gt; u32 master_ctl; @@ -2711,9 +2708,9 @@ __gen11_irq_handler(struct drm_i915_private * const i915, if (!intel_irqs_enabled(i915)) return IRQ_NONE; - master_ctl = intr_disable(regs); + master_ctl = gen11_master_intr_disable(regs); if (!master_ctl) { - intr_enable(regs); + gen11_master_intr_enable(regs); return IRQ_NONE; } @@ -2726,7 +2723,7 @@ __gen11_irq_handler(struct drm_i915_private * const i915, gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); - intr_enable(regs); + gen11_master_intr_enable(regs); gen11_gu_misc_irq_handler(gt, gu_misc_iir); @@ -2735,51 +2732,69 @@ __gen11_irq_handler(struct drm_i915_private * const i915, return IRQ_HANDLED; } -static irqreturn_t gen11_irq_handler(int irq, void *arg) -{ - return __gen11_irq_handler(arg, - gen11_master_intr_disable, - gen11_master_intr_enable); -} - -static u32 dg1_master_intr_disable_and_ack(void __iomem * const regs) +static inline u32 dg1_master_intr_disable(void __iomem * const regs) { u32 val; /* First disable interrupts */ - raw_reg_write(regs, DG1_MSTR_UNIT_INTR, 0); + raw_reg_write(regs, DG1_MSTR_TILE_INTR, 0); /* Get the indication levels and ack the master unit */ - val = raw_reg_read(regs, DG1_MSTR_UNIT_INTR); + val = raw_reg_read(regs, DG1_MSTR_TILE_INTR); if (unlikely(!val)) return 0; - raw_reg_write(regs, DG1_MSTR_UNIT_INTR, val); - - /* - * Now with master disabled, get a sample of level indications - * for this interrupt and ack them right away - we keep GEN11_MASTER_IRQ - * out as this bit doesn't exist anymore for DG1 - */ - val = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ) & ~GEN11_MASTER_IRQ; - if (unlikely(!val)) - return 0; - - raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, val); + raw_reg_write(regs, DG1_MSTR_TILE_INTR, val); return val; } static inline void dg1_master_intr_enable(void __iomem * const regs) { - raw_reg_write(regs, DG1_MSTR_UNIT_INTR, DG1_MSTR_IRQ); + raw_reg_write(regs, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ); } static irqreturn_t dg1_irq_handler(int irq, void *arg) { - return __gen11_irq_handler(arg, - dg1_master_intr_disable_and_ack, - dg1_master_intr_enable); + struct drm_i915_private * const i915 = arg; + struct intel_gt *gt = &i915->gt; + void __iomem * const regs = i915->uncore.regs; + u32 master_tile_ctl, master_ctl; + u32 gu_misc_iir; + + if (!intel_irqs_enabled(i915)) + return IRQ_NONE; + + master_tile_ctl = dg1_master_intr_disable(regs); + if (!master_tile_ctl) { + dg1_master_intr_enable(regs); + return IRQ_NONE; + } + + /* FIXME: we only support tile 0 for now. */ + if (master_tile_ctl & DG1_MSTR_TILE(0)) { + master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ); + raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, master_ctl); + } else { + DRM_ERROR("Tile not supported: 0x%08x\n", master_tile_ctl); + dg1_master_intr_enable(regs); + return IRQ_NONE; + } + + gen11_gt_irq_handler(gt, master_ctl); + + if (master_ctl & GEN11_DISPLAY_IRQ) + gen11_display_irq_handler(i915); + + gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); + + dg1_master_intr_enable(regs); + + gen11_gu_misc_irq_handler(gt, gu_misc_iir); + + pmu_irq_stats(i915, IRQ_HANDLED); + + return IRQ_HANDLED; } /* Called from drm generic code, passed 'crtc' which @@ -3064,24 +3079,6 @@ static void valleyview_irq_reset(struct drm_i915_private *dev_priv) spin_unlock_irq(&dev_priv->irq_lock); } -static void cnp_display_clock_wa(struct drm_i915_private *dev_priv) -{ - struct intel_uncore *uncore = &dev_priv->uncore; - - /* - * Wa_14010685332:cnp/cmp,tgp,adp - * TODO: Clarify which platforms this applies to - * TODO: Figure out if this workaround can be applied in the s0ix suspend/resume handlers as - * on earlier platforms and whether the workaround is also needed for runtime suspend/resume - */ - if (INTEL_PCH_TYPE(dev_priv) == PCH_CNP || - (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP && INTEL_PCH_TYPE(dev_priv) < PCH_DG1)) { - intel_uncore_rmw(uncore, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, - SBCLK_RUN_REFCLK_DIS); - intel_uncore_rmw(uncore, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0); - } -} - static void gen8_display_irq_reset(struct drm_i915_private *dev_priv) { struct intel_uncore *uncore = &dev_priv->uncore; @@ -3115,7 +3112,6 @@ static void gen8_irq_reset(struct drm_i915_private *dev_priv) if (HAS_PCH_SPLIT(dev_priv)) ibx_irq_reset(dev_priv); - cnp_display_clock_wa(dev_priv); } static void gen11_display_irq_reset(struct drm_i915_private *dev_priv) @@ -3159,18 +3155,26 @@ static void gen11_display_irq_reset(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) GEN3_IRQ_RESET(uncore, SDE); - - cnp_display_clock_wa(dev_priv); } static void gen11_irq_reset(struct drm_i915_private *dev_priv) { struct intel_uncore *uncore = &dev_priv->uncore; - if (HAS_MASTER_UNIT_IRQ(dev_priv)) - dg1_master_intr_disable_and_ack(dev_priv->uncore.regs); - else - gen11_master_intr_disable(dev_priv->uncore.regs); + gen11_master_intr_disable(dev_priv->uncore.regs); + + gen11_gt_irq_reset(&dev_priv->gt); + gen11_display_irq_reset(dev_priv); + + GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_); + GEN3_IRQ_RESET(uncore, GEN8_PCU_); +} + +static void dg1_irq_reset(struct drm_i915_private *dev_priv) +{ + struct intel_uncore *uncore = &dev_priv->uncore; + + dg1_master_intr_disable(dev_priv->uncore.regs); gen11_gt_irq_reset(&dev_priv->gt); gen11_display_irq_reset(dev_priv); @@ -3862,13 +3866,28 @@ static void gen11_irq_postinstall(struct drm_i915_private *dev_priv) GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked); - if (HAS_MASTER_UNIT_IRQ(dev_priv)) { - dg1_master_intr_enable(uncore->regs); - intel_uncore_posting_read(&dev_priv->uncore, DG1_MSTR_UNIT_INTR); - } else { - gen11_master_intr_enable(uncore->regs); - intel_uncore_posting_read(&dev_priv->uncore, GEN11_GFX_MSTR_IRQ); + gen11_master_intr_enable(uncore->regs); + intel_uncore_posting_read(&dev_priv->uncore, GEN11_GFX_MSTR_IRQ); +} + +static void dg1_irq_postinstall(struct drm_i915_private *dev_priv) +{ + struct intel_uncore *uncore = &dev_priv->uncore; + u32 gu_misc_masked = GEN11_GU_MISC_GSE; + + gen11_gt_irq_postinstall(&dev_priv->gt); + + GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked); + + if (HAS_DISPLAY(dev_priv)) { + icp_irq_postinstall(dev_priv); + gen8_de_irq_postinstall(dev_priv); + intel_uncore_write(&dev_priv->uncore, GEN11_DISPLAY_INT_CTL, + GEN11_DISPLAY_IRQ_ENABLE); } + + dg1_master_intr_enable(dev_priv->uncore.regs); + intel_uncore_posting_read(&dev_priv->uncore, DG1_MSTR_TILE_INTR); } static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv) @@ -4407,9 +4426,9 @@ static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv) else return i8xx_irq_handler; } else { - if (HAS_MASTER_UNIT_IRQ(dev_priv)) + if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10)) return dg1_irq_handler; - if (GRAPHICS_VER(dev_priv) >= 11) + else if (GRAPHICS_VER(dev_priv) >= 11) return gen11_irq_handler; else if (GRAPHICS_VER(dev_priv) >= 8) return gen8_irq_handler; @@ -4432,7 +4451,9 @@ static void intel_irq_reset(struct drm_i915_private *dev_priv) else i8xx_irq_reset(dev_priv); } else { - if (GRAPHICS_VER(dev_priv) >= 11) + if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10)) + dg1_irq_reset(dev_priv); + else if (GRAPHICS_VER(dev_priv) >= 11) gen11_irq_reset(dev_priv); else if (GRAPHICS_VER(dev_priv) >= 8) gen8_irq_reset(dev_priv); @@ -4455,7 +4476,9 @@ static void intel_irq_postinstall(struct drm_i915_private *dev_priv) else i8xx_irq_postinstall(dev_priv); } else { - if (GRAPHICS_VER(dev_priv) >= 11) + if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10)) + dg1_irq_postinstall(dev_priv); + else if (GRAPHICS_VER(dev_priv) >= 11) gen11_irq_postinstall(dev_priv); else if (GRAPHICS_VER(dev_priv) >= 8) gen8_irq_postinstall(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_module.c b/drivers/gpu/drm/i915/i915_module.c new file mode 100644 index 000000000000..d8b4482c69d0 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_module.c @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2021 Intel Corporation + */ + +#include <linux/console.h> + +#include "gem/i915_gem_context.h" +#include "gem/i915_gem_object.h" +#include "i915_active.h" +#include "i915_buddy.h" +#include "i915_params.h" +#include "i915_pci.h" +#include "i915_perf.h" +#include "i915_request.h" +#include "i915_scheduler.h" +#include "i915_selftest.h" +#include "i915_vma.h" + +static int i915_check_nomodeset(void) +{ + bool use_kms = true; + + /* + * Enable KMS by default, unless explicitly overriden by + * either the i915.modeset prarameter or by the + * vga_text_mode_force boot option. + */ + + if (i915_modparams.modeset == 0) + use_kms = false; + + if (vgacon_text_force() && i915_modparams.modeset == -1) + use_kms = false; + + if (!use_kms) { + /* Silently fail loading to not upset userspace. */ + DRM_DEBUG_DRIVER("KMS disabled.\n"); + return 1; + } + + return 0; +} + +static const struct { + int (*init)(void); + void (*exit)(void); +} init_funcs[] = { + { .init = i915_check_nomodeset }, + { .init = i915_active_module_init, + .exit = i915_active_module_exit }, + { .init = i915_buddy_module_init, + .exit = i915_buddy_module_exit }, + { .init = i915_context_module_init, + .exit = i915_context_module_exit }, + { .init = i915_gem_context_module_init, + .exit = i915_gem_context_module_exit }, + { .init = i915_objects_module_init, + .exit = i915_objects_module_exit }, + { .init = i915_request_module_init, + .exit = i915_request_module_exit }, + { .init = i915_scheduler_module_init, + .exit = i915_scheduler_module_exit }, + { .init = i915_vma_module_init, + .exit = i915_vma_module_exit }, + { .init = i915_mock_selftests }, + { .init = i915_pmu_init, + .exit = i915_pmu_exit }, + { .init = i915_register_pci_driver, + .exit = i915_unregister_pci_driver }, + { .init = i915_perf_sysctl_register, + .exit = i915_perf_sysctl_unregister }, +}; +static int init_progress; + +static int __init i915_init(void) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(init_funcs); i++) { + err = init_funcs[i].init(); + if (err < 0) { + while (i--) { + if (init_funcs[i].exit) + init_funcs[i].exit(); + } + return err; + } else if (err > 0) { + /* + * Early-exit success is reserved for things which + * don't have an exit() function because we have no + * idea how far they got or how to partially tear + * them down. + */ + WARN_ON(init_funcs[i].exit); + break; + } + } + + init_progress = i; + + return 0; +} + +static void __exit i915_exit(void) +{ + int i; + + for (i = init_progress - 1; i >= 0; i--) { + GEM_BUG_ON(i >= ARRAY_SIZE(init_funcs)); + if (init_funcs[i].exit) + init_funcs[i].exit(); + } +} + +module_init(i915_init); +module_exit(i915_exit); + +MODULE_AUTHOR("Tungsten Graphics, Inc."); +MODULE_AUTHOR("Intel Corporation"); + +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 7030e563985c..1bbd09ad5287 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -22,18 +22,13 @@ * */ -#include <linux/console.h> #include <linux/vga_switcheroo.h> #include <drm/drm_drv.h> #include <drm/i915_pciids.h> -#include "display/intel_fbdev.h" - #include "i915_drv.h" -#include "i915_perf.h" -#include "i915_globals.h" -#include "i915_selftest.h" +#include "i915_pci.h" #define PLATFORM(x) .platform = (x) #define GEN(x) \ @@ -787,27 +782,13 @@ static const struct intel_device_info cml_gt2_info = { .gt = 2, }; -#define GEN10_FEATURES \ - GEN9_FEATURES, \ - GEN(10), \ - .dbuf.size = 1024 - 4, /* 4 blocks for bypass path allocation */ \ - .display.has_dsc = 1, \ - .has_coherent_ggtt = false, \ - GLK_COLORS - -static const struct intel_device_info cnl_info = { - GEN10_FEATURES, - PLATFORM(INTEL_CANNONLAKE), - .gt = 2, -}; - #define GEN11_DEFAULT_PAGE_SIZES \ .page_sizes = I915_GTT_PAGE_SIZE_4K | \ I915_GTT_PAGE_SIZE_64K | \ I915_GTT_PAGE_SIZE_2M #define GEN11_FEATURES \ - GEN10_FEATURES, \ + GEN9_FEATURES, \ GEN11_DEFAULT_PAGE_SIZES, \ .abox_mask = BIT(0), \ .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ @@ -830,10 +811,12 @@ static const struct intel_device_info cnl_info = { [TRANSCODER_DSI_1] = TRANSCODER_DSI1_OFFSET, \ }, \ GEN(11), \ + .color = { .degamma_lut_size = 33, .gamma_lut_size = 262145 }, \ .dbuf.size = 2048, \ .dbuf.slice_mask = BIT(DBUF_S1) | BIT(DBUF_S2), \ - .has_logical_ring_elsq = 1, \ - .color = { .degamma_lut_size = 33, .gamma_lut_size = 262145 } + .display.has_dsc = 1, \ + .has_coherent_ggtt = false, \ + .has_logical_ring_elsq = 1 static const struct intel_device_info icl_info = { GEN11_FEATURES, @@ -907,7 +890,6 @@ static const struct intel_device_info rkl_info = { #define DGFX_FEATURES \ .memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ - .has_master_unit_irq = 1, \ .has_llc = 0, \ .has_snoop = 1, \ .is_dgfx = 1 @@ -915,6 +897,7 @@ static const struct intel_device_info rkl_info = { static const struct intel_device_info dg1_info __maybe_unused = { GEN12_FEATURES, DGFX_FEATURES, + .graphics_rel = 10, PLATFORM(INTEL_DG1), .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), .require_force_probe = 1, @@ -934,7 +917,7 @@ static const struct intel_device_info adl_s_info = { .display.has_psr_hw_tracking = 0, .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), - .dma_mask_size = 46, + .dma_mask_size = 39, }; #define XE_LPD_CURSOR_OFFSETS \ @@ -947,7 +930,7 @@ static const struct intel_device_info adl_s_info = { #define XE_LPD_FEATURES \ .abox_mask = GENMASK(1, 0), \ - .color = { .degamma_lut_size = 33, .gamma_lut_size = 262145 }, \ + .color = { .degamma_lut_size = 0, .gamma_lut_size = 0 }, \ .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ BIT(TRANSCODER_C) | BIT(TRANSCODER_D), \ .dbuf.size = 4096, \ @@ -984,8 +967,8 @@ static const struct intel_device_info adl_p_info = { GEN12_FEATURES, XE_LPD_FEATURES, PLATFORM(INTEL_ALDERLAKE_P), - .has_cdclk_crawl = 1, .require_force_probe = 1, + .display.has_cdclk_crawl = 1, .display.has_modular_fia = 1, .display.has_psr_hw_tracking = 0, .platform_engine_mask = @@ -995,6 +978,67 @@ static const struct intel_device_info adl_p_info = { }; #undef GEN + +#define XE_HP_PAGE_SIZES \ + .page_sizes = I915_GTT_PAGE_SIZE_4K | \ + I915_GTT_PAGE_SIZE_64K | \ + I915_GTT_PAGE_SIZE_2M + +#define XE_HP_FEATURES \ + .graphics_ver = 12, \ + .graphics_rel = 50, \ + XE_HP_PAGE_SIZES, \ + .dma_mask_size = 46, \ + .has_64bit_reloc = 1, \ + .has_global_mocs = 1, \ + .has_gt_uc = 1, \ + .has_llc = 1, \ + .has_logical_ring_contexts = 1, \ + .has_logical_ring_elsq = 1, \ + .has_mslices = 1, \ + .has_rc6 = 1, \ + .has_reset_engine = 1, \ + .has_rps = 1, \ + .has_runtime_pm = 1, \ + .ppgtt_size = 48, \ + .ppgtt_type = INTEL_PPGTT_FULL + +#define XE_HPM_FEATURES \ + .media_ver = 12, \ + .media_rel = 50 + +__maybe_unused +static const struct intel_device_info xehpsdv_info = { + XE_HP_FEATURES, + XE_HPM_FEATURES, + DGFX_FEATURES, + PLATFORM(INTEL_XEHPSDV), + .display = { }, + .pipe_mask = 0, + .platform_engine_mask = + BIT(RCS0) | BIT(BCS0) | + BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) | + BIT(VCS0) | BIT(VCS1) | BIT(VCS2) | BIT(VCS3) | + BIT(VCS4) | BIT(VCS5) | BIT(VCS6) | BIT(VCS7), + .require_force_probe = 1, +}; + +__maybe_unused +static const struct intel_device_info dg2_info = { + XE_HP_FEATURES, + XE_HPM_FEATURES, + XE_LPD_FEATURES, + DGFX_FEATURES, + .graphics_rel = 55, + .media_rel = 55, + PLATFORM(INTEL_DG2), + .platform_engine_mask = + BIT(RCS0) | BIT(BCS0) | + BIT(VECS0) | BIT(VECS1) | + BIT(VCS0) | BIT(VCS2), + .require_force_probe = 1, +}; + #undef PLATFORM /* @@ -1064,7 +1108,6 @@ static const struct pci_device_id pciidlist[] = { INTEL_CML_GT2_IDS(&cml_gt2_info), INTEL_CML_U_GT1_IDS(&cml_gt1_info), INTEL_CML_U_GT2_IDS(&cml_gt2_info), - INTEL_CNL_IDS(&cnl_info), INTEL_ICL_11_IDS(&icl_info), INTEL_EHL_IDS(&ehl_info), INTEL_JSL_IDS(&jsl_info), @@ -1191,65 +1234,12 @@ static struct pci_driver i915_pci_driver = { .driver.pm = &i915_pm_ops, }; -static int __init i915_init(void) +int i915_register_pci_driver(void) { - bool use_kms = true; - int err; - - err = i915_globals_init(); - if (err) - return err; - - err = i915_mock_selftests(); - if (err) - return err > 0 ? 0 : err; - - /* - * Enable KMS by default, unless explicitly overriden by - * either the i915.modeset prarameter or by the - * vga_text_mode_force boot option. - */ - - if (i915_modparams.modeset == 0) - use_kms = false; - - if (vgacon_text_force() && i915_modparams.modeset == -1) - use_kms = false; - - if (!use_kms) { - /* Silently fail loading to not upset userspace. */ - DRM_DEBUG_DRIVER("KMS disabled.\n"); - return 0; - } - - i915_pmu_init(); - - err = pci_register_driver(&i915_pci_driver); - if (err) { - i915_pmu_exit(); - return err; - } - - i915_perf_sysctl_register(); - return 0; + return pci_register_driver(&i915_pci_driver); } -static void __exit i915_exit(void) +void i915_unregister_pci_driver(void) { - if (!i915_pci_driver.driver.owner) - return; - - i915_perf_sysctl_unregister(); pci_unregister_driver(&i915_pci_driver); - i915_globals_exit(); - i915_pmu_exit(); } - -module_init(i915_init); -module_exit(i915_exit); - -MODULE_AUTHOR("Tungsten Graphics, Inc."); -MODULE_AUTHOR("Intel Corporation"); - -MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/i915/i915_pci.h b/drivers/gpu/drm/i915/i915_pci.h new file mode 100644 index 000000000000..b386f319f52e --- /dev/null +++ b/drivers/gpu/drm/i915/i915_pci.h @@ -0,0 +1,8 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2021 Intel Corporation + */ + +int i915_register_pci_driver(void); +void i915_unregister_pci_driver(void); diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 9f94914958c3..2f01b8c0284c 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -1256,7 +1256,6 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) case 8: case 9: - case 10: if (intel_engine_uses_guc(ce->engine)) { /* * When using GuC, the context descriptor we write in @@ -1284,17 +1283,26 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) break; case 11: - case 12: { - stream->specific_ctx_id_mask = - ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); - /* - * Pick an unused context id - * 0 - BITS_PER_LONG are used by other contexts - * GEN12_MAX_CONTEXT_HW_ID (0x7ff) is used by idle context - */ - stream->specific_ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); + case 12: + if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 50)) { + stream->specific_ctx_id_mask = + ((1U << XEHP_SW_CTX_ID_WIDTH) - 1) << + (XEHP_SW_CTX_ID_SHIFT - 32); + stream->specific_ctx_id = + (XEHP_MAX_CONTEXT_HW_ID - 1) << + (XEHP_SW_CTX_ID_SHIFT - 32); + } else { + stream->specific_ctx_id_mask = + ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); + /* + * Pick an unused context id + * 0 - BITS_PER_LONG are used by other contexts + * GEN12_MAX_CONTEXT_HW_ID (0x7ff) is used by idle context + */ + stream->specific_ctx_id = + (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); + } break; - } default: MISSING_CASE(GRAPHICS_VER(ce->engine->i915)); @@ -2580,7 +2588,7 @@ static void gen8_disable_metric_set(struct i915_perf_stream *stream) intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, 0); } -static void gen10_disable_metric_set(struct i915_perf_stream *stream) +static void gen11_disable_metric_set(struct i915_perf_stream *stream) { struct intel_uncore *uncore = stream->uncore; @@ -3414,10 +3422,10 @@ i915_perf_open_ioctl_locked(struct i915_perf *perf, struct drm_i915_file_private *file_priv = file->driver_priv; specific_ctx = i915_gem_context_lookup(file_priv, ctx_handle); - if (!specific_ctx) { + if (IS_ERR(specific_ctx)) { DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", ctx_handle); - ret = -ENOENT; + ret = PTR_ERR(specific_ctx); goto err; } } @@ -3887,7 +3895,7 @@ static bool gen8_is_valid_mux_addr(struct i915_perf *perf, u32 addr) REG_IN_RANGE(addr, RPM_CONFIG0, NOA_CONFIG(8)); } -static bool gen10_is_valid_mux_addr(struct i915_perf *perf, u32 addr) +static bool gen11_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { return gen8_is_valid_mux_addr(perf, addr) || REG_EQUAL(addr, GEN10_NOA_WRITE_HIGH) || @@ -4310,7 +4318,6 @@ static void oa_init_supported_formats(struct i915_perf *perf) case INTEL_GEMINILAKE: case INTEL_COFFEELAKE: case INTEL_COMETLAKE: - case INTEL_CANNONLAKE: case INTEL_ICELAKE: case INTEL_ELKHARTLAKE: case INTEL_JASPERLAKE: @@ -4395,27 +4402,23 @@ void i915_perf_init(struct drm_i915_private *i915) perf->gen8_valid_ctx_bit = BIT(16); } - } else if (IS_GRAPHICS_VER(i915, 10, 11)) { + } else if (GRAPHICS_VER(i915) == 11) { perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; perf->ops.is_valid_mux_reg = - gen10_is_valid_mux_addr; + gen11_is_valid_mux_addr; perf->ops.is_valid_flex_reg = gen8_is_valid_flex_addr; perf->ops.oa_enable = gen8_oa_enable; perf->ops.oa_disable = gen8_oa_disable; perf->ops.enable_metric_set = gen8_enable_metric_set; - perf->ops.disable_metric_set = gen10_disable_metric_set; + perf->ops.disable_metric_set = gen11_disable_metric_set; perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; - if (GRAPHICS_VER(i915) == 10) { - perf->ctx_oactxctrl_offset = 0x128; - perf->ctx_flexeu0_offset = 0x3de; - } else { - perf->ctx_oactxctrl_offset = 0x124; - perf->ctx_flexeu0_offset = 0x78e; - } + perf->ctx_oactxctrl_offset = 0x124; + perf->ctx_flexeu0_offset = 0x78e; + perf->gen8_valid_ctx_bit = BIT(16); } else if (GRAPHICS_VER(i915) == 12) { perf->ops.is_valid_b_counter_reg = @@ -4483,9 +4486,10 @@ static int destroy_config(int id, void *p, void *data) return 0; } -void i915_perf_sysctl_register(void) +int i915_perf_sysctl_register(void) { sysctl_header = register_sysctl_table(dev_root); + return 0; } void i915_perf_sysctl_unregister(void) diff --git a/drivers/gpu/drm/i915/i915_perf.h b/drivers/gpu/drm/i915/i915_perf.h index 882fdd0a7680..1d1329e5af3a 100644 --- a/drivers/gpu/drm/i915/i915_perf.h +++ b/drivers/gpu/drm/i915/i915_perf.h @@ -23,7 +23,7 @@ void i915_perf_fini(struct drm_i915_private *i915); void i915_perf_register(struct drm_i915_private *i915); void i915_perf_unregister(struct drm_i915_private *i915); int i915_perf_ioctl_version(void); -void i915_perf_sysctl_register(void); +int i915_perf_sysctl_register(void); void i915_perf_sysctl_unregister(void); int i915_perf_open_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 34d37d46a126..0b488d49694c 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -407,7 +407,7 @@ frequency_sample(struct intel_gt *gt, unsigned int period_ns) if (pmu->enable & config_mask(I915_PMU_REQUESTED_FREQUENCY)) { add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ], - intel_gpu_freq(rps, rps->cur_freq), + intel_rps_get_requested_frequency(rps), period_ns / 1000); } @@ -1088,7 +1088,7 @@ static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; -void i915_pmu_init(void) +int i915_pmu_init(void) { int ret; @@ -1101,6 +1101,8 @@ void i915_pmu_init(void) ret); else cpuhp_slot = ret; + + return 0; } void i915_pmu_exit(void) diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h index 60f9595f902c..449057648f39 100644 --- a/drivers/gpu/drm/i915/i915_pmu.h +++ b/drivers/gpu/drm/i915/i915_pmu.h @@ -147,14 +147,14 @@ struct i915_pmu { }; #ifdef CONFIG_PERF_EVENTS -void i915_pmu_init(void); +int i915_pmu_init(void); void i915_pmu_exit(void); void i915_pmu_register(struct drm_i915_private *i915); void i915_pmu_unregister(struct drm_i915_private *i915); void i915_pmu_gt_parked(struct drm_i915_private *i915); void i915_pmu_gt_unparked(struct drm_i915_private *i915); #else -static inline void i915_pmu_init(void) {} +static inline int i915_pmu_init(void) { return 0; } static inline void i915_pmu_exit(void) {} static inline void i915_pmu_register(struct drm_i915_private *i915) {} static inline void i915_pmu_unregister(struct drm_i915_private *i915) {} diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 16a19239d86d..664970f2bc62 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -395,10 +395,18 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_GRDOM_MEDIA2 (1 << 6) #define GEN11_GRDOM_MEDIA3 (1 << 7) #define GEN11_GRDOM_MEDIA4 (1 << 8) +#define GEN11_GRDOM_MEDIA5 (1 << 9) +#define GEN11_GRDOM_MEDIA6 (1 << 10) +#define GEN11_GRDOM_MEDIA7 (1 << 11) +#define GEN11_GRDOM_MEDIA8 (1 << 12) #define GEN11_GRDOM_VECS (1 << 13) #define GEN11_GRDOM_VECS2 (1 << 14) +#define GEN11_GRDOM_VECS3 (1 << 15) +#define GEN11_GRDOM_VECS4 (1 << 16) #define GEN11_GRDOM_SFC0 (1 << 17) #define GEN11_GRDOM_SFC1 (1 << 18) +#define GEN11_GRDOM_SFC2 (1 << 19) +#define GEN11_GRDOM_SFC3 (1 << 20) #define GEN11_VCS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << ((instance) >> 1)) #define GEN11_VECS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << (instance)) @@ -422,7 +430,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN12_HCP_SFC_LOCK_ACK_BIT REG_BIT(1) #define GEN12_HCP_SFC_USAGE_BIT REG_BIT(0) -#define GEN12_SFC_DONE(n) _MMIO(0x1cc00 + (n) * 0x100) +#define GEN12_SFC_DONE(n) _MMIO(0x1cc000 + (n) * 0x1000) #define GEN12_SFC_DONE_MAX 4 #define RING_PP_DIR_BASE(base) _MMIO((base) + 0x228) @@ -1877,7 +1885,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) /* - * CNL/ICL Port/COMBO-PHY Registers + * ICL Port/COMBO-PHY Registers */ #define _ICL_COMBOPHY_A 0x162000 #define _ICL_COMBOPHY_B 0x6C000 @@ -1891,11 +1899,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) _RKL_COMBOPHY_D, \ _ADL_COMBOPHY_E) -/* CNL/ICL Port CL_DW registers */ +/* ICL Port CL_DW registers */ #define _ICL_PORT_CL_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ 4 * (dw)) -#define CNL_PORT_CL1CM_DW5 _MMIO(0x162014) #define ICL_PORT_CL_DW5(phy) _MMIO(_ICL_PORT_CL_DW(5, phy)) #define CL_POWER_DOWN_ENABLE (1 << 4) #define SUS_CLOCK_CONFIG (3 << 0) @@ -1920,19 +1927,16 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_CL_DW12(phy) _MMIO(_ICL_PORT_CL_DW(12, phy)) #define ICL_LANE_ENABLE_AUX (1 << 0) -/* CNL/ICL Port COMP_DW registers */ +/* ICL Port COMP_DW registers */ #define _ICL_PORT_COMP 0x100 #define _ICL_PORT_COMP_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ _ICL_PORT_COMP + 4 * (dw)) -#define CNL_PORT_COMP_DW0 _MMIO(0x162100) #define ICL_PORT_COMP_DW0(phy) _MMIO(_ICL_PORT_COMP_DW(0, phy)) #define COMP_INIT (1 << 31) -#define CNL_PORT_COMP_DW1 _MMIO(0x162104) #define ICL_PORT_COMP_DW1(phy) _MMIO(_ICL_PORT_COMP_DW(1, phy)) -#define CNL_PORT_COMP_DW3 _MMIO(0x16210c) #define ICL_PORT_COMP_DW3(phy) _MMIO(_ICL_PORT_COMP_DW(3, phy)) #define PROCESS_INFO_DOT_0 (0 << 26) #define PROCESS_INFO_DOT_1 (1 << 26) @@ -1948,38 +1952,11 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_COMP_DW8(phy) _MMIO(_ICL_PORT_COMP_DW(8, phy)) #define IREFGEN (1 << 24) -#define CNL_PORT_COMP_DW9 _MMIO(0x162124) #define ICL_PORT_COMP_DW9(phy) _MMIO(_ICL_PORT_COMP_DW(9, phy)) -#define CNL_PORT_COMP_DW10 _MMIO(0x162128) #define ICL_PORT_COMP_DW10(phy) _MMIO(_ICL_PORT_COMP_DW(10, phy)) -/* CNL/ICL Port PCS registers */ -#define _CNL_PORT_PCS_DW1_GRP_AE 0x162304 -#define _CNL_PORT_PCS_DW1_GRP_B 0x162384 -#define _CNL_PORT_PCS_DW1_GRP_C 0x162B04 -#define _CNL_PORT_PCS_DW1_GRP_D 0x162B84 -#define _CNL_PORT_PCS_DW1_GRP_F 0x162A04 -#define _CNL_PORT_PCS_DW1_LN0_AE 0x162404 -#define _CNL_PORT_PCS_DW1_LN0_B 0x162604 -#define _CNL_PORT_PCS_DW1_LN0_C 0x162C04 -#define _CNL_PORT_PCS_DW1_LN0_D 0x162E04 -#define _CNL_PORT_PCS_DW1_LN0_F 0x162804 -#define CNL_PORT_PCS_DW1_GRP(phy) _MMIO(_PICK(phy, \ - _CNL_PORT_PCS_DW1_GRP_AE, \ - _CNL_PORT_PCS_DW1_GRP_B, \ - _CNL_PORT_PCS_DW1_GRP_C, \ - _CNL_PORT_PCS_DW1_GRP_D, \ - _CNL_PORT_PCS_DW1_GRP_AE, \ - _CNL_PORT_PCS_DW1_GRP_F)) -#define CNL_PORT_PCS_DW1_LN0(phy) _MMIO(_PICK(phy, \ - _CNL_PORT_PCS_DW1_LN0_AE, \ - _CNL_PORT_PCS_DW1_LN0_B, \ - _CNL_PORT_PCS_DW1_LN0_C, \ - _CNL_PORT_PCS_DW1_LN0_D, \ - _CNL_PORT_PCS_DW1_LN0_AE, \ - _CNL_PORT_PCS_DW1_LN0_F)) - +/* ICL Port PCS registers */ #define _ICL_PORT_PCS_AUX 0x300 #define _ICL_PORT_PCS_GRP 0x600 #define _ICL_PORT_PCS_LN(ln) (0x800 + (ln) * 0x100) @@ -1998,34 +1975,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define LATENCY_OPTIM_MASK (0x3 << 2) #define LATENCY_OPTIM_VAL(x) ((x) << 2) -/* CNL/ICL Port TX registers */ -#define _CNL_PORT_TX_AE_GRP_OFFSET 0x162340 -#define _CNL_PORT_TX_B_GRP_OFFSET 0x1623C0 -#define _CNL_PORT_TX_C_GRP_OFFSET 0x162B40 -#define _CNL_PORT_TX_D_GRP_OFFSET 0x162BC0 -#define _CNL_PORT_TX_F_GRP_OFFSET 0x162A40 -#define _CNL_PORT_TX_AE_LN0_OFFSET 0x162440 -#define _CNL_PORT_TX_B_LN0_OFFSET 0x162640 -#define _CNL_PORT_TX_C_LN0_OFFSET 0x162C40 -#define _CNL_PORT_TX_D_LN0_OFFSET 0x162E40 -#define _CNL_PORT_TX_F_LN0_OFFSET 0x162840 -#define _CNL_PORT_TX_DW_GRP(dw, port) (_PICK((port), \ - _CNL_PORT_TX_AE_GRP_OFFSET, \ - _CNL_PORT_TX_B_GRP_OFFSET, \ - _CNL_PORT_TX_B_GRP_OFFSET, \ - _CNL_PORT_TX_D_GRP_OFFSET, \ - _CNL_PORT_TX_AE_GRP_OFFSET, \ - _CNL_PORT_TX_F_GRP_OFFSET) + \ - 4 * (dw)) -#define _CNL_PORT_TX_DW_LN0(dw, port) (_PICK((port), \ - _CNL_PORT_TX_AE_LN0_OFFSET, \ - _CNL_PORT_TX_B_LN0_OFFSET, \ - _CNL_PORT_TX_B_LN0_OFFSET, \ - _CNL_PORT_TX_D_LN0_OFFSET, \ - _CNL_PORT_TX_AE_LN0_OFFSET, \ - _CNL_PORT_TX_F_LN0_OFFSET) + \ - 4 * (dw)) - +/* ICL Port TX registers */ #define _ICL_PORT_TX_AUX 0x380 #define _ICL_PORT_TX_GRP 0x680 #define _ICL_PORT_TX_LN(ln) (0x880 + (ln) * 0x100) @@ -2037,8 +1987,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define _ICL_PORT_TX_DW_LN(dw, ln, phy) (_ICL_COMBOPHY(phy) + \ _ICL_PORT_TX_LN(ln) + 4 * (dw)) -#define CNL_PORT_TX_DW2_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(2, port)) -#define CNL_PORT_TX_DW2_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(2, port)) #define ICL_PORT_TX_DW2_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(2, phy)) #define ICL_PORT_TX_DW2_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(2, phy)) #define ICL_PORT_TX_DW2_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(2, 0, phy)) @@ -2051,13 +1999,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RCOMP_SCALAR(x) ((x) << 0) #define RCOMP_SCALAR_MASK (0xFF << 0) -#define _CNL_PORT_TX_DW4_LN0_AE 0x162450 -#define _CNL_PORT_TX_DW4_LN1_AE 0x1624D0 -#define CNL_PORT_TX_DW4_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(4, (port))) -#define CNL_PORT_TX_DW4_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port))) -#define CNL_PORT_TX_DW4_LN(ln, port) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port)) + \ - ((ln) * (_CNL_PORT_TX_DW4_LN1_AE - \ - _CNL_PORT_TX_DW4_LN0_AE))) #define ICL_PORT_TX_DW4_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(4, phy)) #define ICL_PORT_TX_DW4_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(4, phy)) #define ICL_PORT_TX_DW4_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(4, 0, phy)) @@ -2070,8 +2011,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define CURSOR_COEFF(x) ((x) << 0) #define CURSOR_COEFF_MASK (0x3F << 0) -#define CNL_PORT_TX_DW5_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(5, port)) -#define CNL_PORT_TX_DW5_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(5, port)) #define ICL_PORT_TX_DW5_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(5, phy)) #define ICL_PORT_TX_DW5_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(5, phy)) #define ICL_PORT_TX_DW5_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(5, 0, phy)) @@ -2083,8 +2022,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RTERM_SELECT(x) ((x) << 3) #define RTERM_SELECT_MASK (0x7 << 3) -#define CNL_PORT_TX_DW7_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(7, (port))) -#define CNL_PORT_TX_DW7_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(7, (port))) #define ICL_PORT_TX_DW7_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(7, phy)) #define ICL_PORT_TX_DW7_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(7, phy)) #define ICL_PORT_TX_DW7_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(7, 0, phy)) @@ -2278,6 +2215,68 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MG_DP_MODE_CFG_DP_X2_MODE (1 << 7) #define MG_DP_MODE_CFG_DP_X1_MODE (1 << 6) +/* + * DG2 SNPS PHY registers (TC1 = PHY_E) + */ +#define _SNPS_PHY_A_BASE 0x168000 +#define _SNPS_PHY_B_BASE 0x169000 +#define _SNPS_PHY(phy) _PHY(phy, \ + _SNPS_PHY_A_BASE, \ + _SNPS_PHY_B_BASE) +#define _SNPS2(phy, reg) (_SNPS_PHY(phy) - \ + _SNPS_PHY_A_BASE + (reg)) +#define _MMIO_SNPS(phy, reg) _MMIO(_SNPS2(phy, reg)) +#define _MMIO_SNPS_LN(ln, phy, reg) _MMIO(_SNPS2(phy, \ + (reg) + (ln) * 0x10)) + +#define SNPS_PHY_MPLLB_CP(phy) _MMIO_SNPS(phy, 0x168000) +#define SNPS_PHY_MPLLB_CP_INT REG_GENMASK(31, 25) +#define SNPS_PHY_MPLLB_CP_INT_GS REG_GENMASK(23, 17) +#define SNPS_PHY_MPLLB_CP_PROP REG_GENMASK(15, 9) +#define SNPS_PHY_MPLLB_CP_PROP_GS REG_GENMASK(7, 1) + +#define SNPS_PHY_MPLLB_DIV(phy) _MMIO_SNPS(phy, 0x168004) +#define SNPS_PHY_MPLLB_FORCE_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_DIV5_CLK_EN REG_BIT(29) +#define SNPS_PHY_MPLLB_V2I REG_GENMASK(27, 26) +#define SNPS_PHY_MPLLB_FREQ_VCO REG_GENMASK(25, 24) +#define SNPS_PHY_MPLLB_PMIX_EN REG_BIT(10) +#define SNPS_PHY_MPLLB_TX_CLK_DIV REG_GENMASK(7, 5) + +#define SNPS_PHY_MPLLB_FRACN1(phy) _MMIO_SNPS(phy, 0x168008) +#define SNPS_PHY_MPLLB_FRACN_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN REG_BIT(30) +#define SNPS_PHY_MPLLB_FRACN_DEN REG_GENMASK(15, 0) + +#define SNPS_PHY_MPLLB_FRACN2(phy) _MMIO_SNPS(phy, 0x16800C) +#define SNPS_PHY_MPLLB_FRACN_REM REG_GENMASK(31, 16) +#define SNPS_PHY_MPLLB_FRACN_QUOT REG_GENMASK(15, 0) + +#define SNPS_PHY_MPLLB_SSCEN(phy) _MMIO_SNPS(phy, 0x168014) +#define SNPS_PHY_MPLLB_SSC_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_SSC_UP_SPREAD REG_BIT(30) +#define SNPS_PHY_MPLLB_SSC_PEAK REG_GENMASK(29, 10) + +#define SNPS_PHY_MPLLB_SSCSTEP(phy) _MMIO_SNPS(phy, 0x168018) +#define SNPS_PHY_MPLLB_SSC_STEPSIZE REG_GENMASK(31, 11) + +#define SNPS_PHY_MPLLB_DIV2(phy) _MMIO_SNPS(phy, 0x16801C) +#define SNPS_PHY_MPLLB_HDMI_PIXEL_CLK_DIV REG_GENMASK(19, 18) +#define SNPS_PHY_MPLLB_HDMI_DIV REG_GENMASK(17, 15) +#define SNPS_PHY_MPLLB_REF_CLK_DIV REG_GENMASK(14, 12) +#define SNPS_PHY_MPLLB_MULTIPLIER REG_GENMASK(11, 0) + +#define SNPS_PHY_REF_CONTROL(phy) _MMIO_SNPS(phy, 0x168188) +#define SNPS_PHY_REF_CONTROL_REF_RANGE REG_GENMASK(31, 27) + +#define SNPS_PHY_TX_REQ(phy) _MMIO_SNPS(phy, 0x168200) +#define SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR REG_GENMASK(31, 30) + +#define SNPS_PHY_TX_EQ(ln, phy) _MMIO_SNPS_LN(ln, phy, 0x168300) +#define SNPS_PHY_TX_EQ_MAIN REG_GENMASK(23, 18) +#define SNPS_PHY_TX_EQ_POST REG_GENMASK(15, 10) +#define SNPS_PHY_TX_EQ_PRE REG_GENMASK(7, 2) + /* The spec defines this only for BXT PHY0, but lets assume that this * would exist for PHY1 too if it had a second channel. */ @@ -2516,9 +2515,15 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_BSD2_RING_BASE 0x1c4000 #define GEN11_BSD3_RING_BASE 0x1d0000 #define GEN11_BSD4_RING_BASE 0x1d4000 +#define XEHP_BSD5_RING_BASE 0x1e0000 +#define XEHP_BSD6_RING_BASE 0x1e4000 +#define XEHP_BSD7_RING_BASE 0x1f0000 +#define XEHP_BSD8_RING_BASE 0x1f4000 #define VEBOX_RING_BASE 0x1a000 #define GEN11_VEBOX_RING_BASE 0x1c8000 #define GEN11_VEBOX2_RING_BASE 0x1d8000 +#define XEHP_VEBOX3_RING_BASE 0x1e8000 +#define XEHP_VEBOX4_RING_BASE 0x1f8000 #define BLT_RING_BASE 0x22000 #define RING_TAIL(base) _MMIO((base) + 0x30) #define RING_HEAD(base) _MMIO((base) + 0x34) @@ -2572,7 +2577,16 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ARB_MODE_BWGTLB_DISABLE (1 << 9) #define ARB_MODE_SWIZZLE_BDW (1 << 1) #define RENDER_HWS_PGA_GEN7 _MMIO(0x04080) -#define RING_FAULT_REG(engine) _MMIO(0x4094 + 0x100 * (engine)->hw_id) + +#define _RING_FAULT_REG_RCS 0x4094 +#define _RING_FAULT_REG_VCS 0x4194 +#define _RING_FAULT_REG_BCS 0x4294 +#define _RING_FAULT_REG_VECS 0x4394 +#define RING_FAULT_REG(engine) _MMIO(_PICK((engine)->class, \ + _RING_FAULT_REG_RCS, \ + _RING_FAULT_REG_VCS, \ + _RING_FAULT_REG_VECS, \ + _RING_FAULT_REG_BCS)) #define GEN8_RING_FAULT_REG _MMIO(0x4094) #define GEN12_RING_FAULT_REG _MMIO(0xcec4) #define GEN8_RING_FAULT_ENGINE_ID(x) (((x) >> 12) & 0x7) @@ -2672,6 +2686,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN12_SC_INSTDONE_EXTRA2 _MMIO(0x7108) #define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) #define GEN7_ROW_INSTDONE _MMIO(0xe164) +#define MCFG_MCR_SELECTOR _MMIO(0xfd0) +#define SF_MCR_SELECTOR _MMIO(0xfd8) #define GEN8_MCR_SELECTOR _MMIO(0xfdc) #define GEN8_MCR_SLICE(slice) (((slice) & 3) << 26) #define GEN8_MCR_SLICE_MASK GEN8_MCR_SLICE(3) @@ -3099,6 +3115,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN10_MIRROR_FUSE3 _MMIO(0x9118) #define GEN10_L3BANK_PAIR_COUNT 4 #define GEN10_L3BANK_MASK 0x0F +/* on Xe_HP the same fuses indicates mslices instead of L3 banks */ +#define GEN12_MAX_MSLICES 4 +#define GEN12_MEML3_EN_MASK 0x0F #define GEN8_EU_DISABLE0 _MMIO(0x9134) #define GEN8_EU_DIS0_S0_MASK 0xffffff @@ -3133,6 +3152,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN12_GT_DSS_ENABLE _MMIO(0x913C) +#define XEHP_EU_ENABLE _MMIO(0x9134) +#define XEHP_EU_ENA_MASK 0xFF + #define GEN6_BSD_SLEEP_PSMI_CONTROL _MMIO(0x12050) #define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0) #define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2) @@ -4086,6 +4108,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define BXT_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070) #define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) #define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) +#define RP0_CAP_MASK REG_GENMASK(7, 0) +#define RP1_CAP_MASK REG_GENMASK(15, 8) +#define RPN_CAP_MASK REG_GENMASK(23, 16) #define BXT_RP_STATE_CAP _MMIO(0x138170) #define GEN9_RP_STATE_LIMITS _MMIO(0x138148) @@ -4142,6 +4167,7 @@ enum { FAULT_AND_CONTINUE /* Unsupported */ }; +#define CTX_GTT_ADDRESS_MASK GENMASK(31, 12) #define GEN8_CTX_VALID (1 << 0) #define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) #define GEN8_CTX_FORCE_RESTORE (1 << 2) @@ -4158,6 +4184,11 @@ enum { #define GEN11_ENGINE_INSTANCE_SHIFT 48 #define GEN11_ENGINE_INSTANCE_WIDTH 6 +#define XEHP_SW_CTX_ID_SHIFT 39 +#define XEHP_SW_CTX_ID_WIDTH 16 +#define XEHP_SW_COUNTER_SHIFT 58 +#define XEHP_SW_COUNTER_WIDTH 6 + #define CHV_CLK_CTL1 _MMIO(0x101100) #define VLV_CLK_CTL2 _MMIO(0x101104) #define CLK_CTL2_CZCOUNT_30NS_SHIFT 28 @@ -4586,7 +4617,7 @@ enum { #define _PSR2_CTL_EDP 0x6f900 #define EDP_PSR2_CTL(tran) _MMIO_TRANS2(tran, _PSR2_CTL_A) #define EDP_PSR2_ENABLE (1 << 31) -#define EDP_SU_TRACK_ENABLE (1 << 30) +#define EDP_SU_TRACK_ENABLE (1 << 30) /* up to adl-p */ #define TGL_EDP_PSR2_BLOCK_COUNT_NUM_2 (0 << 28) #define TGL_EDP_PSR2_BLOCK_COUNT_NUM_3 (1 << 28) #define EDP_Y_COORDINATE_ENABLE REG_BIT(25) /* display 10, 11 and 12 */ @@ -4655,17 +4686,23 @@ enum { #define PSR2_SU_STATUS_MASK(frame) (0x3ff << PSR2_SU_STATUS_SHIFT(frame)) #define PSR2_SU_STATUS_FRAMES 8 -#define _PSR2_MAN_TRK_CTL_A 0x60910 -#define _PSR2_MAN_TRK_CTL_EDP 0x6f910 -#define PSR2_MAN_TRK_CTL(tran) _MMIO_TRANS2(tran, _PSR2_MAN_TRK_CTL_A) -#define PSR2_MAN_TRK_CTL_ENABLE REG_BIT(31) -#define PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK REG_GENMASK(30, 21) -#define PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val) REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val) +#define _PSR2_MAN_TRK_CTL_A 0x60910 +#define _PSR2_MAN_TRK_CTL_EDP 0x6f910 +#define PSR2_MAN_TRK_CTL(tran) _MMIO_TRANS2(tran, _PSR2_MAN_TRK_CTL_A) +#define PSR2_MAN_TRK_CTL_ENABLE REG_BIT(31) +#define PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK REG_GENMASK(30, 21) +#define PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val) REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val) #define PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK REG_GENMASK(20, 11) #define PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val) REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val) -#define PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME REG_BIT(3) -#define PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME REG_BIT(2) -#define PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE REG_BIT(1) +#define PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME REG_BIT(3) +#define PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME REG_BIT(2) +#define PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE REG_BIT(1) +#define ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK REG_GENMASK(28, 16) +#define ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val) REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val) +#define ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK REG_GENMASK(12, 0) +#define ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val) REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val) +#define ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME REG_BIT(14) +#define ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME REG_BIT(13) /* Icelake DSC Rate Control Range Parameter Registers */ #define DSCA_RC_RANGE_PARAMETERS_0 _MMIO(0x6B240) @@ -6166,11 +6203,17 @@ enum { #define PIPEMISC_HDR_MODE_PRECISION (1 << 23) /* icl+ */ #define PIPEMISC_OUTPUT_COLORSPACE_YUV (1 << 11) #define PIPEMISC_PIXEL_ROUNDING_TRUNC REG_BIT(8) /* tgl+ */ -#define PIPEMISC_DITHER_BPC_MASK (7 << 5) -#define PIPEMISC_DITHER_8_BPC (0 << 5) -#define PIPEMISC_DITHER_10_BPC (1 << 5) -#define PIPEMISC_DITHER_6_BPC (2 << 5) -#define PIPEMISC_DITHER_12_BPC (3 << 5) +/* + * For Display < 13, Bits 5-7 of PIPE MISC represent DITHER BPC with + * valid values of: 6, 8, 10 BPC. + * ADLP+, the bits 5-7 represent PORT OUTPUT BPC with valid values of: + * 6, 8, 10, 12 BPC. + */ +#define PIPEMISC_BPC_MASK (7 << 5) +#define PIPEMISC_8_BPC (0 << 5) +#define PIPEMISC_10_BPC (1 << 5) +#define PIPEMISC_6_BPC (2 << 5) +#define PIPEMISC_12_BPC_ADLP (4 << 5) /* adlp+ */ #define PIPEMISC_DITHER_ENABLE (1 << 4) #define PIPEMISC_DITHER_TYPE_MASK (3 << 2) #define PIPEMISC_DITHER_TYPE_SP (0 << 2) @@ -7714,11 +7757,11 @@ enum { #define SKL_PS_ECC_STAT(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_ECC_STAT_1A, _PS_ECC_STAT_2A), \ _ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B)) -#define CNL_PS_COEF_INDEX_SET(pipe, id, set) _MMIO_PIPE(pipe, \ +#define GLK_PS_COEF_INDEX_SET(pipe, id, set) _MMIO_PIPE(pipe, \ _ID(id, _PS_COEF_SET0_INDEX_1A, _PS_COEF_SET0_INDEX_2A) + (set) * 8, \ _ID(id, _PS_COEF_SET0_INDEX_1B, _PS_COEF_SET0_INDEX_2B) + (set) * 8) -#define CNL_PS_COEF_DATA_SET(pipe, id, set) _MMIO_PIPE(pipe, \ +#define GLK_PS_COEF_DATA_SET(pipe, id, set) _MMIO_PIPE(pipe, \ _ID(id, _PS_COEF_SET0_DATA_1A, _PS_COEF_SET0_DATA_2A) + (set) * 8, \ _ID(id, _PS_COEF_SET0_DATA_1B, _PS_COEF_SET0_DATA_2B) + (set) * 8) /* legacy palette */ @@ -7933,7 +7976,7 @@ enum { #define DSI1_NON_TE (1 << 31) #define DSI0_NON_TE (1 << 30) #define ICL_AUX_CHANNEL_E (1 << 29) -#define CNL_AUX_CHANNEL_F (1 << 28) +#define ICL_AUX_CHANNEL_F (1 << 28) #define GEN9_AUX_CHANNEL_D (1 << 27) #define GEN9_AUX_CHANNEL_C (1 << 26) #define GEN9_AUX_CHANNEL_B (1 << 25) @@ -7985,9 +8028,9 @@ enum { #define GEN11_GT_DW1_IRQ (1 << 1) #define GEN11_GT_DW0_IRQ (1 << 0) -#define DG1_MSTR_UNIT_INTR _MMIO(0x190008) +#define DG1_MSTR_TILE_INTR _MMIO(0x190008) #define DG1_MSTR_IRQ REG_BIT(31) -#define DG1_MSTR_UNIT(u) REG_BIT(u) +#define DG1_MSTR_TILE(t) REG_BIT(t) #define GEN11_DISPLAY_INT_CTL _MMIO(0x44200) #define GEN11_DISPLAY_IRQ_ENABLE (1 << 31) @@ -8070,7 +8113,10 @@ enum { #define GEN11_BCS_RSVD_INTR_MASK _MMIO(0x1900a0) #define GEN11_VCS0_VCS1_INTR_MASK _MMIO(0x1900a8) #define GEN11_VCS2_VCS3_INTR_MASK _MMIO(0x1900ac) +#define GEN12_VCS4_VCS5_INTR_MASK _MMIO(0x1900b0) +#define GEN12_VCS6_VCS7_INTR_MASK _MMIO(0x1900b4) #define GEN11_VECS0_VECS1_INTR_MASK _MMIO(0x1900d0) +#define GEN12_VECS2_VECS3_INTR_MASK _MMIO(0x1900d4) #define GEN11_GUC_SG_INTR_MASK _MMIO(0x1900e8) #define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) #define GEN11_CRYPTO_RSVD_INTR_MASK _MMIO(0x1900f0) @@ -8123,7 +8169,6 @@ enum { #define KVM_CONFIG_CHANGE_NOTIFICATION_SELECT (1 << 14) #define CHICKEN_MISC_2 _MMIO(0x42084) -#define CNL_COMP_PWR_DOWN (1 << 23) #define KBL_ARB_FILL_SPARE_14 REG_BIT(14) #define KBL_ARB_FILL_SPARE_13 REG_BIT(13) #define GLK_CL2_PWR_DOWN (1 << 12) @@ -8161,15 +8206,16 @@ enum { [TRANSCODER_B] = _CHICKEN_TRANS_B, \ [TRANSCODER_C] = _CHICKEN_TRANS_C, \ [TRANSCODER_D] = _CHICKEN_TRANS_D)) -#define HSW_FRAME_START_DELAY_MASK (3 << 27) -#define HSW_FRAME_START_DELAY(x) ((x) << 27) /* 0-3 */ -#define VSC_DATA_SEL_SOFTWARE_CONTROL (1 << 25) /* GLK and CNL+ */ -#define DDI_TRAINING_OVERRIDE_ENABLE (1 << 19) -#define DDI_TRAINING_OVERRIDE_VALUE (1 << 18) -#define DDIE_TRAINING_OVERRIDE_ENABLE (1 << 17) /* CHICKEN_TRANS_A only */ -#define DDIE_TRAINING_OVERRIDE_VALUE (1 << 16) /* CHICKEN_TRANS_A only */ -#define PSR2_ADD_VERTICAL_LINE_COUNT (1 << 15) -#define PSR2_VSC_ENABLE_PROG_HEADER (1 << 12) +#define HSW_FRAME_START_DELAY_MASK REG_GENMASK(28, 27) +#define HSW_FRAME_START_DELAY(x) REG_FIELD_PREP(HSW_FRAME_START_DELAY_MASK, x) +#define VSC_DATA_SEL_SOFTWARE_CONTROL REG_BIT(25) /* GLK */ +#define FECSTALL_DIS_DPTSTREAM_DPTTG REG_BIT(23) +#define DDI_TRAINING_OVERRIDE_ENABLE REG_BIT(19) +#define DDI_TRAINING_OVERRIDE_VALUE REG_BIT(18) +#define DDIE_TRAINING_OVERRIDE_ENABLE REG_BIT(17) /* CHICKEN_TRANS_A only */ +#define DDIE_TRAINING_OVERRIDE_VALUE REG_BIT(16) /* CHICKEN_TRANS_A only */ +#define PSR2_ADD_VERTICAL_LINE_COUNT REG_BIT(15) +#define PSR2_VSC_ENABLE_PROG_HEADER REG_BIT(12) #define DISP_ARB_CTL _MMIO(0x45000) #define DISP_FBC_MEMORY_WAKE (1 << 31) @@ -8227,9 +8273,8 @@ enum { #define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) #define SKL_SELECT_ALTERNATE_DC_EXIT (1 << 30) -#define CNL_DELAY_PMRSP (1 << 22) +#define ICL_DELAY_PMRSP (1 << 22) #define MASK_WAKEMEM (1 << 13) -#define CNL_DDI_CLOCK_REG_ACCESS_ON (1 << 7) #define GEN11_CHICKEN_DCPR_2 _MMIO(0x46434) #define DCPR_MASK_MAXLATENCY_MEMUP_CLR REG_BIT(27) @@ -8250,10 +8295,9 @@ enum { #define SKL_DFSM_PIPE_B_DISABLE (1 << 21) #define SKL_DFSM_PIPE_C_DISABLE (1 << 28) #define TGL_DFSM_PIPE_D_DISABLE (1 << 22) -#define CNL_DFSM_DISPLAY_DSC_DISABLE (1 << 7) +#define GLK_DFSM_DISPLAY_DSC_DISABLE (1 << 7) #define SKL_DSSM _MMIO(0x51004) -#define CNL_DSSM_CDCLK_PLL_REFCLK_24MHz (1 << 31) #define ICL_DSSM_CDCLK_PLL_REFCLK_MASK (7 << 29) #define ICL_DSSM_CDCLK_PLL_REFCLK_24MHz (0 << 29) #define ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz (1 << 29) @@ -8352,7 +8396,6 @@ enum { /* GEN8 chicken */ #define HDC_CHICKEN0 _MMIO(0x7300) -#define CNL_HDC_CHICKEN0 _MMIO(0xE5F0) #define ICL_HDC_MODE _MMIO(0xE5F4) #define HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE (1 << 15) #define HDC_FENCE_DEST_SLM_DISABLE (1 << 14) @@ -8386,7 +8429,8 @@ enum { #define _PIPEC_CHICKEN 0x72038 #define PIPE_CHICKEN(pipe) _MMIO_PIPE(pipe, _PIPEA_CHICKEN,\ _PIPEB_CHICKEN) -#define UNDERRUN_RECOVERY_DISABLE REG_BIT(30) +#define UNDERRUN_RECOVERY_DISABLE_ADLP REG_BIT(30) +#define UNDERRUN_RECOVERY_ENABLE_DG2 REG_BIT(30) #define PIXEL_ROUNDING_TRUNC_FB_PASSTHRU (1 << 15) #define PER_PIXEL_ALPHA_BYPASS_EN (1 << 7) @@ -9198,6 +9242,8 @@ enum { #define GEN9_FREQUENCY(x) ((x) << 23) #define GEN6_OFFSET(x) ((x) << 19) #define GEN6_AGGRESSIVE_TURBO (0 << 15) +#define GEN9_SW_REQ_UNSLICE_RATIO_SHIFT 23 + #define GEN6_RC_VIDEO_FREQ _MMIO(0xA00C) #define GEN6_RC_CONTROL _MMIO(0xA090) #define GEN6_RC_CTL_RC6pp_ENABLE (1 << 16) @@ -9366,9 +9412,13 @@ enum { #define ICL_PCODE_MEM_SUBSYSYSTEM_INFO 0xd #define ICL_PCODE_MEM_SS_READ_GLOBAL_INFO (0x0 << 8) #define ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point) (((point) << 16) | (0x1 << 8)) +#define ADL_PCODE_MEM_SS_READ_PSF_GV_INFO ((0) | (0x2 << 8)) #define ICL_PCODE_SAGV_DE_MEM_SS_CONFIG 0xe #define ICL_PCODE_POINTS_RESTRICTED 0x0 -#define ICL_PCODE_POINTS_RESTRICTED_MASK 0x1 +#define ICL_PCODE_POINTS_RESTRICTED_MASK 0xf +#define ADLS_PSF_PT_SHIFT 8 +#define ADLS_QGV_PT_MASK REG_GENMASK(7, 0) +#define ADLS_PSF_PT_MASK REG_GENMASK(10, 8) #define GEN6_PCODE_READ_D_COMP 0x10 #define GEN6_PCODE_WRITE_D_COMP 0x11 #define ICL_PCODE_EXIT_TCCOLD 0x12 @@ -9528,7 +9578,6 @@ enum { #define HSW_SAMPLE_C_PERFORMANCE (1 << 9) #define GEN8_CENTROID_PIXEL_OPT_DIS (1 << 8) #define GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC (1 << 5) -#define CNL_FAST_ANISO_L1_BANKING_FIX (1 << 4) #define GEN8_SAMPLER_POWER_BYPASS_DIS (1 << 1) #define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194) @@ -9709,15 +9758,12 @@ enum { /* HSW/BDW power well */ #define HSW_PW_CTL_IDX_GLOBAL 15 -/* SKL/BXT/GLK/CNL power wells */ +/* SKL/BXT/GLK power wells */ #define SKL_PW_CTL_IDX_PW_2 15 #define SKL_PW_CTL_IDX_PW_1 14 -#define CNL_PW_CTL_IDX_AUX_F 12 -#define CNL_PW_CTL_IDX_AUX_D 11 #define GLK_PW_CTL_IDX_AUX_C 10 #define GLK_PW_CTL_IDX_AUX_B 9 #define GLK_PW_CTL_IDX_AUX_A 8 -#define CNL_PW_CTL_IDX_DDI_F 6 #define SKL_PW_CTL_IDX_DDI_D 4 #define SKL_PW_CTL_IDX_DDI_C 3 #define SKL_PW_CTL_IDX_DDI_B 2 @@ -9816,19 +9862,6 @@ enum skl_power_gate { ((pw_idx) - ICL_PW_CTL_IDX_PW_1 + SKL_PG1) #define SKL_FUSE_PG_DIST_STATUS(pg) (1 << (27 - (pg))) -#define _CNL_AUX_REG_IDX(pw_idx) ((pw_idx) - GLK_PW_CTL_IDX_AUX_B) -#define _CNL_AUX_ANAOVRD1_B 0x162250 -#define _CNL_AUX_ANAOVRD1_C 0x162210 -#define _CNL_AUX_ANAOVRD1_D 0x1622D0 -#define _CNL_AUX_ANAOVRD1_F 0x162A90 -#define CNL_AUX_ANAOVRD1(pw_idx) _MMIO(_PICK(_CNL_AUX_REG_IDX(pw_idx), \ - _CNL_AUX_ANAOVRD1_B, \ - _CNL_AUX_ANAOVRD1_C, \ - _CNL_AUX_ANAOVRD1_D, \ - _CNL_AUX_ANAOVRD1_F)) -#define CNL_AUX_ANAOVRD1_ENABLE (1 << 16) -#define CNL_AUX_ANAOVRD1_LDO_BYPASS (1 << 23) - #define _ICL_AUX_REG_IDX(pw_idx) ((pw_idx) - ICL_PW_CTL_IDX_AUX_A) #define _ICL_AUX_ANAOVRD1_A 0x162398 #define _ICL_AUX_ANAOVRD1_B 0x6C398 @@ -10128,11 +10161,11 @@ enum skl_power_gate { #define TRANS_DDI_BPC_10 (1 << 20) #define TRANS_DDI_BPC_6 (2 << 20) #define TRANS_DDI_BPC_12 (3 << 20) -#define TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK REG_GENMASK(19, 18) /* bdw-cnl */ +#define TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK REG_GENMASK(19, 18) #define TRANS_DDI_PORT_SYNC_MASTER_SELECT(x) REG_FIELD_PREP(TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK, (x)) #define TRANS_DDI_PVSYNC (1 << 17) #define TRANS_DDI_PHSYNC (1 << 16) -#define TRANS_DDI_PORT_SYNC_ENABLE REG_BIT(15) /* bdw-cnl */ +#define TRANS_DDI_PORT_SYNC_ENABLE REG_BIT(15) #define TRANS_DDI_EDP_INPUT_MASK (7 << 12) #define TRANS_DDI_EDP_INPUT_A_ON (0 << 12) #define TRANS_DDI_EDP_INPUT_A_ONOFF (4 << 12) @@ -10165,6 +10198,9 @@ enum skl_power_gate { #define PORT_SYNC_MODE_MASTER_SELECT_MASK REG_GENMASK(2, 0) #define PORT_SYNC_MODE_MASTER_SELECT(x) REG_FIELD_PREP(PORT_SYNC_MODE_MASTER_SELECT_MASK, (x)) +#define TRANS_CMTG_CHICKEN _MMIO(0x6fa90) +#define DISABLE_DPT_CLK_GATING REG_BIT(1) + /* DisplayPort Transport Control */ #define _DP_TP_CTL_A 0x64040 #define _DP_TP_CTL_B 0x64140 @@ -10488,17 +10524,6 @@ enum skl_power_gate { #define DPLL_CFGCR1(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR1, _DPLL2_CFGCR1) #define DPLL_CFGCR2(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR2, _DPLL2_CFGCR2) -/* - * CNL Clocks - */ -#define DPCLKA_CFGCR0 _MMIO(0x6C200) -#define DPCLKA_CFGCR0_DDI_CLK_OFF(port) (1 << ((port) == PORT_F ? 23 : \ - (port) + 10)) -#define DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port) ((port) == PORT_F ? 21 : \ - (port) * 2) -#define DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port) (3 << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) -#define DPCLKA_CFGCR0_DDI_CLK_SEL(pll, port) ((pll) << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) - /* ICL Clocks */ #define ICL_DPCLKA_CFGCR0 _MMIO(0x164280) #define ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) (1 << _PICK(phy, 10, 11, 24, 4, 5)) @@ -10554,7 +10579,7 @@ enum skl_power_gate { ADLS_DPCLKA_DDIJ_SEL_MASK, \ ADLS_DPCLKA_DDIK_SEL_MASK) -/* CNL PLL */ +/* ICL PLL */ #define DPLL0_ENABLE 0x46010 #define DPLL1_ENABLE 0x46014 #define _ADLS_DPLL2_ENABLE 0x46018 @@ -10563,9 +10588,14 @@ enum skl_power_gate { #define PLL_LOCK (1 << 30) #define PLL_POWER_ENABLE (1 << 27) #define PLL_POWER_STATE (1 << 26) -#define CNL_DPLL_ENABLE(pll) _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \ +#define ICL_DPLL_ENABLE(pll) _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \ _ADLS_DPLL2_ENABLE, _ADLS_DPLL3_ENABLE) +#define _DG2_PLL3_ENABLE 0x4601C + +#define DG2_PLL_ENABLE(pll) _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \ + _ADLS_DPLL2_ENABLE, _DG2_PLL3_ENABLE) + #define TBT_PLL_ENABLE _MMIO(0x46020) #define _MG_PLL1_ENABLE 0x46030 @@ -10731,60 +10761,52 @@ enum skl_power_gate { _MG_PLL_TDC_COLDST_BIAS_PORT1, \ _MG_PLL_TDC_COLDST_BIAS_PORT2) -#define _CNL_DPLL0_CFGCR0 0x6C000 -#define _CNL_DPLL1_CFGCR0 0x6C080 -#define DPLL_CFGCR0_HDMI_MODE (1 << 30) -#define DPLL_CFGCR0_SSC_ENABLE (1 << 29) -#define DPLL_CFGCR0_SSC_ENABLE_ICL (1 << 25) -#define DPLL_CFGCR0_LINK_RATE_MASK (0xf << 25) -#define DPLL_CFGCR0_LINK_RATE_2700 (0 << 25) -#define DPLL_CFGCR0_LINK_RATE_1350 (1 << 25) -#define DPLL_CFGCR0_LINK_RATE_810 (2 << 25) -#define DPLL_CFGCR0_LINK_RATE_1620 (3 << 25) -#define DPLL_CFGCR0_LINK_RATE_1080 (4 << 25) -#define DPLL_CFGCR0_LINK_RATE_2160 (5 << 25) -#define DPLL_CFGCR0_LINK_RATE_3240 (6 << 25) -#define DPLL_CFGCR0_LINK_RATE_4050 (7 << 25) -#define DPLL_CFGCR0_DCO_FRACTION_MASK (0x7fff << 10) -#define DPLL_CFGCR0_DCO_FRACTION_SHIFT (10) -#define DPLL_CFGCR0_DCO_FRACTION(x) ((x) << 10) -#define DPLL_CFGCR0_DCO_INTEGER_MASK (0x3ff) -#define CNL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _CNL_DPLL0_CFGCR0, _CNL_DPLL1_CFGCR0) - -#define _CNL_DPLL0_CFGCR1 0x6C004 -#define _CNL_DPLL1_CFGCR1 0x6C084 -#define DPLL_CFGCR1_QDIV_RATIO_MASK (0xff << 10) -#define DPLL_CFGCR1_QDIV_RATIO_SHIFT (10) -#define DPLL_CFGCR1_QDIV_RATIO(x) ((x) << 10) -#define DPLL_CFGCR1_QDIV_MODE_SHIFT (9) -#define DPLL_CFGCR1_QDIV_MODE(x) ((x) << 9) -#define DPLL_CFGCR1_KDIV_MASK (7 << 6) -#define DPLL_CFGCR1_KDIV_SHIFT (6) -#define DPLL_CFGCR1_KDIV(x) ((x) << 6) -#define DPLL_CFGCR1_KDIV_1 (1 << 6) -#define DPLL_CFGCR1_KDIV_2 (2 << 6) -#define DPLL_CFGCR1_KDIV_3 (4 << 6) -#define DPLL_CFGCR1_PDIV_MASK (0xf << 2) -#define DPLL_CFGCR1_PDIV_SHIFT (2) -#define DPLL_CFGCR1_PDIV(x) ((x) << 2) -#define DPLL_CFGCR1_PDIV_2 (1 << 2) -#define DPLL_CFGCR1_PDIV_3 (2 << 2) -#define DPLL_CFGCR1_PDIV_5 (4 << 2) -#define DPLL_CFGCR1_PDIV_7 (8 << 2) -#define DPLL_CFGCR1_CENTRAL_FREQ (3 << 0) -#define DPLL_CFGCR1_CENTRAL_FREQ_8400 (3 << 0) -#define TGL_DPLL_CFGCR1_CFSELOVRD_NORMAL_XTAL (0 << 0) -#define CNL_DPLL_CFGCR1(pll) _MMIO_PLL(pll, _CNL_DPLL0_CFGCR1, _CNL_DPLL1_CFGCR1) - #define _ICL_DPLL0_CFGCR0 0x164000 #define _ICL_DPLL1_CFGCR0 0x164080 #define ICL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _ICL_DPLL0_CFGCR0, \ _ICL_DPLL1_CFGCR0) +#define DPLL_CFGCR0_HDMI_MODE (1 << 30) +#define DPLL_CFGCR0_SSC_ENABLE (1 << 29) +#define DPLL_CFGCR0_SSC_ENABLE_ICL (1 << 25) +#define DPLL_CFGCR0_LINK_RATE_MASK (0xf << 25) +#define DPLL_CFGCR0_LINK_RATE_2700 (0 << 25) +#define DPLL_CFGCR0_LINK_RATE_1350 (1 << 25) +#define DPLL_CFGCR0_LINK_RATE_810 (2 << 25) +#define DPLL_CFGCR0_LINK_RATE_1620 (3 << 25) +#define DPLL_CFGCR0_LINK_RATE_1080 (4 << 25) +#define DPLL_CFGCR0_LINK_RATE_2160 (5 << 25) +#define DPLL_CFGCR0_LINK_RATE_3240 (6 << 25) +#define DPLL_CFGCR0_LINK_RATE_4050 (7 << 25) +#define DPLL_CFGCR0_DCO_FRACTION_MASK (0x7fff << 10) +#define DPLL_CFGCR0_DCO_FRACTION_SHIFT (10) +#define DPLL_CFGCR0_DCO_FRACTION(x) ((x) << 10) +#define DPLL_CFGCR0_DCO_INTEGER_MASK (0x3ff) #define _ICL_DPLL0_CFGCR1 0x164004 #define _ICL_DPLL1_CFGCR1 0x164084 #define ICL_DPLL_CFGCR1(pll) _MMIO_PLL(pll, _ICL_DPLL0_CFGCR1, \ _ICL_DPLL1_CFGCR1) +#define DPLL_CFGCR1_QDIV_RATIO_MASK (0xff << 10) +#define DPLL_CFGCR1_QDIV_RATIO_SHIFT (10) +#define DPLL_CFGCR1_QDIV_RATIO(x) ((x) << 10) +#define DPLL_CFGCR1_QDIV_MODE_SHIFT (9) +#define DPLL_CFGCR1_QDIV_MODE(x) ((x) << 9) +#define DPLL_CFGCR1_KDIV_MASK (7 << 6) +#define DPLL_CFGCR1_KDIV_SHIFT (6) +#define DPLL_CFGCR1_KDIV(x) ((x) << 6) +#define DPLL_CFGCR1_KDIV_1 (1 << 6) +#define DPLL_CFGCR1_KDIV_2 (2 << 6) +#define DPLL_CFGCR1_KDIV_3 (4 << 6) +#define DPLL_CFGCR1_PDIV_MASK (0xf << 2) +#define DPLL_CFGCR1_PDIV_SHIFT (2) +#define DPLL_CFGCR1_PDIV(x) ((x) << 2) +#define DPLL_CFGCR1_PDIV_2 (1 << 2) +#define DPLL_CFGCR1_PDIV_3 (2 << 2) +#define DPLL_CFGCR1_PDIV_5 (4 << 2) +#define DPLL_CFGCR1_PDIV_7 (8 << 2) +#define DPLL_CFGCR1_CENTRAL_FREQ (3 << 0) +#define DPLL_CFGCR1_CENTRAL_FREQ_8400 (3 << 0) +#define TGL_DPLL_CFGCR1_CFSELOVRD_NORMAL_XTAL (0 << 0) #define _TGL_DPLL0_CFGCR0 0x164284 #define _TGL_DPLL1_CFGCR0 0x16428C @@ -11004,8 +11026,8 @@ enum skl_power_gate { #define BXT_DE_PLL_LOCK (1 << 30) #define BXT_DE_PLL_FREQ_REQ (1 << 23) #define BXT_DE_PLL_FREQ_REQ_ACK (1 << 22) -#define CNL_CDCLK_PLL_RATIO(x) (x) -#define CNL_CDCLK_PLL_RATIO_MASK 0xff +#define ICL_CDCLK_PLL_RATIO(x) (x) +#define ICL_CDCLK_PLL_RATIO_MASK 0xff /* GEN9 DC */ #define DC_STATE_EN _MMIO(0x45504) @@ -11060,6 +11082,7 @@ enum skl_power_gate { #define SKL_MEMORY_FREQ_MULTIPLIER_HZ 266666666 #define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5E04) #define SKL_REQ_DATA_MASK (0xF << 0) +#define DG1_GEAR_TYPE REG_BIT(16) #define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000) #define SKL_DRAM_DDR_TYPE_MASK (0x3 << 0) @@ -11082,18 +11105,29 @@ enum skl_power_gate { #define SKL_DRAM_RANK_1 (0x0 << 10) #define SKL_DRAM_RANK_2 (0x1 << 10) #define SKL_DRAM_RANK_MASK (0x1 << 10) -#define CNL_DRAM_SIZE_MASK 0x7F -#define CNL_DRAM_WIDTH_MASK (0x3 << 7) -#define CNL_DRAM_WIDTH_SHIFT 7 -#define CNL_DRAM_WIDTH_X8 (0x0 << 7) -#define CNL_DRAM_WIDTH_X16 (0x1 << 7) -#define CNL_DRAM_WIDTH_X32 (0x2 << 7) -#define CNL_DRAM_RANK_MASK (0x3 << 9) -#define CNL_DRAM_RANK_SHIFT 9 -#define CNL_DRAM_RANK_1 (0x0 << 9) -#define CNL_DRAM_RANK_2 (0x1 << 9) -#define CNL_DRAM_RANK_3 (0x2 << 9) -#define CNL_DRAM_RANK_4 (0x3 << 9) +#define ICL_DRAM_SIZE_MASK 0x7F +#define ICL_DRAM_WIDTH_MASK (0x3 << 7) +#define ICL_DRAM_WIDTH_SHIFT 7 +#define ICL_DRAM_WIDTH_X8 (0x0 << 7) +#define ICL_DRAM_WIDTH_X16 (0x1 << 7) +#define ICL_DRAM_WIDTH_X32 (0x2 << 7) +#define ICL_DRAM_RANK_MASK (0x3 << 9) +#define ICL_DRAM_RANK_SHIFT 9 +#define ICL_DRAM_RANK_1 (0x0 << 9) +#define ICL_DRAM_RANK_2 (0x1 << 9) +#define ICL_DRAM_RANK_3 (0x2 << 9) +#define ICL_DRAM_RANK_4 (0x3 << 9) + +#define SA_PERF_STATUS_0_0_0_MCHBAR_PC _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918) +#define DG1_QCLK_RATIO_MASK REG_GENMASK(9, 2) +#define DG1_QCLK_REFERENCE REG_BIT(10) + +#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000) +#define DG1_DRAM_T_RDPRE_MASK REG_GENMASK(16, 11) +#define DG1_DRAM_T_RP_MASK REG_GENMASK(6, 0) +#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4004) +#define DG1_DRAM_T_RCD_MASK REG_GENMASK(15, 9) +#define DG1_DRAM_T_RAS_MASK REG_GENMASK(8, 1) /* * Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register, @@ -12296,6 +12330,7 @@ enum skl_power_gate { /* MOCS (Memory Object Control State) registers */ #define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ +#define GEN9_LNCFCMOCS_REG_COUNT 32 #define __GEN9_RCS0_MOCS0 0xc800 #define GEN9_GFX_MOCS(i) _MMIO(__GEN9_RCS0_MOCS0 + (i) * 4) @@ -12343,6 +12378,7 @@ enum skl_power_gate { _ICL_PHY_MISC_B) #define ICL_PHY_MISC_MUX_DDID (1 << 28) #define ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN (1 << 23) +#define DG2_PHY_DP_TX_ACK_MASK REG_GENMASK(23, 20) /* Icelake Display Stream Compression Registers */ #define DSCA_PICTURE_PARAMETER_SET_0 _MMIO(0x6B200) diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 37aef1308573..ce446716d092 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -42,22 +42,17 @@ #include "i915_active.h" #include "i915_drv.h" -#include "i915_globals.h" #include "i915_trace.h" #include "intel_pm.h" struct execute_cb { struct irq_work work; struct i915_sw_fence *fence; - void (*hook)(struct i915_request *rq, struct dma_fence *signal); struct i915_request *signal; }; -static struct i915_global_request { - struct i915_global base; - struct kmem_cache *slab_requests; - struct kmem_cache *slab_execute_cbs; -} global; +static struct kmem_cache *slab_requests; +static struct kmem_cache *slab_execute_cbs; static const char *i915_fence_get_driver_name(struct dma_fence *fence) { @@ -108,13 +103,16 @@ static signed long i915_fence_wait(struct dma_fence *fence, struct kmem_cache *i915_request_slab_cache(void) { - return global.slab_requests; + return slab_requests; } static void i915_fence_release(struct dma_fence *fence) { struct i915_request *rq = to_request(fence); + GEM_BUG_ON(rq->guc_prio != GUC_PRIO_INIT && + rq->guc_prio != GUC_PRIO_FINI); + /* * The request is put onto a RCU freelist (i.e. the address * is immediately reused), mark the fences as being freed now. @@ -126,41 +124,19 @@ static void i915_fence_release(struct dma_fence *fence) i915_sw_fence_fini(&rq->semaphore); /* - * Keep one request on each engine for reserved use under mempressure - * - * We do not hold a reference to the engine here and so have to be - * very careful in what rq->engine we poke. The virtual engine is - * referenced via the rq->context and we released that ref during - * i915_request_retire(), ergo we must not dereference a virtual - * engine here. Not that we would want to, as the only consumer of - * the reserved engine->request_pool is the power management parking, - * which must-not-fail, and that is only run on the physical engines. - * - * Since the request must have been executed to be have completed, - * we know that it will have been processed by the HW and will - * not be unsubmitted again, so rq->engine and rq->execution_mask - * at this point is stable. rq->execution_mask will be a single - * bit if the last and _only_ engine it could execution on was a - * physical engine, if it's multiple bits then it started on and - * could still be on a virtual engine. Thus if the mask is not a - * power-of-two we assume that rq->engine may still be a virtual - * engine and so a dangling invalid pointer that we cannot dereference - * - * For example, consider the flow of a bonded request through a virtual - * engine. The request is created with a wide engine mask (all engines - * that we might execute on). On processing the bond, the request mask - * is reduced to one or more engines. If the request is subsequently - * bound to a single engine, it will then be constrained to only - * execute on that engine and never returned to the virtual engine - * after timeslicing away, see __unwind_incomplete_requests(). Thus we - * know that if the rq->execution_mask is a single bit, rq->engine - * can be a physical engine with the exact corresponding mask. + * Keep one request on each engine for reserved use under mempressure, + * do not use with virtual engines as this really is only needed for + * kernel contexts. */ - if (is_power_of_2(rq->execution_mask) && - !cmpxchg(&rq->engine->request_pool, NULL, rq)) + if (!intel_engine_is_virtual(rq->engine) && + !cmpxchg(&rq->engine->request_pool, NULL, rq)) { + intel_context_put(rq->context); return; + } + + intel_context_put(rq->context); - kmem_cache_free(global.slab_requests, rq); + kmem_cache_free(slab_requests, rq); } const struct dma_fence_ops i915_fence_ops = { @@ -177,18 +153,7 @@ static void irq_execute_cb(struct irq_work *wrk) struct execute_cb *cb = container_of(wrk, typeof(*cb), work); i915_sw_fence_complete(cb->fence); - kmem_cache_free(global.slab_execute_cbs, cb); -} - -static void irq_execute_cb_hook(struct irq_work *wrk) -{ - struct execute_cb *cb = container_of(wrk, typeof(*cb), work); - - cb->hook(container_of(cb->fence, struct i915_request, submit), - &cb->signal->fence); - i915_request_put(cb->signal); - - irq_execute_cb(wrk); + kmem_cache_free(slab_execute_cbs, cb); } static __always_inline void @@ -216,7 +181,7 @@ static bool irq_work_imm(struct irq_work *wrk) return false; } -static void __notify_execute_cb_imm(struct i915_request *rq) +void i915_request_notify_execute_cb_imm(struct i915_request *rq) { __notify_execute_cb(rq, irq_work_imm); } @@ -272,11 +237,11 @@ i915_request_active_engine(struct i915_request *rq, * check that we have acquired the lock on the final engine. */ locked = READ_ONCE(rq->engine); - spin_lock_irq(&locked->active.lock); + spin_lock_irq(&locked->sched_engine->lock); while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { - spin_unlock(&locked->active.lock); + spin_unlock(&locked->sched_engine->lock); locked = engine; - spin_lock(&locked->active.lock); + spin_lock(&locked->sched_engine->lock); } if (i915_request_is_active(rq)) { @@ -285,42 +250,11 @@ i915_request_active_engine(struct i915_request *rq, ret = true; } - spin_unlock_irq(&locked->active.lock); + spin_unlock_irq(&locked->sched_engine->lock); return ret; } - -static void remove_from_engine(struct i915_request *rq) -{ - struct intel_engine_cs *engine, *locked; - - /* - * Virtual engines complicate acquiring the engine timeline lock, - * as their rq->engine pointer is not stable until under that - * engine lock. The simple ploy we use is to take the lock then - * check that the rq still belongs to the newly locked engine. - */ - locked = READ_ONCE(rq->engine); - spin_lock_irq(&locked->active.lock); - while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { - spin_unlock(&locked->active.lock); - spin_lock(&engine->active.lock); - locked = engine; - } - list_del_init(&rq->sched.link); - - clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); - clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags); - - /* Prevent further __await_execution() registering a cb, then flush */ - set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); - - spin_unlock_irq(&locked->active.lock); - - __notify_execute_cb_imm(rq); -} - static void __rq_init_watchdog(struct i915_request *rq) { rq->watchdog.timer.function = NULL; @@ -417,8 +351,7 @@ bool i915_request_retire(struct i915_request *rq) * after removing the breadcrumb and signaling it, so that we do not * inadvertently attach the breadcrumb to a completed request. */ - if (!list_empty(&rq->sched.link)) - remove_from_engine(rq); + rq->engine->remove_active_request(rq); GEM_BUG_ON(!llist_empty(&rq->execute_cb)); __list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */ @@ -443,6 +376,7 @@ void i915_request_retire_upto(struct i915_request *rq) do { tmp = list_first_entry(&tl->requests, typeof(*tmp), link); + GEM_BUG_ON(!i915_request_completed(tmp)); } while (i915_request_retire(tmp) && tmp != rq); } @@ -517,19 +451,14 @@ static bool __request_in_flight(const struct i915_request *signal) static int __await_execution(struct i915_request *rq, struct i915_request *signal, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal), gfp_t gfp) { struct execute_cb *cb; - if (i915_request_is_active(signal)) { - if (hook) - hook(rq, &signal->fence); + if (i915_request_is_active(signal)) return 0; - } - cb = kmem_cache_alloc(global.slab_execute_cbs, gfp); + cb = kmem_cache_alloc(slab_execute_cbs, gfp); if (!cb) return -ENOMEM; @@ -537,12 +466,6 @@ __await_execution(struct i915_request *rq, i915_sw_fence_await(cb->fence); init_irq_work(&cb->work, irq_execute_cb); - if (hook) { - cb->hook = hook; - cb->signal = i915_request_get(signal); - cb->work.func = irq_execute_cb_hook; - } - /* * Register the callback first, then see if the signaler is already * active. This ensures that if we race with the @@ -559,7 +482,7 @@ __await_execution(struct i915_request *rq, if (llist_add(&cb->work.node.llist, &signal->execute_cb)) { if (i915_request_is_active(signal) || __request_in_flight(signal)) - __notify_execute_cb_imm(signal); + i915_request_notify_execute_cb_imm(signal); } return 0; @@ -637,7 +560,7 @@ bool __i915_request_submit(struct i915_request *request) RQ_TRACE(request, "\n"); GEM_BUG_ON(!irqs_disabled()); - lockdep_assert_held(&engine->active.lock); + lockdep_assert_held(&engine->sched_engine->lock); /* * With the advent of preempt-to-busy, we frequently encounter @@ -649,7 +572,7 @@ bool __i915_request_submit(struct i915_request *request) * * We must remove the request from the caller's priority queue, * and the caller must only call us when the request is in their - * priority queue, under the active.lock. This ensures that the + * priority queue, under the sched_engine->lock. This ensures that the * request has *not* yet been retired and we can safely move * the request into the engine->active.list where it will be * dropped upon retiring. (Otherwise if resubmit a *retired* @@ -690,11 +613,15 @@ bool __i915_request_submit(struct i915_request *request) request->ring->vaddr + request->postfix); trace_i915_request_execute(request); - engine->serial++; + if (engine->bump_serial) + engine->bump_serial(engine); + else + engine->serial++; + result = true; GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); - list_move_tail(&request->sched.link, &engine->active.requests); + engine->add_active_request(request); active: clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags); set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); @@ -724,11 +651,11 @@ void i915_request_submit(struct i915_request *request) unsigned long flags; /* Will be called from irq-context when using foreign fences. */ - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); __i915_request_submit(request); - spin_unlock_irqrestore(&engine->active.lock, flags); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } void __i915_request_unsubmit(struct i915_request *request) @@ -742,7 +669,7 @@ void __i915_request_unsubmit(struct i915_request *request) RQ_TRACE(request, "\n"); GEM_BUG_ON(!irqs_disabled()); - lockdep_assert_held(&engine->active.lock); + lockdep_assert_held(&engine->sched_engine->lock); /* * Before we remove this breadcrumb from the signal list, we have @@ -775,23 +702,11 @@ void i915_request_unsubmit(struct i915_request *request) unsigned long flags; /* Will be called from irq-context when using foreign fences. */ - spin_lock_irqsave(&engine->active.lock, flags); + spin_lock_irqsave(&engine->sched_engine->lock, flags); __i915_request_unsubmit(request); - spin_unlock_irqrestore(&engine->active.lock, flags); -} - -static void __cancel_request(struct i915_request *rq) -{ - struct intel_engine_cs *engine = NULL; - - i915_request_active_engine(rq, &engine); - - if (engine && intel_engine_pulse(engine)) - intel_gt_handle_error(engine->gt, engine->mask, 0, - "request cancellation by %s", - current->comm); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); } void i915_request_cancel(struct i915_request *rq, int error) @@ -801,7 +716,7 @@ void i915_request_cancel(struct i915_request *rq, int error) set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags); - __cancel_request(rq); + intel_context_cancel_request(rq->context, rq); } static int __i915_sw_fence_call @@ -889,7 +804,7 @@ request_alloc_slow(struct intel_timeline *tl, rq = list_first_entry(&tl->requests, typeof(*rq), link); i915_request_retire(rq); - rq = kmem_cache_alloc(global.slab_requests, + rq = kmem_cache_alloc(slab_requests, gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); if (rq) return rq; @@ -902,7 +817,7 @@ request_alloc_slow(struct intel_timeline *tl, retire_requests(tl); out: - return kmem_cache_alloc(global.slab_requests, gfp); + return kmem_cache_alloc(slab_requests, gfp); } static void __i915_request_ctor(void *arg) @@ -963,7 +878,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) * * Do not use kmem_cache_zalloc() here! */ - rq = kmem_cache_alloc(global.slab_requests, + rq = kmem_cache_alloc(slab_requests, gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); if (unlikely(!rq)) { rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp); @@ -973,7 +888,19 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) } } - rq->context = ce; + /* + * Hold a reference to the intel_context over life of an i915_request. + * Without this an i915_request can exist after the context has been + * destroyed (e.g. request retired, context closed, but user space holds + * a reference to the request from an out fence). In the case of GuC + * submission + virtual engine, the engine that the request references + * is also destroyed which can trigger bad pointer dref in fence ops + * (e.g. i915_fence_get_driver_name). We could likely change these + * functions to avoid touching the engine but let's just be safe and + * hold the intel_context reference. In execlist mode the request always + * eventually points to a physical engine so this isn't an issue. + */ + rq->context = intel_context_get(ce); rq->engine = ce->engine; rq->ring = ce->ring; rq->execution_mask = ce->engine->mask; @@ -996,6 +923,8 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */ + rq->guc_prio = GUC_PRIO_INIT; + /* We bump the ref for the fence chain */ i915_sw_fence_reinit(&i915_request_get(rq)->submit); i915_sw_fence_reinit(&i915_request_get(rq)->semaphore); @@ -1050,7 +979,8 @@ err_unwind: GEM_BUG_ON(!list_empty(&rq->sched.waiters_list)); err_free: - kmem_cache_free(global.slab_requests, rq); + intel_context_put(ce); + kmem_cache_free(slab_requests, rq); err_unreserve: intel_context_unpin(ce); return ERR_PTR(ret); @@ -1253,7 +1183,7 @@ emit_semaphore_wait(struct i915_request *to, goto await_fence; /* Only submit our spinner after the signaler is running! */ - if (__await_execution(to, from, NULL, gfp)) + if (__await_execution(to, from, gfp)) goto await_fence; if (__emit_semaphore_wait(to, from, from->fence.seqno)) @@ -1284,16 +1214,14 @@ static int intel_timeline_sync_set_start(struct intel_timeline *tl, static int __i915_request_await_execution(struct i915_request *to, - struct i915_request *from, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal)) + struct i915_request *from) { int err; GEM_BUG_ON(intel_context_is_barrier(from->context)); /* Submit both requests at the same time */ - err = __await_execution(to, from, hook, I915_FENCE_GFP); + err = __await_execution(to, from, I915_FENCE_GFP); if (err) return err; @@ -1343,7 +1271,7 @@ __i915_request_await_execution(struct i915_request *to, } /* Couple the dependency tree for PI on this exposed to->fence */ - if (to->engine->schedule) { + if (to->engine->sched_engine->schedule) { err = i915_sched_node_add_dependency(&to->sched, &from->sched, I915_DEPENDENCY_WEAK); @@ -1406,9 +1334,7 @@ i915_request_await_external(struct i915_request *rq, struct dma_fence *fence) int i915_request_await_execution(struct i915_request *rq, - struct dma_fence *fence, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal)) + struct dma_fence *fence) { struct dma_fence **child = &fence; unsigned int nchild = 1; @@ -1439,8 +1365,7 @@ i915_request_await_execution(struct i915_request *rq, if (dma_fence_is_i915(fence)) ret = __i915_request_await_execution(rq, - to_request(fence), - hook); + to_request(fence)); else ret = i915_request_await_external(rq, fence); if (ret < 0) @@ -1466,7 +1391,7 @@ await_request_submit(struct i915_request *to, struct i915_request *from) &from->submit, I915_FENCE_GFP); else - return __i915_request_await_execution(to, from, NULL); + return __i915_request_await_execution(to, from); } static int @@ -1482,7 +1407,7 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) return 0; } - if (to->engine->schedule) { + if (to->engine->sched_engine->schedule) { ret = i915_sched_node_add_dependency(&to->sched, &from->sched, I915_DEPENDENCY_EXTERNAL); @@ -1490,7 +1415,8 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) return ret; } - if (is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask))) + if (!intel_engine_uses_guc(to->engine) && + is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask))) ret = await_request_submit(to, from); else ret = emit_semaphore_wait(to, from, I915_FENCE_GFP); @@ -1649,6 +1575,8 @@ __i915_request_add_to_timeline(struct i915_request *rq) prev = to_request(__i915_active_fence_set(&timeline->last_request, &rq->fence)); if (prev && !__i915_request_is_complete(prev)) { + bool uses_guc = intel_engine_uses_guc(rq->engine); + /* * The requests are supposed to be kept in order. However, * we need to be wary in case the timeline->last_request @@ -1659,7 +1587,9 @@ __i915_request_add_to_timeline(struct i915_request *rq) i915_seqno_passed(prev->fence.seqno, rq->fence.seqno)); - if (is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask)) + if ((!uses_guc && + is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask)) || + (uses_guc && prev->context == rq->context)) i915_sw_fence_await_sw_fence(&rq->submit, &prev->submit, &rq->submitq); @@ -1667,7 +1597,7 @@ __i915_request_add_to_timeline(struct i915_request *rq) __i915_sw_fence_await_dma_fence(&rq->submit, &prev->fence, &rq->dmaq); - if (rq->engine->schedule) + if (rq->engine->sched_engine->schedule) __i915_sched_node_add_dependency(&rq->sched, &prev->sched, &rq->dep, @@ -1739,8 +1669,8 @@ void __i915_request_queue(struct i915_request *rq, * decide whether to preempt the entire chain so that it is ready to * run at the earliest possible convenience. */ - if (attr && rq->engine->schedule) - rq->engine->schedule(rq, attr); + if (attr && rq->engine->sched_engine->schedule) + rq->engine->sched_engine->schedule(rq, attr); local_bh_disable(); __i915_request_queue_bh(rq); @@ -2100,31 +2030,61 @@ void i915_request_show(struct drm_printer *m, name); } -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/mock_request.c" -#include "selftests/i915_request.c" -#endif +static bool engine_match_ring(struct intel_engine_cs *engine, struct i915_request *rq) +{ + u32 ring = ENGINE_READ(engine, RING_START); + + return ring == i915_ggtt_offset(rq->ring->vma); +} -static void i915_global_request_shrink(void) +static bool match_ring(struct i915_request *rq) { - kmem_cache_shrink(global.slab_execute_cbs); - kmem_cache_shrink(global.slab_requests); + struct intel_engine_cs *engine; + bool found; + int i; + + if (!intel_engine_is_virtual(rq->engine)) + return engine_match_ring(rq->engine, rq); + + found = false; + i = 0; + while ((engine = intel_engine_get_sibling(rq->engine, i++))) { + found = engine_match_ring(engine, rq); + if (found) + break; + } + + return found; } -static void i915_global_request_exit(void) +enum i915_request_state i915_test_request_state(struct i915_request *rq) { - kmem_cache_destroy(global.slab_execute_cbs); - kmem_cache_destroy(global.slab_requests); + if (i915_request_completed(rq)) + return I915_REQUEST_COMPLETE; + + if (!i915_request_started(rq)) + return I915_REQUEST_PENDING; + + if (match_ring(rq)) + return I915_REQUEST_ACTIVE; + + return I915_REQUEST_QUEUED; } -static struct i915_global_request global = { { - .shrink = i915_global_request_shrink, - .exit = i915_global_request_exit, -} }; +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftests/mock_request.c" +#include "selftests/i915_request.c" +#endif + +void i915_request_module_exit(void) +{ + kmem_cache_destroy(slab_execute_cbs); + kmem_cache_destroy(slab_requests); +} -int __init i915_global_request_init(void) +int __init i915_request_module_init(void) { - global.slab_requests = + slab_requests = kmem_cache_create("i915_request", sizeof(struct i915_request), __alignof__(struct i915_request), @@ -2132,20 +2092,19 @@ int __init i915_global_request_init(void) SLAB_RECLAIM_ACCOUNT | SLAB_TYPESAFE_BY_RCU, __i915_request_ctor); - if (!global.slab_requests) + if (!slab_requests) return -ENOMEM; - global.slab_execute_cbs = KMEM_CACHE(execute_cb, + slab_execute_cbs = KMEM_CACHE(execute_cb, SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_TYPESAFE_BY_RCU); - if (!global.slab_execute_cbs) + if (!slab_execute_cbs) goto err_requests; - i915_global_register(&global.base); return 0; err_requests: - kmem_cache_destroy(global.slab_requests); + kmem_cache_destroy(slab_requests); return -ENOMEM; } diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 270f6cd37650..1bc1349ba3c2 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -285,6 +285,23 @@ struct i915_request { struct hrtimer timer; } watchdog; + /* + * Requests may need to be stalled when using GuC submission waiting for + * certain GuC operations to complete. If that is the case, stalled + * requests are added to a per context list of stalled requests. The + * below list_head is the link in that list. + */ + struct list_head guc_fence_link; + + /** + * Priority level while the request is inflight. Differs from i915 + * scheduler priority. See comment above + * I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP for details. + */ +#define GUC_PRIO_INIT 0xff +#define GUC_PRIO_FINI 0xfe + u8 guc_prio; + I915_SELFTEST_DECLARE(struct { struct list_head link; unsigned long delay; @@ -352,9 +369,7 @@ int i915_request_await_object(struct i915_request *to, int i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence); int i915_request_await_execution(struct i915_request *rq, - struct dma_fence *fence, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal)); + struct dma_fence *fence); void i915_request_add(struct i915_request *rq); @@ -613,7 +628,7 @@ i915_request_active_timeline(const struct i915_request *rq) * this submission. */ return rcu_dereference_protected(rq->timeline, - lockdep_is_held(&rq->engine->active.lock)); + lockdep_is_held(&rq->engine->sched_engine->lock)); } static inline u32 @@ -641,4 +656,19 @@ bool i915_request_active_engine(struct i915_request *rq, struct intel_engine_cs **active); +void i915_request_notify_execute_cb_imm(struct i915_request *rq); + +enum i915_request_state { + I915_REQUEST_UNKNOWN = 0, + I915_REQUEST_COMPLETE, + I915_REQUEST_PENDING, + I915_REQUEST_QUEUED, + I915_REQUEST_ACTIVE, +}; + +enum i915_request_state i915_test_request_state(struct i915_request *rq); + +void i915_request_module_exit(void); +int i915_request_module_init(void); + #endif /* I915_REQUEST_H */ diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c index 69e9e6c3135e..4a6712dca838 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.c +++ b/drivers/gpu/drm/i915/i915_scatterlist.c @@ -6,6 +6,9 @@ #include "i915_scatterlist.h" +#include "i915_buddy.h" +#include "i915_ttm_buddy_manager.h" + #include <drm/drm_mm.h> #include <linux/slab.h> @@ -104,6 +107,83 @@ struct sg_table *i915_sg_from_mm_node(const struct drm_mm_node *node, return st; } +/** + * i915_sg_from_buddy_resource - Create an sg_table from a struct + * i915_buddy_block list + * @res: The struct i915_ttm_buddy_resource. + * @region_start: An offset to add to the dma addresses of the sg list. + * + * Create a struct sg_table, initializing it from struct i915_buddy_block list, + * taking a maximum segment length into account, splitting into segments + * if necessary. + * + * Return: A pointer to a kmalloced struct sg_table on success, negative + * error code cast to an error pointer on failure. + */ +struct sg_table *i915_sg_from_buddy_resource(struct ttm_resource *res, + u64 region_start) +{ + struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); + const u64 size = res->num_pages << PAGE_SHIFT; + const u64 max_segment = rounddown(UINT_MAX, PAGE_SIZE); + struct i915_buddy_mm *mm = bman_res->mm; + struct list_head *blocks = &bman_res->blocks; + struct i915_buddy_block *block; + struct scatterlist *sg; + struct sg_table *st; + resource_size_t prev_end; + + GEM_BUG_ON(list_empty(blocks)); + + st = kmalloc(sizeof(*st), GFP_KERNEL); + if (!st) + return ERR_PTR(-ENOMEM); + + if (sg_alloc_table(st, res->num_pages, GFP_KERNEL)) { + kfree(st); + return ERR_PTR(-ENOMEM); + } + + sg = st->sgl; + st->nents = 0; + prev_end = (resource_size_t)-1; + + list_for_each_entry(block, blocks, link) { + u64 block_size, offset; + + block_size = min_t(u64, size, i915_buddy_block_size(mm, block)); + offset = i915_buddy_block_offset(block); + + while (block_size) { + u64 len; + + if (offset != prev_end || sg->length >= max_segment) { + if (st->nents) + sg = __sg_next(sg); + + sg_dma_address(sg) = region_start + offset; + sg_dma_len(sg) = 0; + sg->length = 0; + st->nents++; + } + + len = min(block_size, max_segment - sg->length); + sg->length += len; + sg_dma_len(sg) += len; + + offset += len; + block_size -= len; + + prev_end = offset; + } + } + + sg_mark_end(sg); + i915_sg_trim(st); + + return st; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/scatterlist.c" #endif diff --git a/drivers/gpu/drm/i915/i915_scatterlist.h b/drivers/gpu/drm/i915/i915_scatterlist.h index 5acca45ea981..b8bd5925b03f 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.h +++ b/drivers/gpu/drm/i915/i915_scatterlist.h @@ -14,6 +14,7 @@ #include "i915_gem.h" struct drm_mm_node; +struct ttm_resource; /* * Optimised SGL iterator for GEM objects @@ -145,4 +146,8 @@ bool i915_sg_trim(struct sg_table *orig_st); struct sg_table *i915_sg_from_mm_node(const struct drm_mm_node *node, u64 region_start); + +struct sg_table *i915_sg_from_buddy_resource(struct ttm_resource *res, + u64 region_start); + #endif diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index efa638c3acc7..762127dd56c5 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -7,15 +7,11 @@ #include <linux/mutex.h> #include "i915_drv.h" -#include "i915_globals.h" #include "i915_request.h" #include "i915_scheduler.h" -static struct i915_global_scheduler { - struct i915_global base; - struct kmem_cache *slab_dependencies; - struct kmem_cache *slab_priorities; -} global; +static struct kmem_cache *slab_dependencies; +static struct kmem_cache *slab_priorities; static DEFINE_SPINLOCK(schedule_lock); @@ -40,7 +36,7 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb) return rb_entry(rb, struct i915_priolist, node); } -static void assert_priolists(struct intel_engine_execlists * const execlists) +static void assert_priolists(struct i915_sched_engine * const sched_engine) { struct rb_node *rb; long last_prio; @@ -48,11 +44,11 @@ static void assert_priolists(struct intel_engine_execlists * const execlists) if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) return; - GEM_BUG_ON(rb_first_cached(&execlists->queue) != - rb_first(&execlists->queue.rb_root)); + GEM_BUG_ON(rb_first_cached(&sched_engine->queue) != + rb_first(&sched_engine->queue.rb_root)); last_prio = INT_MAX; - for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) { const struct i915_priolist *p = to_priolist(rb); GEM_BUG_ON(p->priority > last_prio); @@ -61,23 +57,22 @@ static void assert_priolists(struct intel_engine_execlists * const execlists) } struct list_head * -i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio) +i915_sched_lookup_priolist(struct i915_sched_engine *sched_engine, int prio) { - struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_priolist *p; struct rb_node **parent, *rb; bool first = true; - lockdep_assert_held(&engine->active.lock); - assert_priolists(execlists); + lockdep_assert_held(&sched_engine->lock); + assert_priolists(sched_engine); - if (unlikely(execlists->no_priolist)) + if (unlikely(sched_engine->no_priolist)) prio = I915_PRIORITY_NORMAL; find_priolist: /* most positive priority is scheduled first, equal priorities fifo */ rb = NULL; - parent = &execlists->queue.rb_root.rb_node; + parent = &sched_engine->queue.rb_root.rb_node; while (*parent) { rb = *parent; p = to_priolist(rb); @@ -92,9 +87,9 @@ find_priolist: } if (prio == I915_PRIORITY_NORMAL) { - p = &execlists->default_priolist; + p = &sched_engine->default_priolist; } else { - p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC); + p = kmem_cache_alloc(slab_priorities, GFP_ATOMIC); /* Convert an allocation failure to a priority bump */ if (unlikely(!p)) { prio = I915_PRIORITY_NORMAL; /* recurses just once */ @@ -107,7 +102,7 @@ find_priolist: * requests, so if userspace lied about their * dependencies that reordering may be visible. */ - execlists->no_priolist = true; + sched_engine->no_priolist = true; goto find_priolist; } } @@ -116,27 +111,27 @@ find_priolist: INIT_LIST_HEAD(&p->requests); rb_link_node(&p->node, rb, parent); - rb_insert_color_cached(&p->node, &execlists->queue, first); + rb_insert_color_cached(&p->node, &sched_engine->queue, first); return &p->requests; } void __i915_priolist_free(struct i915_priolist *p) { - kmem_cache_free(global.slab_priorities, p); + kmem_cache_free(slab_priorities, p); } struct sched_cache { struct list_head *priolist; }; -static struct intel_engine_cs * -sched_lock_engine(const struct i915_sched_node *node, - struct intel_engine_cs *locked, +static struct i915_sched_engine * +lock_sched_engine(struct i915_sched_node *node, + struct i915_sched_engine *locked, struct sched_cache *cache) { const struct i915_request *rq = node_to_request(node); - struct intel_engine_cs *engine; + struct i915_sched_engine *sched_engine; GEM_BUG_ON(!locked); @@ -146,81 +141,22 @@ sched_lock_engine(const struct i915_sched_node *node, * engine lock. The simple ploy we use is to take the lock then * check that the rq still belongs to the newly locked engine. */ - while (locked != (engine = READ_ONCE(rq->engine))) { - spin_unlock(&locked->active.lock); + while (locked != (sched_engine = READ_ONCE(rq->engine)->sched_engine)) { + spin_unlock(&locked->lock); memset(cache, 0, sizeof(*cache)); - spin_lock(&engine->active.lock); - locked = engine; + spin_lock(&sched_engine->lock); + locked = sched_engine; } - GEM_BUG_ON(locked != engine); + GEM_BUG_ON(locked != sched_engine); return locked; } -static inline int rq_prio(const struct i915_request *rq) -{ - return rq->sched.attr.priority; -} - -static inline bool need_preempt(int prio, int active) -{ - /* - * Allow preemption of low -> normal -> high, but we do - * not allow low priority tasks to preempt other low priority - * tasks under the impression that latency for low priority - * tasks does not matter (as much as background throughput), - * so kiss. - */ - return prio >= max(I915_PRIORITY_NORMAL, active); -} - -static void kick_submission(struct intel_engine_cs *engine, - const struct i915_request *rq, - int prio) -{ - const struct i915_request *inflight; - - /* - * We only need to kick the tasklet once for the high priority - * new context we add into the queue. - */ - if (prio <= engine->execlists.queue_priority_hint) - return; - - rcu_read_lock(); - - /* Nothing currently active? We're overdue for a submission! */ - inflight = execlists_active(&engine->execlists); - if (!inflight) - goto unlock; - - /* - * If we are already the currently executing context, don't - * bother evaluating if we should preempt ourselves. - */ - if (inflight->context == rq->context) - goto unlock; - - ENGINE_TRACE(engine, - "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n", - prio, - rq->fence.context, rq->fence.seqno, - inflight->fence.context, inflight->fence.seqno, - inflight->sched.attr.priority); - - engine->execlists.queue_priority_hint = prio; - if (need_preempt(prio, rq_prio(inflight))) - tasklet_hi_schedule(&engine->execlists.tasklet); - -unlock: - rcu_read_unlock(); -} - static void __i915_schedule(struct i915_sched_node *node, const struct i915_sched_attr *attr) { const int prio = max(attr->priority, node->attr.priority); - struct intel_engine_cs *engine; + struct i915_sched_engine *sched_engine; struct i915_dependency *dep, *p; struct i915_dependency stack; struct sched_cache cache; @@ -295,23 +231,31 @@ static void __i915_schedule(struct i915_sched_node *node, } memset(&cache, 0, sizeof(cache)); - engine = node_to_request(node)->engine; - spin_lock(&engine->active.lock); + sched_engine = node_to_request(node)->engine->sched_engine; + spin_lock(&sched_engine->lock); /* Fifo and depth-first replacement ensure our deps execute before us */ - engine = sched_lock_engine(node, engine, &cache); + sched_engine = lock_sched_engine(node, sched_engine, &cache); list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { + struct i915_request *from = container_of(dep->signaler, + struct i915_request, + sched); INIT_LIST_HEAD(&dep->dfs_link); node = dep->signaler; - engine = sched_lock_engine(node, engine, &cache); - lockdep_assert_held(&engine->active.lock); + sched_engine = lock_sched_engine(node, sched_engine, &cache); + lockdep_assert_held(&sched_engine->lock); /* Recheck after acquiring the engine->timeline.lock */ if (prio <= node->attr.priority || node_signaled(node)) continue; - GEM_BUG_ON(node_to_request(node)->engine != engine); + GEM_BUG_ON(node_to_request(node)->engine->sched_engine != + sched_engine); + + /* Must be called before changing the nodes priority */ + if (sched_engine->bump_inflight_request_prio) + sched_engine->bump_inflight_request_prio(from, prio); WRITE_ONCE(node->attr.priority, prio); @@ -329,16 +273,17 @@ static void __i915_schedule(struct i915_sched_node *node, if (i915_request_in_priority_queue(node_to_request(node))) { if (!cache.priolist) cache.priolist = - i915_sched_lookup_priolist(engine, + i915_sched_lookup_priolist(sched_engine, prio); list_move_tail(&node->link, cache.priolist); } /* Defer (tasklet) submission until after all of our updates. */ - kick_submission(engine, node_to_request(node), prio); + if (sched_engine->kick_backend) + sched_engine->kick_backend(node_to_request(node), prio); } - spin_unlock(&engine->active.lock); + spin_unlock(&sched_engine->lock); } void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr) @@ -371,13 +316,13 @@ void i915_sched_node_reinit(struct i915_sched_node *node) static struct i915_dependency * i915_dependency_alloc(void) { - return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL); + return kmem_cache_alloc(slab_dependencies, GFP_KERNEL); } static void i915_dependency_free(struct i915_dependency *dep) { - kmem_cache_free(global.slab_dependencies, dep); + kmem_cache_free(slab_dependencies, dep); } bool __i915_sched_node_add_dependency(struct i915_sched_node *node, @@ -489,39 +434,78 @@ void i915_request_show_with_schedule(struct drm_printer *m, rcu_read_unlock(); } -static void i915_global_scheduler_shrink(void) +static void default_destroy(struct kref *kref) +{ + struct i915_sched_engine *sched_engine = + container_of(kref, typeof(*sched_engine), ref); + + tasklet_kill(&sched_engine->tasklet); /* flush the callback */ + kfree(sched_engine); +} + +static bool default_disabled(struct i915_sched_engine *sched_engine) { - kmem_cache_shrink(global.slab_dependencies); - kmem_cache_shrink(global.slab_priorities); + return false; } -static void i915_global_scheduler_exit(void) +struct i915_sched_engine * +i915_sched_engine_create(unsigned int subclass) { - kmem_cache_destroy(global.slab_dependencies); - kmem_cache_destroy(global.slab_priorities); + struct i915_sched_engine *sched_engine; + + sched_engine = kzalloc(sizeof(*sched_engine), GFP_KERNEL); + if (!sched_engine) + return NULL; + + kref_init(&sched_engine->ref); + + sched_engine->queue = RB_ROOT_CACHED; + sched_engine->queue_priority_hint = INT_MIN; + sched_engine->destroy = default_destroy; + sched_engine->disabled = default_disabled; + + INIT_LIST_HEAD(&sched_engine->requests); + INIT_LIST_HEAD(&sched_engine->hold); + + spin_lock_init(&sched_engine->lock); + lockdep_set_subclass(&sched_engine->lock, subclass); + + /* + * Due to an interesting quirk in lockdep's internal debug tracking, + * after setting a subclass we must ensure the lock is used. Otherwise, + * nr_unused_locks is incremented once too often. + */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + local_irq_disable(); + lock_map_acquire(&sched_engine->lock.dep_map); + lock_map_release(&sched_engine->lock.dep_map); + local_irq_enable(); +#endif + + return sched_engine; } -static struct i915_global_scheduler global = { { - .shrink = i915_global_scheduler_shrink, - .exit = i915_global_scheduler_exit, -} }; +void i915_scheduler_module_exit(void) +{ + kmem_cache_destroy(slab_dependencies); + kmem_cache_destroy(slab_priorities); +} -int __init i915_global_scheduler_init(void) +int __init i915_scheduler_module_init(void) { - global.slab_dependencies = KMEM_CACHE(i915_dependency, + slab_dependencies = KMEM_CACHE(i915_dependency, SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU); - if (!global.slab_dependencies) + if (!slab_dependencies) return -ENOMEM; - global.slab_priorities = KMEM_CACHE(i915_priolist, 0); - if (!global.slab_priorities) + slab_priorities = KMEM_CACHE(i915_priolist, 0); + if (!slab_priorities) goto err_priorities; - i915_global_register(&global.base); return 0; err_priorities: - kmem_cache_destroy(global.slab_priorities); + kmem_cache_destroy(slab_priorities); return -ENOMEM; } diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index 858a0938f47a..0b9b86af6c7f 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -39,7 +39,7 @@ void i915_schedule(struct i915_request *request, const struct i915_sched_attr *attr); struct list_head * -i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio); +i915_sched_lookup_priolist(struct i915_sched_engine *sched_engine, int prio); void __i915_priolist_free(struct i915_priolist *p); static inline void i915_priolist_free(struct i915_priolist *p) @@ -48,9 +48,61 @@ static inline void i915_priolist_free(struct i915_priolist *p) __i915_priolist_free(p); } +struct i915_sched_engine * +i915_sched_engine_create(unsigned int subclass); + +static inline struct i915_sched_engine * +i915_sched_engine_get(struct i915_sched_engine *sched_engine) +{ + kref_get(&sched_engine->ref); + return sched_engine; +} + +static inline void +i915_sched_engine_put(struct i915_sched_engine *sched_engine) +{ + kref_put(&sched_engine->ref, sched_engine->destroy); +} + +static inline bool +i915_sched_engine_is_empty(struct i915_sched_engine *sched_engine) +{ + return RB_EMPTY_ROOT(&sched_engine->queue.rb_root); +} + +static inline void +i915_sched_engine_reset_on_empty(struct i915_sched_engine *sched_engine) +{ + if (i915_sched_engine_is_empty(sched_engine)) + sched_engine->no_priolist = false; +} + +static inline void +i915_sched_engine_active_lock_bh(struct i915_sched_engine *sched_engine) +{ + local_bh_disable(); /* prevent local softirq and lock recursion */ + tasklet_lock(&sched_engine->tasklet); +} + +static inline void +i915_sched_engine_active_unlock_bh(struct i915_sched_engine *sched_engine) +{ + tasklet_unlock(&sched_engine->tasklet); + local_bh_enable(); /* restore softirq, and kick ksoftirqd! */ +} + void i915_request_show_with_schedule(struct drm_printer *m, const struct i915_request *rq, const char *prefix, int indent); +static inline bool +i915_sched_engine_disabled(struct i915_sched_engine *sched_engine) +{ + return sched_engine->disabled(sched_engine); +} + +void i915_scheduler_module_exit(void); +int i915_scheduler_module_init(void); + #endif /* _I915_SCHEDULER_H_ */ diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h index 343ed44d5ed4..b0a1b58c7893 100644 --- a/drivers/gpu/drm/i915/i915_scheduler_types.h +++ b/drivers/gpu/drm/i915/i915_scheduler_types.h @@ -91,4 +91,115 @@ struct i915_dependency { &(rq__)->sched.signalers_list, \ signal_link) +/** + * struct i915_sched_engine - scheduler engine + * + * A schedule engine represents a submission queue with different priority + * bands. It contains all the common state (relative to the backend) to queue, + * track, and submit a request. + * + * This object at the moment is quite i915 specific but will transition into a + * container for the drm_gpu_scheduler plus a few other variables once the i915 + * is integrated with the DRM scheduler. + */ +struct i915_sched_engine { + /** + * @ref: reference count of schedule engine object + */ + struct kref ref; + + /** + * @lock: protects requests in priority lists, requests, hold and + * tasklet while running + */ + spinlock_t lock; + + /** + * @requests: list of requests inflight on this schedule engine + */ + struct list_head requests; + + /** + * @hold: list of ready requests, but on hold + */ + struct list_head hold; + + /** + * @tasklet: softirq tasklet for submission + */ + struct tasklet_struct tasklet; + + /** + * @default_priolist: priority list for I915_PRIORITY_NORMAL + */ + struct i915_priolist default_priolist; + + /** + * @queue_priority_hint: Highest pending priority. + * + * When we add requests into the queue, or adjust the priority of + * executing requests, we compute the maximum priority of those + * pending requests. We can then use this value to determine if + * we need to preempt the executing requests to service the queue. + * However, since the we may have recorded the priority of an inflight + * request we wanted to preempt but since completed, at the time of + * dequeuing the priority hint may no longer may match the highest + * available request priority. + */ + int queue_priority_hint; + + /** + * @queue: queue of requests, in priority lists + */ + struct rb_root_cached queue; + + /** + * @no_priolist: priority lists disabled + */ + bool no_priolist; + + /** + * @private_data: private data of the submission backend + */ + void *private_data; + + /** + * @destroy: destroy schedule engine / cleanup in backend + */ + void (*destroy)(struct kref *kref); + + /** + * @disabled: check if backend has disabled submission + */ + bool (*disabled)(struct i915_sched_engine *sched_engine); + + /** + * @kick_backend: kick backend after a request's priority has changed + */ + void (*kick_backend)(const struct i915_request *rq, + int prio); + + /** + * @bump_inflight_request_prio: update priority of an inflight request + */ + void (*bump_inflight_request_prio)(struct i915_request *rq, + int prio); + + /** + * @retire_inflight_request_prio: indicate request is retired to + * priority tracking + */ + void (*retire_inflight_request_prio)(struct i915_request *rq); + + /** + * @schedule: adjust priority of request + * + * Call when the priority on a request has changed and it and its + * dependencies may need rescheduling. Note the request itself may + * not be ready to run! + */ + void (*schedule)(struct i915_request *request, + const struct i915_sched_attr *attr); +}; + #endif /* _I915_SCHEDULER_TYPES_H_ */ diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.c b/drivers/gpu/drm/i915/i915_sw_fence_work.c index a3a81bb8f2c3..5b33ef23d54c 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence_work.c +++ b/drivers/gpu/drm/i915/i915_sw_fence_work.c @@ -16,11 +16,8 @@ static void fence_complete(struct dma_fence_work *f) static void fence_work(struct work_struct *work) { struct dma_fence_work *f = container_of(work, typeof(*f), work); - int err; - err = f->ops->work(f); - if (err) - dma_fence_set_error(&f->dma, err); + f->ops->work(f); fence_complete(f); dma_fence_put(&f->dma); diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.h b/drivers/gpu/drm/i915/i915_sw_fence_work.h index 2c409f11c5c5..d56806918d13 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence_work.h +++ b/drivers/gpu/drm/i915/i915_sw_fence_work.h @@ -17,7 +17,7 @@ struct dma_fence_work; struct dma_fence_work_ops { const char *name; - int (*work)(struct dma_fence_work *f); + void (*work)(struct dma_fence_work *f); void (*release)(struct dma_fence_work *f); }; diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 873bf996ceb5..cdf0e9c6fd73 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -272,7 +272,7 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev, struct drm_i915_private *i915 = kdev_minor_to_i915(kdev); struct intel_rps *rps = &i915->gt.rps; - return sysfs_emit(buf, "%d\n", intel_gpu_freq(rps, rps->cur_freq)); + return sysfs_emit(buf, "%d\n", intel_rps_get_requested_frequency(rps)); } static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) @@ -326,9 +326,10 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt.rps; + struct intel_gt *gt = &dev_priv->gt; + struct intel_rps *rps = >->rps; - return sysfs_emit(buf, "%d\n", intel_gpu_freq(rps, rps->max_freq_softlimit)); + return sysfs_emit(buf, "%d\n", intel_rps_get_max_frequency(rps)); } static ssize_t gt_max_freq_mhz_store(struct device *kdev, @@ -336,7 +337,8 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, const char *buf, size_t count) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt.rps; + struct intel_gt *gt = &dev_priv->gt; + struct intel_rps *rps = >->rps; ssize_t ret; u32 val; @@ -344,53 +346,26 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, if (ret) return ret; - mutex_lock(&rps->lock); - - val = intel_freq_opcode(rps, val); - if (val < rps->min_freq || - val > rps->max_freq || - val < rps->min_freq_softlimit) { - ret = -EINVAL; - goto unlock; - } - - if (val > rps->rp0_freq) - DRM_DEBUG("User requested overclocking to %d\n", - intel_gpu_freq(rps, val)); - - rps->max_freq_softlimit = val; - - val = clamp_t(int, rps->cur_freq, - rps->min_freq_softlimit, - rps->max_freq_softlimit); - - /* - * We still need *_set_rps to process the new max_delay and - * update the interrupt limits and PMINTRMSK even though - * frequency request may be unchanged. - */ - intel_rps_set(rps, val); - -unlock: - mutex_unlock(&rps->lock); + ret = intel_rps_set_max_frequency(rps, val); return ret ?: count; } static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { - struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt.rps; + struct drm_i915_private *i915 = kdev_minor_to_i915(kdev); + struct intel_gt *gt = &i915->gt; + struct intel_rps *rps = >->rps; - return sysfs_emit(buf, "%d\n", intel_gpu_freq(rps, rps->min_freq_softlimit)); + return sysfs_emit(buf, "%d\n", intel_rps_get_min_frequency(rps)); } static ssize_t gt_min_freq_mhz_store(struct device *kdev, struct device_attribute *attr, const char *buf, size_t count) { - struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt.rps; + struct drm_i915_private *i915 = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &i915->gt.rps; ssize_t ret; u32 val; @@ -398,31 +373,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, if (ret) return ret; - mutex_lock(&rps->lock); - - val = intel_freq_opcode(rps, val); - if (val < rps->min_freq || - val > rps->max_freq || - val > rps->max_freq_softlimit) { - ret = -EINVAL; - goto unlock; - } - - rps->min_freq_softlimit = val; - - val = clamp_t(int, rps->cur_freq, - rps->min_freq_softlimit, - rps->max_freq_softlimit); - - /* - * We still need *_set_rps to process the new min_delay and - * update the interrupt limits and PMINTRMSK even though - * frequency request may be unchanged. - */ - intel_rps_set(rps, val); - -unlock: - mutex_unlock(&rps->lock); + ret = intel_rps_set_min_frequency(rps, val); return ret ?: count; } @@ -448,11 +399,11 @@ static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr u32 val; if (attr == &dev_attr_gt_RP0_freq_mhz) - val = intel_gpu_freq(rps, rps->rp0_freq); + val = intel_rps_get_rp0_frequency(rps); else if (attr == &dev_attr_gt_RP1_freq_mhz) - val = intel_gpu_freq(rps, rps->rp1_freq); + val = intel_rps_get_rp1_frequency(rps); else if (attr == &dev_attr_gt_RPn_freq_mhz) - val = intel_gpu_freq(rps, rps->min_freq); + val = intel_rps_get_rpn_frequency(rps); else BUG(); diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 6778ad2a14a4..806ad688274b 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -794,30 +794,40 @@ DECLARE_EVENT_CLASS(i915_request, TP_STRUCT__entry( __field(u32, dev) __field(u64, ctx) + __field(u32, guc_id) __field(u16, class) __field(u16, instance) __field(u32, seqno) + __field(u32, tail) ), TP_fast_assign( __entry->dev = rq->engine->i915->drm.primary->index; __entry->class = rq->engine->uabi_class; __entry->instance = rq->engine->uabi_instance; + __entry->guc_id = rq->context->guc_id; __entry->ctx = rq->fence.context; __entry->seqno = rq->fence.seqno; + __entry->tail = rq->tail; ), - TP_printk("dev=%u, engine=%u:%u, ctx=%llu, seqno=%u", + TP_printk("dev=%u, engine=%u:%u, guc_id=%u, ctx=%llu, seqno=%u, tail=%u", __entry->dev, __entry->class, __entry->instance, - __entry->ctx, __entry->seqno) + __entry->guc_id, __entry->ctx, __entry->seqno, + __entry->tail) ); DEFINE_EVENT(i915_request, i915_request_add, - TP_PROTO(struct i915_request *rq), - TP_ARGS(rq) + TP_PROTO(struct i915_request *rq), + TP_ARGS(rq) ); #if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) +DEFINE_EVENT(i915_request, i915_request_guc_submit, + TP_PROTO(struct i915_request *rq), + TP_ARGS(rq) +); + DEFINE_EVENT(i915_request, i915_request_submit, TP_PROTO(struct i915_request *rq), TP_ARGS(rq) @@ -885,9 +895,117 @@ TRACE_EVENT(i915_request_out, __entry->ctx, __entry->seqno, __entry->completed) ); +DECLARE_EVENT_CLASS(intel_context, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce), + + TP_STRUCT__entry( + __field(u32, guc_id) + __field(int, pin_count) + __field(u32, sched_state) + __field(u32, guc_sched_state_no_lock) + __field(u8, guc_prio) + ), + + TP_fast_assign( + __entry->guc_id = ce->guc_id; + __entry->pin_count = atomic_read(&ce->pin_count); + __entry->sched_state = ce->guc_state.sched_state; + __entry->guc_sched_state_no_lock = + atomic_read(&ce->guc_sched_state_no_lock); + __entry->guc_prio = ce->guc_prio; + ), + + TP_printk("guc_id=%d, pin_count=%d sched_state=0x%x,0x%x, guc_prio=%u", + __entry->guc_id, __entry->pin_count, + __entry->sched_state, + __entry->guc_sched_state_no_lock, + __entry->guc_prio) +); + +DEFINE_EVENT(intel_context, intel_context_set_prio, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_reset, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_ban, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_register, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_deregister, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_deregister_done, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_sched_enable, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_sched_disable, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_sched_done, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_create, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_fence_release, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_free, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_steal_guc_id, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_do_pin, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + +DEFINE_EVENT(intel_context, intel_context_do_unpin, + TP_PROTO(struct intel_context *ce), + TP_ARGS(ce) +); + #else #if !defined(TRACE_HEADER_MULTI_READ) static inline void +trace_i915_request_guc_submit(struct i915_request *rq) +{ +} + +static inline void trace_i915_request_submit(struct i915_request *rq) { } @@ -906,6 +1024,81 @@ static inline void trace_i915_request_out(struct i915_request *rq) { } + +static inline void +trace_intel_context_set_prio(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_reset(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_ban(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_register(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_deregister(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_deregister_done(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_sched_enable(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_sched_disable(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_sched_done(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_create(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_fence_release(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_free(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_steal_guc_id(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_do_pin(struct intel_context *ce) +{ +} + +static inline void +trace_intel_context_do_unpin(struct intel_context *ce) +{ +} #endif #endif diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c new file mode 100644 index 000000000000..6877362f6b85 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/slab.h> + +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> + +#include "i915_ttm_buddy_manager.h" + +#include "i915_buddy.h" +#include "i915_gem.h" + +struct i915_ttm_buddy_manager { + struct ttm_resource_manager manager; + struct i915_buddy_mm mm; + struct list_head reserved; + struct mutex lock; + u64 default_page_size; +}; + +static struct i915_ttm_buddy_manager * +to_buddy_manager(struct ttm_resource_manager *man) +{ + return container_of(man, struct i915_ttm_buddy_manager, manager); +} + +static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + struct i915_ttm_buddy_resource *bman_res; + struct i915_buddy_mm *mm = &bman->mm; + unsigned long n_pages; + unsigned int min_order; + u64 min_page_size; + u64 size; + int err; + + GEM_BUG_ON(place->fpfn || place->lpfn); + + bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); + if (!bman_res) + return -ENOMEM; + + ttm_resource_init(bo, place, &bman_res->base); + INIT_LIST_HEAD(&bman_res->blocks); + bman_res->mm = mm; + + GEM_BUG_ON(!bman_res->base.num_pages); + size = bman_res->base.num_pages << PAGE_SHIFT; + + min_page_size = bman->default_page_size; + if (bo->page_alignment) + min_page_size = bo->page_alignment << PAGE_SHIFT; + + GEM_BUG_ON(min_page_size < mm->chunk_size); + min_order = ilog2(min_page_size) - ilog2(mm->chunk_size); + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + size = roundup_pow_of_two(size); + min_order = ilog2(size) - ilog2(mm->chunk_size); + } + + if (size > mm->size) { + err = -E2BIG; + goto err_free_res; + } + + n_pages = size >> ilog2(mm->chunk_size); + + do { + struct i915_buddy_block *block; + unsigned int order; + + order = fls(n_pages) - 1; + GEM_BUG_ON(order > mm->max_order); + GEM_BUG_ON(order < min_order); + + do { + mutex_lock(&bman->lock); + block = i915_buddy_alloc(mm, order); + mutex_unlock(&bman->lock); + if (!IS_ERR(block)) + break; + + if (order-- == min_order) { + err = -ENOSPC; + goto err_free_blocks; + } + } while (1); + + n_pages -= BIT(order); + + list_add_tail(&block->link, &bman_res->blocks); + + if (!n_pages) + break; + } while (1); + + *res = &bman_res->base; + return 0; + +err_free_blocks: + mutex_lock(&bman->lock); + i915_buddy_free_list(mm, &bman_res->blocks); + mutex_unlock(&bman->lock); +err_free_res: + kfree(bman_res); + return err; +} + +static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + + mutex_lock(&bman->lock); + i915_buddy_free_list(&bman->mm, &bman_res->blocks); + mutex_unlock(&bman->lock); + + kfree(bman_res); +} + +static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { + .alloc = i915_ttm_buddy_man_alloc, + .free = i915_ttm_buddy_man_free, +}; + + +/** + * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager + * @bdev: The ttm device + * @type: Memory type we want to manage + * @use_tt: Set use_tt for the manager + * @size: The size in bytes to manage + * @default_page_size: The default minimum page size in bytes for allocations, + * this must be at least as large as @chunk_size, and can be overridden by + * setting the BO page_alignment, to be larger or smaller as needed. + * @chunk_size: The minimum page size in bytes for our allocations i.e + * order-zero + * + * Note that the starting address is assumed to be zero here, since this + * simplifies keeping the property where allocated blocks having natural + * power-of-two alignment. So long as the real starting address is some large + * power-of-two, or naturally start from zero, then this should be fine. Also + * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment + * if say there is some unusable range from the start of the region. We can + * revisit this in the future and make the interface accept an actual starting + * offset and let it take care of the rest. + * + * Note that if the @size is not aligned to the @chunk_size then we perform the + * required rounding to get the usable size. The final size in pages can be + * taken from &ttm_resource_manager.size. + * + * Return: 0 on success, negative error code on failure. + */ +int i915_ttm_buddy_man_init(struct ttm_device *bdev, + unsigned int type, bool use_tt, + u64 size, u64 default_page_size, + u64 chunk_size) +{ + struct ttm_resource_manager *man; + struct i915_ttm_buddy_manager *bman; + int err; + + bman = kzalloc(sizeof(*bman), GFP_KERNEL); + if (!bman) + return -ENOMEM; + + err = i915_buddy_init(&bman->mm, size, chunk_size); + if (err) + goto err_free_bman; + + mutex_init(&bman->lock); + INIT_LIST_HEAD(&bman->reserved); + GEM_BUG_ON(default_page_size < chunk_size); + bman->default_page_size = default_page_size; + + man = &bman->manager; + man->use_tt = use_tt; + man->func = &i915_ttm_buddy_manager_func; + ttm_resource_manager_init(man, bman->mm.size >> PAGE_SHIFT); + + ttm_resource_manager_set_used(man, true); + ttm_set_driver_manager(bdev, type, man); + + return 0; + +err_free_bman: + kfree(bman); + return err; +} + +/** + * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager + * @bdev: The ttm device + * @type: Memory type we want to manage + * + * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will + * also be freed for us here. + * + * Return: 0 on success, negative error code on failure. + */ +int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) +{ + struct ttm_resource_manager *man = ttm_manager_type(bdev, type); + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + struct i915_buddy_mm *mm = &bman->mm; + int ret; + + ttm_resource_manager_set_used(man, false); + + ret = ttm_resource_manager_evict_all(bdev, man); + if (ret) + return ret; + + ttm_set_driver_manager(bdev, type, NULL); + + mutex_lock(&bman->lock); + i915_buddy_free_list(mm, &bman->reserved); + i915_buddy_fini(mm); + mutex_unlock(&bman->lock); + + ttm_resource_manager_cleanup(man); + kfree(bman); + + return 0; +} + +/** + * i915_ttm_buddy_man_reserve - Reserve address range + * @man: The buddy allocator ttm manager + * @start: The offset in bytes, where the region start is assumed to be zero + * @size: The size in bytes + * + * Note that the starting address for the region is always assumed to be zero. + * + * Return: 0 on success, negative error code on failure. + */ +int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, + u64 start, u64 size) +{ + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + struct i915_buddy_mm *mm = &bman->mm; + int ret; + + mutex_lock(&bman->lock); + ret = i915_buddy_alloc_range(mm, &bman->reserved, start, size); + mutex_unlock(&bman->lock); + + return ret; +} + diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h new file mode 100644 index 000000000000..0722d33f3e14 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_TTM_BUDDY_MANAGER_H__ +#define __I915_TTM_BUDDY_MANAGER_H__ + +#include <linux/list.h> +#include <linux/types.h> + +#include <drm/ttm/ttm_resource.h> + +struct ttm_device; +struct ttm_resource_manager; +struct i915_buddy_mm; + +/** + * struct i915_ttm_buddy_resource + * + * @base: struct ttm_resource base class we extend + * @blocks: the list of struct i915_buddy_block for this resource/allocation + * @mm: the struct i915_buddy_mm for this resource + * + * Extends the struct ttm_resource to manage an address space allocation with + * one or more struct i915_buddy_block. + */ +struct i915_ttm_buddy_resource { + struct ttm_resource base; + struct list_head blocks; + struct i915_buddy_mm *mm; +}; + +/** + * to_ttm_buddy_resource + * + * @res: the resource to upcast + * + * Upcast the struct ttm_resource object into a struct i915_ttm_buddy_resource. + */ +static inline struct i915_ttm_buddy_resource * +to_ttm_buddy_resource(struct ttm_resource *res) +{ + return container_of(res, struct i915_ttm_buddy_resource, base); +} + +int i915_ttm_buddy_man_init(struct ttm_device *bdev, + unsigned type, bool use_tt, + u64 size, u64 default_page_size, u64 chunk_size); +int i915_ttm_buddy_man_fini(struct ttm_device *bdev, + unsigned int type); + +int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, + u64 start, u64 size); + +#endif diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 0f227f28b280..4b7fc4647e46 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -34,24 +34,20 @@ #include "gt/intel_gt_requests.h" #include "i915_drv.h" -#include "i915_globals.h" #include "i915_sw_fence_work.h" #include "i915_trace.h" #include "i915_vma.h" -static struct i915_global_vma { - struct i915_global base; - struct kmem_cache *slab_vmas; -} global; +static struct kmem_cache *slab_vmas; struct i915_vma *i915_vma_alloc(void) { - return kmem_cache_zalloc(global.slab_vmas, GFP_KERNEL); + return kmem_cache_zalloc(slab_vmas, GFP_KERNEL); } void i915_vma_free(struct i915_vma *vma) { - return kmem_cache_free(global.slab_vmas, vma); + return kmem_cache_free(slab_vmas, vma); } #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM) @@ -300,14 +296,13 @@ struct i915_vma_work { unsigned int flags; }; -static int __vma_bind(struct dma_fence_work *work) +static void __vma_bind(struct dma_fence_work *work) { struct i915_vma_work *vw = container_of(work, typeof(*vw), base); struct i915_vma *vma = vw->vma; vma->ops->bind_vma(vw->vm, &vw->stash, vma, vw->cache_level, vw->flags); - return 0; } static void __vma_release(struct dma_fence_work *work) @@ -1415,27 +1410,16 @@ void i915_vma_make_purgeable(struct i915_vma *vma) #include "selftests/i915_vma.c" #endif -static void i915_global_vma_shrink(void) +void i915_vma_module_exit(void) { - kmem_cache_shrink(global.slab_vmas); + kmem_cache_destroy(slab_vmas); } -static void i915_global_vma_exit(void) -{ - kmem_cache_destroy(global.slab_vmas); -} - -static struct i915_global_vma global = { { - .shrink = i915_global_vma_shrink, - .exit = i915_global_vma_exit, -} }; - -int __init i915_global_vma_init(void) +int __init i915_vma_module_init(void) { - global.slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); - if (!global.slab_vmas) + slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); + if (!slab_vmas) return -ENOMEM; - i915_global_register(&global.base); return 0; } diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index eca452a9851f..ed69f66c7ab0 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -426,4 +426,7 @@ static inline int i915_vma_sync(struct i915_vma *vma) return i915_active_wait(&vma->active); } +void i915_vma_module_exit(void); +int i915_vma_module_init(void); + #endif diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 7eaa92fee421..305facedd284 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -59,7 +59,6 @@ static const char * const platform_names[] = { PLATFORM_NAME(GEMINILAKE), PLATFORM_NAME(COFFEELAKE), PLATFORM_NAME(COMETLAKE), - PLATFORM_NAME(CANNONLAKE), PLATFORM_NAME(ICELAKE), PLATFORM_NAME(ELKHARTLAKE), PLATFORM_NAME(JASPERLAKE), @@ -68,6 +67,8 @@ static const char * const platform_names[] = { PLATFORM_NAME(DG1), PLATFORM_NAME(ALDERLAKE_S), PLATFORM_NAME(ALDERLAKE_P), + PLATFORM_NAME(XEHPSDV), + PLATFORM_NAME(DG2), }; #undef PLATFORM_NAME @@ -96,9 +97,17 @@ static const char *iommu_name(void) void intel_device_info_print_static(const struct intel_device_info *info, struct drm_printer *p) { - drm_printf(p, "graphics_ver: %u\n", info->graphics_ver); - drm_printf(p, "media_ver: %u\n", info->media_ver); - drm_printf(p, "display_ver: %u\n", info->display.ver); + if (info->graphics_rel) + drm_printf(p, "graphics version: %u.%02u\n", info->graphics_ver, info->graphics_rel); + else + drm_printf(p, "graphics version: %u\n", info->graphics_ver); + + if (info->media_rel) + drm_printf(p, "media version: %u.%02u\n", info->media_ver, info->media_rel); + else + drm_printf(p, "media version: %u\n", info->media_ver); + + drm_printf(p, "display version: %u\n", info->display.ver); drm_printf(p, "gt: %d\n", info->gt); drm_printf(p, "iommu: %s\n", iommu_name()); drm_printf(p, "memory-regions: %x\n", info->memory_regions); @@ -165,7 +174,6 @@ static const u16 subplatform_ulx_ids[] = { }; static const u16 subplatform_portf_ids[] = { - INTEL_CNL_PORT_F_IDS(0), INTEL_ICL_PORT_F_IDS(0), }; @@ -253,14 +261,14 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) struct intel_runtime_info *runtime = RUNTIME_INFO(dev_priv); enum pipe pipe; - /* Wa_14011765242: adl-s A0 */ - if (IS_ADLS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) + /* Wa_14011765242: adl-s A0,A1 */ + if (IS_ADLS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A2)) for_each_pipe(dev_priv, pipe) runtime->num_scalers[pipe] = 0; - else if (GRAPHICS_VER(dev_priv) >= 10) { + else if (DISPLAY_VER(dev_priv) >= 11) { for_each_pipe(dev_priv, pipe) runtime->num_scalers[pipe] = 2; - } else if (GRAPHICS_VER(dev_priv) == 9) { + } else if (DISPLAY_VER(dev_priv) >= 9) { runtime->num_scalers[PIPE_A] = 2; runtime->num_scalers[PIPE_B] = 2; runtime->num_scalers[PIPE_C] = 1; @@ -271,10 +279,10 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (DISPLAY_VER(dev_priv) >= 13 || HAS_D12_PLANE_MINIMIZATION(dev_priv)) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 4; - else if (GRAPHICS_VER(dev_priv) >= 11) + else if (DISPLAY_VER(dev_priv) >= 11) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 6; - else if (GRAPHICS_VER(dev_priv) == 10 || IS_GEMINILAKE(dev_priv)) + else if (DISPLAY_VER(dev_priv) == 10) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 3; else if (IS_BROXTON(dev_priv)) { @@ -293,7 +301,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 2; - } else if (GRAPHICS_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) { + } else if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) { for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 1; } @@ -325,7 +333,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->pipe_mask &= ~BIT(PIPE_C); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } - } else if (HAS_DISPLAY(dev_priv) && GRAPHICS_VER(dev_priv) >= 9) { + } else if (HAS_DISPLAY(dev_priv) && DISPLAY_VER(dev_priv) >= 9) { u32 dfsm = intel_de_read(dev_priv, SKL_DFSM); if (dfsm & SKL_DFSM_PIPE_A_DISABLE) { @@ -340,7 +348,8 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->pipe_mask &= ~BIT(PIPE_C); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } - if (GRAPHICS_VER(dev_priv) >= 12 && + + if (DISPLAY_VER(dev_priv) >= 12 && (dfsm & TGL_DFSM_PIPE_D_DISABLE)) { info->pipe_mask &= ~BIT(PIPE_D); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_D); @@ -352,11 +361,11 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (dfsm & SKL_DFSM_DISPLAY_PM_DISABLE) info->display.has_fbc = 0; - if (GRAPHICS_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) + if (DISPLAY_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) info->display.has_dmc = 0; - if (GRAPHICS_VER(dev_priv) >= 10 && - (dfsm & CNL_DFSM_DISPLAY_DSC_DISABLE)) + if (DISPLAY_VER(dev_priv) >= 10 && + (dfsm & GLK_DFSM_DISPLAY_DSC_DISABLE)) info->display.has_dsc = 0; } diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index b326aff65cd6..d328bb95c49b 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -76,8 +76,6 @@ enum intel_platform { INTEL_GEMINILAKE, INTEL_COFFEELAKE, INTEL_COMETLAKE, - /* gen10 */ - INTEL_CANNONLAKE, /* gen11 */ INTEL_ICELAKE, INTEL_ELKHARTLAKE, @@ -88,6 +86,8 @@ enum intel_platform { INTEL_DG1, INTEL_ALDERLAKE_S, INTEL_ALDERLAKE_P, + INTEL_XEHPSDV, + INTEL_DG2, INTEL_MAX_PLATFORMS }; @@ -103,9 +103,13 @@ enum intel_platform { #define INTEL_SUBPLATFORM_ULT (0) #define INTEL_SUBPLATFORM_ULX (1) -/* CNL/ICL */ +/* ICL */ #define INTEL_SUBPLATFORM_PORTF (0) +/* DG2 */ +#define INTEL_SUBPLATFORM_G10 0 +#define INTEL_SUBPLATFORM_G11 1 + enum intel_ppgtt_type { INTEL_PPGTT_NONE = I915_GEM_PPGTT_NONE, INTEL_PPGTT_ALIASING = I915_GEM_PPGTT_ALIASING, @@ -127,7 +131,7 @@ enum intel_ppgtt_type { func(has_llc); \ func(has_logical_ring_contexts); \ func(has_logical_ring_elsq); \ - func(has_master_unit_irq); \ + func(has_mslices); \ func(has_pooled_eu); \ func(has_rc6); \ func(has_rc6p); \ @@ -141,6 +145,7 @@ enum intel_ppgtt_type { #define DEV_INFO_DISPLAY_FOR_EACH_FLAG(func) \ /* Keep in alphabetical order */ \ func(cursor_needs_physical); \ + func(has_cdclk_crawl); \ func(has_dmc); \ func(has_ddi); \ func(has_dp_mst); \ @@ -162,9 +167,10 @@ enum intel_ppgtt_type { struct intel_device_info { u8 graphics_ver; + u8 graphics_rel; u8 media_ver; + u8 media_rel; - u8 gt; /* GT number, 0 if undefined */ intel_engine_mask_t platform_engine_mask; /* Engines supported by the HW */ enum intel_platform platform; @@ -180,13 +186,13 @@ struct intel_device_info { u32 display_mmio_offset; + u8 gt; /* GT number, 0 if undefined */ + u8 pipe_mask; u8 cpu_transcoder_mask; u8 abox_mask; - u8 has_cdclk_crawl; /* does support CDCLK crawling */ - #define DEFINE_FLAG(name) u8 name:1 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); #undef DEFINE_FLAG diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 879b0f007be3..91866520c173 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -77,21 +77,21 @@ static int skl_get_dimm_ranks(u16 val) } /* Returns total Gb for the whole DIMM */ -static int cnl_get_dimm_size(u16 val) +static int icl_get_dimm_size(u16 val) { - return (val & CNL_DRAM_SIZE_MASK) * 8 / 2; + return (val & ICL_DRAM_SIZE_MASK) * 8 / 2; } -static int cnl_get_dimm_width(u16 val) +static int icl_get_dimm_width(u16 val) { - if (cnl_get_dimm_size(val) == 0) + if (icl_get_dimm_size(val) == 0) return 0; - switch (val & CNL_DRAM_WIDTH_MASK) { - case CNL_DRAM_WIDTH_X8: - case CNL_DRAM_WIDTH_X16: - case CNL_DRAM_WIDTH_X32: - val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT; + switch (val & ICL_DRAM_WIDTH_MASK) { + case ICL_DRAM_WIDTH_X8: + case ICL_DRAM_WIDTH_X16: + case ICL_DRAM_WIDTH_X32: + val = (val & ICL_DRAM_WIDTH_MASK) >> ICL_DRAM_WIDTH_SHIFT; return 8 << val; default: MISSING_CASE(val); @@ -99,12 +99,12 @@ static int cnl_get_dimm_width(u16 val) } } -static int cnl_get_dimm_ranks(u16 val) +static int icl_get_dimm_ranks(u16 val) { - if (cnl_get_dimm_size(val) == 0) + if (icl_get_dimm_size(val) == 0) return 0; - val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT; + val = (val & ICL_DRAM_RANK_MASK) >> ICL_DRAM_RANK_SHIFT; return val + 1; } @@ -121,10 +121,10 @@ skl_dram_get_dimm_info(struct drm_i915_private *i915, struct dram_dimm_info *dimm, int channel, char dimm_name, u16 val) { - if (GRAPHICS_VER(i915) >= 10) { - dimm->size = cnl_get_dimm_size(val); - dimm->width = cnl_get_dimm_width(val); - dimm->ranks = cnl_get_dimm_ranks(val); + if (GRAPHICS_VER(i915) >= 11) { + dimm->size = icl_get_dimm_size(val); + dimm->width = icl_get_dimm_width(val); + dimm->ranks = icl_get_dimm_ranks(val); } else { dimm->size = skl_get_dimm_size(val); dimm->width = skl_get_dimm_width(val); @@ -468,6 +468,7 @@ static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv) dram_info->num_channels = (val & 0xf0) >> 4; dram_info->num_qgv_points = (val & 0xf00) >> 8; + dram_info->num_psf_gv_points = (val & 0x3000) >> 12; return 0; } @@ -494,15 +495,15 @@ void intel_dram_detect(struct drm_i915_private *i915) struct dram_info *dram_info = &i915->dram_info; int ret; + if (GRAPHICS_VER(i915) < 9 || IS_DG2(i915) || !HAS_DISPLAY(i915)) + return; + /* * Assume level 0 watermark latency adjustment is needed until proven * otherwise, this w/a is not needed by bxt/glk. */ dram_info->wm_lv_0_adjust_needed = !IS_GEN9_LP(i915); - if (GRAPHICS_VER(i915) < 9 || !HAS_DISPLAY(i915)) - return; - if (GRAPHICS_VER(i915) >= 12) ret = gen12_get_dram_info(i915); else if (GRAPHICS_VER(i915) >= 11) diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index 12fb5423fd5e..779eb2fa90b6 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -5,6 +5,7 @@ #include "intel_memory_region.h" #include "i915_drv.h" +#include "i915_ttm_buddy_manager.h" static const struct { u16 class; @@ -28,11 +29,6 @@ static const struct { }, }; -struct intel_region_reserve { - struct list_head link; - struct ttm_resource *res; -}; - struct intel_memory_region * intel_memory_region_lookup(struct drm_i915_private *i915, u16 class, u16 instance) @@ -64,27 +60,6 @@ intel_memory_region_by_type(struct drm_i915_private *i915, } /** - * intel_memory_region_unreserve - Unreserve all previously reserved - * ranges - * @mem: The region containing the reserved ranges. - */ -void intel_memory_region_unreserve(struct intel_memory_region *mem) -{ - struct intel_region_reserve *reserve, *next; - - if (!mem->priv_ops || !mem->priv_ops->free) - return; - - mutex_lock(&mem->mm_lock); - list_for_each_entry_safe(reserve, next, &mem->reserved, link) { - list_del(&reserve->link); - mem->priv_ops->free(mem, reserve->res); - kfree(reserve); - } - mutex_unlock(&mem->mm_lock); -} - -/** * intel_memory_region_reserve - Reserve a memory range * @mem: The region for which we want to reserve a range. * @offset: Start of the range to reserve. @@ -96,28 +71,11 @@ int intel_memory_region_reserve(struct intel_memory_region *mem, resource_size_t offset, resource_size_t size) { - int ret; - struct intel_region_reserve *reserve; - - if (!mem->priv_ops || !mem->priv_ops->reserve) - return -EINVAL; - - reserve = kzalloc(sizeof(*reserve), GFP_KERNEL); - if (!reserve) - return -ENOMEM; + struct ttm_resource_manager *man = mem->region_private; - reserve->res = mem->priv_ops->reserve(mem, offset, size); - if (IS_ERR(reserve->res)) { - ret = PTR_ERR(reserve->res); - kfree(reserve); - return ret; - } - - mutex_lock(&mem->mm_lock); - list_add_tail(&reserve->link, &mem->reserved); - mutex_unlock(&mem->mm_lock); + GEM_BUG_ON(mem->is_range_manager); - return 0; + return i915_ttm_buddy_man_reserve(man, offset, size); } struct intel_memory_region * @@ -149,9 +107,6 @@ intel_memory_region_create(struct drm_i915_private *i915, mutex_init(&mem->objects.lock); INIT_LIST_HEAD(&mem->objects.list); - INIT_LIST_HEAD(&mem->reserved); - - mutex_init(&mem->mm_lock); if (ops->init) { err = ops->init(mem); @@ -182,11 +137,9 @@ static void __intel_memory_region_destroy(struct kref *kref) struct intel_memory_region *mem = container_of(kref, typeof(*mem), kref); - intel_memory_region_unreserve(mem); if (mem->ops->release) mem->ops->release(mem); - mutex_destroy(&mem->mm_lock); mutex_destroy(&mem->objects.lock); kfree(mem); } @@ -220,7 +173,12 @@ int intel_memory_regions_hw_probe(struct drm_i915_private *i915) instance = intel_region_map[i].instance; switch (type) { case INTEL_MEMORY_SYSTEM: - mem = i915_gem_shmem_setup(i915, type, instance); + if (IS_DGFX(i915)) + mem = i915_gem_ttm_system_setup(i915, type, + instance); + else + mem = i915_gem_shmem_setup(i915, type, + instance); break; case INTEL_MEMORY_STOLEN_LOCAL: mem = i915_gem_stolen_lmem_setup(i915, type, instance); diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index c7e635d62e1a..1f2b96efa69d 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -40,8 +40,7 @@ enum intel_region_id { #define REGION_STOLEN_SMEM BIT(INTEL_REGION_STOLEN_SMEM) #define REGION_STOLEN_LMEM BIT(INTEL_REGION_STOLEN_LMEM) -#define I915_ALLOC_MIN_PAGE_SIZE BIT(0) -#define I915_ALLOC_CONTIGUOUS BIT(1) +#define I915_ALLOC_CONTIGUOUS BIT(0) #define for_each_memory_region(mr, i915, id) \ for (id = 0; id < ARRAY_SIZE((i915)->mm.regions); id++) \ @@ -56,22 +55,14 @@ struct intel_memory_region_ops { int (*init_object)(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags); }; -struct intel_memory_region_private_ops { - struct ttm_resource *(*reserve)(struct intel_memory_region *mem, - resource_size_t offset, - resource_size_t size); - void (*free)(struct intel_memory_region *mem, - struct ttm_resource *res); -}; - struct intel_memory_region { struct drm_i915_private *i915; const struct intel_memory_region_ops *ops; - const struct intel_memory_region_private_ops *priv_ops; struct io_mapping iomap; struct resource region; @@ -79,8 +70,6 @@ struct intel_memory_region { /* For fake LMEM */ struct drm_mm_node fake_mappable; - struct mutex mm_lock; - struct kref kref; resource_size_t io_start; @@ -94,8 +83,6 @@ struct intel_memory_region { char name[16]; bool private; /* not for userspace */ - struct list_head reserved; - dma_addr_t remap_addr; struct { @@ -103,8 +90,6 @@ struct intel_memory_region { struct list_head list; } objects; - size_t chunk_size; - unsigned int max_order; bool is_range_manager; void *region_private; @@ -138,9 +123,15 @@ __printf(2, 3) void intel_memory_region_set_name(struct intel_memory_region *mem, const char *fmt, ...); -void intel_memory_region_unreserve(struct intel_memory_region *mem); - int intel_memory_region_reserve(struct intel_memory_region *mem, resource_size_t offset, resource_size_t size); + +struct intel_memory_region * +i915_gem_ttm_system_setup(struct drm_i915_private *i915, + u16 type, u16 instance); +struct intel_memory_region * +i915_gem_shmem_setup(struct drm_i915_private *i915, + u16 type, u16 instance); + #endif diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c index 4e92ae19189e..d1d4b97b86f5 100644 --- a/drivers/gpu/drm/i915/intel_pch.c +++ b/drivers/gpu/drm/i915/intel_pch.c @@ -81,7 +81,6 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) case INTEL_PCH_CNP_DEVICE_ID_TYPE: drm_dbg_kms(&dev_priv->drm, "Found Cannon Lake PCH (CNP)\n"); drm_WARN_ON(&dev_priv->drm, - !IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv) && !IS_COMETLAKE(dev_priv)); return PCH_CNP; @@ -89,7 +88,6 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) drm_dbg_kms(&dev_priv->drm, "Found Cannon Lake LP PCH (CNP-LP)\n"); drm_WARN_ON(&dev_priv->drm, - !IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv) && !IS_COMETLAKE(dev_priv)); return PCH_CNP; @@ -171,8 +169,7 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv, id = INTEL_PCH_MCC_DEVICE_ID_TYPE; else if (IS_ICELAKE(dev_priv)) id = INTEL_PCH_ICP_DEVICE_ID_TYPE; - else if (IS_CANNONLAKE(dev_priv) || - IS_COFFEELAKE(dev_priv) || + else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) id = INTEL_PCH_CNP_DEVICE_ID_TYPE; else if (IS_KABYLAKE(dev_priv) || IS_SKYLAKE(dev_priv)) @@ -211,6 +208,9 @@ void intel_detect_pch(struct drm_i915_private *dev_priv) if (IS_DG1(dev_priv)) { dev_priv->pch_type = PCH_DG1; return; + } else if (IS_DG2(dev_priv)) { + dev_priv->pch_type = PCH_DG2; + return; } /* diff --git a/drivers/gpu/drm/i915/intel_pch.h b/drivers/gpu/drm/i915/intel_pch.h index e2f3f30c6445..7c0d83d292dc 100644 --- a/drivers/gpu/drm/i915/intel_pch.h +++ b/drivers/gpu/drm/i915/intel_pch.h @@ -30,6 +30,7 @@ enum intel_pch { /* Fake PCHs, functionality handled on the same PCI dev */ PCH_DG1 = 1024, + PCH_DG2, }; #define INTEL_PCH_DEVICE_ID_MASK 0xff80 @@ -62,6 +63,7 @@ enum intel_pch { #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) #define INTEL_PCH_ID(dev_priv) ((dev_priv)->pch_id) +#define HAS_PCH_DG2(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_DG2) #define HAS_PCH_ADP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_ADP) #define HAS_PCH_DG1(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_DG1) #define HAS_PCH_JSP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_JSP) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 74a8863b94c2..65bc3709f54c 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -4584,6 +4584,117 @@ static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = {} }; +static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = { + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_D), + .dbuf_mask = { + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + {} +}; + static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = { { .active_pipes = BIT(PIPE_A), @@ -4759,12 +4870,19 @@ static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes) return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs); } +static u32 dg2_compute_dbuf_slices(enum pipe pipe, u32 active_pipes) +{ + return compute_dbuf_slices(pipe, active_pipes, dg2_allowed_dbufs); +} + static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (IS_ALDERLAKE_P(dev_priv)) + if (IS_DG2(dev_priv)) + return dg2_compute_dbuf_slices(pipe, active_pipes); + else if (IS_ALDERLAKE_P(dev_priv)) return adlp_compute_dbuf_slices(pipe, active_pipes); else if (DISPLAY_VER(dev_priv) == 12) return tgl_compute_dbuf_slices(pipe, active_pipes); @@ -7345,30 +7463,24 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); - /* This is not an Wa. Enable to reduce Sampler power */ - intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN, - intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); - /*Wa_14010594013:icl, ehl */ intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, - 0, CNL_DELAY_PMRSP); + 0, ICL_DELAY_PMRSP); } static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv) { - /* Wa_1409120013:tgl,rkl,adl_s,dg1 */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, - ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); + /* Wa_1409120013:tgl,rkl,adl-s,dg1 */ + if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) || + IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv)) + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, + ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); /* Wa_1409825376:tgl (pre-prod)*/ - if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1)) + if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) | TGL_VRH_GATING_DIS); - /* Wa_14011059788:tgl,rkl,adl_s,dg1 */ - intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN, - 0, DFR_DISABLE); - /* Wa_14013723622:tgl,rkl,dg1,adl-s */ if (DISPLAY_VER(dev_priv) == 12) intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY, @@ -7388,7 +7500,7 @@ static void dg1_init_clock_gating(struct drm_i915_private *dev_priv) gen12lp_init_clock_gating(dev_priv); /* Wa_1409836686:dg1[a0] */ - if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0)) + if (IS_DG1_GT_STEP(dev_priv, STEP_A0, STEP_B0)) intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) | DPT_GATING_DIS); } @@ -7403,43 +7515,6 @@ static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) CNP_PWM_CGE_GATING_DISABLE); } -static void cnl_init_clock_gating(struct drm_i915_private *dev_priv) -{ - u32 val; - cnp_init_clock_gating(dev_priv); - - /* This is not an Wa. Enable for better image quality */ - intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE)); - - /* WaEnableChickenDCPR:cnl */ - intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, - intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); - - /* - * WaFbcWakeMemOn:cnl - * Display WA #0859: cnl - */ - intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) | - DISP_FBC_MEMORY_WAKE); - - val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE); - /* ReadHitWriteOnlyDisable:cnl */ - val |= RCCUNIT_CLKGATE_DIS; - intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val); - - /* Wa_2201832410:cnl */ - val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE); - val |= GWUNIT_CLKGATE_DIS; - intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val); - - /* WaDisableVFclkgate:cnl */ - /* WaVFUnitClockGatingDisable:cnl */ - val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE); - val |= VFUNIT_CLKGATE_DIS; - intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val); -} - static void cfl_init_clock_gating(struct drm_i915_private *dev_priv) { cnp_init_clock_gating(dev_priv); @@ -7473,12 +7548,12 @@ static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) FBC_LLC_FULLY_OPEN); /* WaDisableSDEUnitClockGating:kbl */ - if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0)) + if (IS_KBL_GT_STEP(dev_priv, 0, STEP_C0)) intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* WaDisableGamClockGating:kbl */ - if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0)) + if (IS_KBL_GT_STEP(dev_priv, 0, STEP_C0)) intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_GAMUNIT_CLOCK_GATE_DISABLE); @@ -7868,8 +7943,6 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = gen12lp_init_clock_gating; else if (GRAPHICS_VER(dev_priv) == 11) dev_priv->display.init_clock_gating = icl_init_clock_gating; - else if (IS_CANNONLAKE(dev_priv)) - dev_priv->display.init_clock_gating = cnl_init_clock_gating; else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) dev_priv->display.init_clock_gating = cfl_init_clock_gating; else if (IS_SKYLAKE(dev_priv)) diff --git a/drivers/gpu/drm/i915/intel_region_ttm.c b/drivers/gpu/drm/i915/intel_region_ttm.c index 27fe0668d094..98c7339bf8ba 100644 --- a/drivers/gpu/drm/i915/intel_region_ttm.c +++ b/drivers/gpu/drm/i915/intel_region_ttm.c @@ -8,6 +8,7 @@ #include "i915_drv.h" #include "i915_scatterlist.h" +#include "i915_ttm_buddy_manager.h" #include "intel_region_ttm.h" @@ -50,12 +51,16 @@ void intel_region_ttm_device_fini(struct drm_i915_private *dev_priv) * driver-private types for now, reserving TTM_PL_VRAM for stolen * memory and TTM_PL_TT for GGTT use if decided to implement this. */ -static int intel_region_to_ttm_type(struct intel_memory_region *mem) +int intel_region_to_ttm_type(const struct intel_memory_region *mem) { int type; GEM_BUG_ON(mem->type != INTEL_MEMORY_LOCAL && - mem->type != INTEL_MEMORY_MOCK); + mem->type != INTEL_MEMORY_MOCK && + mem->type != INTEL_MEMORY_SYSTEM); + + if (mem->type == INTEL_MEMORY_SYSTEM) + return TTM_PL_SYSTEM; type = mem->instance + TTM_PL_PRIV; GEM_BUG_ON(type >= TTM_NUM_MEM_TYPES); @@ -63,72 +68,29 @@ static int intel_region_to_ttm_type(struct intel_memory_region *mem) return type; } -static struct ttm_resource * -intel_region_ttm_node_reserve(struct intel_memory_region *mem, - resource_size_t offset, - resource_size_t size) -{ - struct ttm_resource_manager *man = mem->region_private; - struct ttm_place place = {}; - struct ttm_buffer_object mock_bo = {}; - struct ttm_resource *res; - int ret; - - /* - * Having to use a mock_bo is unfortunate but stems from some - * drivers having private managers that insist to know what the - * allocate memory is intended for, using it to send private - * data to the manager. Also recently the bo has been used to send - * alignment info to the manager. Assume that apart from the latter, - * none of the managers we use will ever access the buffer object - * members, hoping we can pass the alignment info in the - * struct ttm_place in the future. - */ - - place.fpfn = offset >> PAGE_SHIFT; - place.lpfn = place.fpfn + (size >> PAGE_SHIFT); - mock_bo.base.size = size; - ret = man->func->alloc(man, &mock_bo, &place, &res); - if (ret == -ENOSPC) - ret = -ENXIO; - - return ret ? ERR_PTR(ret) : res; -} - /** - * intel_region_ttm_node_free - Free a node allocated from a resource manager - * @mem: The region the node was allocated from. - * @node: The opaque node representing an allocation. + * intel_region_ttm_init - Initialize a memory region for TTM. + * @mem: The region to initialize. + * + * This function initializes a suitable TTM resource manager for the + * region, and if it's a LMEM region type, attaches it to the TTM + * device. MOCK regions are NOT attached to the TTM device, since we don't + * have one for the mock selftests. + * + * Return: 0 on success, negative error code on failure. */ -void intel_region_ttm_node_free(struct intel_memory_region *mem, - struct ttm_resource *res) -{ - struct ttm_resource_manager *man = mem->region_private; - - man->func->free(man, res); -} - -static const struct intel_memory_region_private_ops priv_ops = { - .reserve = intel_region_ttm_node_reserve, - .free = intel_region_ttm_node_free, -}; - int intel_region_ttm_init(struct intel_memory_region *mem) { struct ttm_device *bdev = &mem->i915->bdev; int mem_type = intel_region_to_ttm_type(mem); int ret; - ret = ttm_range_man_init(bdev, mem_type, false, - resource_size(&mem->region) >> PAGE_SHIFT); + ret = i915_ttm_buddy_man_init(bdev, mem_type, false, + resource_size(&mem->region), + mem->min_page_size, PAGE_SIZE); if (ret) return ret; - mem->chunk_size = PAGE_SIZE; - mem->max_order = - get_order(rounddown_pow_of_two(resource_size(&mem->region))); - mem->is_range_manager = true; - mem->priv_ops = &priv_ops; mem->region_private = ttm_manager_type(bdev, mem_type); return 0; @@ -146,17 +108,17 @@ void intel_region_ttm_fini(struct intel_memory_region *mem) { int ret; - ret = ttm_range_man_fini(&mem->i915->bdev, - intel_region_to_ttm_type(mem)); + ret = i915_ttm_buddy_man_fini(&mem->i915->bdev, + intel_region_to_ttm_type(mem)); GEM_WARN_ON(ret); mem->region_private = NULL; } /** - * intel_region_ttm_node_to_st - Convert an opaque TTM resource manager node + * intel_region_ttm_resource_to_st - Convert an opaque TTM resource manager resource * to an sg_table. * @mem: The memory region. - * @node: The resource manager node obtained from the TTM resource manager. + * @res: The resource manager resource obtained from the TTM resource manager. * * The gem backends typically use sg-tables for operations on the underlying * io_memory. So provide a way for the backends to translate the @@ -164,20 +126,23 @@ void intel_region_ttm_fini(struct intel_memory_region *mem) * * Return: A malloced sg_table on success, an error pointer on failure. */ -struct sg_table *intel_region_ttm_node_to_st(struct intel_memory_region *mem, - struct ttm_resource *res) +struct sg_table *intel_region_ttm_resource_to_st(struct intel_memory_region *mem, + struct ttm_resource *res) { - struct ttm_range_mgr_node *range_node = - container_of(res, typeof(*range_node), base); + if (mem->is_range_manager) { + struct ttm_range_mgr_node *range_node = + to_ttm_range_mgr_node(res); - GEM_WARN_ON(!mem->is_range_manager); - return i915_sg_from_mm_node(&range_node->mm_nodes[0], - mem->region.start); + return i915_sg_from_mm_node(&range_node->mm_nodes[0], + mem->region.start); + } else { + return i915_sg_from_buddy_resource(res, mem->region.start); + } } #ifdef CONFIG_DRM_I915_SELFTEST /** - * intel_region_ttm_node_alloc - Allocate memory resources from a region + * intel_region_ttm_resource_alloc - Allocate memory resources from a region * @mem: The memory region, * @size: The requested size in bytes * @flags: Allocation flags @@ -186,15 +151,15 @@ struct sg_table *intel_region_ttm_node_to_st(struct intel_memory_region *mem, * memory from standalone TTM range managers, without the TTM eviction * functionality. Don't use if you are not completely sure that's the * case. The returned opaque node can be converted to an sg_table using - * intel_region_ttm_node_to_st(), and can be freed using - * intel_region_ttm_node_free(). + * intel_region_ttm_resource_to_st(), and can be freed using + * intel_region_ttm_resource_free(). * * Return: A valid pointer on success, an error pointer on failure. */ struct ttm_resource * -intel_region_ttm_node_alloc(struct intel_memory_region *mem, - resource_size_t size, - unsigned int flags) +intel_region_ttm_resource_alloc(struct intel_memory_region *mem, + resource_size_t size, + unsigned int flags) { struct ttm_resource_manager *man = mem->region_private; struct ttm_place place = {}; @@ -202,25 +167,26 @@ intel_region_ttm_node_alloc(struct intel_memory_region *mem, struct ttm_resource *res; int ret; - /* - * We ignore the flags for now since we're using the range - * manager and contigous and min page size would be fulfilled - * by default if size is min page size aligned. - */ mock_bo.base.size = size; - - if (mem->is_range_manager) { - if (size >= SZ_1G) - mock_bo.page_alignment = SZ_1G >> PAGE_SHIFT; - else if (size >= SZ_2M) - mock_bo.page_alignment = SZ_2M >> PAGE_SHIFT; - else if (size >= SZ_64K) - mock_bo.page_alignment = SZ_64K >> PAGE_SHIFT; - } + place.flags = flags; ret = man->func->alloc(man, &mock_bo, &place, &res); if (ret == -ENOSPC) ret = -ENXIO; return ret ? ERR_PTR(ret) : res; } + #endif + +/** + * intel_region_ttm_resource_free - Free a resource allocated from a resource manager + * @mem: The region the resource was allocated from. + * @res: The opaque resource representing an allocation. + */ +void intel_region_ttm_resource_free(struct intel_memory_region *mem, + struct ttm_resource *res) +{ + struct ttm_resource_manager *man = mem->region_private; + + man->func->free(man, res); +} diff --git a/drivers/gpu/drm/i915/intel_region_ttm.h b/drivers/gpu/drm/i915/intel_region_ttm.h index e8cf830fda6f..6f44075920f2 100644 --- a/drivers/gpu/drm/i915/intel_region_ttm.h +++ b/drivers/gpu/drm/i915/intel_region_ttm.h @@ -22,18 +22,20 @@ int intel_region_ttm_init(struct intel_memory_region *mem); void intel_region_ttm_fini(struct intel_memory_region *mem); -struct sg_table *intel_region_ttm_node_to_st(struct intel_memory_region *mem, - struct ttm_resource *res); +struct sg_table *intel_region_ttm_resource_to_st(struct intel_memory_region *mem, + struct ttm_resource *res); -void intel_region_ttm_node_free(struct intel_memory_region *mem, - struct ttm_resource *node); +void intel_region_ttm_resource_free(struct intel_memory_region *mem, + struct ttm_resource *res); + +int intel_region_to_ttm_type(const struct intel_memory_region *mem); struct ttm_device_funcs *i915_ttm_driver(void); #ifdef CONFIG_DRM_I915_SELFTEST struct ttm_resource * -intel_region_ttm_node_alloc(struct intel_memory_region *mem, - resource_size_t size, - unsigned int flags); +intel_region_ttm_resource_alloc(struct intel_memory_region *mem, + resource_size_t size, + unsigned int flags); #endif #endif /* _INTEL_REGION_TTM_H_ */ diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c index f0a82b37bd1a..e304bf44e1ff 100644 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ b/drivers/gpu/drm/i915/intel_sideband.c @@ -556,17 +556,22 @@ out: #undef COND } -void intel_pcode_init(struct drm_i915_private *i915) +int intel_pcode_init(struct drm_i915_private *i915) { - int ret; + int ret = 0; if (!IS_DGFX(i915)) - return; + return ret; ret = skl_pcode_request(i915, DG1_PCODE_STATUS, DG1_UNCORE_GET_INIT_STATUS, DG1_UNCORE_INIT_STATUS_COMPLETE, - DG1_UNCORE_INIT_STATUS_COMPLETE, 50); + DG1_UNCORE_INIT_STATUS_COMPLETE, 180000); + + drm_dbg(&i915->drm, "PCODE init status %d\n", ret); + if (ret) drm_err(&i915->drm, "Pcode did not report uncore initialization completion!\n"); + + return ret; } diff --git a/drivers/gpu/drm/i915/intel_sideband.h b/drivers/gpu/drm/i915/intel_sideband.h index 094c7b19c5d4..d1d14bcb8f56 100644 --- a/drivers/gpu/drm/i915/intel_sideband.h +++ b/drivers/gpu/drm/i915/intel_sideband.h @@ -138,6 +138,6 @@ int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, u32 reply_mask, u32 reply, int timeout_base_ms); -void intel_pcode_init(struct drm_i915_private *i915); +int intel_pcode_init(struct drm_i915_private *i915); #endif /* _INTEL_SIDEBAND_H */ diff --git a/drivers/gpu/drm/i915/intel_step.c b/drivers/gpu/drm/i915/intel_step.c index ba9479a67521..6cf967631395 100644 --- a/drivers/gpu/drm/i915/intel_step.c +++ b/drivers/gpu/drm/i915/intel_step.c @@ -7,16 +7,32 @@ #include "intel_step.h" /* - * KBL revision ID ordering is bizarre; higher revision ID's map to lower - * steppings in some cases. So rather than test against the revision ID - * directly, let's map that into our own range of increasing ID's that we - * can test against in a regular manner. + * Some platforms have unusual ways of mapping PCI revision ID to GT/display + * steppings. E.g., in some cases a higher PCI revision may translate to a + * lower stepping of the GT and/or display IP. This file provides lookup + * tables to map the PCI revision into a standard set of stepping values that + * can be compared numerically. + * + * Also note that some revisions/steppings may have been set aside as + * placeholders but never materialized in real hardware; in those cases there + * may be jumps in the revision IDs or stepping values in the tables below. */ +/* + * Some platforms always have the same stepping value for GT and display; + * use a macro to define these to make it easier to identify the platforms + * where the two steppings can deviate. + */ +#define COMMON_STEP(x) .gt_step = STEP_##x, .display_step = STEP_##x + +static const struct intel_step_info skl_revids[] = { + [0x6] = { COMMON_STEP(G0) }, + [0x7] = { COMMON_STEP(H0) }, + [0x9] = { COMMON_STEP(J0) }, + [0xA] = { COMMON_STEP(I1) }, +}; -/* FIXME: what about REVID_E0 */ static const struct intel_step_info kbl_revids[] = { - [0] = { .gt_step = STEP_A0, .display_step = STEP_A0 }, [1] = { .gt_step = STEP_B0, .display_step = STEP_B0 }, [2] = { .gt_step = STEP_C0, .display_step = STEP_B0 }, [3] = { .gt_step = STEP_D0, .display_step = STEP_B0 }, @@ -26,7 +42,27 @@ static const struct intel_step_info kbl_revids[] = { [7] = { .gt_step = STEP_G0, .display_step = STEP_C0 }, }; -static const struct intel_step_info tgl_uy_revid_step_tbl[] = { +static const struct intel_step_info bxt_revids[] = { + [0xA] = { COMMON_STEP(C0) }, + [0xB] = { COMMON_STEP(C0) }, + [0xC] = { COMMON_STEP(D0) }, + [0xD] = { COMMON_STEP(E0) }, +}; + +static const struct intel_step_info glk_revids[] = { + [3] = { COMMON_STEP(B0) }, +}; + +static const struct intel_step_info icl_revids[] = { + [7] = { COMMON_STEP(D0) }, +}; + +static const struct intel_step_info jsl_ehl_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, +}; + +static const struct intel_step_info tgl_uy_revids[] = { [0] = { .gt_step = STEP_A0, .display_step = STEP_A0 }, [1] = { .gt_step = STEP_B0, .display_step = STEP_C0 }, [2] = { .gt_step = STEP_B1, .display_step = STEP_C0 }, @@ -34,12 +70,23 @@ static const struct intel_step_info tgl_uy_revid_step_tbl[] = { }; /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */ -static const struct intel_step_info tgl_revid_step_tbl[] = { +static const struct intel_step_info tgl_revids[] = { [0] = { .gt_step = STEP_A0, .display_step = STEP_B0 }, [1] = { .gt_step = STEP_B0, .display_step = STEP_D0 }, }; -static const struct intel_step_info adls_revid_step_tbl[] = { +static const struct intel_step_info rkl_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, + [4] = { COMMON_STEP(C0) }, +}; + +static const struct intel_step_info dg1_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, +}; + +static const struct intel_step_info adls_revids[] = { [0x0] = { .gt_step = STEP_A0, .display_step = STEP_A0 }, [0x1] = { .gt_step = STEP_A0, .display_step = STEP_A2 }, [0x4] = { .gt_step = STEP_B0, .display_step = STEP_B0 }, @@ -47,13 +94,33 @@ static const struct intel_step_info adls_revid_step_tbl[] = { [0xC] = { .gt_step = STEP_D0, .display_step = STEP_C0 }, }; -static const struct intel_step_info adlp_revid_step_tbl[] = { +static const struct intel_step_info adlp_revids[] = { [0x0] = { .gt_step = STEP_A0, .display_step = STEP_A0 }, [0x4] = { .gt_step = STEP_B0, .display_step = STEP_B0 }, [0x8] = { .gt_step = STEP_C0, .display_step = STEP_C0 }, [0xC] = { .gt_step = STEP_C0, .display_step = STEP_D0 }, }; +static const struct intel_step_info xehpsdv_revids[] = { + [0x0] = { .gt_step = STEP_A0 }, + [0x1] = { .gt_step = STEP_A1 }, + [0x4] = { .gt_step = STEP_B0 }, + [0x8] = { .gt_step = STEP_C0 }, +}; + +static const struct intel_step_info dg2_g10_revid_step_tbl[] = { + [0x0] = { .gt_step = STEP_A0, .display_step = STEP_A0 }, + [0x1] = { .gt_step = STEP_A1, .display_step = STEP_A0 }, + [0x4] = { .gt_step = STEP_B0, .display_step = STEP_B0 }, + [0x8] = { .gt_step = STEP_C0, .display_step = STEP_C0 }, +}; + +static const struct intel_step_info dg2_g11_revid_step_tbl[] = { + [0x0] = { .gt_step = STEP_A0, .display_step = STEP_B0 }, + [0x4] = { .gt_step = STEP_B0, .display_step = STEP_C0 }, + [0x5] = { .gt_step = STEP_B1, .display_step = STEP_C0 }, +}; + void intel_step_init(struct drm_i915_private *i915) { const struct intel_step_info *revids = NULL; @@ -61,21 +128,51 @@ void intel_step_init(struct drm_i915_private *i915) int revid = INTEL_REVID(i915); struct intel_step_info step = {}; - if (IS_ALDERLAKE_P(i915)) { - revids = adlp_revid_step_tbl; - size = ARRAY_SIZE(adlp_revid_step_tbl); + if (IS_DG2_G10(i915)) { + revids = dg2_g10_revid_step_tbl; + size = ARRAY_SIZE(dg2_g10_revid_step_tbl); + } else if (IS_DG2_G11(i915)) { + revids = dg2_g11_revid_step_tbl; + size = ARRAY_SIZE(dg2_g11_revid_step_tbl); + } else if (IS_XEHPSDV(i915)) { + revids = xehpsdv_revids; + size = ARRAY_SIZE(xehpsdv_revids); + } else if (IS_ALDERLAKE_P(i915)) { + revids = adlp_revids; + size = ARRAY_SIZE(adlp_revids); } else if (IS_ALDERLAKE_S(i915)) { - revids = adls_revid_step_tbl; - size = ARRAY_SIZE(adls_revid_step_tbl); + revids = adls_revids; + size = ARRAY_SIZE(adls_revids); + } else if (IS_DG1(i915)) { + revids = dg1_revids; + size = ARRAY_SIZE(dg1_revids); + } else if (IS_ROCKETLAKE(i915)) { + revids = rkl_revids; + size = ARRAY_SIZE(rkl_revids); } else if (IS_TGL_U(i915) || IS_TGL_Y(i915)) { - revids = tgl_uy_revid_step_tbl; - size = ARRAY_SIZE(tgl_uy_revid_step_tbl); + revids = tgl_uy_revids; + size = ARRAY_SIZE(tgl_uy_revids); } else if (IS_TIGERLAKE(i915)) { - revids = tgl_revid_step_tbl; - size = ARRAY_SIZE(tgl_revid_step_tbl); + revids = tgl_revids; + size = ARRAY_SIZE(tgl_revids); + } else if (IS_JSL_EHL(i915)) { + revids = jsl_ehl_revids; + size = ARRAY_SIZE(jsl_ehl_revids); + } else if (IS_ICELAKE(i915)) { + revids = icl_revids; + size = ARRAY_SIZE(icl_revids); + } else if (IS_GEMINILAKE(i915)) { + revids = glk_revids; + size = ARRAY_SIZE(glk_revids); + } else if (IS_BROXTON(i915)) { + revids = bxt_revids; + size = ARRAY_SIZE(bxt_revids); } else if (IS_KABYLAKE(i915)) { revids = kbl_revids; size = ARRAY_SIZE(kbl_revids); + } else if (IS_SKYLAKE(i915)) { + revids = skl_revids; + size = ARRAY_SIZE(skl_revids); } /* Not using the stepping scheme for the platform yet. */ @@ -114,3 +211,17 @@ void intel_step_init(struct drm_i915_private *i915) RUNTIME_INFO(i915)->step = step; } + +#define STEP_NAME_CASE(name) \ + case STEP_##name: \ + return #name; + +const char *intel_step_name(enum intel_step step) +{ + switch (step) { + STEP_NAME_LIST(STEP_NAME_CASE); + + default: + return "**"; + } +} diff --git a/drivers/gpu/drm/i915/intel_step.h b/drivers/gpu/drm/i915/intel_step.h index 958a8bb5d677..f6641e2a3c77 100644 --- a/drivers/gpu/drm/i915/intel_step.h +++ b/drivers/gpu/drm/i915/intel_step.h @@ -15,26 +15,39 @@ struct intel_step_info { u8 display_step; }; +#define STEP_ENUM_VAL(name) STEP_##name, + +#define STEP_NAME_LIST(func) \ + func(A0) \ + func(A1) \ + func(A2) \ + func(B0) \ + func(B1) \ + func(B2) \ + func(C0) \ + func(C1) \ + func(D0) \ + func(D1) \ + func(E0) \ + func(F0) \ + func(G0) \ + func(H0) \ + func(I0) \ + func(I1) \ + func(J0) + /* * Symbolic steppings that do not match the hardware. These are valid both as gt * and display steppings as symbolic names. */ enum intel_step { STEP_NONE = 0, - STEP_A0, - STEP_A2, - STEP_B0, - STEP_B1, - STEP_C0, - STEP_D0, - STEP_D1, - STEP_E0, - STEP_F0, - STEP_G0, + STEP_NAME_LIST(STEP_ENUM_VAL) STEP_FUTURE, STEP_FOREVER, }; void intel_step_init(struct drm_i915_private *i915); +const char *intel_step_name(enum intel_step step); #endif /* __INTEL_STEP_H__ */ diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 7178bc6f8556..6b38bc2811c1 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -24,6 +24,8 @@ #include <linux/pm_runtime.h> #include <asm/iosf_mbi.h> +#include "gt/intel_lrc_reg.h" /* for shadow reg list */ + #include "i915_drv.h" #include "i915_trace.h" #include "i915_vgpu.h" @@ -68,8 +70,14 @@ static const char * const forcewake_domain_names[] = { "vdbox1", "vdbox2", "vdbox3", + "vdbox4", + "vdbox5", + "vdbox6", + "vdbox7", "vebox0", "vebox1", + "vebox2", + "vebox3", }; const char * @@ -952,30 +960,80 @@ static const i915_reg_t gen8_shadowed_regs[] = { }; static const i915_reg_t gen11_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ - RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ - RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ - RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ - RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ - RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ - RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ + RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ + RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ + GEN6_RPNSWREQ, /* 0xA008 */ + GEN6_RC_VIDEO_FREQ, /* 0xA00C */ + RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ + RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ + RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ + RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ + RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ + RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ + RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ + RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ /* TODO: Other registers are not yet used */ }; static const i915_reg_t gen12_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ - RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ - RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ - RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ - RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ - RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ - RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ + RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ + RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ + GEN6_RPNSWREQ, /* 0xA008 */ + GEN6_RC_VIDEO_FREQ, /* 0xA00C */ + RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ + RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ + RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ + RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ + RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ + RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ + RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ + RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ + /* TODO: Other registers are not yet used */ +}; + +static const i915_reg_t xehp_shadowed_regs[] = { + RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ + RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ + GEN6_RPNSWREQ, /* 0xA008 */ + GEN6_RC_VIDEO_FREQ, /* 0xA00C */ + RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ + RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ + RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ + RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ + RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ + RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ + RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ + RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ + RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ + RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ + RING_TAIL(XEHP_BSD5_RING_BASE), /* 0x1E0000 (base) */ + RING_EXECLIST_CONTROL(XEHP_BSD5_RING_BASE), /* 0x1E0550 */ + RING_TAIL(XEHP_BSD6_RING_BASE), /* 0x1E4000 (base) */ + RING_EXECLIST_CONTROL(XEHP_BSD6_RING_BASE), /* 0x1E4550 */ + RING_TAIL(XEHP_VEBOX3_RING_BASE), /* 0x1E8000 (base) */ + RING_EXECLIST_CONTROL(XEHP_VEBOX3_RING_BASE), /* 0x1E8550 */ + RING_TAIL(XEHP_BSD7_RING_BASE), /* 0x1F0000 (base) */ + RING_EXECLIST_CONTROL(XEHP_BSD7_RING_BASE), /* 0x1F0550 */ + RING_TAIL(XEHP_BSD8_RING_BASE), /* 0x1F4000 (base) */ + RING_EXECLIST_CONTROL(XEHP_BSD8_RING_BASE), /* 0x1F4550 */ + RING_TAIL(XEHP_VEBOX4_RING_BASE), /* 0x1F8000 (base) */ + RING_EXECLIST_CONTROL(XEHP_VEBOX4_RING_BASE), /* 0x1F8550 */ /* TODO: Other registers are not yet used */ }; @@ -991,17 +1049,18 @@ static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) return 0; } -#define __is_genX_shadowed(x) \ -static bool is_gen##x##_shadowed(u32 offset) \ +#define __is_X_shadowed(x) \ +static bool is_##x##_shadowed(u32 offset) \ { \ - const i915_reg_t *regs = gen##x##_shadowed_regs; \ - return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \ + const i915_reg_t *regs = x##_shadowed_regs; \ + return BSEARCH(offset, regs, ARRAY_SIZE(x##_shadowed_regs), \ mmio_reg_cmp); \ } -__is_genX_shadowed(8) -__is_genX_shadowed(11) -__is_genX_shadowed(12) +__is_X_shadowed(gen8) +__is_X_shadowed(gen11) +__is_X_shadowed(gen12) +__is_X_shadowed(xehp) static enum forcewake_domains gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) @@ -1065,6 +1124,15 @@ static const struct intel_forcewake_range __chv_fw_ranges[] = { __fwd; \ }) +#define __xehp_fwtable_reg_write_fw_domains(uncore, offset) \ +({ \ + enum forcewake_domains __fwd = 0; \ + const u32 __offset = (offset); \ + if (!is_xehp_shadowed(__offset)) \ + __fwd = find_fw_domain(uncore, __offset); \ + __fwd; \ +}) + /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ static const struct intel_forcewake_range __gen9_fw_ranges[] = { GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT), @@ -1249,6 +1317,172 @@ static const struct intel_forcewake_range __gen12_fw_ranges[] = { 0x1d3f00 - 0x1d3fff: VD2 */ }; +/* + * Graphics IP version 12.55 brings a slight change to the 0xd800 range, + * switching it from the GT domain to the render domain. + * + * *Must* be sorted by offset ranges! See intel_fw_table_check(). + */ +#define XEHP_FWRANGES(FW_RANGE_D800) \ + GEN_FW_RANGE(0x0, 0x1fff, 0), /* \ + 0x0 - 0xaff: reserved \ + 0xb00 - 0x1fff: always on */ \ + GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT), \ + GEN_FW_RANGE(0x4b00, 0x51ff, 0), /* \ + 0x4b00 - 0x4fff: reserved \ + 0x5000 - 0x51ff: always on */ \ + GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), \ + GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x8160, 0x81ff, 0), /* \ + 0x8160 - 0x817f: reserved \ + 0x8180 - 0x81ff: always on */ \ + GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT), \ + GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /* \ + 0x8500 - 0x87ff: gt \ + 0x8800 - 0x8c7f: reserved \ + 0x8c80 - 0x8cff: gt (DG2 only) */ \ + GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /* \ + 0x8d00 - 0x8dff: render (DG2 only) \ + 0x8e00 - 0x8fff: reserved */ \ + GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /* \ + 0x9000 - 0x947f: gt \ + 0x9480 - 0x94cf: reserved */ \ + GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x9560, 0x967f, 0), /* \ + 0x9560 - 0x95ff: always on \ + 0x9600 - 0x967f: reserved */ \ + GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /* \ + 0x9680 - 0x96ff: render (DG2 only) \ + 0x9700 - 0x97ff: reserved */ \ + GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /* \ + 0x9800 - 0xb4ff: gt \ + 0xb500 - 0xbfff: reserved \ + 0xc000 - 0xcfff: gt */ \ + GEN_FW_RANGE(0xd000, 0xd7ff, 0), \ + GEN_FW_RANGE(0xd800, 0xd87f, FW_RANGE_D800), \ + GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT), \ + GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /* \ + 0xdd00 - 0xddff: gt \ + 0xde00 - 0xde7f: reserved */ \ + GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /* \ + 0xde80 - 0xdfff: render \ + 0xe000 - 0xe0ff: reserved \ + 0xe100 - 0xe8ff: render */ \ + GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /* \ + 0xe900 - 0xe9ff: gt \ + 0xea00 - 0xefff: reserved \ + 0xf000 - 0xffff: gt */ \ + GEN_FW_RANGE(0x10000, 0x12fff, 0), /* \ + 0x10000 - 0x11fff: reserved \ + 0x12000 - 0x127ff: always on \ + 0x12800 - 0x12fff: reserved */ \ + GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), /* DG2 only */ \ + GEN_FW_RANGE(0x13200, 0x13fff, FORCEWAKE_MEDIA_VDBOX2), /* \ + 0x13200 - 0x133ff: VD2 (DG2 only) \ + 0x13400 - 0x13fff: reserved */ \ + GEN_FW_RANGE(0x14000, 0x141ff, FORCEWAKE_MEDIA_VDBOX0), /* XEHPSDV only */ \ + GEN_FW_RANGE(0x14200, 0x143ff, FORCEWAKE_MEDIA_VDBOX2), /* XEHPSDV only */ \ + GEN_FW_RANGE(0x14400, 0x145ff, FORCEWAKE_MEDIA_VDBOX4), /* XEHPSDV only */ \ + GEN_FW_RANGE(0x14600, 0x147ff, FORCEWAKE_MEDIA_VDBOX6), /* XEHPSDV only */ \ + GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /* \ + 0x15000 - 0x15fff: gt (DG2 only) \ + 0x16000 - 0x16dff: reserved */ \ + GEN_FW_RANGE(0x16e00, 0x1ffff, FORCEWAKE_RENDER), \ + GEN_FW_RANGE(0x20000, 0x21fff, FORCEWAKE_MEDIA_VDBOX0), /* \ + 0x20000 - 0x20fff: VD0 (XEHPSDV only) \ + 0x21000 - 0x21fff: reserved */ \ + GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), \ + GEN_FW_RANGE(0x24000, 0x2417f, 0), /* \ + 0x24000 - 0x2407f: always on \ + 0x24080 - 0x2417f: reserved */ \ + GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /* \ + 0x24180 - 0x241ff: gt \ + 0x24200 - 0x249ff: reserved */ \ + GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /* \ + 0x24a00 - 0x24a7f: render \ + 0x24a80 - 0x251ff: reserved */ \ + GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /* \ + 0x25200 - 0x252ff: gt \ + 0x25300 - 0x25fff: reserved */ \ + GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /* \ + 0x26000 - 0x27fff: render \ + 0x28000 - 0x29fff: reserved \ + 0x2a000 - 0x2ffff: undocumented */ \ + GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), \ + GEN_FW_RANGE(0x40000, 0x1bffff, 0), \ + GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /* \ + 0x1c0000 - 0x1c2bff: VD0 \ + 0x1c2c00 - 0x1c2cff: reserved \ + 0x1c2d00 - 0x1c2dff: VD0 \ + 0x1c2e00 - 0x1c3eff: VD0 (DG2 only) \ + 0x1c3f00 - 0x1c3fff: VD0 */ \ + GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /* \ + 0x1c4000 - 0x1c6bff: VD1 \ + 0x1c6c00 - 0x1c6cff: reserved \ + 0x1c6d00 - 0x1c6dff: VD1 \ + 0x1c6e00 - 0x1c7fff: reserved */ \ + GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* \ + 0x1c8000 - 0x1ca0ff: VE0 \ + 0x1ca100 - 0x1cbfff: reserved */ \ + GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0), \ + GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2), \ + GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4), \ + GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6), \ + GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /* \ + 0x1d0000 - 0x1d2bff: VD2 \ + 0x1d2c00 - 0x1d2cff: reserved \ + 0x1d2d00 - 0x1d2dff: VD2 \ + 0x1d2e00 - 0x1d3dff: VD2 (DG2 only) \ + 0x1d3e00 - 0x1d3eff: reserved \ + 0x1d3f00 - 0x1d3fff: VD2 */ \ + GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /* \ + 0x1d4000 - 0x1d6bff: VD3 \ + 0x1d6c00 - 0x1d6cff: reserved \ + 0x1d6d00 - 0x1d6dff: VD3 \ + 0x1d6e00 - 0x1d7fff: reserved */ \ + GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /* \ + 0x1d8000 - 0x1da0ff: VE1 \ + 0x1da100 - 0x1dffff: reserved */ \ + GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /* \ + 0x1e0000 - 0x1e2bff: VD4 \ + 0x1e2c00 - 0x1e2cff: reserved \ + 0x1e2d00 - 0x1e2dff: VD4 \ + 0x1e2e00 - 0x1e3eff: reserved \ + 0x1e3f00 - 0x1e3fff: VD4 */ \ + GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /* \ + 0x1e4000 - 0x1e6bff: VD5 \ + 0x1e6c00 - 0x1e6cff: reserved \ + 0x1e6d00 - 0x1e6dff: VD5 \ + 0x1e6e00 - 0x1e7fff: reserved */ \ + GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /* \ + 0x1e8000 - 0x1ea0ff: VE2 \ + 0x1ea100 - 0x1effff: reserved */ \ + GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /* \ + 0x1f0000 - 0x1f2bff: VD6 \ + 0x1f2c00 - 0x1f2cff: reserved \ + 0x1f2d00 - 0x1f2dff: VD6 \ + 0x1f2e00 - 0x1f3eff: reserved \ + 0x1f3f00 - 0x1f3fff: VD6 */ \ + GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /* \ + 0x1f4000 - 0x1f6bff: VD7 \ + 0x1f6c00 - 0x1f6cff: reserved \ + 0x1f6d00 - 0x1f6dff: VD7 \ + 0x1f6e00 - 0x1f7fff: reserved */ \ + GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3), + +static const struct intel_forcewake_range __xehp_fw_ranges[] = { + XEHP_FWRANGES(FORCEWAKE_GT) +}; + +static const struct intel_forcewake_range __dg2_fw_ranges[] = { + XEHP_FWRANGES(FORCEWAKE_RENDER) +}; + static void ilk_dummy_write(struct intel_uncore *uncore) { @@ -1502,6 +1736,7 @@ __gen_write(func, 8) \ __gen_write(func, 16) \ __gen_write(func, 32) +__gen_reg_write_funcs(xehp_fwtable); __gen_reg_write_funcs(gen12_fwtable); __gen_reg_write_funcs(gen11_fwtable); __gen_reg_write_funcs(fwtable); @@ -1582,8 +1817,14 @@ static int __fw_domain_init(struct intel_uncore *uncore, BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1)); BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2)); BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7)); BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0)); BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2)); + BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3)); d->mask = BIT(domain_id); @@ -1870,36 +2111,40 @@ static int uncore_forcewake_init(struct intel_uncore *uncore) return ret; forcewake_early_sanitize(uncore, 0); - if (IS_GRAPHICS_VER(i915, 6, 7)) { - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); - - if (IS_VALLEYVIEW(i915)) { - ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); - } else { - ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); - } - } else if (GRAPHICS_VER(i915) == 8) { - if (IS_CHERRYVIEW(i915)) { - ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); - } else { - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); - } - } else if (IS_GRAPHICS_VER(i915, 9, 10)) { - ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); - } else if (GRAPHICS_VER(i915) == 11) { - ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable); + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, xehp_fwtable); ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); - } else { + } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, xehp_fwtable); + ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); + } else if (GRAPHICS_VER(i915) >= 12) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen12_fwtable); ASSIGN_READ_MMIO_VFUNCS(uncore, gen12_fwtable); + } else if (GRAPHICS_VER(i915) == 11) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable); + ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); + } else if (IS_GRAPHICS_VER(i915, 9, 10)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); + ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); + } else if (IS_CHERRYVIEW(i915)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); + ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); + } else if (GRAPHICS_VER(i915) == 8) { + ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8); + ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); + } else if (IS_VALLEYVIEW(i915)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); + ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); + } else if (IS_GRAPHICS_VER(i915, 6, 7)) { + ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); + ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); } uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier; @@ -1988,6 +2233,22 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, if (HAS_ENGINE(gt, _VCS(i))) continue; + /* + * Starting with XeHP, the power well for an even-numbered + * VDBOX is also used for shared units within the + * media slice such as SFC. So even if the engine + * itself is fused off, we still need to initialize + * the forcewake domain if any of the other engines + * in the same media slice are present. + */ + if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 50) && i % 2 == 0) { + if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1))) + continue; + + if (HAS_ENGINE(gt, _VECS(i / 2))) + continue; + } + if (fw_domains & BIT(domain_id)) fw_domain_fini(uncore, domain_id); } @@ -2277,6 +2538,61 @@ intel_uncore_forcewake_for_reg(struct intel_uncore *uncore, return fw_domains; } +u32 intel_uncore_read_with_mcr_steering_fw(struct intel_uncore *uncore, + i915_reg_t reg, + int slice, int subslice) +{ + u32 mcr_mask, mcr_ss, mcr, old_mcr, val; + + lockdep_assert_held(&uncore->lock); + + if (GRAPHICS_VER(uncore->i915) >= 11) { + mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; + mcr_ss = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); + } else { + mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; + mcr_ss = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); + } + + old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); + + mcr &= ~mcr_mask; + mcr |= mcr_ss; + intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); + + val = intel_uncore_read_fw(uncore, reg); + + mcr &= ~mcr_mask; + mcr |= old_mcr & mcr_mask; + + intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); + + return val; +} + +u32 intel_uncore_read_with_mcr_steering(struct intel_uncore *uncore, + i915_reg_t reg, int slice, int subslice) +{ + enum forcewake_domains fw_domains; + u32 val; + + fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, + FW_REG_READ); + fw_domains |= intel_uncore_forcewake_for_reg(uncore, + GEN8_MCR_SELECTOR, + FW_REG_READ | FW_REG_WRITE); + + spin_lock_irq(&uncore->lock); + intel_uncore_forcewake_get__locked(uncore, fw_domains); + + val = intel_uncore_read_with_mcr_steering_fw(uncore, reg, slice, subslice); + + intel_uncore_forcewake_put__locked(uncore, fw_domains); + spin_unlock_irq(&uncore->lock); + + return val; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/mock_uncore.c" #include "selftests/intel_uncore.c" diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index 59f0da8f1fbb..3c0b0a8b5250 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -52,8 +52,14 @@ enum forcewake_domain_id { FW_DOMAIN_ID_MEDIA_VDBOX1, FW_DOMAIN_ID_MEDIA_VDBOX2, FW_DOMAIN_ID_MEDIA_VDBOX3, + FW_DOMAIN_ID_MEDIA_VDBOX4, + FW_DOMAIN_ID_MEDIA_VDBOX5, + FW_DOMAIN_ID_MEDIA_VDBOX6, + FW_DOMAIN_ID_MEDIA_VDBOX7, FW_DOMAIN_ID_MEDIA_VEBOX0, FW_DOMAIN_ID_MEDIA_VEBOX1, + FW_DOMAIN_ID_MEDIA_VEBOX2, + FW_DOMAIN_ID_MEDIA_VEBOX3, FW_DOMAIN_ID_COUNT }; @@ -66,10 +72,16 @@ enum forcewake_domains { FORCEWAKE_MEDIA_VDBOX1 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX1), FORCEWAKE_MEDIA_VDBOX2 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX2), FORCEWAKE_MEDIA_VDBOX3 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX3), + FORCEWAKE_MEDIA_VDBOX4 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX4), + FORCEWAKE_MEDIA_VDBOX5 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX5), + FORCEWAKE_MEDIA_VDBOX6 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX6), + FORCEWAKE_MEDIA_VDBOX7 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX7), FORCEWAKE_MEDIA_VEBOX0 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX0), FORCEWAKE_MEDIA_VEBOX1 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX1), + FORCEWAKE_MEDIA_VEBOX2 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX2), + FORCEWAKE_MEDIA_VEBOX3 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX3), - FORCEWAKE_ALL = BIT(FW_DOMAIN_ID_COUNT) - 1 + FORCEWAKE_ALL = BIT(FW_DOMAIN_ID_COUNT) - 1, }; struct intel_uncore_funcs { @@ -182,6 +194,12 @@ intel_uncore_has_fifo(const struct intel_uncore *uncore) return uncore->flags & UNCORE_HAS_FIFO; } +u32 intel_uncore_read_with_mcr_steering_fw(struct intel_uncore *uncore, + i915_reg_t reg, + int slice, int subslice); +u32 intel_uncore_read_with_mcr_steering(struct intel_uncore *uncore, + i915_reg_t reg, int slice, int subslice); + void intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug); void intel_uncore_init_early(struct intel_uncore *uncore, diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c index 8309455f13ea..5e511bb891f9 100644 --- a/drivers/gpu/drm/i915/intel_wopcm.c +++ b/drivers/gpu/drm/i915/intel_wopcm.c @@ -56,8 +56,8 @@ /* 24KB at the end of WOPCM is reserved for RC6 CTX on BXT. */ #define BXT_WOPCM_RC6_CTX_RESERVED (SZ_16K + SZ_8K) -/* 36KB WOPCM reserved at the end of WOPCM on CNL. */ -#define CNL_WOPCM_HW_CTX_RESERVED (SZ_32K + SZ_4K) +/* 36KB WOPCM reserved at the end of WOPCM on ICL. */ +#define ICL_WOPCM_HW_CTX_RESERVED (SZ_32K + SZ_4K) /* 128KB from GUC_WOPCM_RESERVED is reserved for FW on Gen9. */ #define GEN9_GUC_FW_RESERVED SZ_128K @@ -93,8 +93,8 @@ static u32 context_reserved_size(struct drm_i915_private *i915) { if (IS_GEN9_LP(i915)) return BXT_WOPCM_RC6_CTX_RESERVED; - else if (GRAPHICS_VER(i915) >= 10) - return CNL_WOPCM_HW_CTX_RESERVED; + else if (GRAPHICS_VER(i915) >= 11) + return ICL_WOPCM_HW_CTX_RESERVED; else return 0; } @@ -126,7 +126,7 @@ static bool gen9_check_huc_fw_fits(struct drm_i915_private *i915, u32 guc_wopcm_size, u32 huc_fw_size) { /* - * On Gen9 & CNL A0, hardware requires the total available GuC WOPCM + * On Gen9, hardware requires the total available GuC WOPCM * size to be larger than or equal to HuC firmware size. Otherwise, * firmware uploading would fail. */ diff --git a/drivers/gpu/drm/i915/selftests/i915_buddy.c b/drivers/gpu/drm/i915/selftests/i915_buddy.c new file mode 100644 index 000000000000..d61ec9c951bf --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/i915_buddy.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include <linux/prime_numbers.h> + +#include "../i915_selftest.h" +#include "i915_random.h" + +static void __igt_dump_block(struct i915_buddy_mm *mm, + struct i915_buddy_block *block, + bool buddy) +{ + pr_err("block info: header=%llx, state=%u, order=%d, offset=%llx size=%llx root=%s buddy=%s\n", + block->header, + i915_buddy_block_state(block), + i915_buddy_block_order(block), + i915_buddy_block_offset(block), + i915_buddy_block_size(mm, block), + yesno(!block->parent), + yesno(buddy)); +} + +static void igt_dump_block(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + struct i915_buddy_block *buddy; + + __igt_dump_block(mm, block, false); + + buddy = get_buddy(block); + if (buddy) + __igt_dump_block(mm, buddy, true); +} + +static int igt_check_block(struct i915_buddy_mm *mm, + struct i915_buddy_block *block) +{ + struct i915_buddy_block *buddy; + unsigned int block_state; + u64 block_size; + u64 offset; + int err = 0; + + block_state = i915_buddy_block_state(block); + + if (block_state != I915_BUDDY_ALLOCATED && + block_state != I915_BUDDY_FREE && + block_state != I915_BUDDY_SPLIT) { + pr_err("block state mismatch\n"); + err = -EINVAL; + } + + block_size = i915_buddy_block_size(mm, block); + offset = i915_buddy_block_offset(block); + + if (block_size < mm->chunk_size) { + pr_err("block size smaller than min size\n"); + err = -EINVAL; + } + + if (!is_power_of_2(block_size)) { + pr_err("block size not power of two\n"); + err = -EINVAL; + } + + if (!IS_ALIGNED(block_size, mm->chunk_size)) { + pr_err("block size not aligned to min size\n"); + err = -EINVAL; + } + + if (!IS_ALIGNED(offset, mm->chunk_size)) { + pr_err("block offset not aligned to min size\n"); + err = -EINVAL; + } + + if (!IS_ALIGNED(offset, block_size)) { + pr_err("block offset not aligned to block size\n"); + err = -EINVAL; + } + + buddy = get_buddy(block); + + if (!buddy && block->parent) { + pr_err("buddy has gone fishing\n"); + err = -EINVAL; + } + + if (buddy) { + if (i915_buddy_block_offset(buddy) != (offset ^ block_size)) { + pr_err("buddy has wrong offset\n"); + err = -EINVAL; + } + + if (i915_buddy_block_size(mm, buddy) != block_size) { + pr_err("buddy size mismatch\n"); + err = -EINVAL; + } + + if (i915_buddy_block_state(buddy) == block_state && + block_state == I915_BUDDY_FREE) { + pr_err("block and its buddy are free\n"); + err = -EINVAL; + } + } + + return err; +} + +static int igt_check_blocks(struct i915_buddy_mm *mm, + struct list_head *blocks, + u64 expected_size, + bool is_contiguous) +{ + struct i915_buddy_block *block; + struct i915_buddy_block *prev; + u64 total; + int err = 0; + + block = NULL; + prev = NULL; + total = 0; + + list_for_each_entry(block, blocks, link) { + err = igt_check_block(mm, block); + + if (!i915_buddy_block_is_allocated(block)) { + pr_err("block not allocated\n"), + err = -EINVAL; + } + + if (is_contiguous && prev) { + u64 prev_block_size; + u64 prev_offset; + u64 offset; + + prev_offset = i915_buddy_block_offset(prev); + prev_block_size = i915_buddy_block_size(mm, prev); + offset = i915_buddy_block_offset(block); + + if (offset != (prev_offset + prev_block_size)) { + pr_err("block offset mismatch\n"); + err = -EINVAL; + } + } + + if (err) + break; + + total += i915_buddy_block_size(mm, block); + prev = block; + } + + if (!err) { + if (total != expected_size) { + pr_err("size mismatch, expected=%llx, found=%llx\n", + expected_size, total); + err = -EINVAL; + } + return err; + } + + if (prev) { + pr_err("prev block, dump:\n"); + igt_dump_block(mm, prev); + } + + pr_err("bad block, dump:\n"); + igt_dump_block(mm, block); + + return err; +} + +static int igt_check_mm(struct i915_buddy_mm *mm) +{ + struct i915_buddy_block *root; + struct i915_buddy_block *prev; + unsigned int i; + u64 total; + int err = 0; + + if (!mm->n_roots) { + pr_err("n_roots is zero\n"); + return -EINVAL; + } + + if (mm->n_roots != hweight64(mm->size)) { + pr_err("n_roots mismatch, n_roots=%u, expected=%lu\n", + mm->n_roots, hweight64(mm->size)); + return -EINVAL; + } + + root = NULL; + prev = NULL; + total = 0; + + for (i = 0; i < mm->n_roots; ++i) { + struct i915_buddy_block *block; + unsigned int order; + + root = mm->roots[i]; + if (!root) { + pr_err("root(%u) is NULL\n", i); + err = -EINVAL; + break; + } + + err = igt_check_block(mm, root); + + if (!i915_buddy_block_is_free(root)) { + pr_err("root not free\n"); + err = -EINVAL; + } + + order = i915_buddy_block_order(root); + + if (!i) { + if (order != mm->max_order) { + pr_err("max order root missing\n"); + err = -EINVAL; + } + } + + if (prev) { + u64 prev_block_size; + u64 prev_offset; + u64 offset; + + prev_offset = i915_buddy_block_offset(prev); + prev_block_size = i915_buddy_block_size(mm, prev); + offset = i915_buddy_block_offset(root); + + if (offset != (prev_offset + prev_block_size)) { + pr_err("root offset mismatch\n"); + err = -EINVAL; + } + } + + block = list_first_entry_or_null(&mm->free_list[order], + struct i915_buddy_block, + link); + if (block != root) { + pr_err("root mismatch at order=%u\n", order); + err = -EINVAL; + } + + if (err) + break; + + prev = root; + total += i915_buddy_block_size(mm, root); + } + + if (!err) { + if (total != mm->size) { + pr_err("expected mm size=%llx, found=%llx\n", mm->size, + total); + err = -EINVAL; + } + return err; + } + + if (prev) { + pr_err("prev root(%u), dump:\n", i - 1); + igt_dump_block(mm, prev); + } + + if (root) { + pr_err("bad root(%u), dump:\n", i); + igt_dump_block(mm, root); + } + + return err; +} + +static void igt_mm_config(u64 *size, u64 *chunk_size) +{ + I915_RND_STATE(prng); + u32 s, ms; + + /* Nothing fancy, just try to get an interesting bit pattern */ + + prandom_seed_state(&prng, i915_selftest.random_seed); + + /* Let size be a random number of pages up to 8 GB (2M pages) */ + s = 1 + i915_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng); + /* Let the chunk size be a random power of 2 less than size */ + ms = BIT(i915_prandom_u32_max_state(ilog2(s), &prng)); + /* Round size down to the chunk size */ + s &= -ms; + + /* Convert from pages to bytes */ + *chunk_size = (u64)ms << 12; + *size = (u64)s << 12; +} + +static int igt_buddy_alloc_smoke(void *arg) +{ + struct i915_buddy_mm mm; + IGT_TIMEOUT(end_time); + I915_RND_STATE(prng); + u64 chunk_size; + u64 mm_size; + int *order; + int err, i; + + igt_mm_config(&mm_size, &chunk_size); + + pr_info("buddy_init with size=%llx, chunk_size=%llx\n", mm_size, chunk_size); + + err = i915_buddy_init(&mm, mm_size, chunk_size); + if (err) { + pr_err("buddy_init failed(%d)\n", err); + return err; + } + + order = i915_random_order(mm.max_order + 1, &prng); + if (!order) + goto out_fini; + + for (i = 0; i <= mm.max_order; ++i) { + struct i915_buddy_block *block; + int max_order = order[i]; + bool timeout = false; + LIST_HEAD(blocks); + int order; + u64 total; + + err = igt_check_mm(&mm); + if (err) { + pr_err("pre-mm check failed, abort\n"); + break; + } + + pr_info("filling from max_order=%u\n", max_order); + + order = max_order; + total = 0; + + do { +retry: + block = i915_buddy_alloc(&mm, order); + if (IS_ERR(block)) { + err = PTR_ERR(block); + if (err == -ENOMEM) { + pr_info("buddy_alloc hit -ENOMEM with order=%d\n", + order); + } else { + if (order--) { + err = 0; + goto retry; + } + + pr_err("buddy_alloc with order=%d failed(%d)\n", + order, err); + } + + break; + } + + list_add_tail(&block->link, &blocks); + + if (i915_buddy_block_order(block) != order) { + pr_err("buddy_alloc order mismatch\n"); + err = -EINVAL; + break; + } + + total += i915_buddy_block_size(&mm, block); + + if (__igt_timeout(end_time, NULL)) { + timeout = true; + break; + } + } while (total < mm.size); + + if (!err) + err = igt_check_blocks(&mm, &blocks, total, false); + + i915_buddy_free_list(&mm, &blocks); + + if (!err) { + err = igt_check_mm(&mm); + if (err) + pr_err("post-mm check failed\n"); + } + + if (err || timeout) + break; + + cond_resched(); + } + + if (err == -ENOMEM) + err = 0; + + kfree(order); +out_fini: + i915_buddy_fini(&mm); + + return err; +} + +static int igt_buddy_alloc_pessimistic(void *arg) +{ + const unsigned int max_order = 16; + struct i915_buddy_block *block, *bn; + struct i915_buddy_mm mm; + unsigned int order; + LIST_HEAD(blocks); + int err; + + /* + * Create a pot-sized mm, then allocate one of each possible + * order within. This should leave the mm with exactly one + * page left. + */ + + err = i915_buddy_init(&mm, PAGE_SIZE << max_order, PAGE_SIZE); + if (err) { + pr_err("buddy_init failed(%d)\n", err); + return err; + } + GEM_BUG_ON(mm.max_order != max_order); + + for (order = 0; order < max_order; order++) { + block = i915_buddy_alloc(&mm, order); + if (IS_ERR(block)) { + pr_info("buddy_alloc hit -ENOMEM with order=%d\n", + order); + err = PTR_ERR(block); + goto err; + } + + list_add_tail(&block->link, &blocks); + } + + /* And now the last remaining block available */ + block = i915_buddy_alloc(&mm, 0); + if (IS_ERR(block)) { + pr_info("buddy_alloc hit -ENOMEM on final alloc\n"); + err = PTR_ERR(block); + goto err; + } + list_add_tail(&block->link, &blocks); + + /* Should be completely full! */ + for (order = max_order; order--; ) { + block = i915_buddy_alloc(&mm, order); + if (!IS_ERR(block)) { + pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!", + order); + list_add_tail(&block->link, &blocks); + err = -EINVAL; + goto err; + } + } + + block = list_last_entry(&blocks, typeof(*block), link); + list_del(&block->link); + i915_buddy_free(&mm, block); + + /* As we free in increasing size, we make available larger blocks */ + order = 1; + list_for_each_entry_safe(block, bn, &blocks, link) { + list_del(&block->link); + i915_buddy_free(&mm, block); + + block = i915_buddy_alloc(&mm, order); + if (IS_ERR(block)) { + pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n", + order); + err = PTR_ERR(block); + goto err; + } + i915_buddy_free(&mm, block); + order++; + } + + /* To confirm, now the whole mm should be available */ + block = i915_buddy_alloc(&mm, max_order); + if (IS_ERR(block)) { + pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n", + max_order); + err = PTR_ERR(block); + goto err; + } + i915_buddy_free(&mm, block); + +err: + i915_buddy_free_list(&mm, &blocks); + i915_buddy_fini(&mm); + return err; +} + +static int igt_buddy_alloc_optimistic(void *arg) +{ + const int max_order = 16; + struct i915_buddy_block *block; + struct i915_buddy_mm mm; + LIST_HEAD(blocks); + int order; + int err; + + /* + * Create a mm with one block of each order available, and + * try to allocate them all. + */ + + err = i915_buddy_init(&mm, + PAGE_SIZE * ((1 << (max_order + 1)) - 1), + PAGE_SIZE); + if (err) { + pr_err("buddy_init failed(%d)\n", err); + return err; + } + GEM_BUG_ON(mm.max_order != max_order); + + for (order = 0; order <= max_order; order++) { + block = i915_buddy_alloc(&mm, order); + if (IS_ERR(block)) { + pr_info("buddy_alloc hit -ENOMEM with order=%d\n", + order); + err = PTR_ERR(block); + goto err; + } + + list_add_tail(&block->link, &blocks); + } + + /* Should be completely full! */ + block = i915_buddy_alloc(&mm, 0); + if (!IS_ERR(block)) { + pr_info("buddy_alloc unexpectedly succeeded, it should be full!"); + list_add_tail(&block->link, &blocks); + err = -EINVAL; + goto err; + } + +err: + i915_buddy_free_list(&mm, &blocks); + i915_buddy_fini(&mm); + return err; +} + +static int igt_buddy_alloc_pathological(void *arg) +{ + const int max_order = 16; + struct i915_buddy_block *block; + struct i915_buddy_mm mm; + LIST_HEAD(blocks); + LIST_HEAD(holes); + int order, top; + int err; + + /* + * Create a pot-sized mm, then allocate one of each possible + * order within. This should leave the mm with exactly one + * page left. Free the largest block, then whittle down again. + * Eventually we will have a fully 50% fragmented mm. + */ + + err = i915_buddy_init(&mm, PAGE_SIZE << max_order, PAGE_SIZE); + if (err) { + pr_err("buddy_init failed(%d)\n", err); + return err; + } + GEM_BUG_ON(mm.max_order != max_order); + + for (top = max_order; top; top--) { + /* Make room by freeing the largest allocated block */ + block = list_first_entry_or_null(&blocks, typeof(*block), link); + if (block) { + list_del(&block->link); + i915_buddy_free(&mm, block); + } + + for (order = top; order--; ) { + block = i915_buddy_alloc(&mm, order); + if (IS_ERR(block)) { + pr_info("buddy_alloc hit -ENOMEM with order=%d, top=%d\n", + order, top); + err = PTR_ERR(block); + goto err; + } + list_add_tail(&block->link, &blocks); + } + + /* There should be one final page for this sub-allocation */ + block = i915_buddy_alloc(&mm, 0); + if (IS_ERR(block)) { + pr_info("buddy_alloc hit -ENOMEM for hole\n"); + err = PTR_ERR(block); + goto err; + } + list_add_tail(&block->link, &holes); + + block = i915_buddy_alloc(&mm, top); + if (!IS_ERR(block)) { + pr_info("buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!", + top, max_order); + list_add_tail(&block->link, &blocks); + err = -EINVAL; + goto err; + } + } + + i915_buddy_free_list(&mm, &holes); + + /* Nothing larger than blocks of chunk_size now available */ + for (order = 1; order <= max_order; order++) { + block = i915_buddy_alloc(&mm, order); + if (!IS_ERR(block)) { + pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!", + order); + list_add_tail(&block->link, &blocks); + err = -EINVAL; + goto err; + } + } + +err: + list_splice_tail(&holes, &blocks); + i915_buddy_free_list(&mm, &blocks); + i915_buddy_fini(&mm); + return err; +} + +static int igt_buddy_alloc_range(void *arg) +{ + struct i915_buddy_mm mm; + unsigned long page_num; + LIST_HEAD(blocks); + u64 chunk_size; + u64 offset; + u64 size; + u64 rem; + int err; + + igt_mm_config(&size, &chunk_size); + + pr_info("buddy_init with size=%llx, chunk_size=%llx\n", size, chunk_size); + + err = i915_buddy_init(&mm, size, chunk_size); + if (err) { + pr_err("buddy_init failed(%d)\n", err); + return err; + } + + err = igt_check_mm(&mm); + if (err) { + pr_err("pre-mm check failed, abort, abort, abort!\n"); + goto err_fini; + } + + rem = mm.size; + offset = 0; + + for_each_prime_number_from(page_num, 1, ULONG_MAX - 1) { + struct i915_buddy_block *block; + LIST_HEAD(tmp); + + size = min(page_num * mm.chunk_size, rem); + + err = i915_buddy_alloc_range(&mm, &tmp, offset, size); + if (err) { + if (err == -ENOMEM) { + pr_info("alloc_range hit -ENOMEM with size=%llx\n", + size); + } else { + pr_err("alloc_range with offset=%llx, size=%llx failed(%d)\n", + offset, size, err); + } + + break; + } + + block = list_first_entry_or_null(&tmp, + struct i915_buddy_block, + link); + if (!block) { + pr_err("alloc_range has no blocks\n"); + err = -EINVAL; + break; + } + + if (i915_buddy_block_offset(block) != offset) { + pr_err("alloc_range start offset mismatch, found=%llx, expected=%llx\n", + i915_buddy_block_offset(block), offset); + err = -EINVAL; + } + + if (!err) + err = igt_check_blocks(&mm, &tmp, size, true); + + list_splice_tail(&tmp, &blocks); + + if (err) + break; + + offset += size; + + rem -= size; + if (!rem) + break; + + cond_resched(); + } + + if (err == -ENOMEM) + err = 0; + + i915_buddy_free_list(&mm, &blocks); + + if (!err) { + err = igt_check_mm(&mm); + if (err) + pr_err("post-mm check failed\n"); + } + +err_fini: + i915_buddy_fini(&mm); + + return err; +} + +static int igt_buddy_alloc_limit(void *arg) +{ + struct i915_buddy_block *block; + struct i915_buddy_mm mm; + const u64 size = U64_MAX; + int err; + + err = i915_buddy_init(&mm, size, PAGE_SIZE); + if (err) + return err; + + if (mm.max_order != I915_BUDDY_MAX_ORDER) { + pr_err("mm.max_order(%d) != %d\n", + mm.max_order, I915_BUDDY_MAX_ORDER); + err = -EINVAL; + goto out_fini; + } + + block = i915_buddy_alloc(&mm, mm.max_order); + if (IS_ERR(block)) { + err = PTR_ERR(block); + goto out_fini; + } + + if (i915_buddy_block_order(block) != mm.max_order) { + pr_err("block order(%d) != %d\n", + i915_buddy_block_order(block), mm.max_order); + err = -EINVAL; + goto out_free; + } + + if (i915_buddy_block_size(&mm, block) != + BIT_ULL(mm.max_order) * PAGE_SIZE) { + pr_err("block size(%llu) != %llu\n", + i915_buddy_block_size(&mm, block), + BIT_ULL(mm.max_order) * PAGE_SIZE); + err = -EINVAL; + goto out_free; + } + +out_free: + i915_buddy_free(&mm, block); +out_fini: + i915_buddy_fini(&mm); + return err; +} + +int i915_buddy_mock_selftests(void) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_buddy_alloc_pessimistic), + SUBTEST(igt_buddy_alloc_optimistic), + SUBTEST(igt_buddy_alloc_pathological), + SUBTEST(igt_buddy_alloc_smoke), + SUBTEST(igt_buddy_alloc_range), + SUBTEST(igt_buddy_alloc_limit), + }; + + return i915_subtests(tests, NULL); +} diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index a92c0e9b7e6b..cfa5c4165a4f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -26,6 +26,7 @@ selftest(gt_mocs, intel_mocs_live_selftests) selftest(gt_pm, intel_gt_pm_live_selftests) selftest(gt_heartbeat, intel_heartbeat_live_selftests) selftest(requests, i915_request_live_selftests) +selftest(migrate, intel_migrate_live_selftests) selftest(active, i915_active_live_selftests) selftest(objects, i915_gem_object_live_selftests) selftest(mman, i915_gem_mman_live_selftests) @@ -37,14 +38,14 @@ selftest(gem, i915_gem_live_selftests) selftest(evict, i915_gem_evict_live_selftests) selftest(hugepages, i915_gem_huge_page_live_selftests) selftest(gem_contexts, i915_gem_context_live_selftests) -selftest(gem_execbuf, i915_gem_execbuffer_live_selftests) -selftest(blt, i915_gem_object_blt_live_selftests) selftest(client, i915_gem_client_blt_live_selftests) +selftest(gem_migrate, i915_gem_migrate_live_selftests) selftest(reset, intel_reset_live_selftests) selftest(memory_region, intel_memory_region_live_selftests) selftest(hangcheck, intel_hangcheck_live_selftests) selftest(execlists, intel_execlists_live_selftests) selftest(ring_submission, intel_ring_submission_live_selftests) selftest(perf, i915_perf_live_selftests) +selftest(slpc, intel_slpc_live_selftests) /* Here be dragons: keep last to run last! */ selftest(late_gt_pm, intel_gt_pm_late_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 34e5caf38093..793fb28a770d 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -32,5 +32,5 @@ selftest(vma, i915_vma_mock_selftests) selftest(evict, i915_gem_evict_mock_selftests) selftest(gtt, i915_gem_gtt_mock_selftests) selftest(hugepages, i915_gem_huge_page_mock_selftests) -selftest(contexts, i915_gem_context_mock_selftests) selftest(memory_region, intel_memory_region_mock_selftests) +selftest(buddy, i915_buddy_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h index c2389f8a257d..058450d351f7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h @@ -17,5 +17,5 @@ */ selftest(engine_cs, intel_engine_cs_perf_selftests) selftest(request, i915_request_perf_selftests) -selftest(blt, i915_gem_object_blt_perf_selftests) +selftest(migrate, intel_migrate_perf_selftests) selftest(region, intel_memory_region_perf_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index bd5c96a77ba3..d67710d10615 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -1313,7 +1313,7 @@ static int __live_parallel_engine1(void *arg) i915_request_add(rq); err = 0; - if (i915_request_wait(rq, 0, HZ / 5) < 0) + if (i915_request_wait(rq, 0, HZ) < 0) err = -ETIME; i915_request_put(rq); if (err) @@ -1419,7 +1419,7 @@ static int __live_parallel_spin(void *arg) } igt_spinner_end(&spin); - if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) + if (err == 0 && i915_request_wait(rq, 0, HZ) < 0) err = -EIO; i915_request_put(rq); diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index 1bc11c09faef..484759c9409c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -187,7 +187,7 @@ int i915_mock_selftests(void) err = run_selftests(mock, NULL); if (err) { i915_selftest.mock = err; - return err; + return 1; } if (i915_selftest.mock < 0) { @@ -430,7 +430,7 @@ module_param_named(st_timeout, i915_selftest.timeout_ms, uint, 0400); module_param_named(st_filter, i915_selftest.filter, charp, 0400); module_param_named_unsafe(mock_selftests, i915_selftest.mock, int, 0400); -MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then exit module)"); +MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then leave dummy module)"); module_param_named_unsafe(live_selftests, i915_selftest.live, int, 0400); MODULE_PARM_DESC(live_selftests, "Run selftests after driver initialisation on the live system (0:disabled [default], 1:run tests then continue, -1:run tests then exit module)"); diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c index 7b0939e3f007..a6c71fca61aa 100644 --- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c @@ -19,7 +19,7 @@ int igt_flush_test(struct drm_i915_private *i915) cond_resched(); - if (intel_gt_wait_for_idle(gt, HZ / 5) == -ETIME) { + if (intel_gt_wait_for_idle(gt, HZ) == -ETIME) { pr_err("%pS timed out, cancelling all further testing.\n", __builtin_return_address(0)); diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.c b/drivers/gpu/drm/i915/selftests/igt_live_test.c index c130010a7033..1c721542e277 100644 --- a/drivers/gpu/drm/i915/selftests/igt_live_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_live_test.c @@ -5,7 +5,7 @@ */ #include "i915_drv.h" -#include "gt/intel_gt_requests.h" +#include "gt/intel_gt.h" #include "../i915_selftest.h" #include "igt_flush_test.h" diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index c85d516b85cd..418caae84759 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -15,12 +15,15 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" -#include "gem/i915_gem_object_blt.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" +#include "gt/intel_engine_pm.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" +#include "i915_buddy.h" +#include "gt/intel_migrate.h" #include "i915_memcpy.h" +#include "i915_ttm_buddy_manager.h" #include "selftests/igt_flush_test.h" #include "selftests/i915_random.h" @@ -57,16 +60,15 @@ static int igt_mock_fill(void *arg) LIST_HEAD(objects); int err = 0; - page_size = mem->chunk_size; + page_size = PAGE_SIZE; + max_pages = div64_u64(total, page_size); rem = total; -retry: - max_pages = div64_u64(rem, page_size); for_each_prime_number_from(page_num, 1, max_pages) { resource_size_t size = page_num * page_size; struct drm_i915_gem_object *obj; - obj = i915_gem_object_create_region(mem, size, 0); + obj = i915_gem_object_create_region(mem, size, 0, 0); if (IS_ERR(obj)) { err = PTR_ERR(obj); break; @@ -86,11 +88,6 @@ retry: err = 0; if (err == -ENXIO) { if (page_num * page_size <= rem) { - if (mem->is_range_manager && max_pages > 1) { - max_pages >>= 1; - goto retry; - } - pr_err("%s failed, space still left in region\n", __func__); err = -EINVAL; @@ -113,7 +110,7 @@ igt_object_create(struct intel_memory_region *mem, struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mem, size, flags); + obj = i915_gem_object_create_region(mem, size, 0, flags); if (IS_ERR(obj)) return obj; @@ -157,6 +154,7 @@ static bool is_contiguous(struct drm_i915_gem_object *obj) static int igt_mock_reserve(void *arg) { struct intel_memory_region *mem = arg; + struct drm_i915_private *i915 = mem->i915; resource_size_t avail = resource_size(&mem->region); struct drm_i915_gem_object *obj; const u32 chunk_size = SZ_32M; @@ -166,16 +164,18 @@ static int igt_mock_reserve(void *arg) LIST_HEAD(objects); int err = 0; - if (!list_empty(&mem->reserved)) { - pr_err("%s region reserved list is not empty\n", __func__); - return -EINVAL; - } - count = avail / chunk_size; order = i915_random_order(count, &prng); if (!order) return 0; + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); + if (IS_ERR(mem)) { + pr_err("failed to create memory region\n"); + err = PTR_ERR(mem); + goto out_free_order; + } + /* Reserve a bunch of ranges within the region */ for (i = 0; i < count; ++i) { u64 start = order[i] * chunk_size; @@ -205,18 +205,12 @@ static int igt_mock_reserve(void *arg) do { u32 size = i915_prandom_u32_max_state(cur_avail, &prng); -retry: size = max_t(u32, round_up(size, PAGE_SIZE), PAGE_SIZE); obj = igt_object_create(mem, &objects, size, 0); if (IS_ERR(obj)) { - if (PTR_ERR(obj) == -ENXIO) { - if (mem->is_range_manager && - size > mem->chunk_size) { - size >>= 1; - goto retry; - } + if (PTR_ERR(obj) == -ENXIO) break; - } + err = PTR_ERR(obj); goto out_close; } @@ -230,9 +224,10 @@ retry: } out_close: - kfree(order); close_objects(mem, &objects); - intel_memory_region_unreserve(mem); + intel_memory_region_put(mem); +out_free_order: + kfree(order); return err; } @@ -252,7 +247,7 @@ static int igt_mock_contiguous(void *arg) total = resource_size(&mem->region); /* Min size */ - obj = igt_object_create(mem, &objects, mem->chunk_size, + obj = igt_object_create(mem, &objects, PAGE_SIZE, I915_BO_ALLOC_CONTIGUOUS); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -333,17 +328,15 @@ static int igt_mock_contiguous(void *arg) min = target; target = total >> 1; - if (!mem->is_range_manager) { - /* Make sure we can still allocate all the fragmented space */ - obj = igt_object_create(mem, &objects, target, 0); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_close_objects; - } - - igt_object_release(obj); + /* Make sure we can still allocate all the fragmented space */ + obj = igt_object_create(mem, &objects, target, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto err_close_objects; } + igt_object_release(obj); + /* * Even though we have enough free space, we don't have a big enough * contiguous block. Make sure that holds true. @@ -362,7 +355,7 @@ static int igt_mock_contiguous(void *arg) } target >>= 1; - } while (target >= mem->chunk_size); + } while (target >= PAGE_SIZE); err_close_objects: list_splice_tail(&holes, &objects); @@ -374,7 +367,9 @@ static int igt_mock_splintered_region(void *arg) { struct intel_memory_region *mem = arg; struct drm_i915_private *i915 = mem->i915; + struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; + struct i915_buddy_mm *mm; unsigned int expected_order; LIST_HEAD(objects); u64 size; @@ -382,7 +377,7 @@ static int igt_mock_splintered_region(void *arg) /* * Sanity check we can still allocate everything even if the - * max_order != mm.size. i.e our starting address space size is not a + * mm.max_order != mm.size. i.e our starting address space size is not a * power-of-two. */ @@ -391,20 +386,29 @@ static int igt_mock_splintered_region(void *arg) if (IS_ERR(mem)) return PTR_ERR(mem); - expected_order = get_order(rounddown_pow_of_two(size)); - if (mem->max_order != expected_order) { - pr_err("%s order mismatch(%u != %u)\n", - __func__, mem->max_order, expected_order); - err = -EINVAL; - goto out_put; - } - obj = igt_object_create(mem, &objects, size, 0); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto out_close; } + res = to_ttm_buddy_resource(obj->mm.res); + mm = res->mm; + if (mm->size != size) { + pr_err("%s size mismatch(%llu != %llu)\n", + __func__, mm->size, size); + err = -EINVAL; + goto out_put; + } + + expected_order = get_order(rounddown_pow_of_two(size)); + if (mm->max_order != expected_order) { + pr_err("%s order mismatch(%u != %u)\n", + __func__, mm->max_order, expected_order); + err = -EINVAL; + goto out_put; + } + close_objects(mem, &objects); /* @@ -415,15 +419,12 @@ static int igt_mock_splintered_region(void *arg) * sure that does indeed hold true. */ - if (!mem->is_range_manager) { - obj = igt_object_create(mem, &objects, size, - I915_BO_ALLOC_CONTIGUOUS); - if (!IS_ERR(obj)) { - pr_err("%s too large contiguous allocation was not rejected\n", - __func__); - err = -EINVAL; - goto out_close; - } + obj = igt_object_create(mem, &objects, size, I915_BO_ALLOC_CONTIGUOUS); + if (!IS_ERR(obj)) { + pr_err("%s too large contiguous allocation was not rejected\n", + __func__); + err = -EINVAL; + goto out_close; } obj = igt_object_create(mem, &objects, rounddown_pow_of_two(size), @@ -442,6 +443,74 @@ out_put: return err; } +#ifndef SZ_8G +#define SZ_8G BIT_ULL(33) +#endif + +static int igt_mock_max_segment(void *arg) +{ + const unsigned int max_segment = rounddown(UINT_MAX, PAGE_SIZE); + struct intel_memory_region *mem = arg; + struct drm_i915_private *i915 = mem->i915; + struct i915_ttm_buddy_resource *res; + struct drm_i915_gem_object *obj; + struct i915_buddy_block *block; + struct i915_buddy_mm *mm; + struct list_head *blocks; + struct scatterlist *sg; + LIST_HEAD(objects); + u64 size; + int err = 0; + + /* + * While we may create very large contiguous blocks, we may need + * to break those down for consumption elsewhere. In particular, + * dma-mapping with scatterlist elements have an implicit limit of + * UINT_MAX on each element. + */ + + size = SZ_8G; + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); + if (IS_ERR(mem)) + return PTR_ERR(mem); + + obj = igt_object_create(mem, &objects, size, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto out_put; + } + + res = to_ttm_buddy_resource(obj->mm.res); + blocks = &res->blocks; + mm = res->mm; + size = 0; + list_for_each_entry(block, blocks, link) { + if (i915_buddy_block_size(mm, block) > size) + size = i915_buddy_block_size(mm, block); + } + if (size < max_segment) { + pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n", + __func__, max_segment, size); + err = -EINVAL; + goto out_close; + } + + for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { + if (sg->length > max_segment) { + pr_err("%s: Created an oversized scatterlist entry, %u > %u\n", + __func__, sg->length, max_segment); + err = -EINVAL; + goto out_close; + } + } + +out_close: + close_objects(mem, &objects); +out_put: + intel_memory_region_put(mem); + return err; +} + static int igt_gpu_write_dw(struct intel_context *ce, struct i915_vma *vma, u32 dword, @@ -579,6 +648,62 @@ out_put: return err; } +static int igt_lmem_create_with_ps(void *arg) +{ + struct drm_i915_private *i915 = arg; + int err = 0; + u32 ps; + + for (ps = PAGE_SIZE; ps <= SZ_1G; ps <<= 1) { + struct drm_i915_gem_object *obj; + dma_addr_t daddr; + + obj = __i915_gem_object_create_lmem_with_ps(i915, ps, ps, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + if (err == -ENXIO || err == -E2BIG) { + pr_info("%s not enough lmem for ps(%u) err=%d\n", + __func__, ps, err); + err = 0; + } + + break; + } + + if (obj->base.size != ps) { + pr_err("%s size(%zu) != ps(%u)\n", + __func__, obj->base.size, ps); + err = -EINVAL; + goto out_put; + } + + i915_gem_object_lock(obj, NULL); + err = i915_gem_object_pin_pages(obj); + if (err) + goto out_put; + + daddr = i915_gem_object_get_dma_address(obj, 0); + if (!IS_ALIGNED(daddr, ps)) { + pr_err("%s daddr(%pa) not aligned with ps(%u)\n", + __func__, &daddr, ps); + err = -EINVAL; + goto out_unpin; + } + +out_unpin: + i915_gem_object_unpin_pages(obj); + __i915_gem_object_put_pages(obj); +out_put: + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + + if (err) + break; + } + + return err; +} + static int igt_lmem_create_cleared_cpu(void *arg) { struct drm_i915_private *i915 = arg; @@ -741,6 +866,7 @@ static int igt_lmem_write_cpu(void *arg) PAGE_SIZE - 64, }; struct intel_engine_cs *engine; + struct i915_request *rq; u32 *vaddr; u32 sz; u32 i; @@ -767,15 +893,20 @@ static int igt_lmem_write_cpu(void *arg) goto out_put; } + i915_gem_object_lock(obj, NULL); /* Put the pages into a known state -- from the gpu for added fun */ intel_engine_pm_get(engine); - err = i915_gem_object_fill_blt(obj, engine->kernel_context, 0xdeadbeaf); - intel_engine_pm_put(engine); - if (err) - goto out_unpin; + err = intel_context_migrate_clear(engine->gt->migrate.context, NULL, + obj->mm.pages->sgl, I915_CACHE_NONE, + true, 0xdeadbeaf, &rq); + if (rq) { + dma_resv_add_excl_fence(obj->base.resv, &rq->fence); + i915_request_put(rq); + } - i915_gem_object_lock(obj, NULL); - err = i915_gem_object_set_to_wc_domain(obj, true); + intel_engine_pm_put(engine); + if (!err) + err = i915_gem_object_set_to_wc_domain(obj, true); i915_gem_object_unlock(obj); if (err) goto out_unpin; @@ -858,7 +989,7 @@ create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, struct drm_i915_gem_object *obj; void *addr; - obj = i915_gem_object_create_region(mr, size, 0); + obj = i915_gem_object_create_region(mr, size, 0, 0); if (IS_ERR(obj)) { if (PTR_ERR(obj) == -ENOSPC) /* Stolen memory */ return ERR_PTR(-ENODEV); @@ -1046,6 +1177,7 @@ int intel_memory_region_mock_selftests(void) SUBTEST(igt_mock_fill), SUBTEST(igt_mock_contiguous), SUBTEST(igt_mock_splintered_region), + SUBTEST(igt_mock_max_segment), }; struct intel_memory_region *mem; struct drm_i915_private *i915; @@ -1074,6 +1206,7 @@ int intel_memory_region_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_lmem_create), + SUBTEST(igt_lmem_create_with_ps), SUBTEST(igt_lmem_create_cleared_cpu), SUBTEST(igt_lmem_write_cpu), SUBTEST(igt_lmem_write_gpu), diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c new file mode 100644 index 000000000000..4b328346b48a --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +//#include "gt/intel_engine_user.h" +#include "gt/intel_gt.h" +#include "i915_drv.h" +#include "i915_selftest.h" + +#include "selftests/intel_scheduler_helpers.h" + +#define REDUCED_TIMESLICE 5 +#define REDUCED_PREEMPT 10 +#define WAIT_FOR_RESET_TIME 10000 + +int intel_selftest_modify_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved, + u32 modify_type) + +{ + int err; + + saved->reset = engine->i915->params.reset; + saved->flags = engine->flags; + saved->timeslice = engine->props.timeslice_duration_ms; + saved->preempt_timeout = engine->props.preempt_timeout_ms; + + switch (modify_type) { + case SELFTEST_SCHEDULER_MODIFY_FAST_RESET: + /* + * Enable force pre-emption on time slice expiration + * together with engine reset on pre-emption timeout. + * This is required to make the GuC notice and reset + * the single hanging context. + * Also, reduce the preemption timeout to something + * small to speed the test up. + */ + engine->i915->params.reset = 2; + engine->flags |= I915_ENGINE_WANT_FORCED_PREEMPTION; + engine->props.timeslice_duration_ms = REDUCED_TIMESLICE; + engine->props.preempt_timeout_ms = REDUCED_PREEMPT; + break; + + case SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK: + engine->props.preempt_timeout_ms = 0; + break; + + default: + pr_err("Invalid scheduler policy modification type: %d!\n", modify_type); + return -EINVAL; + } + + if (!intel_engine_uses_guc(engine)) + return 0; + + err = intel_guc_global_policies_update(&engine->gt->uc.guc); + if (err) + intel_selftest_restore_policy(engine, saved); + + return err; +} + +int intel_selftest_restore_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved) +{ + /* Restore the original policies */ + engine->i915->params.reset = saved->reset; + engine->flags = saved->flags; + engine->props.timeslice_duration_ms = saved->timeslice; + engine->props.preempt_timeout_ms = saved->preempt_timeout; + + if (!intel_engine_uses_guc(engine)) + return 0; + + return intel_guc_global_policies_update(&engine->gt->uc.guc); +} + +int intel_selftest_wait_for_rq(struct i915_request *rq) +{ + long ret; + + ret = i915_request_wait(rq, 0, WAIT_FOR_RESET_TIME); + if (ret < 0) + return ret; + + return 0; +} diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h new file mode 100644 index 000000000000..35c098601ac0 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_SELFTEST_SCHEDULER_HELPERS_H_ +#define _INTEL_SELFTEST_SCHEDULER_HELPERS_H_ + +#include <linux/types.h> + +struct i915_request; +struct intel_engine_cs; + +struct intel_selftest_saved_policy { + u32 flags; + u32 reset; + u64 timeslice; + u64 preempt_timeout; +}; + +enum selftest_scheduler_modify { + SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK = 0, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET, +}; + +int intel_selftest_modify_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved, + enum selftest_scheduler_modify modify_type); +int intel_selftest_restore_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved); +int intel_selftest_wait_for_rq(struct i915_request *rq); + +#endif diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index 8ef9e6a4ad05..720b60853f8b 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -68,6 +68,7 @@ static int intel_shadow_table_check(void) { gen8_shadowed_regs, ARRAY_SIZE(gen8_shadowed_regs) }, { gen11_shadowed_regs, ARRAY_SIZE(gen11_shadowed_regs) }, { gen12_shadowed_regs, ARRAY_SIZE(gen12_shadowed_regs) }, + { xehp_shadowed_regs, ARRAY_SIZE(xehp_shadowed_regs) }, }; const i915_reg_t *reg; unsigned int i, j; @@ -103,6 +104,7 @@ int intel_uncore_mock_selftests(void) { __gen9_fw_ranges, ARRAY_SIZE(__gen9_fw_ranges), true }, { __gen11_fw_ranges, ARRAY_SIZE(__gen11_fw_ranges), true }, { __gen12_fw_ranges, ARRAY_SIZE(__gen12_fw_ranges), true }, + { __xehp_fw_ranges, ARRAY_SIZE(__xehp_fw_ranges), true }, }; int err, i; diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index d189c4bd4bef..4f8180146888 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -52,7 +52,8 @@ void mock_device_flush(struct drm_i915_private *i915) do { for_each_engine(engine, gt, id) mock_engine_flush(engine); - } while (intel_gt_retire_requests_timeout(gt, MAX_SCHEDULE_TIMEOUT)); + } while (intel_gt_retire_requests_timeout(gt, MAX_SCHEDULE_TIMEOUT, + NULL)); } static void mock_device_release(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index 5c7ae40bba63..cc047ec594f9 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -73,7 +73,6 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) ppgtt->vm.gt = &i915->gt; ppgtt->vm.i915 = i915; ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE); - ppgtt->vm.file = ERR_PTR(-ENODEV); ppgtt->vm.dma = i915->drm.dev; i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index eafc5a04975c..efa86dffe3c6 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -3,6 +3,7 @@ * Copyright © 2019-2021 Intel Corporation */ +#include <drm/ttm/ttm_placement.h> #include <linux/scatterlist.h> #include <drm/ttm/ttm_placement.h> @@ -16,7 +17,7 @@ static void mock_region_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { - intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node); + intel_region_ttm_resource_free(obj->mm.region, obj->mm.res); sg_free_table(pages); kfree(pages); } @@ -25,26 +26,31 @@ static int mock_region_get_pages(struct drm_i915_gem_object *obj) { unsigned int flags; struct sg_table *pages; + int err; - flags = I915_ALLOC_MIN_PAGE_SIZE; + flags = 0; if (obj->flags & I915_BO_ALLOC_CONTIGUOUS) - flags |= I915_ALLOC_CONTIGUOUS; + flags |= TTM_PL_FLAG_CONTIGUOUS; - obj->mm.st_mm_node = intel_region_ttm_node_alloc(obj->mm.region, - obj->base.size, - flags); - if (IS_ERR(obj->mm.st_mm_node)) - return PTR_ERR(obj->mm.st_mm_node); + obj->mm.res = intel_region_ttm_resource_alloc(obj->mm.region, + obj->base.size, + flags); + if (IS_ERR(obj->mm.res)) + return PTR_ERR(obj->mm.res); - pages = intel_region_ttm_node_to_st(obj->mm.region, obj->mm.st_mm_node); + pages = intel_region_ttm_resource_to_st(obj->mm.region, obj->mm.res); if (IS_ERR(pages)) { - intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node); - return PTR_ERR(pages); + err = PTR_ERR(pages); + goto err_free_resource; } __i915_gem_object_set_pages(obj, pages, i915_sg_dma_sizes(pages->sgl)); return 0; + +err_free_resource: + intel_region_ttm_resource_free(obj->mm.region, obj->mm.res); + return err; } static const struct drm_i915_gem_object_ops mock_region_obj_ops = { @@ -57,6 +63,7 @@ static const struct drm_i915_gem_object_ops mock_region_obj_ops = { static int mock_object_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, resource_size_t size, + resource_size_t page_size, unsigned int flags) { static struct lock_class_key lock_class; diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c index ef114b6aa691..846c1aae69c8 100644 --- a/drivers/gpu/drm/imx/ipuv3-plane.c +++ b/drivers/gpu/drm/imx/ipuv3-plane.c @@ -683,7 +683,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, break; } - ipu_dmfc_config_wait4eot(ipu_plane->dmfc, drm_rect_width(dst)); + ipu_dmfc_config_wait4eot(ipu_plane->dmfc, ALIGN(drm_rect_width(dst), 8)); width = ipu_src_rect_width(new_state); height = drm_rect_height(&new_state->src) >> 16; diff --git a/drivers/gpu/drm/kmb/kmb_drv.c b/drivers/gpu/drm/kmb/kmb_drv.c index be192f6da9f1..1c2f4799f421 100644 --- a/drivers/gpu/drm/kmb/kmb_drv.c +++ b/drivers/gpu/drm/kmb/kmb_drv.c @@ -202,6 +202,7 @@ static irqreturn_t handle_lcd_irq(struct drm_device *dev) unsigned long status, val, val1; int plane_id, dma0_state, dma1_state; struct kmb_drm_private *kmb = to_kmb(dev); + u32 ctrl = 0; status = kmb_read_lcd(kmb, LCD_INT_STATUS); @@ -226,6 +227,19 @@ static irqreturn_t handle_lcd_irq(struct drm_device *dev) kmb_clr_bitmask_lcd(kmb, LCD_CONTROL, kmb->plane_status[plane_id].ctrl); + ctrl = kmb_read_lcd(kmb, LCD_CONTROL); + if (!(ctrl & (LCD_CTRL_VL1_ENABLE | + LCD_CTRL_VL2_ENABLE | + LCD_CTRL_GL1_ENABLE | + LCD_CTRL_GL2_ENABLE))) { + /* If no LCD layers are using DMA, + * then disable DMA pipelined AXI read + * transactions. + */ + kmb_clr_bitmask_lcd(kmb, LCD_CONTROL, + LCD_CTRL_PIPELINE_DMA); + } + kmb->plane_status[plane_id].disable = false; } } @@ -425,10 +439,10 @@ static const struct drm_driver kmb_driver = { .fops = &fops, DRM_GEM_CMA_DRIVER_OPS_VMAP, .name = "kmb-drm", - .desc = "KEEMBAY DISPLAY DRIVER ", - .date = "20201008", - .major = 1, - .minor = 0, + .desc = "KEEMBAY DISPLAY DRIVER", + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, }; static int kmb_remove(struct platform_device *pdev) diff --git a/drivers/gpu/drm/kmb/kmb_drv.h b/drivers/gpu/drm/kmb/kmb_drv.h index 02e806712a64..ebbaa5f422d5 100644 --- a/drivers/gpu/drm/kmb/kmb_drv.h +++ b/drivers/gpu/drm/kmb/kmb_drv.h @@ -15,6 +15,11 @@ #define KMB_MAX_HEIGHT 1080 /*Max height in pixels */ #define KMB_MIN_WIDTH 1920 /*Max width in pixels */ #define KMB_MIN_HEIGHT 1080 /*Max height in pixels */ + +#define DRIVER_DATE "20210223" +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 1 + #define KMB_LCD_DEFAULT_CLK 200000000 #define KMB_SYS_CLK_MHZ 500 diff --git a/drivers/gpu/drm/kmb/kmb_plane.c b/drivers/gpu/drm/kmb/kmb_plane.c index d5b6195856d1..ecee6782612d 100644 --- a/drivers/gpu/drm/kmb/kmb_plane.c +++ b/drivers/gpu/drm/kmb/kmb_plane.c @@ -427,8 +427,14 @@ static void kmb_plane_atomic_update(struct drm_plane *plane, kmb_set_bitmask_lcd(kmb, LCD_CONTROL, ctrl); - /* FIXME no doc on how to set output format,these values are - * taken from the Myriadx tests + /* Enable pipeline AXI read transactions for the DMA + * after setting graphics layers. This must be done + * in a separate write cycle. + */ + kmb_set_bitmask_lcd(kmb, LCD_CONTROL, LCD_CTRL_PIPELINE_DMA); + + /* FIXME no doc on how to set output format, these values are taken + * from the Myriadx tests */ out_format |= LCD_OUTF_FORMAT_RGB888; @@ -526,6 +532,11 @@ struct kmb_plane *kmb_plane_init(struct drm_device *drm) plane->id = i; } + /* Disable pipeline AXI read transactions for the DMA + * prior to setting graphics layers + */ + kmb_clr_bitmask_lcd(kmb, LCD_CONTROL, LCD_CTRL_PIPELINE_DMA); + return primary; cleanup: drmm_kfree(drm, plane); diff --git a/drivers/gpu/drm/mediatek/Makefile b/drivers/gpu/drm/mediatek/Makefile index dc54a7a69005..29098d7c8307 100644 --- a/drivers/gpu/drm/mediatek/Makefile +++ b/drivers/gpu/drm/mediatek/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -mediatek-drm-y := mtk_disp_ccorr.o \ +mediatek-drm-y := mtk_disp_aal.o \ + mtk_disp_ccorr.o \ mtk_disp_color.o \ mtk_disp_gamma.o \ mtk_disp_ovl.o \ diff --git a/drivers/gpu/drm/mediatek/mtk_disp_aal.c b/drivers/gpu/drm/mediatek/mtk_disp_aal.c new file mode 100644 index 000000000000..f46d4ab73d6a --- /dev/null +++ b/drivers/gpu/drm/mediatek/mtk_disp_aal.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021 MediaTek Inc. + */ + +#include <linux/clk.h> +#include <linux/component.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/soc/mediatek/mtk-cmdq.h> + +#include "mtk_disp_drv.h" +#include "mtk_drm_crtc.h" +#include "mtk_drm_ddp_comp.h" + +#define DISP_AAL_EN 0x0000 +#define AAL_EN BIT(0) +#define DISP_AAL_SIZE 0x0030 +#define DISP_AAL_OUTPUT_SIZE 0x04d8 + + +struct mtk_disp_aal_data { + bool has_gamma; +}; + +/** + * struct mtk_disp_aal - DISP_AAL driver structure + * @ddp_comp - structure containing type enum and hardware resources + * @crtc - associated crtc to report irq events to + */ +struct mtk_disp_aal { + struct clk *clk; + void __iomem *regs; + struct cmdq_client_reg cmdq_reg; + const struct mtk_disp_aal_data *data; +}; + +int mtk_aal_clk_enable(struct device *dev) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + return clk_prepare_enable(aal->clk); +} + +void mtk_aal_clk_disable(struct device *dev) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + clk_disable_unprepare(aal->clk); +} + +void mtk_aal_config(struct device *dev, unsigned int w, + unsigned int h, unsigned int vrefresh, + unsigned int bpc, struct cmdq_pkt *cmdq_pkt) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + mtk_ddp_write(cmdq_pkt, w << 16 | h, &aal->cmdq_reg, aal->regs, DISP_AAL_SIZE); + mtk_ddp_write(cmdq_pkt, w << 16 | h, &aal->cmdq_reg, aal->regs, DISP_AAL_OUTPUT_SIZE); +} + +void mtk_aal_gamma_set(struct device *dev, struct drm_crtc_state *state) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + if (aal->data && aal->data->has_gamma) + mtk_gamma_set_common(aal->regs, state); +} + +void mtk_aal_start(struct device *dev) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + writel(AAL_EN, aal->regs + DISP_AAL_EN); +} + +void mtk_aal_stop(struct device *dev) +{ + struct mtk_disp_aal *aal = dev_get_drvdata(dev); + + writel_relaxed(0x0, aal->regs + DISP_AAL_EN); +} + +static int mtk_disp_aal_bind(struct device *dev, struct device *master, + void *data) +{ + return 0; +} + +static void mtk_disp_aal_unbind(struct device *dev, struct device *master, + void *data) +{ +} + +static const struct component_ops mtk_disp_aal_component_ops = { + .bind = mtk_disp_aal_bind, + .unbind = mtk_disp_aal_unbind, +}; + +static int mtk_disp_aal_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mtk_disp_aal *priv; + struct resource *res; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->clk = devm_clk_get(dev, NULL); + if (IS_ERR(priv->clk)) { + dev_err(dev, "failed to get aal clk\n"); + return PTR_ERR(priv->clk); + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + priv->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(priv->regs)) { + dev_err(dev, "failed to ioremap aal\n"); + return PTR_ERR(priv->regs); + } + +#if IS_REACHABLE(CONFIG_MTK_CMDQ) + ret = cmdq_dev_get_client_reg(dev, &priv->cmdq_reg, 0); + if (ret) + dev_dbg(dev, "get mediatek,gce-client-reg fail!\n"); +#endif + + priv->data = of_device_get_match_data(dev); + platform_set_drvdata(pdev, priv); + + ret = component_add(dev, &mtk_disp_aal_component_ops); + if (ret) + dev_err(dev, "Failed to add component: %d\n", ret); + + return ret; +} + +static int mtk_disp_aal_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &mtk_disp_aal_component_ops); + + return 0; +} + +static const struct mtk_disp_aal_data mt8173_aal_driver_data = { + .has_gamma = true, +}; + +static const struct of_device_id mtk_disp_aal_driver_dt_match[] = { + { .compatible = "mediatek,mt8173-disp-aal", + .data = &mt8173_aal_driver_data}, + { .compatible = "mediatek,mt8183-disp-aal"}, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_disp_aal_driver_dt_match); + +struct platform_driver mtk_disp_aal_driver = { + .probe = mtk_disp_aal_probe, + .remove = mtk_disp_aal_remove, + .driver = { + .name = "mediatek-disp-aal", + .owner = THIS_MODULE, + .of_match_table = mtk_disp_aal_driver_dt_match, + }, +}; diff --git a/drivers/gpu/drm/mediatek/mtk_disp_color.c b/drivers/gpu/drm/mediatek/mtk_disp_color.c index 6f4c80bbc0eb..473f5bb5cbad 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_color.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_color.c @@ -133,6 +133,8 @@ static int mtk_disp_color_probe(struct platform_device *pdev) static int mtk_disp_color_remove(struct platform_device *pdev) { + component_del(&pdev->dev, &mtk_disp_color_component_ops); + return 0; } diff --git a/drivers/gpu/drm/mediatek/mtk_disp_drv.h b/drivers/gpu/drm/mediatek/mtk_disp_drv.h index cafd9df2d63b..86c3068894b1 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_drv.h +++ b/drivers/gpu/drm/mediatek/mtk_disp_drv.h @@ -9,6 +9,15 @@ #include <linux/soc/mediatek/mtk-cmdq.h> #include "mtk_drm_plane.h" +int mtk_aal_clk_enable(struct device *dev); +void mtk_aal_clk_disable(struct device *dev); +void mtk_aal_config(struct device *dev, unsigned int w, + unsigned int h, unsigned int vrefresh, + unsigned int bpc, struct cmdq_pkt *cmdq_pkt); +void mtk_aal_gamma_set(struct device *dev, struct drm_crtc_state *state); +void mtk_aal_start(struct device *dev); +void mtk_aal_stop(struct device *dev); + void mtk_ccorr_ctm_set(struct device *dev, struct drm_crtc_state *state); int mtk_ccorr_clk_enable(struct device *dev); void mtk_ccorr_clk_disable(struct device *dev); diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c index fa9d79963cd3..5326989d5206 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c @@ -423,6 +423,8 @@ static int mtk_disp_ovl_probe(struct platform_device *pdev) static int mtk_disp_ovl_remove(struct platform_device *pdev) { + component_del(&pdev->dev, &mtk_disp_ovl_component_ops); + return 0; } diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c index 705f28ceb4dd..75d7f45579e2 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c @@ -162,10 +162,10 @@ void mtk_rdma_config(struct device *dev, unsigned int width, /* * Enable FIFO underflow since DSI and DPI can't be blocked. * Keep the FIFO pseudo size reset default of 8 KiB. Set the - * output threshold to 6 microseconds with 7/6 overhead to - * account for blanking, and with a pixel depth of 4 bytes: + * output threshold to 70% of max fifo size to make sure the + * threhold will not overflow */ - threshold = width * height * vrefresh * 4 * 7 / 1000000; + threshold = rdma_fifo_size * 7 / 10; reg = RDMA_FIFO_UNDERFLOW_EN | RDMA_FIFO_PSEUDO_SIZE(rdma_fifo_size) | RDMA_OUTPUT_VALID_FIFO_THRESHOLD(threshold); diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c index 96d35770c844..4554e2de1430 100644 --- a/drivers/gpu/drm/mediatek/mtk_dpi.c +++ b/drivers/gpu/drm/mediatek/mtk_dpi.c @@ -605,11 +605,15 @@ static int mtk_dpi_bridge_atomic_check(struct drm_bridge *bridge, struct drm_crtc_state *crtc_state, struct drm_connector_state *conn_state) { - struct mtk_dpi *dpi = bridge->driver_private; + struct mtk_dpi *dpi = bridge_to_dpi(bridge); unsigned int out_bus_format; out_bus_format = bridge_state->output_bus_cfg.format; + if (out_bus_format == MEDIA_BUS_FMT_FIXED) + if (dpi->conf->num_output_fmts) + out_bus_format = dpi->conf->output_fmts[0]; + dev_dbg(dpi->dev, "input format 0x%04x, output format 0x%04x\n", bridge_state->input_bus_cfg.format, bridge_state->output_bus_cfg.format); diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c index 474efb844249..5f81489fc60c 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c @@ -4,6 +4,8 @@ */ #include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/mailbox_controller.h> #include <linux/pm_runtime.h> #include <linux/soc/mediatek/mtk-cmdq.h> #include <linux/soc/mediatek/mtk-mmsys.h> @@ -50,8 +52,11 @@ struct mtk_drm_crtc { bool pending_async_planes; #if IS_REACHABLE(CONFIG_MTK_CMDQ) - struct cmdq_client *cmdq_client; + struct mbox_client cmdq_cl; + struct mbox_chan *cmdq_chan; + struct cmdq_pkt cmdq_handle; u32 cmdq_event; + u32 cmdq_vblank_cnt; #endif struct device *mmsys_dev; @@ -222,9 +227,79 @@ struct mtk_ddp_comp *mtk_drm_ddp_comp_for_plane(struct drm_crtc *crtc, } #if IS_REACHABLE(CONFIG_MTK_CMDQ) -static void ddp_cmdq_cb(struct cmdq_cb_data data) +static int mtk_drm_cmdq_pkt_create(struct mbox_chan *chan, struct cmdq_pkt *pkt, + size_t size) { - cmdq_pkt_destroy(data.data); + struct device *dev; + dma_addr_t dma_addr; + + pkt->va_base = kzalloc(size, GFP_KERNEL); + if (!pkt->va_base) { + kfree(pkt); + return -ENOMEM; + } + pkt->buf_size = size; + + dev = chan->mbox->dev; + dma_addr = dma_map_single(dev, pkt->va_base, pkt->buf_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "dma map failed, size=%u\n", (u32)(u64)size); + kfree(pkt->va_base); + kfree(pkt); + return -ENOMEM; + } + + pkt->pa_base = dma_addr; + + return 0; +} + +static void mtk_drm_cmdq_pkt_destroy(struct mbox_chan *chan, struct cmdq_pkt *pkt) +{ + dma_unmap_single(chan->mbox->dev, pkt->pa_base, pkt->buf_size, + DMA_TO_DEVICE); + kfree(pkt->va_base); + kfree(pkt); +} + +static void ddp_cmdq_cb(struct mbox_client *cl, void *mssg) +{ + struct mtk_drm_crtc *mtk_crtc = container_of(cl, struct mtk_drm_crtc, cmdq_cl); + struct cmdq_cb_data *data = mssg; + struct mtk_crtc_state *state; + unsigned int i; + + state = to_mtk_crtc_state(mtk_crtc->base.state); + + state->pending_config = false; + + if (mtk_crtc->pending_planes) { + for (i = 0; i < mtk_crtc->layer_nr; i++) { + struct drm_plane *plane = &mtk_crtc->planes[i]; + struct mtk_plane_state *plane_state; + + plane_state = to_mtk_plane_state(plane->state); + + plane_state->pending.config = false; + } + mtk_crtc->pending_planes = false; + } + + if (mtk_crtc->pending_async_planes) { + for (i = 0; i < mtk_crtc->layer_nr; i++) { + struct drm_plane *plane = &mtk_crtc->planes[i]; + struct mtk_plane_state *plane_state; + + plane_state = to_mtk_plane_state(plane->state); + + plane_state->pending.async_config = false; + } + mtk_crtc->pending_async_planes = false; + } + + mtk_crtc->cmdq_vblank_cnt = 0; + mtk_drm_cmdq_pkt_destroy(mtk_crtc->cmdq_chan, data->pkt); } #endif @@ -378,7 +453,8 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc, state->pending_vrefresh, 0, cmdq_handle); - state->pending_config = false; + if (!cmdq_handle) + state->pending_config = false; } if (mtk_crtc->pending_planes) { @@ -398,9 +474,12 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc, mtk_ddp_comp_layer_config(comp, local_layer, plane_state, cmdq_handle); - plane_state->pending.config = false; + if (!cmdq_handle) + plane_state->pending.config = false; } - mtk_crtc->pending_planes = false; + + if (!cmdq_handle) + mtk_crtc->pending_planes = false; } if (mtk_crtc->pending_async_planes) { @@ -420,9 +499,12 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc, mtk_ddp_comp_layer_config(comp, local_layer, plane_state, cmdq_handle); - plane_state->pending.async_config = false; + if (!cmdq_handle) + plane_state->pending.async_config = false; } - mtk_crtc->pending_async_planes = false; + + if (!cmdq_handle) + mtk_crtc->pending_async_planes = false; } } @@ -430,7 +512,7 @@ static void mtk_drm_crtc_update_config(struct mtk_drm_crtc *mtk_crtc, bool needs_vblank) { #if IS_REACHABLE(CONFIG_MTK_CMDQ) - struct cmdq_pkt *cmdq_handle; + struct cmdq_pkt *cmdq_handle = &mtk_crtc->cmdq_handle; #endif struct drm_crtc *crtc = &mtk_crtc->base; struct mtk_drm_private *priv = crtc->dev->dev_private; @@ -468,14 +550,24 @@ static void mtk_drm_crtc_update_config(struct mtk_drm_crtc *mtk_crtc, mtk_mutex_release(mtk_crtc->mutex); } #if IS_REACHABLE(CONFIG_MTK_CMDQ) - if (mtk_crtc->cmdq_client) { - mbox_flush(mtk_crtc->cmdq_client->chan, 2000); - cmdq_handle = cmdq_pkt_create(mtk_crtc->cmdq_client, PAGE_SIZE); + if (mtk_crtc->cmdq_chan) { + mbox_flush(mtk_crtc->cmdq_chan, 2000); + cmdq_handle->cmd_buf_size = 0; cmdq_pkt_clear_event(cmdq_handle, mtk_crtc->cmdq_event); cmdq_pkt_wfe(cmdq_handle, mtk_crtc->cmdq_event, false); mtk_crtc_ddp_config(crtc, cmdq_handle); cmdq_pkt_finalize(cmdq_handle); - cmdq_pkt_flush_async(cmdq_handle, ddp_cmdq_cb, cmdq_handle); + dma_sync_single_for_device(mtk_crtc->cmdq_chan->mbox->dev, + cmdq_handle->pa_base, + cmdq_handle->cmd_buf_size, + DMA_TO_DEVICE); + /* + * CMDQ command should execute in next vblank, + * If it fail to execute in next 2 vblank, timeout happen. + */ + mtk_crtc->cmdq_vblank_cnt = 2; + mbox_send_message(mtk_crtc->cmdq_chan, cmdq_handle); + mbox_client_txdone(mtk_crtc->cmdq_chan, 0); } #endif mtk_crtc->config_updating = false; @@ -489,12 +581,15 @@ static void mtk_crtc_ddp_irq(void *data) struct mtk_drm_private *priv = crtc->dev->dev_private; #if IS_REACHABLE(CONFIG_MTK_CMDQ) - if (!priv->data->shadow_register && !mtk_crtc->cmdq_client) + if (!priv->data->shadow_register && !mtk_crtc->cmdq_chan) + mtk_crtc_ddp_config(crtc, NULL); + else if (mtk_crtc->cmdq_vblank_cnt > 0 && --mtk_crtc->cmdq_vblank_cnt == 0) + DRM_ERROR("mtk_crtc %d CMDQ execute command timeout!\n", + drm_crtc_index(&mtk_crtc->base)); #else if (!priv->data->shadow_register) -#endif mtk_crtc_ddp_config(crtc, NULL); - +#endif mtk_drm_finish_page_flip(mtk_crtc); } @@ -532,13 +627,10 @@ void mtk_drm_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane, struct drm_atomic_state *state) { struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc); - const struct drm_plane_helper_funcs *plane_helper_funcs = - plane->helper_private; if (!mtk_crtc->enabled) return; - plane_helper_funcs->atomic_update(plane, state); mtk_drm_crtc_update_config(mtk_crtc, false); } @@ -755,14 +847,22 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev, for (i = 0; i < path_len; i++) { enum mtk_ddp_comp_id comp_id = path[i]; struct device_node *node; + struct mtk_ddp_comp *comp; node = priv->comp_node[comp_id]; + comp = &priv->ddp_comp[comp_id]; + if (!node) { dev_info(dev, "Not creating crtc %d because component %d is disabled or missing\n", pipe, comp_id); return 0; } + + if (!comp->dev) { + dev_err(dev, "Component %pOF not initialized\n", node); + return -ENODEV; + } } mtk_crtc = devm_kzalloc(dev, sizeof(*mtk_crtc), GFP_KERNEL); @@ -787,16 +887,8 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev, for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) { enum mtk_ddp_comp_id comp_id = path[i]; struct mtk_ddp_comp *comp; - struct device_node *node; - node = priv->comp_node[comp_id]; comp = &priv->ddp_comp[comp_id]; - if (!comp) { - dev_err(dev, "Component %pOF not initialized\n", node); - ret = -ENODEV; - return ret; - } - mtk_crtc->ddp_comp[i] = comp; if (comp->funcs) { @@ -832,16 +924,20 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev, mutex_init(&mtk_crtc->hw_lock); #if IS_REACHABLE(CONFIG_MTK_CMDQ) - mtk_crtc->cmdq_client = - cmdq_mbox_create(mtk_crtc->mmsys_dev, - drm_crtc_index(&mtk_crtc->base)); - if (IS_ERR(mtk_crtc->cmdq_client)) { + mtk_crtc->cmdq_cl.dev = mtk_crtc->mmsys_dev; + mtk_crtc->cmdq_cl.tx_block = false; + mtk_crtc->cmdq_cl.knows_txdone = true; + mtk_crtc->cmdq_cl.rx_callback = ddp_cmdq_cb; + mtk_crtc->cmdq_chan = + mbox_request_channel(&mtk_crtc->cmdq_cl, + drm_crtc_index(&mtk_crtc->base)); + if (IS_ERR(mtk_crtc->cmdq_chan)) { dev_dbg(dev, "mtk_crtc %d failed to create mailbox client, writing register by CPU now\n", drm_crtc_index(&mtk_crtc->base)); - mtk_crtc->cmdq_client = NULL; + mtk_crtc->cmdq_chan = NULL; } - if (mtk_crtc->cmdq_client) { + if (mtk_crtc->cmdq_chan) { ret = of_property_read_u32_index(priv->mutex_node, "mediatek,gce-events", drm_crtc_index(&mtk_crtc->base), @@ -849,8 +945,18 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev, if (ret) { dev_dbg(dev, "mtk_crtc %d failed to get mediatek,gce-events property\n", drm_crtc_index(&mtk_crtc->base)); - cmdq_mbox_destroy(mtk_crtc->cmdq_client); - mtk_crtc->cmdq_client = NULL; + mbox_free_channel(mtk_crtc->cmdq_chan); + mtk_crtc->cmdq_chan = NULL; + } else { + ret = mtk_drm_cmdq_pkt_create(mtk_crtc->cmdq_chan, + &mtk_crtc->cmdq_handle, + PAGE_SIZE); + if (ret) { + dev_dbg(dev, "mtk_crtc %d failed to create cmdq packet\n", + drm_crtc_index(&mtk_crtc->base)); + mbox_free_channel(mtk_crtc->cmdq_chan); + mtk_crtc->cmdq_chan = NULL; + } } } #endif diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c index 75bc00e17fc4..99cbf44463e4 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c @@ -32,9 +32,6 @@ #define DISP_REG_UFO_START 0x0000 -#define DISP_AAL_EN 0x0000 -#define DISP_AAL_SIZE 0x0030 - #define DISP_DITHER_EN 0x0000 #define DITHER_EN BIT(0) #define DISP_DITHER_CFG 0x0020 @@ -48,8 +45,6 @@ #define UFO_BYPASS BIT(2) -#define AAL_EN BIT(0) - #define DISP_DITHERING BIT(2) #define DITHER_LSB_ERR_SHIFT_R(x) (((x) & 0x7) << 28) #define DITHER_OVFLW_BIT_R(x) (((x) & 0x7) << 24) @@ -190,36 +185,6 @@ static void mtk_ufoe_start(struct device *dev) writel(UFO_BYPASS, priv->regs + DISP_REG_UFO_START); } -static void mtk_aal_config(struct device *dev, unsigned int w, - unsigned int h, unsigned int vrefresh, - unsigned int bpc, struct cmdq_pkt *cmdq_pkt) -{ - struct mtk_ddp_comp_dev *priv = dev_get_drvdata(dev); - - mtk_ddp_write(cmdq_pkt, w << 16 | h, &priv->cmdq_reg, priv->regs, DISP_AAL_SIZE); -} - -static void mtk_aal_gamma_set(struct device *dev, struct drm_crtc_state *state) -{ - struct mtk_ddp_comp_dev *priv = dev_get_drvdata(dev); - - mtk_gamma_set_common(priv->regs, state); -} - -static void mtk_aal_start(struct device *dev) -{ - struct mtk_ddp_comp_dev *priv = dev_get_drvdata(dev); - - writel(AAL_EN, priv->regs + DISP_AAL_EN); -} - -static void mtk_aal_stop(struct device *dev) -{ - struct mtk_ddp_comp_dev *priv = dev_get_drvdata(dev); - - writel_relaxed(0x0, priv->regs + DISP_AAL_EN); -} - static void mtk_dither_config(struct device *dev, unsigned int w, unsigned int h, unsigned int vrefresh, unsigned int bpc, struct cmdq_pkt *cmdq_pkt) @@ -247,8 +212,8 @@ static void mtk_dither_stop(struct device *dev) } static const struct mtk_ddp_comp_funcs ddp_aal = { - .clk_enable = mtk_ddp_clk_enable, - .clk_disable = mtk_ddp_clk_disable, + .clk_enable = mtk_aal_clk_enable, + .clk_disable = mtk_aal_clk_disable, .gamma_set = mtk_aal_gamma_set, .config = mtk_aal_config, .start = mtk_aal_start, @@ -505,7 +470,8 @@ int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp, return ret; } - if (type == MTK_DISP_BLS || + if (type == MTK_DISP_AAL || + type == MTK_DISP_BLS || type == MTK_DISP_CCORR || type == MTK_DISP_COLOR || type == MTK_DISP_GAMMA || diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index 9b60bec33d3b..aec39724ebeb 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -110,6 +110,17 @@ static const enum mtk_ddp_comp_id mt2712_mtk_ddp_third[] = { DDP_COMPONENT_PWM2, }; +static enum mtk_ddp_comp_id mt8167_mtk_ddp_main[] = { + DDP_COMPONENT_OVL0, + DDP_COMPONENT_COLOR0, + DDP_COMPONENT_CCORR, + DDP_COMPONENT_AAL0, + DDP_COMPONENT_GAMMA, + DDP_COMPONENT_DITHER, + DDP_COMPONENT_RDMA0, + DDP_COMPONENT_DSI0, +}; + static const enum mtk_ddp_comp_id mt8173_mtk_ddp_main[] = { DDP_COMPONENT_OVL0, DDP_COMPONENT_COLOR0, @@ -172,6 +183,11 @@ static const struct mtk_mmsys_driver_data mt2712_mmsys_driver_data = { .third_len = ARRAY_SIZE(mt2712_mtk_ddp_third), }; +static const struct mtk_mmsys_driver_data mt8167_mmsys_driver_data = { + .main_path = mt8167_mtk_ddp_main, + .main_len = ARRAY_SIZE(mt8167_mtk_ddp_main), +}; + static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = { .main_path = mt8173_mtk_ddp_main, .main_len = ARRAY_SIZE(mt8173_mtk_ddp_main), @@ -294,16 +310,7 @@ static void mtk_drm_kms_deinit(struct drm_device *drm) component_unbind_all(drm->dev, drm); } -static const struct file_operations mtk_drm_fops = { - .owner = THIS_MODULE, - .open = drm_open, - .release = drm_release, - .unlocked_ioctl = drm_ioctl, - .mmap = mtk_drm_gem_mmap, - .poll = drm_poll, - .read = drm_read, - .compat_ioctl = drm_compat_ioctl, -}; +DEFINE_DRM_GEM_FOPS(mtk_drm_fops); /* * We need to override this because the device used to import the memory is @@ -326,7 +333,7 @@ static const struct drm_driver mtk_drm_driver = { .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import = mtk_drm_gem_prime_import, .gem_prime_import_sg_table = mtk_gem_prime_import_sg_table, - .gem_prime_mmap = mtk_drm_gem_mmap_buf, + .gem_prime_mmap = drm_gem_prime_mmap, .fops = &mtk_drm_fops, .name = DRIVER_NAME, @@ -392,6 +399,8 @@ static const struct component_master_ops mtk_drm_ops = { static const struct of_device_id mtk_ddp_comp_dt_ids[] = { { .compatible = "mediatek,mt2701-disp-ovl", .data = (void *)MTK_DISP_OVL }, + { .compatible = "mediatek,mt8167-disp-ovl", + .data = (void *)MTK_DISP_OVL }, { .compatible = "mediatek,mt8173-disp-ovl", .data = (void *)MTK_DISP_OVL }, { .compatible = "mediatek,mt8183-disp-ovl", @@ -400,30 +409,46 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = { .data = (void *)MTK_DISP_OVL_2L }, { .compatible = "mediatek,mt2701-disp-rdma", .data = (void *)MTK_DISP_RDMA }, + { .compatible = "mediatek,mt8167-disp-rdma", + .data = (void *)MTK_DISP_RDMA }, { .compatible = "mediatek,mt8173-disp-rdma", .data = (void *)MTK_DISP_RDMA }, { .compatible = "mediatek,mt8183-disp-rdma", .data = (void *)MTK_DISP_RDMA }, { .compatible = "mediatek,mt8173-disp-wdma", .data = (void *)MTK_DISP_WDMA }, + { .compatible = "mediatek,mt8167-disp-ccorr", + .data = (void *)MTK_DISP_CCORR }, { .compatible = "mediatek,mt8183-disp-ccorr", .data = (void *)MTK_DISP_CCORR }, { .compatible = "mediatek,mt2701-disp-color", .data = (void *)MTK_DISP_COLOR }, + { .compatible = "mediatek,mt8167-disp-color", + .data = (void *)MTK_DISP_COLOR }, { .compatible = "mediatek,mt8173-disp-color", .data = (void *)MTK_DISP_COLOR }, + { .compatible = "mediatek,mt8167-disp-aal", + .data = (void *)MTK_DISP_AAL}, { .compatible = "mediatek,mt8173-disp-aal", .data = (void *)MTK_DISP_AAL}, + { .compatible = "mediatek,mt8183-disp-aal", + .data = (void *)MTK_DISP_AAL}, + { .compatible = "mediatek,mt8167-disp-gamma", + .data = (void *)MTK_DISP_GAMMA, }, { .compatible = "mediatek,mt8173-disp-gamma", .data = (void *)MTK_DISP_GAMMA, }, { .compatible = "mediatek,mt8183-disp-gamma", .data = (void *)MTK_DISP_GAMMA, }, + { .compatible = "mediatek,mt8167-disp-dither", + .data = (void *)MTK_DISP_DITHER }, { .compatible = "mediatek,mt8183-disp-dither", .data = (void *)MTK_DISP_DITHER }, { .compatible = "mediatek,mt8173-disp-ufoe", .data = (void *)MTK_DISP_UFOE }, { .compatible = "mediatek,mt2701-dsi", .data = (void *)MTK_DSI }, + { .compatible = "mediatek,mt8167-dsi", + .data = (void *)MTK_DSI }, { .compatible = "mediatek,mt8173-dsi", .data = (void *)MTK_DSI }, { .compatible = "mediatek,mt8183-dsi", @@ -438,12 +463,16 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = { .data = (void *)MTK_DISP_MUTEX }, { .compatible = "mediatek,mt2712-disp-mutex", .data = (void *)MTK_DISP_MUTEX }, + { .compatible = "mediatek,mt8167-disp-mutex", + .data = (void *)MTK_DISP_MUTEX }, { .compatible = "mediatek,mt8173-disp-mutex", .data = (void *)MTK_DISP_MUTEX }, { .compatible = "mediatek,mt8183-disp-mutex", .data = (void *)MTK_DISP_MUTEX }, { .compatible = "mediatek,mt2701-disp-pwm", .data = (void *)MTK_DISP_BLS }, + { .compatible = "mediatek,mt8167-disp-pwm", + .data = (void *)MTK_DISP_PWM }, { .compatible = "mediatek,mt8173-disp-pwm", .data = (void *)MTK_DISP_PWM }, { .compatible = "mediatek,mt8173-disp-od", @@ -458,6 +487,8 @@ static const struct of_device_id mtk_drm_of_ids[] = { .data = &mt7623_mmsys_driver_data}, { .compatible = "mediatek,mt2712-mmsys", .data = &mt2712_mmsys_driver_data}, + { .compatible = "mediatek,mt8167-mmsys", + .data = &mt8167_mmsys_driver_data}, { .compatible = "mediatek,mt8173-mmsys", .data = &mt8173_mmsys_driver_data}, { .compatible = "mediatek,mt8183-mmsys", @@ -526,11 +557,12 @@ static int mtk_drm_probe(struct platform_device *pdev) private->comp_node[comp_id] = of_node_get(node); /* - * Currently only the CCORR, COLOR, GAMMA, OVL, RDMA, DSI, and DPI + * Currently only the AAL, CCORR, COLOR, GAMMA, OVL, RDMA, DSI, and DPI * blocks have separate component platform drivers and initialize their own * DDP component structure. The others are initialized here. */ - if (comp_type == MTK_DISP_CCORR || + if (comp_type == MTK_DISP_AAL || + comp_type == MTK_DISP_CCORR || comp_type == MTK_DISP_COLOR || comp_type == MTK_DISP_GAMMA || comp_type == MTK_DISP_OVL || @@ -630,6 +662,7 @@ static struct platform_driver mtk_drm_platform_driver = { }; static struct platform_driver * const mtk_drm_drivers[] = { + &mtk_disp_aal_driver, &mtk_disp_ccorr_driver, &mtk_disp_color_driver, &mtk_disp_gamma_driver, diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h index 637f5669e895..3e7d1e6fbe01 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h @@ -46,6 +46,7 @@ struct mtk_drm_private { struct drm_atomic_state *suspend_state; }; +extern struct platform_driver mtk_disp_aal_driver; extern struct platform_driver mtk_disp_ccorr_driver; extern struct platform_driver mtk_disp_color_driver; extern struct platform_driver mtk_disp_gamma_driver; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c index 280ea0d5e840..d0544962cfc1 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c @@ -14,11 +14,14 @@ #include "mtk_drm_drv.h" #include "mtk_drm_gem.h" +static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma); + static const struct drm_gem_object_funcs mtk_drm_gem_object_funcs = { .free = mtk_drm_gem_free_object, .get_sg_table = mtk_gem_prime_get_sg_table, .vmap = mtk_drm_gem_prime_vmap, .vunmap = mtk_drm_gem_prime_vunmap, + .mmap = mtk_drm_gem_object_mmap, .vm_ops = &drm_gem_cma_vm_ops, }; @@ -146,10 +149,18 @@ static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj, struct mtk_drm_private *priv = obj->dev->dev_private; /* + * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the + * whole buffer from the start. + */ + vma->vm_pgoff = 0; + + /* * dma_alloc_attrs() allocated a struct page table for mtk_gem, so clear * VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap(). */ - vma->vm_flags &= ~VM_PFNMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; + vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); ret = dma_mmap_attrs(priv->dma_dev, vma, mtk_gem->cookie, mtk_gem->dma_addr, obj->size, mtk_gem->dma_attrs); @@ -159,37 +170,6 @@ static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj, return ret; } -int mtk_drm_gem_mmap_buf(struct drm_gem_object *obj, struct vm_area_struct *vma) -{ - int ret; - - ret = drm_gem_mmap_obj(obj, obj->size, vma); - if (ret) - return ret; - - return mtk_drm_gem_object_mmap(obj, vma); -} - -int mtk_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct drm_gem_object *obj; - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) - return ret; - - obj = vma->vm_private_data; - - /* - * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the - * whole buffer from the start. - */ - vma->vm_pgoff = 0; - - return mtk_drm_gem_object_mmap(obj, vma); -} - /* * Allocate a sg_table for this GEM object. * Note: Both the table's contents, and the sg_table itself must be freed by diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.h b/drivers/gpu/drm/mediatek/mtk_drm_gem.h index 6da5ccb4b933..9a359a06cb73 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_gem.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.h @@ -39,9 +39,6 @@ struct mtk_drm_gem_obj *mtk_drm_gem_create(struct drm_device *dev, size_t size, bool alloc_kmap); int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); -int mtk_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma); -int mtk_drm_gem_mmap_buf(struct drm_gem_object *obj, - struct vm_area_struct *vma); struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj); struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sg); diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c index 1667a7e7de38..734a1fb052df 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c @@ -110,6 +110,35 @@ static int mtk_plane_atomic_async_check(struct drm_plane *plane, true, true); } +static void mtk_plane_update_new_state(struct drm_plane_state *new_state, + struct mtk_plane_state *mtk_plane_state) +{ + struct drm_framebuffer *fb = new_state->fb; + struct drm_gem_object *gem; + struct mtk_drm_gem_obj *mtk_gem; + unsigned int pitch, format; + dma_addr_t addr; + + gem = fb->obj[0]; + mtk_gem = to_mtk_gem_obj(gem); + addr = mtk_gem->dma_addr; + pitch = fb->pitches[0]; + format = fb->format->format; + + addr += (new_state->src.x1 >> 16) * fb->format->cpp[0]; + addr += (new_state->src.y1 >> 16) * pitch; + + mtk_plane_state->pending.enable = true; + mtk_plane_state->pending.pitch = pitch; + mtk_plane_state->pending.format = format; + mtk_plane_state->pending.addr = addr; + mtk_plane_state->pending.x = new_state->dst.x1; + mtk_plane_state->pending.y = new_state->dst.y1; + mtk_plane_state->pending.width = drm_rect_width(&new_state->dst); + mtk_plane_state->pending.height = drm_rect_height(&new_state->dst); + mtk_plane_state->pending.rotation = new_state->rotation; +} + static void mtk_plane_atomic_async_update(struct drm_plane *plane, struct drm_atomic_state *state) { @@ -126,8 +155,10 @@ static void mtk_plane_atomic_async_update(struct drm_plane *plane, plane->state->src_h = new_state->src_h; plane->state->src_w = new_state->src_w; swap(plane->state->fb, new_state->fb); - new_plane_state->pending.async_dirty = true; + mtk_plane_update_new_state(new_state, new_plane_state); + wmb(); /* Make sure the above parameters are set before update */ + new_plane_state->pending.async_dirty = true; mtk_drm_crtc_async_update(new_state->crtc, plane, state); } @@ -189,14 +220,8 @@ static void mtk_plane_atomic_update(struct drm_plane *plane, struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane); struct mtk_plane_state *mtk_plane_state = to_mtk_plane_state(new_state); - struct drm_crtc *crtc = new_state->crtc; - struct drm_framebuffer *fb = new_state->fb; - struct drm_gem_object *gem; - struct mtk_drm_gem_obj *mtk_gem; - unsigned int pitch, format; - dma_addr_t addr; - if (!crtc || WARN_ON(!fb)) + if (!new_state->crtc || WARN_ON(!new_state->fb)) return; if (!new_state->visible) { @@ -204,24 +229,7 @@ static void mtk_plane_atomic_update(struct drm_plane *plane, return; } - gem = fb->obj[0]; - mtk_gem = to_mtk_gem_obj(gem); - addr = mtk_gem->dma_addr; - pitch = fb->pitches[0]; - format = fb->format->format; - - addr += (new_state->src.x1 >> 16) * fb->format->cpp[0]; - addr += (new_state->src.y1 >> 16) * pitch; - - mtk_plane_state->pending.enable = true; - mtk_plane_state->pending.pitch = pitch; - mtk_plane_state->pending.format = format; - mtk_plane_state->pending.addr = addr; - mtk_plane_state->pending.x = new_state->dst.x1; - mtk_plane_state->pending.y = new_state->dst.y1; - mtk_plane_state->pending.width = drm_rect_width(&new_state->dst); - mtk_plane_state->pending.height = drm_rect_height(&new_state->dst); - mtk_plane_state->pending.rotation = new_state->rotation; + mtk_plane_update_new_state(new_state, mtk_plane_state); wmb(); /* Make sure the above parameters are set before update */ mtk_plane_state->pending.dirty = true; } diff --git a/drivers/gpu/drm/meson/meson_registers.h b/drivers/gpu/drm/meson/meson_registers.h index 446e7961da48..0f3cafab8860 100644 --- a/drivers/gpu/drm/meson/meson_registers.h +++ b/drivers/gpu/drm/meson/meson_registers.h @@ -634,6 +634,11 @@ #define VPP_WRAP_OSD3_MATRIX_PRE_OFFSET2 0x3dbc #define VPP_WRAP_OSD3_MATRIX_EN_CTRL 0x3dbd +/* osd1 HDR */ +#define OSD1_HDR2_CTRL 0x38a0 +#define OSD1_HDR2_CTRL_VDIN0_HDR2_TOP_EN BIT(13) +#define OSD1_HDR2_CTRL_REG_ONLY_MAT BIT(16) + /* osd2 scaler */ #define OSD2_VSC_PHASE_STEP 0x3d00 #define OSD2_VSC_INI_PHASE 0x3d01 diff --git a/drivers/gpu/drm/meson/meson_viu.c b/drivers/gpu/drm/meson/meson_viu.c index aede0c67a57f..259f3e6bec90 100644 --- a/drivers/gpu/drm/meson/meson_viu.c +++ b/drivers/gpu/drm/meson/meson_viu.c @@ -425,9 +425,14 @@ void meson_viu_init(struct meson_drm *priv) if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXM) || meson_vpu_is_compatible(priv, VPU_COMPATIBLE_GXL)) meson_viu_load_matrix(priv); - else if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_G12A)) + else if (meson_vpu_is_compatible(priv, VPU_COMPATIBLE_G12A)) { meson_viu_set_g12a_osd1_matrix(priv, RGB709_to_YUV709l_coeff, true); + /* fix green/pink color distortion from vendor u-boot */ + writel_bits_relaxed(OSD1_HDR2_CTRL_REG_ONLY_MAT | + OSD1_HDR2_CTRL_VDIN0_HDR2_TOP_EN, 0, + priv->io_base + _REG(OSD1_HDR2_CTRL)); + } /* Initialize OSD1 fifo control register */ reg = VIU_OSD_DDR_PRIORITY_URGENT | diff --git a/drivers/gpu/drm/mgag200/mgag200_pll.c b/drivers/gpu/drm/mgag200/mgag200_pll.c index 7c903cf19c0d..e9ae22b4f813 100644 --- a/drivers/gpu/drm/mgag200/mgag200_pll.c +++ b/drivers/gpu/drm/mgag200/mgag200_pll.c @@ -124,6 +124,7 @@ static int mgag200_pixpll_compute_g200se_00(struct mgag200_pll *pixpll, long clo unsigned int computed; m = n = p = s = 0; + delta = 0xffffffff; permitteddelta = clock * 5 / 1000; for (testp = 8; testp > 0; testp /= 2) { diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig index dc7f3e40850b..e9c6af78b1d7 100644 --- a/drivers/gpu/drm/msm/Kconfig +++ b/drivers/gpu/drm/msm/Kconfig @@ -116,9 +116,9 @@ config DRM_MSM_DSI_10NM_PHY Choose this option if DSI PHY on SDM845 is used on the platform. config DRM_MSM_DSI_7NM_PHY - bool "Enable DSI 7nm PHY driver in MSM DRM (used by SM8150/SM8250)" + bool "Enable DSI 7nm PHY driver in MSM DRM" depends on DRM_MSM_DSI default y help - Choose this option if DSI PHY on SM8150/SM8250 is used on the - platform. + Choose this option if DSI PHY on SM8150/SM8250/SC7280 is used on + the platform. diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index 0a93ed1d6b06..5e2750eb3810 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -18,6 +18,18 @@ static void a5xx_dump(struct msm_gpu *gpu); #define GPU_PAS_ID 13 +static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); + + if (a5xx_gpu->has_whereami) { + OUT_PKT7(ring, CP_WHERE_AM_I, 2); + OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring))); + OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring))); + } +} + void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, bool sync) { @@ -30,11 +42,8 @@ void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, * Most flush operations need to issue a WHERE_AM_I opcode to sync up * the rptr shadow */ - if (a5xx_gpu->has_whereami && sync) { - OUT_PKT7(ring, CP_WHERE_AM_I, 2); - OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring))); - OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring))); - } + if (sync) + update_shadow_rptr(gpu, ring); spin_lock_irqsave(&ring->preempt_lock, flags); @@ -168,6 +177,16 @@ static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) ibs++; break; } + + /* + * Periodically update shadow-wptr if needed, so that we + * can see partial progress of submits with large # of + * cmds.. otherwise we could needlessly stall waiting for + * ringbuffer state, simply due to looking at a shadow + * rptr value that has not been updated + */ + if ((ibs % 32) == 0) + update_shadow_rptr(gpu, ring); } /* diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index d7cec7f0dde0..a7c58018959f 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -519,9 +519,9 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) if (!pdcptr) goto err; - if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) + if (adreno_is_a650(adreno_gpu) || adreno_is_a660_family(adreno_gpu)) pdc_in_aop = true; - else if (adreno_is_a618(adreno_gpu) || adreno_is_a640(adreno_gpu)) + else if (adreno_is_a618(adreno_gpu) || adreno_is_a640_family(adreno_gpu)) pdc_address_offset = 0x30090; else pdc_address_offset = 0x30080; @@ -933,6 +933,7 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) /* Use a known rate to bring up the GMU */ clk_set_rate(gmu->core_clk, 200000000); + clk_set_rate(gmu->hub_clk, 150000000); ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); if (ret) { pm_runtime_put(gmu->gxpd); @@ -1393,6 +1394,9 @@ static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks, gmu->nr_clocks, "gmu"); + gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks, + gmu->nr_clocks, "hub"); + return 0; } @@ -1504,7 +1508,7 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) * are otherwise unused by a660. */ gmu->dummy.size = SZ_4K; - if (adreno_is_a660(adreno_gpu)) { + if (adreno_is_a660_family(adreno_gpu)) { ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7, 0x60400000); if (ret) goto err_memory; @@ -1522,7 +1526,7 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) SZ_16M - SZ_16K, 0x04000); if (ret) goto err_memory; - } else if (adreno_is_a640(adreno_gpu)) { + } else if (adreno_is_a640_family(adreno_gpu)) { ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, SZ_256K - SZ_16K, 0x04000); if (ret) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 71dfa60070cc..3c74f64e3126 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -66,6 +66,7 @@ struct a6xx_gmu { int nr_clocks; struct clk_bulk_data *clocks; struct clk *core_clk; + struct clk *hub_clk; /* current performance index set externally */ int current_perf_index; diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 989301230f14..40c9fef457a4 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -52,21 +52,25 @@ static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) return true; } -static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) +static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); - uint32_t wptr; - unsigned long flags; /* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */ if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) { - struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); - OUT_PKT7(ring, CP_WHERE_AM_I, 2); OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring))); OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring))); } +} + +static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) +{ + uint32_t wptr; + unsigned long flags; + + update_shadow_rptr(gpu, ring); spin_lock_irqsave(&ring->preempt_lock, flags); @@ -145,7 +149,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); struct msm_ringbuffer *ring = submit->ring; - unsigned int i; + unsigned int i, ibs = 0; a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx); @@ -181,8 +185,19 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); OUT_RING(ring, submit->cmd[i].size); + ibs++; break; } + + /* + * Periodically update shadow-wptr if needed, so that we + * can see partial progress of submits with large # of + * cmds.. otherwise we could needlessly stall waiting for + * ringbuffer state, simply due to looking at a shadow + * rptr value that has not been updated + */ + if ((ibs % 32) == 0) + update_shadow_rptr(gpu, ring); } get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), @@ -652,7 +667,7 @@ static void a6xx_set_cp_protect(struct msm_gpu *gpu) regs = a650_protect; count = ARRAY_SIZE(a650_protect); count_max = 48; - } else if (adreno_is_a660(adreno_gpu)) { + } else if (adreno_is_a660_family(adreno_gpu)) { regs = a660_protect; count = ARRAY_SIZE(a660_protect); count_max = 48; @@ -683,7 +698,7 @@ static void a6xx_set_ubwc_config(struct msm_gpu *gpu) if (adreno_is_a618(adreno_gpu)) return; - if (adreno_is_a640(adreno_gpu)) + if (adreno_is_a640_family(adreno_gpu)) amsbc = 1; if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) { @@ -694,6 +709,13 @@ static void a6xx_set_ubwc_config(struct msm_gpu *gpu) uavflagprd_inv = 2; } + if (adreno_is_7c3(adreno_gpu)) { + lower_bit = 1; + amsbc = 1; + rgb565_predicator = 1; + uavflagprd_inv = 2; + } + gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, rgb565_predicator << 11 | amsbc << 4 | lower_bit << 1); gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, lower_bit << 1); @@ -740,6 +762,7 @@ static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu, { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct msm_gpu *gpu = &adreno_gpu->base; + const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE]; u32 *buf = msm_gem_get_vaddr(obj); bool ret = false; @@ -756,8 +779,7 @@ static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu, * * a660 targets have all the critical security fixes from the start */ - if (adreno_is_a618(adreno_gpu) || adreno_is_a630(adreno_gpu) || - adreno_is_a640(adreno_gpu)) { + if (!strcmp(sqe_name, "a630_sqe.fw")) { /* * If the lowest nibble is 0xa that is an indication that this * microcode has been patched. The actual version is in dword @@ -778,7 +800,7 @@ static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu, DRM_DEV_ERROR(&gpu->pdev->dev, "a630 SQE ucode is too old. Have version %x need at least %x\n", buf[0] & 0xfff, 0x190); - } else if (adreno_is_a650(adreno_gpu)) { + } else if (!strcmp(sqe_name, "a650_sqe.fw")) { if ((buf[0] & 0xfff) >= 0x095) { ret = true; goto out; @@ -787,7 +809,7 @@ static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu, DRM_DEV_ERROR(&gpu->pdev->dev, "a650 SQE ucode is too old. Have version %x need at least %x\n", buf[0] & 0xfff, 0x095); - } else if (adreno_is_a660(adreno_gpu)) { + } else if (!strcmp(sqe_name, "a660_sqe.fw")) { ret = true; } else { DRM_DEV_ERROR(&gpu->pdev->dev, @@ -897,7 +919,8 @@ static int a6xx_hw_init(struct msm_gpu *gpu) a6xx_set_hwcg(gpu, true); /* VBIF/GBIF start*/ - if (adreno_is_a640(adreno_gpu) || adreno_is_a650_family(adreno_gpu)) { + if (adreno_is_a640_family(adreno_gpu) || + adreno_is_a650_family(adreno_gpu)) { gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620); gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620); gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620); @@ -935,13 +958,14 @@ static int a6xx_hw_init(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804); gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4); - if (adreno_is_a640(adreno_gpu) || adreno_is_a650_family(adreno_gpu)) + if (adreno_is_a640_family(adreno_gpu) || + adreno_is_a650_family(adreno_gpu)) gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140); else gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); - if (adreno_is_a660(adreno_gpu)) + if (adreno_is_a660_family(adreno_gpu)) gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020); /* Setting the mem pool size */ @@ -952,8 +976,10 @@ static int a6xx_hw_init(struct msm_gpu *gpu) */ if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200); - else if (adreno_is_a640(adreno_gpu)) + else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu)) gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200); + else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) + gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200); else gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000); @@ -990,13 +1016,15 @@ static int a6xx_hw_init(struct msm_gpu *gpu) /* Protect registers from the CP */ a6xx_set_cp_protect(gpu); - if (adreno_is_a660(adreno_gpu)) { + if (adreno_is_a660_family(adreno_gpu)) { gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1); gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0); - /* Set dualQ + disable afull for A660 GPU but not for A635 */ - gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906); } + /* Set dualQ + disable afull for A660 GPU */ + if (adreno_is_a660(adreno_gpu)) + gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906); + /* Enable expanded apriv for targets that support it */ if (gpu->hw_apriv) { gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL, @@ -1383,13 +1411,13 @@ static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu) { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct msm_gpu *gpu = &adreno_gpu->base; - u32 cntl1_regval = 0; + u32 gpu_scid, cntl1_regval = 0; if (IS_ERR(a6xx_gpu->llc_mmio)) return; if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { - u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); + gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); gpu_scid &= 0x1f; cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) | @@ -1409,26 +1437,34 @@ static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu) } } - if (cntl1_regval) { + if (!cntl1_regval) + return; + + /* + * Program the slice IDs for the various GPU blocks and GPU MMU + * pagetables + */ + if (!a6xx_gpu->have_mmu500) { + a6xx_llc_write(a6xx_gpu, + REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval); + /* - * Program the slice IDs for the various GPU blocks and GPU MMU - * pagetables + * Program cacheability overrides to not allocate cache + * lines on a write miss */ - if (a6xx_gpu->have_mmu500) - gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), - cntl1_regval); - else { - a6xx_llc_write(a6xx_gpu, - REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval); - - /* - * Program cacheability overrides to not allocate cache - * lines on a write miss - */ - a6xx_llc_rmw(a6xx_gpu, - REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03); - } + a6xx_llc_rmw(a6xx_gpu, + REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03); + return; } + + gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval); + + /* On A660, the SCID programming for UCHE traffic is done in + * A6XX_GBIF_SCACHE_CNTL0[14:10] + */ + if (adreno_is_a660_family(adreno_gpu)) + gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) | + (1 << 8), (gpu_scid << 10) | (1 << 8)); } static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu) @@ -1667,11 +1703,11 @@ static u32 a618_get_speed_bin(u32 fuse) return UINT_MAX; } -static u32 fuse_to_supp_hw(struct device *dev, u32 revn, u32 fuse) +static u32 fuse_to_supp_hw(struct device *dev, struct adreno_rev rev, u32 fuse) { u32 val = UINT_MAX; - if (revn == 618) + if (adreno_cmp_rev(ADRENO_REV(6, 1, 8, ANY_ID), rev)) val = a618_get_speed_bin(fuse); if (val == UINT_MAX) { @@ -1684,14 +1720,13 @@ static u32 fuse_to_supp_hw(struct device *dev, u32 revn, u32 fuse) return (1 << val); } -static int a6xx_set_supported_hw(struct device *dev, struct a6xx_gpu *a6xx_gpu, - u32 revn) +static int a6xx_set_supported_hw(struct device *dev, struct adreno_rev rev) { u32 supp_hw = UINT_MAX; - u16 speedbin; + u32 speedbin; int ret; - ret = nvmem_cell_read_u16(dev, "speed_bin", &speedbin); + ret = nvmem_cell_read_variable_le_u32(dev, "speed_bin", &speedbin); /* * -ENOENT means that the platform doesn't support speedbin which is * fine @@ -1704,9 +1739,8 @@ static int a6xx_set_supported_hw(struct device *dev, struct a6xx_gpu *a6xx_gpu, ret); goto done; } - speedbin = le16_to_cpu(speedbin); - supp_hw = fuse_to_supp_hw(dev, revn, speedbin); + supp_hw = fuse_to_supp_hw(dev, rev, speedbin); done: ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1); @@ -1772,12 +1806,13 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) */ info = adreno_info(config->rev); - if (info && (info->revn == 650 || info->revn == 660)) + if (info && (info->revn == 650 || info->revn == 660 || + adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), info->rev))) adreno_gpu->base.hw_apriv = true; a6xx_llc_slices_init(pdev, a6xx_gpu); - ret = a6xx_set_supported_hw(&pdev->dev, a6xx_gpu, info->revn); + ret = a6xx_set_supported_hw(&pdev->dev, config->rev); if (ret) { a6xx_destroy(&(a6xx_gpu->base.base)); return ERR_PTR(ret); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c index 919433732b43..d4c65bf0a1b7 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c @@ -382,6 +382,36 @@ static void a660_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) msg->cnoc_cmds_data[1][0] = 0x60000001; } +static void adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) +{ + /* + * Send a single "off" entry just to get things running + * TODO: bus scaling + */ + msg->bw_level_num = 1; + + msg->ddr_cmds_num = 3; + msg->ddr_wait_bitmask = 0x07; + + msg->ddr_cmds_addrs[0] = 0x50004; + msg->ddr_cmds_addrs[1] = 0x50000; + msg->ddr_cmds_addrs[2] = 0x50088; + + msg->ddr_cmds_data[0][0] = 0x40000000; + msg->ddr_cmds_data[0][1] = 0x40000000; + msg->ddr_cmds_data[0][2] = 0x40000000; + + /* + * These are the CX (CNOC) votes - these are used by the GMU but the + * votes are known and fixed for the target + */ + msg->cnoc_cmds_num = 1; + msg->cnoc_wait_bitmask = 0x01; + + msg->cnoc_cmds_addrs[0] = 0x5006c; + msg->cnoc_cmds_data[0][0] = 0x40000000; + msg->cnoc_cmds_data[1][0] = 0x60000001; +} static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) { /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */ @@ -428,10 +458,12 @@ static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu) if (adreno_is_a618(adreno_gpu)) a618_build_bw_table(&msg); - else if (adreno_is_a640(adreno_gpu)) + else if (adreno_is_a640_family(adreno_gpu)) a640_build_bw_table(&msg); else if (adreno_is_a650(adreno_gpu)) a650_build_bw_table(&msg); + else if (adreno_is_7c3(adreno_gpu)) + adreno_7c3_build_bw_table(&msg); else if (adreno_is_a660(adreno_gpu)) a660_build_bw_table(&msg); else diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 6dad8015c9a1..2a6ce76656aa 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -8,8 +8,6 @@ #include "adreno_gpu.h" -#define ANY_ID 0xff - bool hang_debug = false; MODULE_PARM_DESC(hang_debug, "Dump registers when hang is detected (can be slow!)"); module_param_named(hang_debug, hang_debug, bool, 0600); @@ -300,6 +298,30 @@ static const struct adreno_info gpulist[] = { .init = a6xx_gpu_init, .zapfw = "a660_zap.mdt", .hwcg = a660_hwcg, + }, { + .rev = ADRENO_REV(6, 3, 5, ANY_ID), + .name = "Adreno 7c Gen 3", + .fw = { + [ADRENO_FW_SQE] = "a660_sqe.fw", + [ADRENO_FW_GMU] = "a660_gmu.bin", + }, + .gmem = SZ_512K, + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a6xx_gpu_init, + .hwcg = a660_hwcg, + }, { + .rev = ADRENO_REV(6, 8, 0, ANY_ID), + .revn = 680, + .name = "A680", + .fw = { + [ADRENO_FW_SQE] = "a630_sqe.fw", + [ADRENO_FW_GMU] = "a640_gmu.bin", + }, + .gmem = SZ_2M, + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a6xx_gpu_init, + .zapfw = "a640_zap.mdt", + .hwcg = a640_hwcg, }, }; @@ -325,6 +347,15 @@ static inline bool _rev_match(uint8_t entry, uint8_t id) return (entry == ANY_ID) || (entry == id); } +bool adreno_cmp_rev(struct adreno_rev rev1, struct adreno_rev rev2) +{ + + return _rev_match(rev1.core, rev2.core) && + _rev_match(rev1.major, rev2.major) && + _rev_match(rev1.minor, rev2.minor) && + _rev_match(rev1.patchid, rev2.patchid); +} + const struct adreno_info *adreno_info(struct adreno_rev rev) { int i; @@ -332,10 +363,7 @@ const struct adreno_info *adreno_info(struct adreno_rev rev) /* identify gpu: */ for (i = 0; i < ARRAY_SIZE(gpulist); i++) { const struct adreno_info *info = &gpulist[i]; - if (_rev_match(info->rev.core, rev.core) && - _rev_match(info->rev.major, rev.major) && - _rev_match(info->rev.minor, rev.minor) && - _rev_match(info->rev.patchid, rev.patchid)) + if (adreno_cmp_rev(info->rev, rev)) return info; } diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 8dbe0d157520..225c277a6223 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -42,6 +42,8 @@ struct adreno_rev { uint8_t patchid; }; +#define ANY_ID 0xff + #define ADRENO_REV(core, major, minor, patchid) \ ((struct adreno_rev){ core, major, minor, patchid }) @@ -141,6 +143,8 @@ struct adreno_platform_config { __ret; \ }) +bool adreno_cmp_rev(struct adreno_rev rev1, struct adreno_rev rev2); + static inline bool adreno_is_a2xx(struct adreno_gpu *gpu) { return (gpu->revn < 300); @@ -237,9 +241,9 @@ static inline int adreno_is_a630(struct adreno_gpu *gpu) return gpu->revn == 630; } -static inline int adreno_is_a640(struct adreno_gpu *gpu) +static inline int adreno_is_a640_family(struct adreno_gpu *gpu) { - return gpu->revn == 640; + return (gpu->revn == 640) || (gpu->revn == 680); } static inline int adreno_is_a650(struct adreno_gpu *gpu) @@ -247,15 +251,27 @@ static inline int adreno_is_a650(struct adreno_gpu *gpu) return gpu->revn == 650; } +static inline int adreno_is_7c3(struct adreno_gpu *gpu) +{ + /* The order of args is important here to handle ANY_ID correctly */ + return adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), gpu->rev); +} + static inline int adreno_is_a660(struct adreno_gpu *gpu) { return gpu->revn == 660; } +static inline int adreno_is_a660_family(struct adreno_gpu *gpu) +{ + return adreno_is_a660(gpu) || adreno_is_7c3(gpu); +} + /* check for a650, a660, or any derivatives */ static inline int adreno_is_a650_family(struct adreno_gpu *gpu) { - return gpu->revn == 650 || gpu->revn == 620 || gpu->revn == 660; + return gpu->revn == 650 || gpu->revn == 620 || + adreno_is_a660_family(gpu); } int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index 9a5c70c87cc8..768012243b44 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -30,12 +30,6 @@ #include "dpu_core_perf.h" #include "dpu_trace.h" -#define DPU_DRM_BLEND_OP_NOT_DEFINED 0 -#define DPU_DRM_BLEND_OP_OPAQUE 1 -#define DPU_DRM_BLEND_OP_PREMULTIPLIED 2 -#define DPU_DRM_BLEND_OP_COVERAGE 3 -#define DPU_DRM_BLEND_OP_MAX 4 - /* layer mixer index on dpu_crtc */ #define LEFT_MIXER 0 #define RIGHT_MIXER 1 @@ -146,20 +140,43 @@ static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer, { struct dpu_hw_mixer *lm = mixer->hw_lm; uint32_t blend_op; + uint32_t fg_alpha, bg_alpha; - /* default to opaque blending */ - blend_op = DPU_BLEND_FG_ALPHA_FG_CONST | - DPU_BLEND_BG_ALPHA_BG_CONST; + fg_alpha = pstate->base.alpha >> 8; + bg_alpha = 0xff - fg_alpha; - if (format->alpha_enable) { + /* default to opaque blending */ + if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE || + !format->alpha_enable) { + blend_op = DPU_BLEND_FG_ALPHA_FG_CONST | + DPU_BLEND_BG_ALPHA_BG_CONST; + } else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) { + blend_op = DPU_BLEND_FG_ALPHA_FG_CONST | + DPU_BLEND_BG_ALPHA_FG_PIXEL; + if (fg_alpha != 0xff) { + bg_alpha = fg_alpha; + blend_op |= DPU_BLEND_BG_MOD_ALPHA | + DPU_BLEND_BG_INV_MOD_ALPHA; + } else { + blend_op |= DPU_BLEND_BG_INV_ALPHA; + } + } else { /* coverage blending */ blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL | - DPU_BLEND_BG_ALPHA_FG_PIXEL | - DPU_BLEND_BG_INV_ALPHA; + DPU_BLEND_BG_ALPHA_FG_PIXEL; + if (fg_alpha != 0xff) { + bg_alpha = fg_alpha; + blend_op |= DPU_BLEND_FG_MOD_ALPHA | + DPU_BLEND_FG_INV_MOD_ALPHA | + DPU_BLEND_BG_MOD_ALPHA | + DPU_BLEND_BG_INV_MOD_ALPHA; + } else { + blend_op |= DPU_BLEND_BG_INV_ALPHA; + } } lm->ops.setup_blend_config(lm, pstate->stage, - 0xFF, 0, blend_op); + fg_alpha, bg_alpha, blend_op); DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n", &format->base.pixel_format, format->alpha_enable, blend_op); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 1c04b7cce43e..0e9d3fa1544b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -274,20 +274,20 @@ int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc, /* return EWOULDBLOCK since we know the wait isn't necessary */ if (phys_enc->enable_state == DPU_ENC_DISABLED) { - DRM_ERROR("encoder is disabled id=%u, intr=%d, irq=%d", + DRM_ERROR("encoder is disabled id=%u, intr=%d, irq=%d\n", DRMID(phys_enc->parent), intr_idx, irq->irq_idx); return -EWOULDBLOCK; } if (irq->irq_idx < 0) { - DRM_DEBUG_KMS("skip irq wait id=%u, intr=%d, irq=%s", + DRM_DEBUG_KMS("skip irq wait id=%u, intr=%d, irq=%s\n", DRMID(phys_enc->parent), intr_idx, irq->name); return 0; } - DRM_DEBUG_KMS("id=%u, intr=%d, irq=%d, pp=%d, pending_cnt=%d", + DRM_DEBUG_KMS("id=%u, intr=%d, irq=%d, pp=%d, pending_cnt=%d\n", DRMID(phys_enc->parent), intr_idx, irq->irq_idx, phys_enc->hw_pp->idx - PINGPONG_0, atomic_read(wait_info->atomic_cnt)); @@ -303,8 +303,7 @@ int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc, if (irq_status) { unsigned long flags; - DRM_DEBUG_KMS("irq not triggered id=%u, intr=%d, " - "irq=%d, pp=%d, atomic_cnt=%d", + DRM_DEBUG_KMS("irq not triggered id=%u, intr=%d, irq=%d, pp=%d, atomic_cnt=%d\n", DRMID(phys_enc->parent), intr_idx, irq->irq_idx, phys_enc->hw_pp->idx - PINGPONG_0, @@ -315,8 +314,7 @@ int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc, ret = 0; } else { ret = -ETIMEDOUT; - DRM_DEBUG_KMS("irq timeout id=%u, intr=%d, " - "irq=%d, pp=%d, atomic_cnt=%d", + DRM_DEBUG_KMS("irq timeout id=%u, intr=%d, irq=%d, pp=%d, atomic_cnt=%d\n", DRMID(phys_enc->parent), intr_idx, irq->irq_idx, phys_enc->hw_pp->idx - PINGPONG_0, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index d01c4c919504..b131fd376192 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -296,7 +296,7 @@ static const struct dpu_mdp_cfg sc7180_mdp[] = { static const struct dpu_mdp_cfg sm8250_mdp[] = { { .name = "top_0", .id = MDP_TOP, - .base = 0x0, .len = 0x45C, + .base = 0x0, .len = 0x494, .features = 0, .highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */ .clk_ctrls[DPU_CLK_CTRL_VIG0] = { @@ -974,6 +974,7 @@ static const struct dpu_perf_cfg sdm845_perf_data = { .amortizable_threshold = 25, .min_prefill_lines = 24, .danger_lut_tbl = {0xf, 0xffff, 0x0}, + .safe_lut_tbl = {0xfff0, 0xf000, 0xffff}, .qos_lut_tbl = { {.nentry = ARRAY_SIZE(sdm845_qos_linear), .entries = sdm845_qos_linear @@ -1001,6 +1002,7 @@ static const struct dpu_perf_cfg sc7180_perf_data = { .min_dram_ib = 1600000, .min_prefill_lines = 24, .danger_lut_tbl = {0xff, 0xffff, 0x0}, + .safe_lut_tbl = {0xfff0, 0xff00, 0xffff}, .qos_lut_tbl = { {.nentry = ARRAY_SIZE(sc7180_qos_linear), .entries = sc7180_qos_linear @@ -1028,6 +1030,7 @@ static const struct dpu_perf_cfg sm8150_perf_data = { .min_dram_ib = 800000, .min_prefill_lines = 24, .danger_lut_tbl = {0xf, 0xffff, 0x0}, + .safe_lut_tbl = {0xfff8, 0xf000, 0xffff}, .qos_lut_tbl = { {.nentry = ARRAY_SIZE(sm8150_qos_linear), .entries = sm8150_qos_linear @@ -1056,6 +1059,7 @@ static const struct dpu_perf_cfg sm8250_perf_data = { .min_dram_ib = 800000, .min_prefill_lines = 35, .danger_lut_tbl = {0xf, 0xffff, 0x0}, + .safe_lut_tbl = {0xfff0, 0xff00, 0xffff}, .qos_lut_tbl = { {.nentry = ARRAY_SIZE(sc7180_qos_linear), .entries = sc7180_qos_linear @@ -1084,6 +1088,7 @@ static const struct dpu_perf_cfg sc7280_perf_data = { .min_dram_ib = 1600000, .min_prefill_lines = 24, .danger_lut_tbl = {0xffff, 0xffff, 0x0}, + .safe_lut_tbl = {0xff00, 0xff00, 0xffff}, .qos_lut_tbl = { {.nentry = ARRAY_SIZE(sc7180_qos_macrotile), .entries = sc7180_qos_macrotile diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c index f8a74f6cdc4c..64740ddb983e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c @@ -345,10 +345,12 @@ static void dpu_hw_ctl_clear_all_blendstages(struct dpu_hw_ctl *ctx) int i; for (i = 0; i < ctx->mixer_count; i++) { - DPU_REG_WRITE(c, CTL_LAYER(LM_0 + i), 0); - DPU_REG_WRITE(c, CTL_LAYER_EXT(LM_0 + i), 0); - DPU_REG_WRITE(c, CTL_LAYER_EXT2(LM_0 + i), 0); - DPU_REG_WRITE(c, CTL_LAYER_EXT3(LM_0 + i), 0); + enum dpu_lm mixer_id = ctx->mixer_hw_caps[i].id; + + DPU_REG_WRITE(c, CTL_LAYER(mixer_id), 0); + DPU_REG_WRITE(c, CTL_LAYER_EXT(mixer_id), 0); + DPU_REG_WRITE(c, CTL_LAYER_EXT2(mixer_id), 0); + DPU_REG_WRITE(c, CTL_LAYER_EXT3(mixer_id), 0); } DPU_REG_WRITE(c, CTL_FETCH_PIPE_ACTIVE, 0); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 4fd913522931..ae48f41821cf 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -471,30 +471,68 @@ static int _dpu_kms_initialize_dsi(struct drm_device *dev, struct dpu_kms *dpu_kms) { struct drm_encoder *encoder = NULL; + struct msm_display_info info; int i, rc = 0; if (!(priv->dsi[0] || priv->dsi[1])) return rc; - /*TODO: Support two independent DSI connectors */ - encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DSI); - if (IS_ERR(encoder)) { - DPU_ERROR("encoder init failed for dsi display\n"); - return PTR_ERR(encoder); - } - - priv->encoders[priv->num_encoders++] = encoder; - + /* + * We support following confiurations: + * - Single DSI host (dsi0 or dsi1) + * - Two independent DSI hosts + * - Bonded DSI0 and DSI1 hosts + * + * TODO: Support swapping DSI0 and DSI1 in the bonded setup. + */ for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) { + int other = (i + 1) % 2; + if (!priv->dsi[i]) continue; + if (msm_dsi_is_bonded_dsi(priv->dsi[i]) && + !msm_dsi_is_master_dsi(priv->dsi[i])) + continue; + + encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DSI); + if (IS_ERR(encoder)) { + DPU_ERROR("encoder init failed for dsi display\n"); + return PTR_ERR(encoder); + } + + priv->encoders[priv->num_encoders++] = encoder; + + memset(&info, 0, sizeof(info)); + info.intf_type = encoder->encoder_type; + rc = msm_dsi_modeset_init(priv->dsi[i], dev, encoder); if (rc) { DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n", i, rc); break; } + + info.h_tile_instance[info.num_of_h_tiles++] = i; + info.capabilities = msm_dsi_is_cmd_mode(priv->dsi[i]) ? + MSM_DISPLAY_CAP_CMD_MODE : + MSM_DISPLAY_CAP_VID_MODE; + + if (msm_dsi_is_bonded_dsi(priv->dsi[i]) && priv->dsi[other]) { + rc = msm_dsi_modeset_init(priv->dsi[other], dev, encoder); + if (rc) { + DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n", + other, rc); + break; + } + + info.h_tile_instance[info.num_of_h_tiles++] = other; + } + + rc = dpu_encoder_setup(dev, encoder, &info); + if (rc) + DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n", + encoder->base.id, rc); } return rc; @@ -505,6 +543,7 @@ static int _dpu_kms_initialize_displayport(struct drm_device *dev, struct dpu_kms *dpu_kms) { struct drm_encoder *encoder = NULL; + struct msm_display_info info; int rc = 0; if (!priv->dp) @@ -516,6 +555,7 @@ static int _dpu_kms_initialize_displayport(struct drm_device *dev, return PTR_ERR(encoder); } + memset(&info, 0, sizeof(info)); rc = msm_dp_modeset_init(priv->dp, dev, encoder); if (rc) { DPU_ERROR("modeset_init failed for DP, rc = %d\n", rc); @@ -524,6 +564,14 @@ static int _dpu_kms_initialize_displayport(struct drm_device *dev, } priv->encoders[priv->num_encoders++] = encoder; + + info.num_of_h_tiles = 1; + info.capabilities = MSM_DISPLAY_CAP_VID_MODE; + info.intf_type = encoder->encoder_type; + rc = dpu_encoder_setup(dev, encoder, &info); + if (rc) + DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n", + encoder->base.id, rc); return rc; } @@ -726,41 +774,6 @@ static void dpu_kms_destroy(struct msm_kms *kms) msm_kms_destroy(&dpu_kms->base); } -static void _dpu_kms_set_encoder_mode(struct msm_kms *kms, - struct drm_encoder *encoder, - bool cmd_mode) -{ - struct msm_display_info info; - struct msm_drm_private *priv = encoder->dev->dev_private; - int i, rc = 0; - - memset(&info, 0, sizeof(info)); - - info.intf_type = encoder->encoder_type; - info.capabilities = cmd_mode ? MSM_DISPLAY_CAP_CMD_MODE : - MSM_DISPLAY_CAP_VID_MODE; - - switch (info.intf_type) { - case DRM_MODE_ENCODER_DSI: - /* TODO: No support for DSI swap */ - for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) { - if (priv->dsi[i]) { - info.h_tile_instance[info.num_of_h_tiles] = i; - info.num_of_h_tiles++; - } - } - break; - case DRM_MODE_ENCODER_TMDS: - info.num_of_h_tiles = 1; - break; - } - - rc = dpu_encoder_setup(encoder->dev, encoder, &info); - if (rc) - DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n", - encoder->base.id, rc); -} - static irqreturn_t dpu_irq(struct msm_kms *kms) { struct dpu_kms *dpu_kms = to_dpu_kms(kms); @@ -863,7 +876,6 @@ static const struct msm_kms_funcs kms_funcs = { .get_format = dpu_get_msm_format, .round_pixclk = dpu_kms_round_pixclk, .destroy = dpu_kms_destroy, - .set_encoder_mode = _dpu_kms_set_encoder_mode, .snapshot = dpu_kms_mdp_snapshot, #ifdef CONFIG_DEBUG_FS .debugfs_init = dpu_kms_debugfs_init, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c index 6b0a7bc87eb7..b466784d9822 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c @@ -45,20 +45,13 @@ static void dpu_mdss_irq(struct irq_desc *desc) while (interrupts) { irq_hw_number_t hwirq = fls(interrupts) - 1; - unsigned int mapping; int rc; - mapping = irq_find_mapping(dpu_mdss->irq_controller.domain, - hwirq); - if (mapping == 0) { - DRM_ERROR("couldn't find irq mapping for %lu\n", hwirq); - break; - } - - rc = generic_handle_irq(mapping); + rc = generic_handle_domain_irq(dpu_mdss->irq_controller.domain, + hwirq); if (rc < 0) { - DRM_ERROR("handle irq fail: irq=%lu mapping=%u rc=%d\n", - hwirq, mapping, rc); + DRM_ERROR("handle irq fail: irq=%lu rc=%d\n", + hwirq, rc); break; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index ec4a6f04394a..c989621209aa 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -1339,9 +1339,7 @@ static void dpu_plane_reset(struct drm_plane *plane) return; } - pstate->base.plane = plane; - - plane->state = &pstate->base; + __drm_atomic_helper_plane_reset(plane, &pstate->base); } #ifdef CONFIG_DEBUG_FS @@ -1647,6 +1645,12 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, if (ret) DPU_ERROR("failed to install zpos property, rc = %d\n", ret); + drm_plane_create_alpha_property(plane); + drm_plane_create_blend_mode_property(plane, + BIT(DRM_MODE_BLEND_PIXEL_NONE) | + BIT(DRM_MODE_BLEND_PREMULTI) | + BIT(DRM_MODE_BLEND_COVERAGE)); + drm_plane_create_rotation_property(plane, DRM_MODE_ROTATE_0, DRM_MODE_ROTATE_0 | diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c index 4a5b518288b0..cdcaf470f148 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c @@ -19,30 +19,12 @@ static int mdp4_hw_init(struct msm_kms *kms) { struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms)); struct drm_device *dev = mdp4_kms->dev; - uint32_t version, major, minor, dmap_cfg, vg_cfg; + u32 dmap_cfg, vg_cfg; unsigned long clk; int ret = 0; pm_runtime_get_sync(dev->dev); - mdp4_enable(mdp4_kms); - version = mdp4_read(mdp4_kms, REG_MDP4_VERSION); - mdp4_disable(mdp4_kms); - - major = FIELD(version, MDP4_VERSION_MAJOR); - minor = FIELD(version, MDP4_VERSION_MINOR); - - DBG("found MDP4 version v%d.%d", major, minor); - - if (major != 4) { - DRM_DEV_ERROR(dev->dev, "unexpected MDP version: v%d.%d\n", - major, minor); - ret = -ENXIO; - goto out; - } - - mdp4_kms->rev = minor; - if (mdp4_kms->rev > 1) { mdp4_write(mdp4_kms, REG_MDP4_CS_CONTROLLER0, 0x0707ffff); mdp4_write(mdp4_kms, REG_MDP4_CS_CONTROLLER1, 0x03073f3f); @@ -88,7 +70,6 @@ static int mdp4_hw_init(struct msm_kms *kms) if (mdp4_kms->rev > 1) mdp4_write(mdp4_kms, REG_MDP4_RESET_STATUS, 1); -out: pm_runtime_put_sync(dev->dev); return ret; @@ -108,13 +89,6 @@ static void mdp4_disable_commit(struct msm_kms *kms) static void mdp4_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state) { - int i; - struct drm_crtc *crtc; - struct drm_crtc_state *crtc_state; - - /* see 119ecb7fd */ - for_each_new_crtc_in_state(state, crtc, crtc_state, i) - drm_crtc_vblank_get(crtc); } static void mdp4_flush_commit(struct msm_kms *kms, unsigned crtc_mask) @@ -133,12 +107,6 @@ static void mdp4_wait_flush(struct msm_kms *kms, unsigned crtc_mask) static void mdp4_complete_commit(struct msm_kms *kms, unsigned crtc_mask) { - struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms)); - struct drm_crtc *crtc; - - /* see 119ecb7fd */ - for_each_crtc_mask(mdp4_kms->dev, crtc, crtc_mask) - drm_crtc_vblank_put(crtc); } static long mdp4_round_pixclk(struct msm_kms *kms, unsigned long rate, @@ -411,14 +379,32 @@ fail: return ret; } +static void read_mdp_hw_revision(struct mdp4_kms *mdp4_kms, + u32 *major, u32 *minor) +{ + struct drm_device *dev = mdp4_kms->dev; + u32 version; + + mdp4_enable(mdp4_kms); + version = mdp4_read(mdp4_kms, REG_MDP4_VERSION); + mdp4_disable(mdp4_kms); + + *major = FIELD(version, MDP4_VERSION_MAJOR); + *minor = FIELD(version, MDP4_VERSION_MINOR); + + DRM_DEV_INFO(dev->dev, "MDP4 version v%d.%d", *major, *minor); +} + struct msm_kms *mdp4_kms_init(struct drm_device *dev) { struct platform_device *pdev = to_platform_device(dev->dev); struct mdp4_platform_config *config = mdp4_get_config(pdev); + struct msm_drm_private *priv = dev->dev_private; struct mdp4_kms *mdp4_kms; struct msm_kms *kms = NULL; struct msm_gem_address_space *aspace; int irq, ret; + u32 major, minor; mdp4_kms = kzalloc(sizeof(*mdp4_kms), GFP_KERNEL); if (!mdp4_kms) { @@ -433,7 +419,8 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev) goto fail; } - kms = &mdp4_kms->base.base; + priv->kms = &mdp4_kms->base.base; + kms = priv->kms; mdp4_kms->dev = dev; @@ -479,15 +466,6 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev) if (IS_ERR(mdp4_kms->pclk)) mdp4_kms->pclk = NULL; - if (mdp4_kms->rev >= 2) { - mdp4_kms->lut_clk = devm_clk_get(&pdev->dev, "lut_clk"); - if (IS_ERR(mdp4_kms->lut_clk)) { - DRM_DEV_ERROR(dev->dev, "failed to get lut_clk\n"); - ret = PTR_ERR(mdp4_kms->lut_clk); - goto fail; - } - } - mdp4_kms->axi_clk = devm_clk_get(&pdev->dev, "bus_clk"); if (IS_ERR(mdp4_kms->axi_clk)) { DRM_DEV_ERROR(dev->dev, "failed to get axi_clk\n"); @@ -496,8 +474,27 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev) } clk_set_rate(mdp4_kms->clk, config->max_clk); - if (mdp4_kms->lut_clk) + + read_mdp_hw_revision(mdp4_kms, &major, &minor); + + if (major != 4) { + DRM_DEV_ERROR(dev->dev, "unexpected MDP version: v%d.%d\n", + major, minor); + ret = -ENXIO; + goto fail; + } + + mdp4_kms->rev = minor; + + if (mdp4_kms->rev >= 2) { + mdp4_kms->lut_clk = devm_clk_get(&pdev->dev, "lut_clk"); + if (IS_ERR(mdp4_kms->lut_clk)) { + DRM_DEV_ERROR(dev->dev, "failed to get lut_clk\n"); + ret = PTR_ERR(mdp4_kms->lut_clk); + goto fail; + } clk_set_rate(mdp4_kms->lut_clk, config->max_clk); + } pm_runtime_enable(dev->dev); mdp4_kms->rpm_enabled = true; diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_ctl.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_ctl.c index 81b0c7cf954e..1220f2b20e05 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_ctl.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_ctl.c @@ -737,7 +737,7 @@ struct mdp5_ctl_manager *mdp5_ctlm_init(struct drm_device *dev, } /* - * In Dual DSI case, CTL0 and CTL1 are always assigned to two DSI + * In bonded DSI case, CTL0 and CTL1 are always assigned to two DSI * interfaces to support single FLUSH feature (Flush CTL0 and CTL1 when * only write into CTL0's FLUSH register) to keep two DSI pipes in sync. * Single FLUSH is supported from hw rev v3.0. diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index 15aed45022bc..b3b42672b2d4 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -209,13 +209,6 @@ static int mdp5_set_split_display(struct msm_kms *kms, slave_encoder); } -static void mdp5_set_encoder_mode(struct msm_kms *kms, - struct drm_encoder *encoder, - bool cmd_mode) -{ - mdp5_encoder_set_intf_mode(encoder, cmd_mode); -} - static void mdp5_kms_destroy(struct msm_kms *kms) { struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms)); @@ -287,7 +280,6 @@ static const struct mdp_kms_funcs kms_funcs = { .get_format = mdp_get_format, .round_pixclk = mdp5_round_pixclk, .set_split_display = mdp5_set_split_display, - .set_encoder_mode = mdp5_set_encoder_mode, .destroy = mdp5_kms_destroy, #ifdef CONFIG_DEBUG_FS .debugfs_init = mdp5_kms_debugfs_init, @@ -448,6 +440,9 @@ static int modeset_init_intf(struct mdp5_kms *mdp5_kms, } ret = msm_dsi_modeset_init(priv->dsi[dsi_id], dev, encoder); + if (!ret) + mdp5_encoder_set_intf_mode(encoder, msm_dsi_is_cmd_mode(priv->dsi[dsi_id])); + break; } default: diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c index 09bd46ad820b..2f4895bcb0b0 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c @@ -50,8 +50,7 @@ static irqreturn_t mdss_irq(int irq, void *arg) while (intr) { irq_hw_number_t hwirq = fls(intr) - 1; - generic_handle_irq(irq_find_mapping( - mdp5_mdss->irqcontroller.domain, hwirq)); + generic_handle_domain_irq(mdp5_mdss->irqcontroller.domain, hwirq); intr &= ~(1 << hwirq); } diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h index c92a9508c8d3..4c619307612c 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h @@ -16,7 +16,6 @@ #include <linux/delay.h> #include <linux/spinlock.h> #include <linux/ktime.h> -#include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/dma-buf.h> #include <linux/slab.h> @@ -25,7 +24,6 @@ #include <linux/pm_runtime.h> #include <linux/kthread.h> #include <linux/devcoredump.h> -#include <stdarg.h> #include "msm_kms.h" #define MSM_DISP_SNAPSHOT_MAX_BLKS 10 diff --git a/drivers/gpu/drm/msm/dp/dp_aux.c b/drivers/gpu/drm/msm/dp/dp_aux.c index 4a3293b590b0..eb40d8413bca 100644 --- a/drivers/gpu/drm/msm/dp/dp_aux.c +++ b/drivers/gpu/drm/msm/dp/dp_aux.c @@ -353,6 +353,9 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux, if (!(aux->retry_cnt % MAX_AUX_RETRIES)) dp_catalog_aux_update_cfg(aux->catalog); } + /* reset aux if link is in connected state */ + if (dp_catalog_link_is_connected(aux->catalog)) + dp_catalog_aux_reset(aux->catalog); } else { aux->retry_cnt = 0; switch (aux->aux_error_num) { diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c index ca96e3514790..cc2bb8295329 100644 --- a/drivers/gpu/drm/msm/dp/dp_catalog.c +++ b/drivers/gpu/drm/msm/dp/dp_catalog.c @@ -372,6 +372,7 @@ void dp_catalog_ctrl_mainlink_ctrl(struct dp_catalog *dp_catalog, struct dp_catalog_private *catalog = container_of(dp_catalog, struct dp_catalog_private, dp_catalog); + DRM_DEBUG_DP("enable=%d\n", enable); if (enable) { /* * To make sure link reg writes happens before other operation, @@ -580,6 +581,7 @@ void dp_catalog_hpd_config_intr(struct dp_catalog *dp_catalog, config = (en ? config | intr_mask : config & ~intr_mask); + DRM_DEBUG_DP("intr_mask=%#x config=%#x\n", intr_mask, config); dp_write_aux(catalog, REG_DP_DP_HPD_INT_MASK, config & DP_DP_HPD_INT_MASK); } @@ -610,6 +612,7 @@ u32 dp_catalog_link_is_connected(struct dp_catalog *dp_catalog) u32 status; status = dp_read_aux(catalog, REG_DP_DP_HPD_INT_STATUS); + DRM_DEBUG_DP("aux status: %#x\n", status); status >>= DP_DP_HPD_STATE_STATUS_BITS_SHIFT; status &= DP_DP_HPD_STATE_STATUS_BITS_MASK; @@ -685,6 +688,7 @@ void dp_catalog_ctrl_send_phy_pattern(struct dp_catalog *dp_catalog, /* Make sure to clear the current pattern before starting a new one */ dp_write_link(catalog, REG_DP_STATE_CTRL, 0x0); + DRM_DEBUG_DP("pattern: %#x\n", pattern); switch (pattern) { case DP_PHY_TEST_PATTERN_D10_2: dp_write_link(catalog, REG_DP_STATE_CTRL, @@ -745,7 +749,7 @@ void dp_catalog_ctrl_send_phy_pattern(struct dp_catalog *dp_catalog, DP_STATE_CTRL_LINK_TRAINING_PATTERN4); break; default: - DRM_DEBUG_DP("No valid test pattern requested:0x%x\n", pattern); + DRM_DEBUG_DP("No valid test pattern requested: %#x\n", pattern); break; } } @@ -771,6 +775,7 @@ int dp_catalog_panel_timing_cfg(struct dp_catalog *dp_catalog) dp_write_link(catalog, REG_DP_HSYNC_VSYNC_WIDTH_POLARITY, dp_catalog->width_blanking); dp_write_link(catalog, REG_DP_ACTIVE_HOR_VER, dp_catalog->dp_active); + dp_write_p0(catalog, MMSS_DP_INTF_CONFIG, 0); return 0; } @@ -928,7 +933,7 @@ void dp_catalog_audio_config_acr(struct dp_catalog *dp_catalog) select = dp_catalog->audio_data; acr_ctrl = select << 4 | BIT(31) | BIT(8) | BIT(14); - DRM_DEBUG_DP("select = 0x%x, acr_ctrl = 0x%x\n", select, acr_ctrl); + DRM_DEBUG_DP("select: %#x, acr_ctrl: %#x\n", select, acr_ctrl); dp_write_link(catalog, MMSS_DP_AUDIO_ACR_CTRL, acr_ctrl); } diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index ee221d835fa0..62e75dc8afc6 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -81,13 +81,6 @@ struct dp_ctrl_private { struct completion video_comp; }; -struct dp_cr_status { - u8 lane_0_1; - u8 lane_2_3; -}; - -#define DP_LANE0_1_CR_DONE 0x11 - static int dp_aux_link_configure(struct drm_dp_aux *aux, struct dp_link_info *link) { @@ -120,7 +113,7 @@ void dp_ctrl_push_idle(struct dp_ctrl *dp_ctrl) IDLE_PATTERN_COMPLETION_TIMEOUT_JIFFIES)) pr_warn("PUSH_IDLE pattern timedout\n"); - pr_debug("mainlink off done\n"); + DRM_DEBUG_DP("mainlink off done\n"); } static void dp_ctrl_config_ctrl(struct dp_ctrl_private *ctrl) @@ -1011,6 +1004,8 @@ static int dp_ctrl_update_vx_px(struct dp_ctrl_private *ctrl) u32 voltage_swing_level = link->phy_params.v_level; u32 pre_emphasis_level = link->phy_params.p_level; + DRM_DEBUG_DP("voltage level: %d emphasis level: %d\n", voltage_swing_level, + pre_emphasis_level); ret = dp_catalog_ctrl_update_vx_px(ctrl->catalog, voltage_swing_level, pre_emphasis_level); @@ -1078,7 +1073,7 @@ static int dp_ctrl_read_link_status(struct dp_ctrl_private *ctrl, } static int dp_ctrl_link_train_1(struct dp_ctrl_private *ctrl, - struct dp_cr_status *cr, int *training_step) + int *training_step) { int tries, old_v_level, ret = 0; u8 link_status[DP_LINK_STATUS_SIZE]; @@ -1107,9 +1102,6 @@ static int dp_ctrl_link_train_1(struct dp_ctrl_private *ctrl, if (ret) return ret; - cr->lane_0_1 = link_status[0]; - cr->lane_2_3 = link_status[1]; - if (drm_dp_clock_recovery_ok(link_status, ctrl->link->link_params.num_lanes)) { return 0; @@ -1186,7 +1178,7 @@ static void dp_ctrl_clear_training_pattern(struct dp_ctrl_private *ctrl) } static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, - struct dp_cr_status *cr, int *training_step) + int *training_step) { int tries = 0, ret = 0; char pattern; @@ -1202,10 +1194,6 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, else pattern = DP_TRAINING_PATTERN_2; - ret = dp_ctrl_update_vx_px(ctrl); - if (ret) - return ret; - ret = dp_catalog_ctrl_set_pattern(ctrl->catalog, pattern); if (ret) return ret; @@ -1218,8 +1206,6 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, ret = dp_ctrl_read_link_status(ctrl, link_status); if (ret) return ret; - cr->lane_0_1 = link_status[0]; - cr->lane_2_3 = link_status[1]; if (drm_dp_channel_eq_ok(link_status, ctrl->link->link_params.num_lanes)) { @@ -1239,7 +1225,7 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, static int dp_ctrl_reinitialize_mainlink(struct dp_ctrl_private *ctrl); static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, - struct dp_cr_status *cr, int *training_step) + int *training_step) { int ret = 0; u8 encoding = DP_SET_ANSI_8B10B; @@ -1255,7 +1241,7 @@ static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, drm_dp_dpcd_write(ctrl->aux, DP_MAIN_LINK_CHANNEL_CODING_SET, &encoding, 1); - ret = dp_ctrl_link_train_1(ctrl, cr, training_step); + ret = dp_ctrl_link_train_1(ctrl, training_step); if (ret) { DRM_ERROR("link training #1 failed. ret=%d\n", ret); goto end; @@ -1264,7 +1250,7 @@ static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, /* print success info as this is a result of user initiated action */ DRM_DEBUG_DP("link training #1 successful\n"); - ret = dp_ctrl_link_train_2(ctrl, cr, training_step); + ret = dp_ctrl_link_train_2(ctrl, training_step); if (ret) { DRM_ERROR("link training #2 failed. ret=%d\n", ret); goto end; @@ -1280,7 +1266,7 @@ end: } static int dp_ctrl_setup_main_link(struct dp_ctrl_private *ctrl, - struct dp_cr_status *cr, int *training_step) + int *training_step) { int ret = 0; @@ -1295,7 +1281,7 @@ static int dp_ctrl_setup_main_link(struct dp_ctrl_private *ctrl, * a link training pattern, we have to first do soft reset. */ - ret = dp_ctrl_link_train(ctrl, cr, training_step); + ret = dp_ctrl_link_train(ctrl, training_step); return ret; } @@ -1382,6 +1368,7 @@ int dp_ctrl_host_init(struct dp_ctrl *dp_ctrl, bool flip, bool reset) if (reset) dp_catalog_ctrl_reset(ctrl->catalog); + DRM_DEBUG_DP("flip=%d\n", flip); dp_catalog_ctrl_phy_reset(ctrl->catalog); phy_init(phy); dp_catalog_ctrl_enable_irq(ctrl->catalog, true); @@ -1492,14 +1479,16 @@ static int dp_ctrl_deinitialize_mainlink(struct dp_ctrl_private *ctrl) static int dp_ctrl_link_maintenance(struct dp_ctrl_private *ctrl) { int ret = 0; - struct dp_cr_status cr; int training_step = DP_TRAINING_NONE; dp_ctrl_push_idle(&ctrl->dp_ctrl); + ctrl->link->phy_params.p_level = 0; + ctrl->link->phy_params.v_level = 0; + ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; - ret = dp_ctrl_setup_main_link(ctrl, &cr, &training_step); + ret = dp_ctrl_setup_main_link(ctrl, &training_step); if (ret) goto end; @@ -1526,7 +1515,7 @@ static int dp_ctrl_process_phy_test_request(struct dp_ctrl_private *ctrl) * running. Add the global reset just before disabling the * link clocks and core clocks. */ - ret = dp_ctrl_off(&ctrl->dp_ctrl); + ret = dp_ctrl_off_link_stream(&ctrl->dp_ctrl); if (ret) { DRM_ERROR("failed to disable DP controller\n"); return ret; @@ -1630,6 +1619,35 @@ void dp_ctrl_handle_sink_request(struct dp_ctrl *dp_ctrl) } } +static bool dp_ctrl_clock_recovery_any_ok( + const u8 link_status[DP_LINK_STATUS_SIZE], + int lane_count) +{ + int reduced_cnt; + + if (lane_count <= 1) + return false; + + /* + * only interested in the lane number after reduced + * lane_count = 4, then only interested in 2 lanes + * lane_count = 2, then only interested in 1 lane + */ + reduced_cnt = lane_count >> 1; + + return drm_dp_clock_recovery_ok(link_status, reduced_cnt); +} + +static bool dp_ctrl_channel_eq_ok(struct dp_ctrl_private *ctrl) +{ + u8 link_status[DP_LINK_STATUS_SIZE]; + int num_lanes = ctrl->link->link_params.num_lanes; + + dp_ctrl_read_link_status(ctrl, link_status); + + return drm_dp_channel_eq_ok(link_status, num_lanes); +} + int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) { int rc = 0; @@ -1637,7 +1655,7 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) u32 rate = 0; int link_train_max_retries = 5; u32 const phy_cts_pixel_clk_khz = 148500; - struct dp_cr_status cr; + u8 link_status[DP_LINK_STATUS_SIZE]; unsigned int training_step; if (!dp_ctrl) @@ -1664,6 +1682,9 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) ctrl->link->link_params.rate, ctrl->link->link_params.num_lanes, ctrl->dp_ctrl.pixel_rate); + ctrl->link->phy_params.p_level = 0; + ctrl->link->phy_params.v_level = 0; + rc = dp_ctrl_enable_mainlink_clocks(ctrl); if (rc) return rc; @@ -1677,19 +1698,21 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) } training_step = DP_TRAINING_NONE; - rc = dp_ctrl_setup_main_link(ctrl, &cr, &training_step); + rc = dp_ctrl_setup_main_link(ctrl, &training_step); if (rc == 0) { /* training completed successfully */ break; } else if (training_step == DP_TRAINING_1) { /* link train_1 failed */ - if (!dp_catalog_link_is_connected(ctrl->catalog)) { + if (!dp_catalog_link_is_connected(ctrl->catalog)) break; - } + + dp_ctrl_read_link_status(ctrl, link_status); rc = dp_ctrl_link_rate_down_shift(ctrl); if (rc < 0) { /* already in RBR = 1.6G */ - if (cr.lane_0_1 & DP_LANE0_1_CR_DONE) { + if (dp_ctrl_clock_recovery_any_ok(link_status, + ctrl->link->link_params.num_lanes)) { /* * some lanes are ready, * reduce lane number @@ -1705,12 +1728,18 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) } } } else if (training_step == DP_TRAINING_2) { - /* link train_2 failed, lower lane rate */ - if (!dp_catalog_link_is_connected(ctrl->catalog)) { + /* link train_2 failed */ + if (!dp_catalog_link_is_connected(ctrl->catalog)) break; - } - rc = dp_ctrl_link_lane_down_shift(ctrl); + dp_ctrl_read_link_status(ctrl, link_status); + + if (!drm_dp_clock_recovery_ok(link_status, + ctrl->link->link_params.num_lanes)) + rc = dp_ctrl_link_rate_down_shift(ctrl); + else + rc = dp_ctrl_link_lane_down_shift(ctrl); + if (rc < 0) { /* end with failure */ break; /* lane == 1 already */ @@ -1721,17 +1750,19 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) if (ctrl->link->sink_request & DP_TEST_LINK_PHY_TEST_PATTERN) return rc; - /* stop txing train pattern */ - dp_ctrl_clear_training_pattern(ctrl); + if (rc == 0) { /* link train successfully */ + /* + * do not stop train pattern here + * stop link training at on_stream + * to pass compliance test + */ + } else { + /* + * link training failed + * end txing train pattern here + */ + dp_ctrl_clear_training_pattern(ctrl); - /* - * keep transmitting idle pattern until video ready - * to avoid main link from loss of sync - */ - if (rc == 0) /* link train successfully */ - dp_ctrl_push_idle(dp_ctrl); - else { - /* link training failed */ dp_ctrl_deinitialize_mainlink(ctrl); rc = -ECONNRESET; } @@ -1739,9 +1770,15 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) return rc; } +static int dp_ctrl_link_retrain(struct dp_ctrl_private *ctrl) +{ + int training_step = DP_TRAINING_NONE; + + return dp_ctrl_setup_main_link(ctrl, &training_step); +} + int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl) { - u32 rate = 0; int ret = 0; bool mainlink_ready = false; struct dp_ctrl_private *ctrl; @@ -1751,10 +1788,6 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl) ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl); - rate = ctrl->panel->link_info.rate; - - ctrl->link->link_params.rate = rate; - ctrl->link->link_params.num_lanes = ctrl->panel->link_info.num_lanes; ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; DRM_DEBUG_DP("rate=%d, num_lanes=%d, pixel_rate=%d\n", @@ -1769,6 +1802,12 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl) } } + if (!dp_ctrl_channel_eq_ok(ctrl)) + dp_ctrl_link_retrain(ctrl); + + /* stop txing train pattern to end link training */ + dp_ctrl_clear_training_pattern(ctrl); + ret = dp_ctrl_enable_stream_clocks(ctrl); if (ret) { DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 051c1be1de7e..fbe4c2cd52a3 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -55,7 +55,6 @@ enum { EV_HPD_INIT_SETUP, EV_HPD_PLUG_INT, EV_IRQ_HPD_INT, - EV_HPD_REPLUG_INT, EV_HPD_UNPLUG_INT, EV_USER_NOTIFICATION, EV_CONNECT_PENDING_TIMEOUT, @@ -102,8 +101,6 @@ struct dp_display_private { struct dp_display_mode dp_mode; struct msm_dp dp_display; - bool encoder_mode_set; - /* wait for audio signaling */ struct completion audio_comp; @@ -219,6 +216,7 @@ static int dp_display_bind(struct device *dev, struct device *master, goto end; } + dp->aux->drm_dev = drm; rc = dp_aux_register(dp->aux); if (rc) { DRM_ERROR("DRM DP AUX register failed\n"); @@ -267,6 +265,8 @@ static bool dp_display_is_ds_bridge(struct dp_panel *panel) static bool dp_display_is_sink_count_zero(struct dp_display_private *dp) { + DRM_DEBUG_DP("present=%#x sink_count=%d\n", dp->panel->dpcd[DP_DOWNSTREAMPORT_PRESENT], + dp->link->sink_count); return dp_display_is_ds_bridge(dp->panel) && (dp->link->sink_count == 0); } @@ -283,20 +283,6 @@ static void dp_display_send_hpd_event(struct msm_dp *dp_display) } -static void dp_display_set_encoder_mode(struct dp_display_private *dp) -{ - struct msm_drm_private *priv = dp->dp_display.drm_dev->dev_private; - struct msm_kms *kms = priv->kms; - - if (!dp->encoder_mode_set && dp->dp_display.encoder && - kms->funcs->set_encoder_mode) { - kms->funcs->set_encoder_mode(kms, - dp->dp_display.encoder, false); - - dp->encoder_mode_set = true; - } -} - static int dp_display_send_hpd_notification(struct dp_display_private *dp, bool hpd) { @@ -312,6 +298,7 @@ static int dp_display_send_hpd_notification(struct dp_display_private *dp, dp->dp_display.is_connected = hpd; + DRM_DEBUG_DP("hpd=%d\n", hpd); dp_display_send_hpd_event(&dp->dp_display); return 0; @@ -361,6 +348,7 @@ static void dp_display_host_init(struct dp_display_private *dp, int reset) { bool flip = false; + DRM_DEBUG_DP("core_initialized=%d\n", dp->core_initialized); if (dp->core_initialized) { DRM_DEBUG_DP("DP core already initialized\n"); return; @@ -369,8 +357,6 @@ static void dp_display_host_init(struct dp_display_private *dp, int reset) if (dp->usbpd->orientation == ORIENTATION_CC2) flip = true; - dp_display_set_encoder_mode(dp); - dp_power_init(dp->power, flip); dp_ctrl_host_init(dp->ctrl, flip, reset); dp_aux_init(dp->aux); @@ -465,8 +451,10 @@ static int dp_display_handle_irq_hpd(struct dp_display_private *dp) { u32 sink_request = dp->link->sink_request; + DRM_DEBUG_DP("%d\n", sink_request); if (dp->hpd_state == ST_DISCONNECTED) { if (sink_request & DP_LINK_STATUS_UPDATED) { + DRM_DEBUG_DP("Disconnected sink_request: %d\n", sink_request); DRM_ERROR("Disconnected, no DP_LINK_STATUS_UPDATED\n"); return -EINVAL; } @@ -498,6 +486,7 @@ static int dp_display_usbpd_attention_cb(struct device *dev) rc = dp_link_process_request(dp->link); if (!rc) { sink_request = dp->link->sink_request; + DRM_DEBUG_DP("hpd_state=%d sink_request=%d\n", dp->hpd_state, sink_request); if (sink_request & DS_PORT_STATUS_CHANGED) rc = dp_display_handle_port_ststus_changed(dp); else @@ -520,6 +509,7 @@ static int dp_hpd_plug_handle(struct dp_display_private *dp, u32 data) mutex_lock(&dp->event_mutex); state = dp->hpd_state; + DRM_DEBUG_DP("hpd_state=%d\n", state); if (state == ST_DISPLAY_OFF || state == ST_SUSPENDED) { mutex_unlock(&dp->event_mutex); return 0; @@ -655,6 +645,7 @@ static int dp_hpd_unplug_handle(struct dp_display_private *dp, u32 data) /* start sentinel checking in case of missing uevent */ dp_add_event(dp, EV_DISCONNECT_PENDING_TIMEOUT, 0, DP_TIMEOUT_5_SECOND); + DRM_DEBUG_DP("hpd_state=%d\n", state); /* signal the disconnect event early to ensure proper teardown */ dp_display_handle_plugged_change(g_dp_display, false); @@ -713,6 +704,7 @@ static int dp_irq_hpd_handle(struct dp_display_private *dp, u32 data) if (ret == -ECONNRESET) { /* cable unplugged */ dp->core_initialized = false; } + DRM_DEBUG_DP("hpd_state=%d\n", state); mutex_unlock(&dp->event_mutex); @@ -854,6 +846,7 @@ static int dp_display_enable(struct dp_display_private *dp, u32 data) dp_display = g_dp_display; + DRM_DEBUG_DP("sink_count=%d\n", dp->link->sink_count); if (dp_display->power_on) { DRM_DEBUG_DP("Link already setup, return\n"); return 0; @@ -915,6 +908,7 @@ static int dp_display_disable(struct dp_display_private *dp, u32 data) dp_display->power_on = false; + DRM_DEBUG_DP("sink count: %d\n", dp->link->sink_count); return 0; } @@ -1014,10 +1008,8 @@ int dp_display_get_test_bpp(struct msm_dp *dp) void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp) { struct dp_display_private *dp_display; - struct drm_device *drm; dp_display = container_of(dp, struct dp_display_private, dp_display); - drm = dp->drm_dev; /* * if we are reading registers we need the link clocks to be on @@ -1118,9 +1110,6 @@ static int hpd_event_thread(void *data) case EV_IRQ_HPD_INT: dp_irq_hpd_handle(dp_priv, todo->data); break; - case EV_HPD_REPLUG_INT: - /* do nothing */ - break; case EV_USER_NOTIFICATION: dp_display_send_hpd_notification(dp_priv, todo->data); @@ -1162,12 +1151,11 @@ static irqreturn_t dp_display_irq_handler(int irq, void *dev_id) hpd_isr_status = dp_catalog_hpd_get_intr_status(dp->catalog); + DRM_DEBUG_DP("hpd isr status=%#x\n", hpd_isr_status); if (hpd_isr_status & 0x0F) { /* hpd related interrupts */ - if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK || - hpd_isr_status & DP_DP_HPD_REPLUG_INT_MASK) { + if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK) dp_add_event(dp, EV_HPD_PLUG_INT, 0, 0); - } if (hpd_isr_status & DP_DP_IRQ_HPD_INT_MASK) { /* stop sentinel connect pending checking */ @@ -1175,8 +1163,10 @@ static irqreturn_t dp_display_irq_handler(int irq, void *dev_id) dp_add_event(dp, EV_IRQ_HPD_INT, 0, 0); } - if (hpd_isr_status & DP_DP_HPD_REPLUG_INT_MASK) - dp_add_event(dp, EV_HPD_REPLUG_INT, 0, 0); + if (hpd_isr_status & DP_DP_HPD_REPLUG_INT_MASK) { + dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0); + dp_add_event(dp, EV_HPD_PLUG_INT, 0, 3); + } if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK) dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0); @@ -1285,12 +1275,15 @@ static int dp_pm_resume(struct device *dev) struct platform_device *pdev = to_platform_device(dev); struct msm_dp *dp_display = platform_get_drvdata(pdev); struct dp_display_private *dp; - u32 status; + int sink_count = 0; dp = container_of(dp_display, struct dp_display_private, dp_display); mutex_lock(&dp->event_mutex); + DRM_DEBUG_DP("Before, core_inited=%d power_on=%d\n", + dp->core_initialized, dp_display->power_on); + /* start from disconnected state */ dp->hpd_state = ST_DISCONNECTED; @@ -1299,18 +1292,36 @@ static int dp_pm_resume(struct device *dev) dp_catalog_ctrl_hpd_config(dp->catalog); - status = dp_catalog_link_is_connected(dp->catalog); + /* + * set sink to normal operation mode -- D0 + * before dpcd read + */ + dp_link_psm_config(dp->link, &dp->panel->link_info, false); + + if (dp_catalog_link_is_connected(dp->catalog)) { + sink_count = drm_dp_read_sink_count(dp->aux); + if (sink_count < 0) + sink_count = 0; + } + dp->link->sink_count = sink_count; /* * can not declared display is connected unless * HDMI cable is plugged in and sink_count of * dongle become 1 */ - if (status && dp->link->sink_count) + if (dp->link->sink_count) dp->dp_display.is_connected = true; else dp->dp_display.is_connected = false; + dp_display_handle_plugged_change(g_dp_display, + dp->dp_display.is_connected); + + DRM_DEBUG_DP("After, sink_count=%d is_connected=%d core_inited=%d power_on=%d\n", + dp->link->sink_count, dp->dp_display.is_connected, + dp->core_initialized, dp_display->power_on); + mutex_unlock(&dp->event_mutex); return 0; @@ -1326,6 +1337,9 @@ static int dp_pm_suspend(struct device *dev) mutex_lock(&dp->event_mutex); + DRM_DEBUG_DP("Before, core_inited=%d power_on=%d\n", + dp->core_initialized, dp_display->power_on); + if (dp->core_initialized == true) { /* mainlink enabled */ if (dp_power_clk_status(dp->power, DP_CTRL_PM)) @@ -1339,6 +1353,9 @@ static int dp_pm_suspend(struct device *dev) /* host_init will be called at pm_resume */ dp->core_initialized = false; + DRM_DEBUG_DP("After, core_inited=%d power_on=%d\n", + dp->core_initialized, dp_display->power_on); + mutex_unlock(&dp->event_mutex); return 0; diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c index 1195044a7a3b..a5bdfc5029de 100644 --- a/drivers/gpu/drm/msm/dp/dp_link.c +++ b/drivers/gpu/drm/msm/dp/dp_link.c @@ -1027,43 +1027,29 @@ int dp_link_process_request(struct dp_link *dp_link) if (link->request.test_requested == DP_TEST_LINK_EDID_READ) { dp_link->sink_request |= DP_TEST_LINK_EDID_READ; - return ret; - } - - ret = dp_link_process_ds_port_status_change(link); - if (!ret) { + } else if (!dp_link_process_ds_port_status_change(link)) { dp_link->sink_request |= DS_PORT_STATUS_CHANGED; - return ret; - } - - ret = dp_link_process_link_training_request(link); - if (!ret) { + } else if (!dp_link_process_link_training_request(link)) { dp_link->sink_request |= DP_TEST_LINK_TRAINING; - return ret; - } - - ret = dp_link_process_phy_test_pattern_request(link); - if (!ret) { + } else if (!dp_link_process_phy_test_pattern_request(link)) { dp_link->sink_request |= DP_TEST_LINK_PHY_TEST_PATTERN; - return ret; - } - - ret = dp_link_process_link_status_update(link); - if (!ret) { - dp_link->sink_request |= DP_LINK_STATUS_UPDATED; - return ret; - } - - if (dp_link_is_video_pattern_requested(link)) { - ret = 0; - dp_link->sink_request |= DP_TEST_LINK_VIDEO_PATTERN; - } - - if (dp_link_is_audio_pattern_requested(link)) { - dp_link->sink_request |= DP_TEST_LINK_AUDIO_PATTERN; - return -EINVAL; + } else { + ret = dp_link_process_link_status_update(link); + if (!ret) { + dp_link->sink_request |= DP_LINK_STATUS_UPDATED; + } else { + if (dp_link_is_video_pattern_requested(link)) { + ret = 0; + dp_link->sink_request |= DP_TEST_LINK_VIDEO_PATTERN; + } + if (dp_link_is_audio_pattern_requested(link)) { + dp_link->sink_request |= DP_TEST_LINK_AUDIO_PATTERN; + ret = -EINVAL; + } + } } + DRM_DEBUG_DP("sink request=%#x", dp_link->sink_request); return ret; } diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index 440b32753430..2181b60e1d1d 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -271,7 +271,7 @@ static u8 dp_panel_get_edid_checksum(struct edid *edid) { struct edid *last_block; u8 *raw_edid; - bool is_edid_corrupt; + bool is_edid_corrupt = false; if (!edid) { DRM_ERROR("invalid edid input\n"); @@ -303,7 +303,12 @@ void dp_panel_handle_sink_request(struct dp_panel *dp_panel) panel = container_of(dp_panel, struct dp_panel_private, dp_panel); if (panel->link->sink_request & DP_TEST_LINK_EDID_READ) { - u8 checksum = dp_panel_get_edid_checksum(dp_panel->edid); + u8 checksum; + + if (dp_panel->edid) + checksum = dp_panel_get_edid_checksum(dp_panel->edid); + else + checksum = dp_panel->connector->real_edid_checksum; dp_link_send_edid_checksum(panel->link, checksum); dp_link_send_test_response(panel->link); diff --git a/drivers/gpu/drm/msm/dp/dp_power.c b/drivers/gpu/drm/msm/dp/dp_power.c index 3961ba4efc3c..b48b45e92bfa 100644 --- a/drivers/gpu/drm/msm/dp/dp_power.c +++ b/drivers/gpu/drm/msm/dp/dp_power.c @@ -208,6 +208,9 @@ static int dp_power_clk_set_rate(struct dp_power_private *power, int dp_power_clk_status(struct dp_power *dp_power, enum dp_pm_type pm_type) { + DRM_DEBUG_DP("core_clk_on=%d link_clk_on=%d stream_clk_on=%d\n", + dp_power->core_clks_on, dp_power->link_clks_on, dp_power->stream_clks_on); + if (pm_type == DP_CORE_PM) return dp_power->core_clks_on; diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c index 75afc12a7b25..614dc7f26f2c 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.c +++ b/drivers/gpu/drm/msm/dsi/dsi.c @@ -13,6 +13,13 @@ struct drm_encoder *msm_dsi_get_encoder(struct msm_dsi *msm_dsi) return msm_dsi->encoder; } +bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi) +{ + unsigned long host_flags = msm_dsi_host_get_mode_flags(msm_dsi->host); + + return !(host_flags & MIPI_DSI_MODE_VIDEO); +} + static int dsi_get_phy(struct msm_dsi *msm_dsi) { struct platform_device *pdev = msm_dsi->pdev; @@ -26,8 +33,10 @@ static int dsi_get_phy(struct msm_dsi *msm_dsi) } phy_pdev = of_find_device_by_node(phy_node); - if (phy_pdev) + if (phy_pdev) { msm_dsi->phy = platform_get_drvdata(phy_pdev); + msm_dsi->phy_dev = &phy_pdev->dev; + } of_node_put(phy_node); @@ -36,8 +45,6 @@ static int dsi_get_phy(struct msm_dsi *msm_dsi) return -EPROBE_DEFER; } - msm_dsi->phy_dev = get_device(&phy_pdev->dev); - return 0; } @@ -244,8 +251,6 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev, goto fail; } - msm_dsi_manager_setup_encoder(msm_dsi->id); - priv->bridges[priv->num_bridges++] = msm_dsi->bridge; priv->connectors[priv->num_connectors++] = msm_dsi->connector; diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h index 9b8e9b07eced..b50db91cb8a7 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.h +++ b/drivers/gpu/drm/msm/dsi/dsi.h @@ -80,10 +80,10 @@ struct drm_connector *msm_dsi_manager_connector_init(u8 id); struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id); int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg); bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len); -void msm_dsi_manager_setup_encoder(int id); int msm_dsi_manager_register(struct msm_dsi *msm_dsi); void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi); bool msm_dsi_manager_validate_current_config(u8 id); +void msm_dsi_manager_tpg_enable(void); /* msm dsi */ static inline bool msm_dsi_device_connected(struct msm_dsi *msm_dsi) @@ -109,7 +109,7 @@ int msm_dsi_host_enable(struct mipi_dsi_host *host); int msm_dsi_host_disable(struct mipi_dsi_host *host); int msm_dsi_host_power_on(struct mipi_dsi_host *host, struct msm_dsi_phy_shared_timings *phy_shared_timings, - bool is_dual_dsi); + bool is_bonded_dsi, struct msm_dsi_phy *phy); int msm_dsi_host_power_off(struct mipi_dsi_host *host); int msm_dsi_host_set_display_mode(struct mipi_dsi_host *host, const struct drm_display_mode *mode); @@ -123,7 +123,7 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host, void msm_dsi_host_reset_phy(struct mipi_dsi_host *host); void msm_dsi_host_get_phy_clk_req(struct mipi_dsi_host *host, struct msm_dsi_phy_clk_request *clk_req, - bool is_dual_dsi); + bool is_bonded_dsi); void msm_dsi_host_destroy(struct mipi_dsi_host *host); int msm_dsi_host_modeset_init(struct mipi_dsi_host *host, struct drm_device *dev); @@ -145,9 +145,11 @@ int dsi_dma_base_get_6g(struct msm_dsi_host *msm_host, uint64_t *iova); int dsi_dma_base_get_v2(struct msm_dsi_host *msm_host, uint64_t *iova); int dsi_clk_init_v2(struct msm_dsi_host *msm_host); int dsi_clk_init_6g_v2(struct msm_dsi_host *msm_host); -int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_dual_dsi); -int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_dual_dsi); +int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_bonded_dsi); +int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi); void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_host *host); +void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host); + /* dsi phy */ struct msm_dsi_phy; struct msm_dsi_phy_shared_timings { @@ -164,10 +166,9 @@ struct msm_dsi_phy_clk_request { void msm_dsi_phy_driver_register(void); void msm_dsi_phy_driver_unregister(void); int msm_dsi_phy_enable(struct msm_dsi_phy *phy, - struct msm_dsi_phy_clk_request *clk_req); + struct msm_dsi_phy_clk_request *clk_req, + struct msm_dsi_phy_shared_timings *shared_timings); void msm_dsi_phy_disable(struct msm_dsi_phy *phy); -void msm_dsi_phy_get_shared_timings(struct msm_dsi_phy *phy, - struct msm_dsi_phy_shared_timings *shared_timing); void msm_dsi_phy_set_usecase(struct msm_dsi_phy *phy, enum msm_dsi_phy_usecase uc); int msm_dsi_phy_get_clk_provider(struct msm_dsi_phy *phy, @@ -175,6 +176,7 @@ int msm_dsi_phy_get_clk_provider(struct msm_dsi_phy *phy, void msm_dsi_phy_pll_save_state(struct msm_dsi_phy *phy); int msm_dsi_phy_pll_restore_state(struct msm_dsi_phy *phy); void msm_dsi_phy_snapshot(struct msm_disp_state *disp_state, struct msm_dsi_phy *phy); +bool msm_dsi_phy_set_continuous_clock(struct msm_dsi_phy *phy, bool enable); #endif /* __DSI_CONNECTOR_H__ */ diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h index eadbcc78fd72..49b551ad1bff 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.xml.h +++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h @@ -105,6 +105,32 @@ enum dsi_lane_swap { LANE_SWAP_3210 = 7, }; +enum video_config_bpp { + VIDEO_CONFIG_18BPP = 0, + VIDEO_CONFIG_24BPP = 1, +}; + +enum video_pattern_sel { + VID_PRBS = 0, + VID_INCREMENTAL = 1, + VID_FIXED = 2, + VID_MDSS_GENERAL_PATTERN = 3, +}; + +enum cmd_mdp_stream0_pattern_sel { + CMD_MDP_PRBS = 0, + CMD_MDP_INCREMENTAL = 1, + CMD_MDP_FIXED = 2, + CMD_MDP_MDSS_GENERAL_PATTERN = 3, +}; + +enum cmd_dma_pattern_sel { + CMD_DMA_PRBS = 0, + CMD_DMA_INCREMENTAL = 1, + CMD_DMA_FIXED = 2, + CMD_DMA_CUSTOM_PATTERN_DMA_FIFO = 3, +}; + #define DSI_IRQ_CMD_DMA_DONE 0x00000001 #define DSI_IRQ_MASK_CMD_DMA_DONE 0x00000002 #define DSI_IRQ_CMD_MDP_DONE 0x00000100 @@ -518,6 +544,7 @@ static inline uint32_t DSI_CLKOUT_TIMING_CTRL_T_CLK_POST(uint32_t val) #define DSI_LANE_STATUS_DLN0_DIRECTION 0x00010000 #define REG_DSI_LANE_CTRL 0x000000a8 +#define DSI_LANE_CTRL_HS_REQ_SEL_PHY 0x01000000 #define DSI_LANE_CTRL_CLKLN_HS_FORCE_REQUEST 0x10000000 #define REG_DSI_LANE_SWAP_CTRL 0x000000ac @@ -564,6 +591,53 @@ static inline uint32_t DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL(enum dsi_lane_swap val) #define REG_DSI_PHY_RESET 0x00000128 #define DSI_PHY_RESET_RESET 0x00000001 +#define REG_DSI_TEST_PATTERN_GEN_VIDEO_INIT_VAL 0x00000160 + +#define REG_DSI_TPG_MAIN_CONTROL 0x00000198 +#define DSI_TPG_MAIN_CONTROL_CHECKERED_RECTANGLE_PATTERN 0x00000100 + +#define REG_DSI_TPG_VIDEO_CONFIG 0x000001a0 +#define DSI_TPG_VIDEO_CONFIG_BPP__MASK 0x00000003 +#define DSI_TPG_VIDEO_CONFIG_BPP__SHIFT 0 +static inline uint32_t DSI_TPG_VIDEO_CONFIG_BPP(enum video_config_bpp val) +{ + return ((val) << DSI_TPG_VIDEO_CONFIG_BPP__SHIFT) & DSI_TPG_VIDEO_CONFIG_BPP__MASK; +} +#define DSI_TPG_VIDEO_CONFIG_RGB 0x00000004 + +#define REG_DSI_TEST_PATTERN_GEN_CTRL 0x00000158 +#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__MASK 0x00030000 +#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__SHIFT 16 +static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL(enum cmd_dma_pattern_sel val) +{ + return ((val) << DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__MASK; +} +#define DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__MASK 0x00000300 +#define DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__SHIFT 8 +static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL(enum cmd_mdp_stream0_pattern_sel val) +{ + return ((val) << DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__MASK; +} +#define DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__MASK 0x00000030 +#define DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__SHIFT 4 +static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL(enum video_pattern_sel val) +{ + return ((val) << DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__MASK; +} +#define DSI_TEST_PATTERN_GEN_CTRL_TPG_DMA_FIFO_MODE 0x00000004 +#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_TPG_EN 0x00000002 +#define DSI_TEST_PATTERN_GEN_CTRL_EN 0x00000001 + +#define REG_DSI_TEST_PATTERN_GEN_CMD_MDP_INIT_VAL0 0x00000168 + +#define REG_DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER 0x00000180 +#define DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER_SW_TRIGGER 0x00000001 + +#define REG_DSI_TPG_MAIN_CONTROL2 0x0000019c +#define DSI_TPG_MAIN_CONTROL2_CMD_MDP0_CHECKERED_RECTANGLE_PATTERN 0x00000080 +#define DSI_TPG_MAIN_CONTROL2_CMD_MDP1_CHECKERED_RECTANGLE_PATTERN 0x00010000 +#define DSI_TPG_MAIN_CONTROL2_CMD_MDP2_CHECKERED_RECTANGLE_PATTERN 0x02000000 + #define REG_DSI_T_CLK_PRE_EXTEND 0x0000017c #define DSI_T_CLK_PRE_EXTEND_INC_BY_2_BYTECLK 0x00000001 diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c index f3f1c03c7db9..96bbc8b6d009 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c @@ -32,9 +32,8 @@ static const char * const dsi_6g_bus_clk_names[] = { static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, .reg_cfg = { - .num = 4, + .num = 3, .regs = { - {"gdsc", -1, -1}, {"vdd", 150000, 100}, /* 3.0 V */ {"vdda", 100000, 100}, /* 1.2 V */ {"vddio", 100000, 100}, /* 1.8 V */ @@ -53,9 +52,8 @@ static const char * const dsi_8916_bus_clk_names[] = { static const struct msm_dsi_config msm8916_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, .reg_cfg = { - .num = 3, + .num = 2, .regs = { - {"gdsc", -1, -1}, {"vdda", 100000, 100}, /* 1.2 V */ {"vddio", 100000, 100}, /* 1.8 V */ }, @@ -73,9 +71,8 @@ static const char * const dsi_8976_bus_clk_names[] = { static const struct msm_dsi_config msm8976_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, .reg_cfg = { - .num = 3, + .num = 2, .regs = { - {"gdsc", -1, -1}, {"vdda", 100000, 100}, /* 1.2 V */ {"vddio", 100000, 100}, /* 1.8 V */ }, @@ -89,9 +86,8 @@ static const struct msm_dsi_config msm8976_dsi_cfg = { static const struct msm_dsi_config msm8994_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, .reg_cfg = { - .num = 7, + .num = 6, .regs = { - {"gdsc", -1, -1}, {"vdda", 100000, 100}, /* 1.25 V */ {"vddio", 100000, 100}, /* 1.8 V */ {"vcca", 10000, 100}, /* 1.0 V */ @@ -154,7 +150,6 @@ static const struct msm_dsi_config sdm660_dsi_cfg = { .reg_cfg = { .num = 2, .regs = { - {"vdd", 73400, 32 }, /* 0.9 V */ {"vdda", 12560, 4 }, /* 1.2 V */ }, }, @@ -200,6 +195,24 @@ static const struct msm_dsi_config sc7180_dsi_cfg = { .num_dsi = 1, }; +static const char * const dsi_sc7280_bus_clk_names[] = { + "iface", "bus", +}; + +static const struct msm_dsi_config sc7280_dsi_cfg = { + .io_offset = DSI_6G_REG_SHIFT, + .reg_cfg = { + .num = 1, + .regs = { + {"vdda", 8350, 0 }, /* 1.2 V */ + }, + }, + .bus_clk_names = dsi_sc7280_bus_clk_names, + .num_bus_clks = ARRAY_SIZE(dsi_sc7280_bus_clk_names), + .io_start = { 0xae94000 }, + .num_dsi = 1, +}; + static const struct msm_dsi_host_cfg_ops msm_dsi_v2_host_ops = { .link_clk_set_rate = dsi_link_clk_set_rate_v2, .link_clk_enable = dsi_link_clk_enable_v2, @@ -267,6 +280,8 @@ static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = { &sdm845_dsi_cfg, &msm_dsi_6g_v2_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_4_1, &sc7180_dsi_cfg, &msm_dsi_6g_v2_host_ops}, + {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_5_0, + &sc7280_dsi_cfg, &msm_dsi_6g_v2_host_ops}, }; const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor) diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h index ade9b609c7d9..41e99a9fb5de 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h @@ -24,6 +24,7 @@ #define MSM_DSI_6G_VER_MINOR_V2_3_0 0x20030000 #define MSM_DSI_6G_VER_MINOR_V2_4_0 0x20040000 #define MSM_DSI_6G_VER_MINOR_V2_4_1 0x20040001 +#define MSM_DSI_6G_VER_MINOR_V2_5_0 0x20050000 #define MSM_DSI_V2_VER_MINOR_8064 0x0 @@ -47,7 +48,7 @@ struct msm_dsi_host_cfg_ops { void* (*tx_buf_get)(struct msm_dsi_host *msm_host); void (*tx_buf_put)(struct msm_dsi_host *msm_host); int (*dma_base_get)(struct msm_dsi_host *msm_host, uint64_t *iova); - int (*calc_clk_rate)(struct msm_dsi_host *msm_host, bool is_dual_dsi); + int (*calc_clk_rate)(struct msm_dsi_host *msm_host, bool is_bonded_dsi); }; struct msm_dsi_cfg_handler { diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index b466a4af7c3e..e269df285136 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -27,6 +27,7 @@ #include "dsi_cfg.h" #include "msm_kms.h" #include "msm_gem.h" +#include "phy/dsi_phy.h" #define DSI_RESET_TOGGLE_DELAY_MS 20 @@ -167,6 +168,9 @@ struct msm_dsi_host { int dlane_swap; int num_data_lanes; + /* from phy DT */ + bool cphy_mode; + u32 dma_cmd_ctrl_restore; bool registered; @@ -203,35 +207,22 @@ static const struct msm_dsi_cfg_handler *dsi_get_config( { const struct msm_dsi_cfg_handler *cfg_hnd = NULL; struct device *dev = &msm_host->pdev->dev; - struct regulator *gdsc_reg; struct clk *ahb_clk; int ret; u32 major = 0, minor = 0; - gdsc_reg = regulator_get(dev, "gdsc"); - if (IS_ERR(gdsc_reg)) { - pr_err("%s: cannot get gdsc\n", __func__); - goto exit; - } - ahb_clk = msm_clk_get(msm_host->pdev, "iface"); if (IS_ERR(ahb_clk)) { pr_err("%s: cannot get interface clock\n", __func__); - goto put_gdsc; + goto exit; } pm_runtime_get_sync(dev); - ret = regulator_enable(gdsc_reg); - if (ret) { - pr_err("%s: unable to enable gdsc\n", __func__); - goto put_gdsc; - } - ret = clk_prepare_enable(ahb_clk); if (ret) { pr_err("%s: unable to enable ahb_clk\n", __func__); - goto disable_gdsc; + goto runtime_put; } ret = dsi_get_version(msm_host->ctrl_base, &major, &minor); @@ -246,11 +237,8 @@ static const struct msm_dsi_cfg_handler *dsi_get_config( disable_clks: clk_disable_unprepare(ahb_clk); -disable_gdsc: - regulator_disable(gdsc_reg); +runtime_put: pm_runtime_put_sync(dev); -put_gdsc: - regulator_put(gdsc_reg); exit: return cfg_hnd; } @@ -510,6 +498,7 @@ int msm_dsi_runtime_resume(struct device *dev) int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host) { + u32 byte_intf_rate; int ret; DBG("Set clk rates: pclk=%d, byteclk=%d", @@ -529,8 +518,13 @@ int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host) } if (msm_host->byte_intf_clk) { - ret = clk_set_rate(msm_host->byte_intf_clk, - msm_host->byte_clk_rate / 2); + /* For CPHY, byte_intf_clk is same as byte_clk */ + if (msm_host->cphy_mode) + byte_intf_rate = msm_host->byte_clk_rate; + else + byte_intf_rate = msm_host->byte_clk_rate / 2; + + ret = clk_set_rate(msm_host->byte_intf_clk, byte_intf_rate); if (ret) { pr_err("%s: Failed to set rate byte intf clk, %d\n", __func__, ret); @@ -679,7 +673,7 @@ void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host) clk_disable_unprepare(msm_host->byte_clk); } -static u32 dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_dual_dsi) +static u32 dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { struct drm_display_mode *mode = msm_host->mode; u32 pclk_rate; @@ -687,22 +681,22 @@ static u32 dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_dual_dsi) pclk_rate = mode->clock * 1000; /* - * For dual DSI mode, the current DRM mode has the complete width of the + * For bonded DSI mode, the current DRM mode has the complete width of the * panel. Since, the complete panel is driven by two DSI controllers, * the clock rates have to be split between the two dsi controllers. * Adjust the byte and pixel clock rates for each dsi host accordingly. */ - if (is_dual_dsi) + if (is_bonded_dsi) pclk_rate /= 2; return pclk_rate; } -static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_dual_dsi) +static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { u8 lanes = msm_host->lanes; u32 bpp = dsi_get_bpp(msm_host->format); - u32 pclk_rate = dsi_get_pclk_rate(msm_host, is_dual_dsi); + u32 pclk_rate = dsi_get_pclk_rate(msm_host, is_bonded_dsi); u64 pclk_bpp = (u64)pclk_rate * bpp; if (lanes == 0) { @@ -710,7 +704,11 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_dual_dsi) lanes = 1; } - do_div(pclk_bpp, (8 * lanes)); + /* CPHY "byte_clk" is in units of 16 bits */ + if (msm_host->cphy_mode) + do_div(pclk_bpp, (16 * lanes)); + else + do_div(pclk_bpp, (8 * lanes)); msm_host->pixel_clk_rate = pclk_rate; msm_host->byte_clk_rate = pclk_bpp; @@ -720,28 +718,28 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_dual_dsi) } -int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_dual_dsi) +int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { if (!msm_host->mode) { pr_err("%s: mode not set\n", __func__); return -EINVAL; } - dsi_calc_pclk(msm_host, is_dual_dsi); + dsi_calc_pclk(msm_host, is_bonded_dsi); msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk); return 0; } -int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_dual_dsi) +int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { u32 bpp = dsi_get_bpp(msm_host->format); u64 pclk_bpp; unsigned int esc_mhz, esc_div; unsigned long byte_mhz; - dsi_calc_pclk(msm_host, is_dual_dsi); + dsi_calc_pclk(msm_host, is_bonded_dsi); - pclk_bpp = (u64)dsi_get_pclk_rate(msm_host, is_dual_dsi) * bpp; + pclk_bpp = (u64)dsi_get_pclk_rate(msm_host, is_bonded_dsi) * bpp; do_div(pclk_bpp, 8); msm_host->src_clk_rate = pclk_bpp; @@ -834,7 +832,7 @@ static inline enum dsi_cmd_dst_format dsi_get_cmd_fmt( } static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable, - struct msm_dsi_phy_shared_timings *phy_shared_timings) + struct msm_dsi_phy_shared_timings *phy_shared_timings, struct msm_dsi_phy *phy) { u32 flags = msm_host->mode_flags; enum mipi_dsi_pixel_format mipi_fmt = msm_host->format; @@ -929,6 +927,10 @@ static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable, if (!(flags & MIPI_DSI_CLOCK_NON_CONTINUOUS)) { lane_ctrl = dsi_read(msm_host, REG_DSI_LANE_CTRL); + + if (msm_dsi_phy_set_continuous_clock(phy, enable)) + lane_ctrl &= ~DSI_LANE_CTRL_HS_REQ_SEL_PHY; + dsi_write(msm_host, REG_DSI_LANE_CTRL, lane_ctrl | DSI_LANE_CTRL_CLKLN_HS_FORCE_REQUEST); } @@ -936,9 +938,12 @@ static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable, data |= DSI_CTRL_ENABLE; dsi_write(msm_host, REG_DSI_CTRL, data); + + if (msm_host->cphy_mode) + dsi_write(msm_host, REG_DSI_CPHY_MODE_CTRL, BIT(0)); } -static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_dual_dsi) +static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { struct drm_display_mode *mode = msm_host->mode; u32 hs_start = 0, vs_start = 0; /* take sync start as 0 */ @@ -956,13 +961,13 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_dual_dsi) DBG(""); /* - * For dual DSI mode, the current DRM mode has + * For bonded DSI mode, the current DRM mode has * the complete width of the panel. Since, the complete * panel is driven by two DSI controllers, the horizontal * timings have to be split between the two dsi controllers. * Adjust the DSI host timing values accordingly. */ - if (is_dual_dsi) { + if (is_bonded_dsi) { h_total /= 2; hs_end /= 2; ha_start /= 2; @@ -2226,6 +2231,8 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host, struct clk *byte_clk_provider, *pixel_clk_provider; int ret; + msm_host->cphy_mode = src_phy->cphy_mode; + ret = msm_dsi_phy_get_clk_provider(src_phy, &byte_clk_provider, &pixel_clk_provider); if (ret) { @@ -2285,19 +2292,26 @@ void msm_dsi_host_reset_phy(struct mipi_dsi_host *host) void msm_dsi_host_get_phy_clk_req(struct mipi_dsi_host *host, struct msm_dsi_phy_clk_request *clk_req, - bool is_dual_dsi) + bool is_bonded_dsi) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd; int ret; - ret = cfg_hnd->ops->calc_clk_rate(msm_host, is_dual_dsi); + ret = cfg_hnd->ops->calc_clk_rate(msm_host, is_bonded_dsi); if (ret) { pr_err("%s: unable to calc clk rate, %d\n", __func__, ret); return; } - clk_req->bitclk_rate = msm_host->byte_clk_rate * 8; + /* CPHY transmits 16 bits over 7 clock cycles + * "byte_clk" is in units of 16-bits (see dsi_calc_pclk), + * so multiply by 7 to get the "bitclk rate" + */ + if (msm_host->cphy_mode) + clk_req->bitclk_rate = msm_host->byte_clk_rate * 7; + else + clk_req->bitclk_rate = msm_host->byte_clk_rate * 8; clk_req->escclk_rate = msm_host->esc_clk_rate; } @@ -2354,7 +2368,7 @@ static void msm_dsi_sfpb_config(struct msm_dsi_host *msm_host, bool enable) int msm_dsi_host_power_on(struct mipi_dsi_host *host, struct msm_dsi_phy_shared_timings *phy_shared_timings, - bool is_dual_dsi) + bool is_bonded_dsi, struct msm_dsi_phy *phy) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd; @@ -2392,9 +2406,9 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host, goto fail_disable_clk; } - dsi_timing_setup(msm_host, is_dual_dsi); + dsi_timing_setup(msm_host, is_bonded_dsi); dsi_sw_reset(msm_host); - dsi_ctrl_config(msm_host, true, phy_shared_timings); + dsi_ctrl_config(msm_host, true, phy_shared_timings, phy); if (msm_host->disp_en_gpio) gpiod_set_value(msm_host->disp_en_gpio, 1); @@ -2425,7 +2439,7 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host) goto unlock_ret; } - dsi_ctrl_config(msm_host, false, NULL); + dsi_ctrl_config(msm_host, false, NULL, NULL); if (msm_host->disp_en_gpio) gpiod_set_value(msm_host->disp_en_gpio, 0); @@ -2495,3 +2509,64 @@ void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_ho pm_runtime_put_sync(&msm_host->pdev->dev); } + +static void msm_dsi_host_video_test_pattern_setup(struct msm_dsi_host *msm_host) +{ + u32 reg; + + reg = dsi_read(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL); + + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_VIDEO_INIT_VAL, 0xff); + /* draw checkered rectangle pattern */ + dsi_write(msm_host, REG_DSI_TPG_MAIN_CONTROL, + DSI_TPG_MAIN_CONTROL_CHECKERED_RECTANGLE_PATTERN); + /* use 24-bit RGB test pttern */ + dsi_write(msm_host, REG_DSI_TPG_VIDEO_CONFIG, + DSI_TPG_VIDEO_CONFIG_BPP(VIDEO_CONFIG_24BPP) | + DSI_TPG_VIDEO_CONFIG_RGB); + + reg |= DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL(VID_MDSS_GENERAL_PATTERN); + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL, reg); + + DBG("Video test pattern setup done\n"); +} + +static void msm_dsi_host_cmd_test_pattern_setup(struct msm_dsi_host *msm_host) +{ + u32 reg; + + reg = dsi_read(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL); + + /* initial value for test pattern */ + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_CMD_MDP_INIT_VAL0, 0xff); + + reg |= DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL(CMD_MDP_MDSS_GENERAL_PATTERN); + + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL, reg); + /* draw checkered rectangle pattern */ + dsi_write(msm_host, REG_DSI_TPG_MAIN_CONTROL2, + DSI_TPG_MAIN_CONTROL2_CMD_MDP0_CHECKERED_RECTANGLE_PATTERN); + + DBG("Cmd test pattern setup done\n"); +} + +void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host) +{ + struct msm_dsi_host *msm_host = to_msm_dsi_host(host); + bool is_video_mode = !!(msm_host->mode_flags & MIPI_DSI_MODE_VIDEO); + u32 reg; + + if (is_video_mode) + msm_dsi_host_video_test_pattern_setup(msm_host); + else + msm_dsi_host_cmd_test_pattern_setup(msm_host); + + reg = dsi_read(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL); + /* enable the test pattern generator */ + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_CTRL, (reg | DSI_TEST_PATTERN_GEN_CTRL_EN)); + + /* for command mode need to trigger one frame from tpg */ + if (!is_video_mode) + dsi_write(msm_host, REG_DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER, + DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER_SW_TRIGGER); +} diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c index 4ebfedc4a9ac..c41d39f5b7cf 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_manager.c +++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c @@ -21,14 +21,14 @@ struct msm_dsi_manager { struct msm_dsi *dsi[DSI_MAX]; - bool is_dual_dsi; + bool is_bonded_dsi; bool is_sync_needed; int master_dsi_link_id; }; static struct msm_dsi_manager msm_dsim_glb; -#define IS_DUAL_DSI() (msm_dsim_glb.is_dual_dsi) +#define IS_BONDED_DSI() (msm_dsim_glb.is_bonded_dsi) #define IS_SYNC_NEEDED() (msm_dsim_glb.is_sync_needed) #define IS_MASTER_DSI_LINK(id) (msm_dsim_glb.master_dsi_link_id == id) @@ -42,18 +42,17 @@ static inline struct msm_dsi *dsi_mgr_get_other_dsi(int id) return msm_dsim_glb.dsi[(id + 1) % DSI_MAX]; } -static int dsi_mgr_parse_dual_dsi(struct device_node *np, int id) +static int dsi_mgr_parse_of(struct device_node *np, int id) { struct msm_dsi_manager *msm_dsim = &msm_dsim_glb; - /* We assume 2 dsi nodes have the same information of dual-dsi and - * sync-mode, and only one node specifies master in case of dual mode. + /* We assume 2 dsi nodes have the same information of bonded dsi and + * sync-mode, and only one node specifies master in case of bonded mode. */ - if (!msm_dsim->is_dual_dsi) - msm_dsim->is_dual_dsi = of_property_read_bool( - np, "qcom,dual-dsi-mode"); + if (!msm_dsim->is_bonded_dsi) + msm_dsim->is_bonded_dsi = of_property_read_bool(np, "qcom,dual-dsi-mode"); - if (msm_dsim->is_dual_dsi) { + if (msm_dsim->is_bonded_dsi) { if (of_property_read_bool(np, "qcom,master-dsi")) msm_dsim->master_dsi_link_id = id; if (!msm_dsim->is_sync_needed) @@ -72,7 +71,7 @@ static int dsi_mgr_setup_components(int id) struct msm_dsi *clk_slave_dsi = dsi_mgr_get_dsi(DSI_CLOCK_SLAVE); int ret; - if (!IS_DUAL_DSI()) { + if (!IS_BONDED_DSI()) { ret = msm_dsi_host_register(msm_dsi->host, true); if (ret) return ret; @@ -100,7 +99,7 @@ static int dsi_mgr_setup_components(int id) if (ret) return ret; - /* PLL0 is to drive both 2 DSI link clocks in Dual DSI mode. */ + /* PLL0 is to drive both 2 DSI link clocks in bonded DSI mode. */ msm_dsi_phy_set_usecase(clk_master_dsi->phy, MSM_DSI_PHY_MASTER); msm_dsi_phy_set_usecase(clk_slave_dsi->phy, @@ -119,12 +118,11 @@ static int enable_phy(struct msm_dsi *msm_dsi, { struct msm_dsi_phy_clk_request clk_req; int ret; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); - msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req, is_dual_dsi); + msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req, is_bonded_dsi); - ret = msm_dsi_phy_enable(msm_dsi->phy, &clk_req); - msm_dsi_phy_get_shared_timings(msm_dsi->phy, shared_timings); + ret = msm_dsi_phy_enable(msm_dsi->phy, &clk_req, shared_timings); return ret; } @@ -138,12 +136,12 @@ dsi_mgr_phy_enable(int id, struct msm_dsi *sdsi = dsi_mgr_get_dsi(DSI_CLOCK_SLAVE); int ret; - /* In case of dual DSI, some registers in PHY1 have been programmed + /* In case of bonded DSI, some registers in PHY1 have been programmed * during PLL0 clock's set_rate. The PHY1 reset called by host1 here * will silently reset those PHY1 registers. Therefore we need to reset * and enable both PHYs before any PLL clock operation. */ - if (IS_DUAL_DSI() && mdsi && sdsi) { + if (IS_BONDED_DSI() && mdsi && sdsi) { if (!mdsi->phy_enabled && !sdsi->phy_enabled) { msm_dsi_host_reset_phy(mdsi->host); msm_dsi_host_reset_phy(sdsi->host); @@ -178,11 +176,11 @@ static void dsi_mgr_phy_disable(int id) struct msm_dsi *sdsi = dsi_mgr_get_dsi(DSI_CLOCK_SLAVE); /* disable DSI phy - * In dual-dsi configuration, the phy should be disabled for the + * In bonded dsi configuration, the phy should be disabled for the * first controller only when the second controller is disabled. */ msm_dsi->phy_enabled = false; - if (IS_DUAL_DSI() && mdsi && sdsi) { + if (IS_BONDED_DSI() && mdsi && sdsi) { if (!mdsi->phy_enabled && !sdsi->phy_enabled) { msm_dsi_phy_disable(sdsi->phy); msm_dsi_phy_disable(mdsi->phy); @@ -217,24 +215,6 @@ static int dsi_mgr_bridge_get_id(struct drm_bridge *bridge) return dsi_bridge->id; } -static bool dsi_mgr_is_cmd_mode(struct msm_dsi *msm_dsi) -{ - unsigned long host_flags = msm_dsi_host_get_mode_flags(msm_dsi->host); - return !(host_flags & MIPI_DSI_MODE_VIDEO); -} - -void msm_dsi_manager_setup_encoder(int id) -{ - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - struct msm_drm_private *priv = msm_dsi->dev->dev_private; - struct msm_kms *kms = priv->kms; - struct drm_encoder *encoder = msm_dsi_get_encoder(msm_dsi); - - if (encoder && kms->funcs->set_encoder_mode) - kms->funcs->set_encoder_mode(kms, encoder, - dsi_mgr_is_cmd_mode(msm_dsi)); -} - static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) { struct msm_drm_private *priv = conn->dev->dev_private; @@ -244,7 +224,7 @@ static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) struct msm_dsi *master_dsi, *slave_dsi; struct drm_panel *panel; - if (IS_DUAL_DSI() && !IS_MASTER_DSI_LINK(id)) { + if (IS_BONDED_DSI() && !IS_MASTER_DSI_LINK(id)) { master_dsi = other_dsi; slave_dsi = msm_dsi; } else { @@ -253,7 +233,7 @@ static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) } /* - * There is only 1 panel in the global panel list for dual DSI mode. + * There is only 1 panel in the global panel list for bonded DSI mode. * Therefore slave dsi should get the drm_panel instance from master * dsi. */ @@ -264,20 +244,20 @@ static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) return PTR_ERR(panel); } - if (!panel || !IS_DUAL_DSI()) + if (!panel || !IS_BONDED_DSI()) goto out; drm_object_attach_property(&conn->base, conn->dev->mode_config.tile_property, 0); /* - * Set split display info to kms once dual DSI panel is connected to + * Set split display info to kms once bonded DSI panel is connected to * both hosts. */ if (other_dsi && other_dsi->panel && kms->funcs->set_split_display) { kms->funcs->set_split_display(kms, master_dsi->encoder, slave_dsi->encoder, - dsi_mgr_is_cmd_mode(msm_dsi)); + msm_dsi_is_cmd_mode(msm_dsi)); } out: @@ -317,7 +297,7 @@ static int dsi_mgr_connector_get_modes(struct drm_connector *connector) return 0; /* - * In dual DSI mode, we have one connector that can be + * In bonded DSI mode, we have one connector that can be * attached to the drm_panel. */ num = drm_panel_get_modes(panel, connector); @@ -366,30 +346,30 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge) struct mipi_dsi_host *host = msm_dsi->host; struct drm_panel *panel = msm_dsi->panel; struct msm_dsi_phy_shared_timings phy_shared_timings[DSI_MAX]; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); int ret; DBG("id=%d", id); if (!msm_dsi_device_connected(msm_dsi)) return; - /* Do nothing with the host if it is slave-DSI in case of dual DSI */ - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) + /* Do nothing with the host if it is slave-DSI in case of bonded DSI */ + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) return; ret = dsi_mgr_phy_enable(id, phy_shared_timings); if (ret) goto phy_en_fail; - ret = msm_dsi_host_power_on(host, &phy_shared_timings[id], is_dual_dsi); + ret = msm_dsi_host_power_on(host, &phy_shared_timings[id], is_bonded_dsi, msm_dsi->phy); if (ret) { pr_err("%s: power on host %d failed, %d\n", __func__, id, ret); goto host_on_fail; } - if (is_dual_dsi && msm_dsi1) { + if (is_bonded_dsi && msm_dsi1) { ret = msm_dsi_host_power_on(msm_dsi1->host, - &phy_shared_timings[DSI_1], is_dual_dsi); + &phy_shared_timings[DSI_1], is_bonded_dsi, msm_dsi1->phy); if (ret) { pr_err("%s: power on host1 failed, %d\n", __func__, ret); @@ -415,7 +395,7 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge) goto host_en_fail; } - if (is_dual_dsi && msm_dsi1) { + if (is_bonded_dsi && msm_dsi1) { ret = msm_dsi_host_enable(msm_dsi1->host); if (ret) { pr_err("%s: enable host1 failed, %d\n", __func__, ret); @@ -431,7 +411,7 @@ host_en_fail: if (panel) drm_panel_unprepare(panel); panel_prep_fail: - if (is_dual_dsi && msm_dsi1) + if (is_bonded_dsi && msm_dsi1) msm_dsi_host_power_off(msm_dsi1->host); host1_on_fail: msm_dsi_host_power_off(host); @@ -441,20 +421,33 @@ phy_en_fail: return; } +void msm_dsi_manager_tpg_enable(void) +{ + struct msm_dsi *m_dsi = dsi_mgr_get_dsi(DSI_0); + struct msm_dsi *s_dsi = dsi_mgr_get_dsi(DSI_1); + + /* if dual dsi, trigger tpg on master first then slave */ + if (m_dsi) { + msm_dsi_host_test_pattern_en(m_dsi->host); + if (IS_BONDED_DSI() && s_dsi) + msm_dsi_host_test_pattern_en(s_dsi->host); + } +} + static void dsi_mgr_bridge_enable(struct drm_bridge *bridge) { int id = dsi_mgr_bridge_get_id(bridge); struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct drm_panel *panel = msm_dsi->panel; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); int ret; DBG("id=%d", id); if (!msm_dsi_device_connected(msm_dsi)) return; - /* Do nothing with the host if it is slave-DSI in case of dual DSI */ - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) + /* Do nothing with the host if it is slave-DSI in case of bonded DSI */ + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) return; if (panel) { @@ -471,15 +464,15 @@ static void dsi_mgr_bridge_disable(struct drm_bridge *bridge) int id = dsi_mgr_bridge_get_id(bridge); struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct drm_panel *panel = msm_dsi->panel; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); int ret; DBG("id=%d", id); if (!msm_dsi_device_connected(msm_dsi)) return; - /* Do nothing with the host if it is slave-DSI in case of dual DSI */ - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) + /* Do nothing with the host if it is slave-DSI in case of bonded DSI */ + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) return; if (panel) { @@ -497,7 +490,7 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge) struct msm_dsi *msm_dsi1 = dsi_mgr_get_dsi(DSI_1); struct mipi_dsi_host *host = msm_dsi->host; struct drm_panel *panel = msm_dsi->panel; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); int ret; DBG("id=%d", id); @@ -506,18 +499,18 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge) return; /* - * Do nothing with the host if it is slave-DSI in case of dual DSI. + * Do nothing with the host if it is slave-DSI in case of bonded DSI. * It is safe to call dsi_mgr_phy_disable() here because a single PHY * won't be diabled until both PHYs request disable. */ - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) goto disable_phy; ret = msm_dsi_host_disable(host); if (ret) pr_err("%s: host %d disable failed, %d\n", __func__, id, ret); - if (is_dual_dsi && msm_dsi1) { + if (is_bonded_dsi && msm_dsi1) { ret = msm_dsi_host_disable(msm_dsi1->host); if (ret) pr_err("%s: host1 disable failed, %d\n", __func__, ret); @@ -537,7 +530,7 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge) if (ret) pr_err("%s: host %d power off failed,%d\n", __func__, id, ret); - if (is_dual_dsi && msm_dsi1) { + if (is_bonded_dsi && msm_dsi1) { ret = msm_dsi_host_power_off(msm_dsi1->host); if (ret) pr_err("%s: host1 power off failed, %d\n", @@ -556,15 +549,15 @@ static void dsi_mgr_bridge_mode_set(struct drm_bridge *bridge, struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct msm_dsi *other_dsi = dsi_mgr_get_other_dsi(id); struct mipi_dsi_host *host = msm_dsi->host; - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); DBG("set mode: " DRM_MODE_FMT, DRM_MODE_ARG(mode)); - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) return; msm_dsi_host_set_display_mode(host, adjusted_mode); - if (is_dual_dsi && other_dsi) + if (is_bonded_dsi && other_dsi) msm_dsi_host_set_display_mode(other_dsi->host, adjusted_mode); } @@ -640,15 +633,15 @@ fail: bool msm_dsi_manager_validate_current_config(u8 id) { - bool is_dual_dsi = IS_DUAL_DSI(); + bool is_bonded_dsi = IS_BONDED_DSI(); /* - * For dual DSI, we only have one drm panel. For this + * For bonded DSI, we only have one drm panel. For this * use case, we register only one bridge/connector. * Skip bridge/connector initialisation if it is - * slave-DSI for dual DSI configuration. + * slave-DSI for bonded DSI configuration. */ - if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) { + if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) { DBG("Skip bridge registration for slave DSI->id: %d\n", id); return false; } @@ -740,7 +733,7 @@ int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg) if (!msg->tx_buf || !msg->tx_len) return 0; - /* In dual master case, panel requires the same commands sent to + /* In bonded master case, panel requires the same commands sent to * both DSI links. Host issues the command trigger to both links * when DSI_1 calls the cmd transfer function, no matter it happens * before or after DSI_0 cmd transfer. @@ -809,9 +802,9 @@ int msm_dsi_manager_register(struct msm_dsi *msm_dsi) msm_dsim->dsi[id] = msm_dsi; - ret = dsi_mgr_parse_dual_dsi(msm_dsi->pdev->dev.of_node, id); + ret = dsi_mgr_parse_of(msm_dsi->pdev->dev.of_node, id); if (ret) { - pr_err("%s: failed to parse dual DSI info\n", __func__); + pr_err("%s: failed to parse OF DSI info\n", __func__); goto fail; } @@ -840,3 +833,12 @@ void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi) msm_dsim->dsi[msm_dsi->id] = NULL; } +bool msm_dsi_is_bonded_dsi(struct msm_dsi *msm_dsi) +{ + return IS_BONDED_DSI(); +} + +bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi) +{ + return IS_MASTER_DSI_LINK(msm_dsi->id); +} diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c index 6ca6bfd4809b..8c65ef6968ca 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c @@ -5,6 +5,7 @@ #include <linux/clk-provider.h> #include <linux/platform_device.h> +#include <dt-bindings/phy/phy.h> #include "dsi_phy.h" @@ -461,6 +462,51 @@ int msm_dsi_dphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing, return 0; } +int msm_dsi_cphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing, + struct msm_dsi_phy_clk_request *clk_req) +{ + const unsigned long bit_rate = clk_req->bitclk_rate; + const unsigned long esc_rate = clk_req->escclk_rate; + s32 ui, ui_x7; + s32 tmax, tmin; + s32 coeff = 1000; /* Precision, should avoid overflow */ + s32 temp; + + if (!bit_rate || !esc_rate) + return -EINVAL; + + ui = mult_frac(NSEC_PER_MSEC, coeff, bit_rate / 1000); + ui_x7 = ui * 7; + + temp = S_DIV_ROUND_UP(38 * coeff, ui_x7); + tmin = max_t(s32, temp, 0); + temp = (95 * coeff) / ui_x7; + tmax = max_t(s32, temp, 0); + timing->clk_prepare = linear_inter(tmax, tmin, 50, 0, false); + + tmin = DIV_ROUND_UP(50 * coeff, ui_x7); + tmax = 255; + timing->hs_rqst = linear_inter(tmax, tmin, 1, 0, false); + + tmin = DIV_ROUND_UP(100 * coeff, ui_x7) - 1; + tmax = 255; + timing->hs_exit = linear_inter(tmax, tmin, 10, 0, false); + + tmin = 1; + tmax = 32; + timing->shared_timings.clk_post = linear_inter(tmax, tmin, 80, 0, false); + + tmin = min_t(s32, 64, S_DIV_ROUND_UP(262 * coeff, ui_x7) - 1); + tmax = 64; + timing->shared_timings.clk_pre = linear_inter(tmax, tmin, 20, 0, false); + + DBG("%d, %d, %d, %d, %d", + timing->shared_timings.clk_pre, timing->shared_timings.clk_post, + timing->clk_prepare, timing->hs_exit, timing->hs_rqst); + + return 0; +} + static int dsi_phy_regulator_init(struct msm_dsi_phy *phy) { struct regulator_bulk_data *s = phy->supplies; @@ -593,6 +639,8 @@ static const struct of_device_id dsi_phy_dt_match[] = { .data = &dsi_phy_7nm_cfgs }, { .compatible = "qcom,dsi-phy-7nm-8150", .data = &dsi_phy_7nm_8150_cfgs }, + { .compatible = "qcom,sc7280-dsi-phy-7nm", + .data = &dsi_phy_7nm_7280_cfgs }, #endif {} }; @@ -625,17 +673,13 @@ static int dsi_phy_driver_probe(struct platform_device *pdev) { struct msm_dsi_phy *phy; struct device *dev = &pdev->dev; - const struct of_device_id *match; + u32 phy_type; int ret; phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); if (!phy) return -ENOMEM; - match = of_match_node(dsi_phy_dt_match, dev->of_node); - if (!match) - return -ENODEV; - phy->provided_clocks = devm_kzalloc(dev, struct_size(phy->provided_clocks, hws, NUM_PROVIDED_CLKS), GFP_KERNEL); @@ -644,7 +688,10 @@ static int dsi_phy_driver_probe(struct platform_device *pdev) phy->provided_clocks->num = NUM_PROVIDED_CLKS; - phy->cfg = match->data; + phy->cfg = of_device_get_match_data(&pdev->dev); + if (!phy->cfg) + return -ENODEV; + phy->pdev = pdev; phy->id = dsi_phy_get_id(phy); @@ -657,6 +704,8 @@ static int dsi_phy_driver_probe(struct platform_device *pdev) phy->regulator_ldo_mode = of_property_read_bool(dev->of_node, "qcom,dsi-phy-regulator-ldo-mode"); + if (!of_property_read_u32(dev->of_node, "phy-type", &phy_type)) + phy->cphy_mode = (phy_type == PHY_TYPE_CPHY); phy->base = msm_ioremap_size(pdev, "dsi_phy", "DSI_PHY", &phy->base_size); if (IS_ERR(phy->base)) { @@ -754,7 +803,8 @@ void __exit msm_dsi_phy_driver_unregister(void) } int msm_dsi_phy_enable(struct msm_dsi_phy *phy, - struct msm_dsi_phy_clk_request *clk_req) + struct msm_dsi_phy_clk_request *clk_req, + struct msm_dsi_phy_shared_timings *shared_timings) { struct device *dev = &phy->pdev->dev; int ret; @@ -782,6 +832,9 @@ int msm_dsi_phy_enable(struct msm_dsi_phy *phy, goto phy_en_fail; } + memcpy(shared_timings, &phy->timing.shared_timings, + sizeof(*shared_timings)); + /* * Resetting DSI PHY silently changes its PLL registers to reset status, * which will confuse clock driver and result in wrong output rate of @@ -821,13 +874,6 @@ void msm_dsi_phy_disable(struct msm_dsi_phy *phy) dsi_phy_disable_resource(phy); } -void msm_dsi_phy_get_shared_timings(struct msm_dsi_phy *phy, - struct msm_dsi_phy_shared_timings *shared_timings) -{ - memcpy(shared_timings, &phy->timing.shared_timings, - sizeof(*shared_timings)); -} - void msm_dsi_phy_set_usecase(struct msm_dsi_phy *phy, enum msm_dsi_phy_usecase uc) { @@ -835,6 +881,15 @@ void msm_dsi_phy_set_usecase(struct msm_dsi_phy *phy, phy->usecase = uc; } +/* Returns true if we have to clear DSI_LANE_CTRL.HS_REQ_SEL_PHY */ +bool msm_dsi_phy_set_continuous_clock(struct msm_dsi_phy *phy, bool enable) +{ + if (!phy || !phy->cfg->ops.set_continuous_clock) + return false; + + return phy->cfg->ops.set_continuous_clock(phy, enable); +} + int msm_dsi_phy_get_clk_provider(struct msm_dsi_phy *phy, struct clk **byte_clk_provider, struct clk **pixel_clk_provider) { diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index 5b0feef87127..b91303ada74f 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -24,6 +24,7 @@ struct msm_dsi_phy_ops { void (*disable)(struct msm_dsi_phy *phy); void (*save_pll_state)(struct msm_dsi_phy *phy); int (*restore_pll_state)(struct msm_dsi_phy *phy); + bool (*set_continuous_clock)(struct msm_dsi_phy *phy, bool enable); }; struct msm_dsi_phy_cfg { @@ -51,6 +52,7 @@ extern const struct msm_dsi_phy_cfg dsi_phy_10nm_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_10nm_8998_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs; extern const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs; +extern const struct msm_dsi_phy_cfg dsi_phy_7nm_7280_cfgs; struct msm_dsi_dphy_timing { u32 clk_zero; @@ -99,6 +101,7 @@ struct msm_dsi_phy { enum msm_dsi_phy_usecase usecase; bool regulator_ldo_mode; + bool cphy_mode; struct clk_hw *vco_hw; bool pll_on; @@ -119,5 +122,7 @@ int msm_dsi_dphy_timing_calc_v3(struct msm_dsi_dphy_timing *timing, struct msm_dsi_phy_clk_request *clk_req); int msm_dsi_dphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing, struct msm_dsi_phy_clk_request *clk_req); +int msm_dsi_cphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing, + struct msm_dsi_phy_clk_request *clk_req); #endif /* __DSI_PHY_H__ */ diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c index e46b10fc793a..d8128f50b0dd 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c @@ -84,7 +84,7 @@ struct dsi_pll_10nm { #define to_pll_10nm(x) container_of(x, struct dsi_pll_10nm, clk_hw) /* - * Global list of private DSI PLL struct pointers. We need this for Dual DSI + * Global list of private DSI PLL struct pointers. We need this for bonded DSI * mode, where the master PLL's clk_ops needs access the slave's private data */ static struct dsi_pll_10nm *pll_10nm_list[DSI_MAX]; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index a34cf151c517..d13552b2213b 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -86,7 +86,7 @@ struct dsi_pll_14nm { /* * Private struct for N1/N2 post-divider clocks. These clocks are similar to * the generic clk_divider class of clocks. The only difference is that it - * also sets the slave DSI PLL's post-dividers if in Dual DSI mode + * also sets the slave DSI PLL's post-dividers if in bonded DSI mode */ struct dsi_pll_14nm_postdiv { struct clk_hw hw; @@ -102,7 +102,7 @@ struct dsi_pll_14nm_postdiv { #define to_pll_14nm_postdiv(_hw) container_of(_hw, struct dsi_pll_14nm_postdiv, hw) /* - * Global list of private DSI PLL struct pointers. We need this for Dual DSI + * Global list of private DSI PLL struct pointers. We need this for bonded DSI * mode, where the master PLL's clk_ops needs access the slave's private data */ static struct dsi_pll_14nm *pll_14nm_list[DSI_MAX]; @@ -658,7 +658,7 @@ static int dsi_pll_14nm_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, val |= value << shift; dsi_phy_write(base + REG_DSI_14nm_PHY_CMN_CLK_CFG0, val); - /* If we're master in dual DSI mode, then the slave PLL's post-dividers + /* If we're master in bonded DSI mode, then the slave PLL's post-dividers * follow the master's post dividers */ if (pll_14nm->phy->usecase == MSM_DSI_PHY_MASTER) { @@ -1050,7 +1050,7 @@ const struct msm_dsi_phy_cfg dsi_phy_14nm_660_cfgs = { .reg_cfg = { .num = 1, .regs = { - {"vcca", 17000, 32}, + {"vcca", 73400, 32}, }, }, .ops = { diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index 7c23d4c47338..cb297b08458e 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -83,7 +83,7 @@ struct dsi_pll_7nm { #define to_pll_7nm(x) container_of(x, struct dsi_pll_7nm, clk_hw) /* - * Global list of private DSI PLL struct pointers. We need this for Dual DSI + * Global list of private DSI PLL struct pointers. We need this for bonded DSI * mode, where the master PLL's clk_ops needs access the slave's private data */ static struct dsi_pll_7nm *pll_7nm_list[DSI_MAX]; @@ -256,7 +256,7 @@ static void dsi_pll_commit(struct dsi_pll_7nm *pll, struct dsi_pll_config *confi (config->frac_div_start & 0x30000) >> 16); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCKDET_RATE_1, 0x40); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCK_DELAY, 0x06); - dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1, 0x10); /* TODO: 0x00 for CPHY */ + dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1, pll->phy->cphy_mode ? 0x00 : 0x10); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS, config->pll_clock_inverters); } @@ -642,7 +642,8 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide /* DSI Byte clock = VCO_CLK / OUT_DIV / BIT_DIV / 8 */ hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - CLK_SET_RATE_PARENT, 1, 8); + CLK_SET_RATE_PARENT, 1, + pll_7nm->phy->cphy_mode ? 7 : 8); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; @@ -663,32 +664,47 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide snprintf(clk_name, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - 0, 1, 4); + if (pll_7nm->phy->cphy_mode) + hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 2, 7); + else + hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 1, 4); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; } - snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); - snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id); - snprintf(parent3, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); - snprintf(parent4, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); - - hw = devm_clk_hw_register_mux(dev, clk_name, - ((const char *[]){ - parent, parent2, parent3, parent4 - }), 4, 0, pll_7nm->phy->base + - REG_DSI_7nm_PHY_CMN_CLK_CFG1, - 0, 2, 0, NULL); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); - goto fail; + /* in CPHY mode, pclk_mux will always have post_out_div as parent + * don't register a pclk_mux clock and just use post_out_div instead + */ + if (pll_7nm->phy->cphy_mode) { + u32 data; + + data = dsi_phy_read(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1); + dsi_phy_write(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1, data | 3); + + snprintf(parent, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); + } else { + snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); + snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); + snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id); + snprintf(parent3, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); + snprintf(parent4, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); + + hw = devm_clk_hw_register_mux(dev, clk_name, + ((const char *[]){ + parent, parent2, parent3, parent4 + }), 4, 0, pll_7nm->phy->base + + REG_DSI_7nm_PHY_CMN_CLK_CFG1, + 0, 2, 0, NULL); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto fail; + } + + snprintf(parent, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); } snprintf(clk_name, 32, "dsi%d_phy_pll_out_dsiclk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); /* PIX CLK DIV : DIV_CTRL_7_4*/ hw = devm_clk_hw_register_divider(dev, clk_name, parent, @@ -813,15 +829,21 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, struct msm_dsi_dphy_timing *timing = &phy->timing; void __iomem *base = phy->base; bool less_than_1500_mhz; - u32 vreg_ctrl_0, glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0; + u32 vreg_ctrl_0, vreg_ctrl_1, lane_ctrl0; + u32 glbl_pemph_ctrl_0; + u32 glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0; u32 glbl_rescode_top_ctrl, glbl_rescode_bot_ctrl; u32 data; DBG(""); - if (msm_dsi_dphy_timing_calc_v4(timing, clk_req)) { + if (phy->cphy_mode) + ret = msm_dsi_cphy_timing_calc_v4(timing, clk_req); + else + ret = msm_dsi_dphy_timing_calc_v4(timing, clk_req); + if (ret) { DRM_DEV_ERROR(&phy->pdev->dev, - "%s: D-PHY timing calculation failed\n", __func__); + "%s: PHY timing calculation failed\n", __func__); return -EINVAL; } @@ -842,6 +864,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, /* Alter PHY configurations if data rate less than 1.5GHZ*/ less_than_1500_mhz = (clk_req->bitclk_rate <= 1500000000); + /* For C-PHY, no low power settings for lower clk rate */ + if (phy->cphy_mode) + less_than_1500_mhz = false; + if (phy->cfg->quirks & DSI_PHY_7NM_QUIRK_V4_1) { vreg_ctrl_0 = less_than_1500_mhz ? 0x53 : 0x52; glbl_rescode_top_ctrl = less_than_1500_mhz ? 0x3d : 0x00; @@ -856,6 +882,17 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, glbl_rescode_bot_ctrl = 0x3c; } + if (phy->cphy_mode) { + vreg_ctrl_0 = 0x51; + vreg_ctrl_1 = 0x55; + glbl_pemph_ctrl_0 = 0x11; + lane_ctrl0 = 0x17; + } else { + vreg_ctrl_1 = 0x5c; + glbl_pemph_ctrl_0 = 0x00; + lane_ctrl0 = 0x1f; + } + /* de-assert digital and pll power down */ data = BIT(6) | BIT(5); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_CTRL_0, data); @@ -876,15 +913,22 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_LANE_CFG0, 0x21); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_LANE_CFG1, 0x84); + if (phy->cphy_mode) + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_CTRL, BIT(6)); + /* Enable LDO */ dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_VREG_CTRL_0, vreg_ctrl_0); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_VREG_CTRL_1, 0x5c); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_VREG_CTRL_1, vreg_ctrl_1); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_CTRL_3, 0x00); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_STR_SWI_CAL_SEL_CTRL, glbl_str_swi_cal_sel_ctrl); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_HSTX_STR_CTRL_0, glbl_hstx_str_ctrl_0); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_PEMPH_CTRL_0, 0x00); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_PEMPH_CTRL_0, + glbl_pemph_ctrl_0); + if (phy->cphy_mode) + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_PEMPH_CTRL_1, 0x01); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_RESCODE_OFFSET_TOP_CTRL, glbl_rescode_top_ctrl); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_GLBL_RESCODE_OFFSET_BOT_CTRL, @@ -894,10 +938,11 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, /* Remove power down from all blocks */ dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_CTRL_0, 0x7f); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0, 0x1f); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0, lane_ctrl0); /* Select full-rate mode */ - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_CTRL_2, 0x40); + if (!phy->cphy_mode) + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_CTRL_2, 0x40); ret = dsi_7nm_set_usecase(phy); if (ret) { @@ -907,22 +952,36 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, } /* DSI PHY timings */ - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0, 0x00); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_1, timing->clk_zero); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_2, timing->clk_prepare); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_3, timing->clk_trail); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4, timing->hs_exit); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5, timing->hs_zero); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6, timing->hs_prepare); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7, timing->hs_trail); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8, timing->hs_rqst); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9, 0x02); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11, 0x00); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_12, - timing->shared_timings.clk_pre); - dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_13, - timing->shared_timings.clk_post); + if (phy->cphy_mode) { + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0, 0x00); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4, timing->hs_exit); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5, + timing->shared_timings.clk_pre); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6, timing->clk_prepare); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7, + timing->shared_timings.clk_post); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8, timing->hs_rqst); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9, 0x02); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11, 0x00); + } else { + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0, 0x00); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_1, timing->clk_zero); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_2, timing->clk_prepare); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_3, timing->clk_trail); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4, timing->hs_exit); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5, timing->hs_zero); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6, timing->hs_prepare); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7, timing->hs_trail); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8, timing->hs_rqst); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9, 0x02); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11, 0x00); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_12, + timing->shared_timings.clk_pre); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_13, + timing->shared_timings.clk_post); + } /* DSI lane settings */ dsi_phy_hw_v4_0_lane_settings(phy); @@ -932,6 +991,21 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, return 0; } +static bool dsi_7nm_set_continuous_clock(struct msm_dsi_phy *phy, bool enable) +{ + void __iomem *base = phy->base; + u32 data; + + data = dsi_phy_read(base + REG_DSI_7nm_PHY_CMN_LANE_CTRL1); + if (enable) + data |= BIT(5) | BIT(6); + else + data &= ~(BIT(5) | BIT(6)); + dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_LANE_CTRL1, data); + + return enable; +} + static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) { void __iomem *base = phy->base; @@ -972,6 +1046,7 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs = { .pll_init = dsi_pll_7nm_init, .save_pll_state = dsi_7nm_pll_save_state, .restore_pll_state = dsi_7nm_pll_restore_state, + .set_continuous_clock = dsi_7nm_set_continuous_clock, }, .min_pll_rate = 600000000UL, #ifdef CONFIG_64BIT @@ -998,9 +1073,36 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs = { .pll_init = dsi_pll_7nm_init, .save_pll_state = dsi_7nm_pll_save_state, .restore_pll_state = dsi_7nm_pll_restore_state, + .set_continuous_clock = dsi_7nm_set_continuous_clock, }, .min_pll_rate = 1000000000UL, .max_pll_rate = 3500000000UL, .io_start = { 0xae94400, 0xae96400 }, .num_dsi_phy = 2, }; + +const struct msm_dsi_phy_cfg dsi_phy_7nm_7280_cfgs = { + .has_phy_lane = true, + .reg_cfg = { + .num = 1, + .regs = { + {"vdds", 37550, 0}, + }, + }, + .ops = { + .enable = dsi_7nm_phy_enable, + .disable = dsi_7nm_phy_disable, + .pll_init = dsi_pll_7nm_init, + .save_pll_state = dsi_7nm_pll_save_state, + .restore_pll_state = dsi_7nm_pll_restore_state, + }, + .min_pll_rate = 600000000UL, +#ifdef CONFIG_64BIT + .max_pll_rate = 5000000000ULL, +#else + .max_pll_rate = ULONG_MAX, +#endif + .io_start = { 0xae94400 }, + .num_dsi_phy = 1, + .quirks = DSI_PHY_7NM_QUIRK_V4_1, +}; diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index a332b09a5a11..2e6fc185e54d 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -603,6 +603,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) if (IS_ERR(priv->event_thread[i].worker)) { ret = PTR_ERR(priv->event_thread[i].worker); DRM_DEV_ERROR(dev, "failed to create crtc_event kthread\n"); + ret = PTR_ERR(priv->event_thread[i].worker); goto err_msm_uninit; } @@ -1057,17 +1058,7 @@ static const struct drm_ioctl_desc msm_ioctls[] = { DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_QUERY, msm_ioctl_submitqueue_query, DRM_RENDER_ALLOW), }; -static const struct file_operations fops = { - .owner = THIS_MODULE, - .open = drm_open, - .release = drm_release, - .unlocked_ioctl = drm_ioctl, - .compat_ioctl = drm_compat_ioctl, - .poll = drm_poll, - .read = drm_read, - .llseek = no_llseek, - .mmap = msm_gem_mmap, -}; +DEFINE_DRM_GEM_FOPS(fops); static const struct drm_driver msm_driver = { .driver_features = DRIVER_GEM | @@ -1083,7 +1074,7 @@ static const struct drm_driver msm_driver = { .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, .gem_prime_import_sg_table = msm_gem_prime_import_sg_table, - .gem_prime_mmap = msm_gem_prime_mmap, + .gem_prime_mmap = drm_gem_prime_mmap, #ifdef CONFIG_DEBUG_FS .debugfs_init = msm_debugfs_init, #endif diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 1a48a709ffb3..8b005d1ac899 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -309,7 +309,6 @@ void msm_gem_shrinker_cleanup(struct drm_device *dev); struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj); int msm_gem_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map); void msm_gem_prime_vunmap(struct drm_gem_object *obj, struct dma_buf_map *map); -int msm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma); struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sg); int msm_gem_prime_pin(struct drm_gem_object *obj); @@ -350,7 +349,9 @@ void __exit msm_dsi_unregister(void); int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev, struct drm_encoder *encoder); void msm_dsi_snapshot(struct msm_disp_state *disp_state, struct msm_dsi *msm_dsi); - +bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi); +bool msm_dsi_is_bonded_dsi(struct msm_dsi *msm_dsi); +bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi); #else static inline void __init msm_dsi_register(void) { @@ -367,7 +368,18 @@ static inline int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, static inline void msm_dsi_snapshot(struct msm_disp_state *disp_state, struct msm_dsi *msm_dsi) { } - +static inline bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi) +{ + return false; +} +static inline bool msm_dsi_is_bonded_dsi(struct msm_dsi *msm_dsi) +{ + return false; +} +static inline bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi) +{ + return false; +} #endif #ifdef CONFIG_DRM_MSM_DP diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c index 67fae60f2fa5..0daaeb54ff6f 100644 --- a/drivers/gpu/drm/msm/msm_fbdev.c +++ b/drivers/gpu/drm/msm/msm_fbdev.c @@ -8,13 +8,12 @@ #include <drm/drm_crtc.h> #include <drm/drm_fb_helper.h> #include <drm/drm_fourcc.h> +#include <drm/drm_prime.h> #include "msm_drv.h" #include "msm_gem.h" #include "msm_kms.h" -extern int msm_gem_mmap_obj(struct drm_gem_object *obj, - struct vm_area_struct *vma); static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma); /* @@ -48,15 +47,8 @@ static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma) struct drm_fb_helper *helper = (struct drm_fb_helper *)info->par; struct msm_fbdev *fbdev = to_msm_fbdev(helper); struct drm_gem_object *bo = msm_framebuffer_bo(fbdev->fb, 0); - int ret = 0; - ret = drm_gem_mmap_obj(bo, bo->size, vma); - if (ret) { - pr_err("%s:drm_gem_mmap_obj fail\n", __func__); - return ret; - } - - return msm_gem_mmap_obj(bo, vma); + return drm_gem_prime_mmap(bo, vma); } static int msm_fbdev_create(struct drm_fb_helper *helper, diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 5db07fc287ad..22308a1b66fc 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -217,31 +217,6 @@ static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot) return prot; } -int msm_gem_mmap_obj(struct drm_gem_object *obj, - struct vm_area_struct *vma) -{ - struct msm_gem_object *msm_obj = to_msm_bo(obj); - - vma->vm_flags &= ~VM_PFNMAP; - vma->vm_flags |= VM_MIXEDMAP; - vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags)); - - return 0; -} - -int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma) -{ - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) { - DBG("mmap failed: %d", ret); - return ret; - } - - return msm_gem_mmap_obj(vma->vm_private_data, vma); -} - static vm_fault_t msm_gem_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -817,8 +792,7 @@ void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu) mutex_lock(&priv->mm_lock); if (msm_obj->evictable) mark_unevictable(msm_obj); - list_del(&msm_obj->mm_list); - list_add_tail(&msm_obj->mm_list, &gpu->active_list); + list_move_tail(&msm_obj->mm_list, &gpu->active_list); mutex_unlock(&priv->mm_lock); } } @@ -1077,6 +1051,17 @@ void msm_gem_free_object(struct drm_gem_object *obj) kfree(msm_obj); } +static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct msm_gem_object *msm_obj = to_msm_bo(obj); + + vma->vm_flags &= ~VM_PFNMAP; + vma->vm_flags |= VM_MIXEDMAP | VM_DONTEXPAND; + vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags)); + + return 0; +} + /* convenience method to construct a GEM buffer object, and userspace handle */ int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, uint32_t size, uint32_t flags, uint32_t *handle, @@ -1114,6 +1099,7 @@ static const struct drm_gem_object_funcs msm_gem_object_funcs = { .get_sg_table = msm_gem_prime_get_sg_table, .vmap = msm_gem_prime_vmap, .vunmap = msm_gem_prime_vunmap, + .mmap = msm_gem_object_mmap, .vm_ops = &vm_ops, }; diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 8bf0ac707fd7..54ca0817d807 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -106,9 +106,6 @@ struct msm_gem_object { }; #define to_msm_bo(x) container_of(x, struct msm_gem_object, base) -int msm_gem_mmap_obj(struct drm_gem_object *obj, - struct vm_area_struct *vma); -int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma); uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj); int msm_gem_get_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova); diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c index 9880348a4dc7..fc94e061d6a7 100644 --- a/drivers/gpu/drm/msm/msm_gem_prime.c +++ b/drivers/gpu/drm/msm/msm_gem_prime.c @@ -39,17 +39,6 @@ void msm_gem_prime_vunmap(struct drm_gem_object *obj, struct dma_buf_map *map) msm_gem_put_vaddr(obj); } -int msm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) -{ - int ret; - - ret = drm_gem_mmap_obj(obj, obj->size, vma); - if (ret < 0) - return ret; - - return msm_gem_mmap_obj(vma->vm_private_data, vma); -} - struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sg) { diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c index eed2a762e9dd..bcaddbba564d 100644 --- a/drivers/gpu/drm/msm/msm_iommu.c +++ b/drivers/gpu/drm/msm/msm_iommu.c @@ -142,6 +142,9 @@ static const struct iommu_flush_ops null_tlb_ops = { .tlb_add_page = msm_iommu_tlb_add_page, }; +static int msm_fault_handler(struct iommu_domain *domain, struct device *dev, + unsigned long iova, int flags, void *arg); + struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent) { struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev); @@ -157,6 +160,13 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent) if (!ttbr1_cfg) return ERR_PTR(-ENODEV); + /* + * Defer setting the fault handler until we have a valid adreno_smmu + * to avoid accidentially installing a GPU specific fault handler for + * the display's iommu + */ + iommu_set_fault_handler(iommu->domain, msm_fault_handler, iommu); + pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL); if (!pagetable) return ERR_PTR(-ENOMEM); @@ -300,7 +310,6 @@ struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain) iommu->domain = domain; msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU); - iommu_set_fault_handler(domain, msm_fault_handler, iommu); atomic_set(&iommu->pagetables, 0); diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h index 9de7c42e1071..de2bc3467bb5 100644 --- a/drivers/gpu/drm/msm/msm_kms.h +++ b/drivers/gpu/drm/msm/msm_kms.h @@ -117,9 +117,6 @@ struct msm_kms_funcs { struct drm_encoder *encoder, struct drm_encoder *slave_encoder, bool is_cmd_mode); - void (*set_encoder_mode)(struct msm_kms *kms, - struct drm_encoder *encoder, - bool cmd_mode); /* cleanup: */ void (*destroy)(struct msm_kms *kms); diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index bb5d1cacd97e..d7b9f7f8c9e3 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -2268,6 +2268,33 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) interlock[NV50_DISP_INTERLOCK_CORE] = 0; } + /* Finish updating head(s)... + * + * NVD is rather picky about both where window assignments can change, + * *and* about certain core and window channel states matching. + * + * The EFI GOP driver on newer GPUs configures window channels with a + * different output format to what we do, and the core channel update + * in the assign_windows case above would result in a state mismatch. + * + * Delay some of the head update until after that point to workaround + * the issue. This only affects the initial modeset. + * + * TODO: handle this better when adding flexible window mapping + */ + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { + struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state); + struct nv50_head *head = nv50_head(crtc); + + NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name, + asyh->set.mask, asyh->clr.mask); + + if (asyh->set.mask) { + nv50_head_flush_set_wndw(head, asyh); + interlock[NV50_DISP_INTERLOCK_CORE] = 1; + } + } + /* Update plane(s). */ for_each_new_plane_in_state(state, plane, new_plane_state, i) { struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state); diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c index ec361d17e900..d66f97280282 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/head.c +++ b/drivers/gpu/drm/nouveau/dispnv50/head.c @@ -50,11 +50,8 @@ nv50_head_flush_clr(struct nv50_head *head, } void -nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh) +nv50_head_flush_set_wndw(struct nv50_head *head, struct nv50_head_atom *asyh) { - if (asyh->set.view ) head->func->view (head, asyh); - if (asyh->set.mode ) head->func->mode (head, asyh); - if (asyh->set.core ) head->func->core_set(head, asyh); if (asyh->set.olut ) { asyh->olut.offset = nv50_lut_load(&head->olut, asyh->olut.buffer, @@ -62,6 +59,14 @@ nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh) asyh->olut.load); head->func->olut_set(head, asyh); } +} + +void +nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + if (asyh->set.view ) head->func->view (head, asyh); + if (asyh->set.mode ) head->func->mode (head, asyh); + if (asyh->set.core ) head->func->core_set(head, asyh); if (asyh->set.curs ) head->func->curs_set(head, asyh); if (asyh->set.base ) head->func->base (head, asyh); if (asyh->set.ovly ) head->func->ovly (head, asyh); diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.h b/drivers/gpu/drm/nouveau/dispnv50/head.h index dae841dc05fd..0bac6be9ba34 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/head.h +++ b/drivers/gpu/drm/nouveau/dispnv50/head.h @@ -21,6 +21,7 @@ struct nv50_head { struct nv50_head *nv50_head_create(struct drm_device *, int index); void nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh); +void nv50_head_flush_set_wndw(struct nv50_head *head, struct nv50_head_atom *asyh); void nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool flush); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h index 0b86c44878e0..59759c4fb62e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h @@ -4,7 +4,8 @@ struct nv_device_v0 { __u8 version; - __u8 pad01[7]; + __u8 priv; + __u8 pad02[6]; __u64 device; /* device identifier, ~0 for client default */ }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index ba2c28ea43d2..c68cc957248e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -61,8 +61,6 @@ #define NV10_CHANNEL_DMA /* cl506b.h */ 0x0000006e #define NV17_CHANNEL_DMA /* cl506b.h */ 0x0000176e #define NV40_CHANNEL_DMA /* cl506b.h */ 0x0000406e -#define NV50_CHANNEL_DMA /* cl506e.h */ 0x0000506e -#define G82_CHANNEL_DMA /* cl826e.h */ 0x0000826e #define NV50_CHANNEL_GPFIFO /* cl506f.h */ 0x0000506f #define G82_CHANNEL_GPFIFO /* cl826f.h */ 0x0000826f diff --git a/drivers/gpu/drm/nouveau/include/nvif/client.h b/drivers/gpu/drm/nouveau/include/nvif/client.h index 347d2c020bd1..5d9395e651b6 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/client.h +++ b/drivers/gpu/drm/nouveau/include/nvif/client.h @@ -9,7 +9,6 @@ struct nvif_client { const struct nvif_driver *driver; u64 version; u8 route; - bool super; }; int nvif_client_ctor(struct nvif_client *parent, const char *name, u64 device, diff --git a/drivers/gpu/drm/nouveau/include/nvif/driver.h b/drivers/gpu/drm/nouveau/include/nvif/driver.h index 8e85b936eaa0..7a3af05f7f98 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/driver.h +++ b/drivers/gpu/drm/nouveau/include/nvif/driver.h @@ -11,7 +11,7 @@ struct nvif_driver { void (*fini)(void *priv); int (*suspend)(void *priv); int (*resume)(void *priv); - int (*ioctl)(void *priv, bool super, void *data, u32 size, void **hack); + int (*ioctl)(void *priv, void *data, u32 size, void **hack); void __iomem *(*map)(void *priv, u64 handle, u32 size); void (*unmap)(void *priv, void __iomem *ptr, u32 size); bool keep; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h index 5d7017fe5039..2f86606e708c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h @@ -13,7 +13,6 @@ struct nvkm_client { struct nvkm_client_notify *notify[32]; struct rb_root objroot; - bool super; void *data; int (*ntfy)(const void *, u32, const void *, u32); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/ioctl.h b/drivers/gpu/drm/nouveau/include/nvkm/core/ioctl.h index 71ed147ad077..f52918a43246 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/ioctl.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/ioctl.h @@ -4,5 +4,5 @@ #include <core/os.h> struct nvkm_client; -int nvkm_ioctl(struct nvkm_client *, bool, void *, u32, void **); +int nvkm_ioctl(struct nvkm_client *, void *, u32, void **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 0911e73f7424..70e7887ef4b4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -15,7 +15,6 @@ struct nvkm_vma { u8 refd:3; /* Current page type (index, or NONE for unreferenced). */ bool used:1; /* Region allocated. */ bool part:1; /* Region was split from an allocated region by map(). */ - bool user:1; /* Region user-allocated. */ bool busy:1; /* Region busy (for temporarily preventing user access). */ bool mapped:1; /* Region contains valid pages. */ struct nvkm_memory *memory; /* Memory currently mapped into VMA. */ diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index b45ec3086285..4107b7006539 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -570,11 +570,9 @@ nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS) } client->route = NVDRM_OBJECT_ABI16; - client->super = true; ret = nvif_object_ctor(&chan->chan->user, "abi16Ntfy", info->handle, NV_DMA_IN_MEMORY, &args, sizeof(args), &ntfy->object); - client->super = false; client->route = NVDRM_OBJECT_NVIF; if (ret) goto done; diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 40362600eed2..80099ef75702 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -86,12 +86,6 @@ nouveau_channel_del(struct nouveau_channel **pchan) struct nouveau_channel *chan = *pchan; if (chan) { struct nouveau_cli *cli = (void *)chan->user.client; - bool super; - - if (cli) { - super = cli->base.super; - cli->base.super = true; - } if (chan->fence) nouveau_fence(chan->drm)->context_del(chan); @@ -111,9 +105,6 @@ nouveau_channel_del(struct nouveau_channel **pchan) nouveau_bo_unpin(chan->push.buffer); nouveau_bo_ref(NULL, &chan->push.buffer); kfree(chan); - - if (cli) - cli->base.super = super; } *pchan = NULL; } @@ -512,20 +503,16 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, struct nouveau_channel **pchan) { struct nouveau_cli *cli = (void *)device->object.client; - bool super; int ret; /* hack until fencenv50 is fixed, and agp access relaxed */ - super = cli->base.super; - cli->base.super = true; - ret = nouveau_channel_ind(drm, device, arg0, priv, pchan); if (ret) { NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret); ret = nouveau_channel_dma(drm, device, pchan); if (ret) { NV_PRINTK(dbg, cli, "dma channel create, %d\n", ret); - goto done; + return ret; } } @@ -533,15 +520,13 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, if (ret) { NV_PRINTK(err, cli, "channel failed to initialise, %d\n", ret); nouveau_channel_del(pchan); - goto done; + return ret; } ret = nouveau_svmm_join((*pchan)->vmm->svmm, (*pchan)->inst); if (ret) nouveau_channel_del(pchan); -done: - cli->base.super = super; return ret; } diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 5e1ff870823b..1f828c9f691c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -244,6 +244,7 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, ret = nvif_device_ctor(&cli->base.object, "drmDevice", 0, NV_DEVICE, &(struct nv_device_v0) { .device = ~0, + .priv = true, }, sizeof(struct nv_device_v0), &cli->device); if (ret) { @@ -1083,8 +1084,6 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) if (ret) goto done; - cli->base.super = false; - fpriv->driver_priv = cli; mutex_lock(&drm->client.mutex); diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 0de6549fb875..2ca3207c13fc 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -41,8 +41,6 @@ nouveau_mem_map(struct nouveau_mem *mem, struct gf100_vmm_map_v0 gf100; } args; u32 argc = 0; - bool super; - int ret; switch (vmm->object.oclass) { case NVIF_CLASS_VMM_NV04: @@ -73,12 +71,7 @@ nouveau_mem_map(struct nouveau_mem *mem, return -ENOSYS; } - super = vmm->object.client->super; - vmm->object.client->super = true; - ret = nvif_vmm_map(vmm, vma->addr, mem->mem.size, &args, argc, - &mem->mem, 0); - vmm->object.client->super = super; - return ret; + return nvif_vmm_map(vmm, vma->addr, mem->mem.size, &args, argc, &mem->mem, 0); } void @@ -99,7 +92,6 @@ nouveau_mem_host(struct ttm_resource *reg, struct ttm_tt *tt) struct nouveau_drm *drm = cli->drm; struct nvif_mmu *mmu = &cli->mmu; struct nvif_mem_ram_v0 args = {}; - bool super = cli->base.super; u8 type; int ret; @@ -122,11 +114,9 @@ nouveau_mem_host(struct ttm_resource *reg, struct ttm_tt *tt) args.dma = tt->dma_address; mutex_lock(&drm->master.lock); - cli->base.super = true; ret = nvif_mem_ctor_type(mmu, "ttmHostMem", cli->mem->oclass, type, PAGE_SHIFT, reg->num_pages << PAGE_SHIFT, &args, sizeof(args), &mem->mem); - cli->base.super = super; mutex_unlock(&drm->master.lock); return ret; } @@ -138,12 +128,10 @@ nouveau_mem_vram(struct ttm_resource *reg, bool contig, u8 page) struct nouveau_cli *cli = mem->cli; struct nouveau_drm *drm = cli->drm; struct nvif_mmu *mmu = &cli->mmu; - bool super = cli->base.super; u64 size = ALIGN(reg->num_pages << PAGE_SHIFT, 1 << page); int ret; mutex_lock(&drm->master.lock); - cli->base.super = true; switch (cli->mem->oclass) { case NVIF_CLASS_MEM_GF100: ret = nvif_mem_ctor_type(mmu, "ttmVram", cli->mem->oclass, @@ -167,7 +155,6 @@ nouveau_mem_vram(struct ttm_resource *reg, bool contig, u8 page) WARN_ON(1); break; } - cli->base.super = super; mutex_unlock(&drm->master.lock); reg->start = mem->mem.addr >> PAGE_SHIFT; diff --git a/drivers/gpu/drm/nouveau/nouveau_nvif.c b/drivers/gpu/drm/nouveau/nouveau_nvif.c index b3f29b1ce9ea..52f5793b7274 100644 --- a/drivers/gpu/drm/nouveau/nouveau_nvif.c +++ b/drivers/gpu/drm/nouveau/nouveau_nvif.c @@ -52,9 +52,9 @@ nvkm_client_map(void *priv, u64 handle, u32 size) } static int -nvkm_client_ioctl(void *priv, bool super, void *data, u32 size, void **hack) +nvkm_client_ioctl(void *priv, void *data, u32 size, void **hack) { - return nvkm_ioctl(priv, super, data, size, hack); + return nvkm_ioctl(priv, data, size, hack); } static int diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index 82b583f5fca8..b0c3422cb01f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -237,14 +237,11 @@ void nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit) { if (limit > start) { - bool super = svmm->vmm->vmm.object.client->super; - svmm->vmm->vmm.object.client->super = true; nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR, &(struct nvif_vmm_pfnclr_v0) { .addr = start, .size = limit - start, }, sizeof(struct nvif_vmm_pfnclr_v0)); - svmm->vmm->vmm.object.client->super = super; } } @@ -634,9 +631,7 @@ static int nouveau_atomic_range_fault(struct nouveau_svmm *svmm, NVIF_VMM_PFNMAP_V0_A | NVIF_VMM_PFNMAP_V0_HOST; - svmm->vmm->vmm.object.client->super = true; ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL); - svmm->vmm->vmm.object.client->super = false; mutex_unlock(&svmm->mutex); unlock_page(page); @@ -702,9 +697,7 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, nouveau_hmm_convert_pfn(drm, &range, args); - svmm->vmm->vmm.object.client->super = true; ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL); - svmm->vmm->vmm.object.client->super = false; mutex_unlock(&svmm->mutex); out: @@ -928,10 +921,8 @@ nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm, mutex_lock(&svmm->mutex); - svmm->vmm->vmm.object.client->super = true; ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, sizeof(*args) + npages * sizeof(args->p.phys[0]), NULL); - svmm->vmm->vmm.object.client->super = false; mutex_unlock(&svmm->mutex); } diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index f4c2e46b6fe1..2ca9d9a9e5d5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -276,7 +276,7 @@ nouveau_ttm_init(struct nouveau_drm *drm) } #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86) - need_swiotlb = is_swiotlb_active(); + need_swiotlb = is_swiotlb_active(dev->dev); #endif ret = ttm_device_init(&drm->ttm.bdev, &nouveau_bo_driver, drm->dev->dev, diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.c b/drivers/gpu/drm/nouveau/nouveau_usif.c index 9dc10b17ad34..5da1f4d223d7 100644 --- a/drivers/gpu/drm/nouveau/nouveau_usif.c +++ b/drivers/gpu/drm/nouveau/nouveau_usif.c @@ -32,6 +32,9 @@ #include <nvif/event.h> #include <nvif/ioctl.h> +#include <nvif/class.h> +#include <nvif/cl0080.h> + struct usif_notify_p { struct drm_pending_event base; struct { @@ -261,7 +264,7 @@ usif_object_dtor(struct usif_object *object) } static int -usif_object_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc) +usif_object_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc, bool parent_abi16) { struct nouveau_cli *cli = nouveau_cli(f); struct nvif_client *client = &cli->base; @@ -271,23 +274,48 @@ usif_object_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc) struct usif_object *object; int ret = -ENOSYS; + if ((ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) + return ret; + + switch (args->v0.oclass) { + case NV_DMA_FROM_MEMORY: + case NV_DMA_TO_MEMORY: + case NV_DMA_IN_MEMORY: + return -EINVAL; + case NV_DEVICE: { + union { + struct nv_device_v0 v0; + } *args = data; + + if ((ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) + return ret; + + args->v0.priv = false; + break; + } + default: + if (!parent_abi16) + return -EINVAL; + break; + } + if (!(object = kmalloc(sizeof(*object), GFP_KERNEL))) return -ENOMEM; list_add(&object->head, &cli->objects); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - object->route = args->v0.route; - object->token = args->v0.token; - args->v0.route = NVDRM_OBJECT_USIF; - args->v0.token = (unsigned long)(void *)object; - ret = nvif_client_ioctl(client, argv, argc); - args->v0.token = object->token; - args->v0.route = object->route; + object->route = args->v0.route; + object->token = args->v0.token; + args->v0.route = NVDRM_OBJECT_USIF; + args->v0.token = (unsigned long)(void *)object; + ret = nvif_client_ioctl(client, argv, argc); + if (ret) { + usif_object_dtor(object); + return ret; } - if (ret) - usif_object_dtor(object); - return ret; + args->v0.token = object->token; + args->v0.route = object->route; + return 0; } int @@ -301,6 +329,7 @@ usif_ioctl(struct drm_file *filp, void __user *user, u32 argc) struct nvif_ioctl_v0 v0; } *argv = data; struct usif_object *object; + bool abi16 = false; u8 owner; int ret; @@ -331,11 +360,13 @@ usif_ioctl(struct drm_file *filp, void __user *user, u32 argc) mutex_unlock(&cli->mutex); goto done; } + + abi16 = true; } switch (argv->v0.type) { case NVIF_IOCTL_V0_NEW: - ret = usif_object_new(filp, data, size, argv, argc); + ret = usif_object_new(filp, data, size, argv, argc, abi16); break; case NVIF_IOCTL_V0_NTFY_NEW: ret = usif_notify_new(filp, data, size, argv, argc); diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c index 12644f811b3e..a3264a0e933a 100644 --- a/drivers/gpu/drm/nouveau/nvif/client.c +++ b/drivers/gpu/drm/nouveau/nvif/client.c @@ -32,7 +32,7 @@ int nvif_client_ioctl(struct nvif_client *client, void *data, u32 size) { - return client->driver->ioctl(client->object.priv, client->super, data, size, NULL); + return client->driver->ioctl(client->object.priv, data, size, NULL); } int @@ -80,7 +80,6 @@ nvif_client_ctor(struct nvif_client *parent, const char *name, u64 device, client->object.client = client; client->object.handle = ~0; client->route = NVIF_IOCTL_V0_ROUTE_NVIF; - client->super = true; client->driver = parent->driver; if (ret == 0) { diff --git a/drivers/gpu/drm/nouveau/nvif/object.c b/drivers/gpu/drm/nouveau/nvif/object.c index 671a5c0199e0..dce1ecee2af5 100644 --- a/drivers/gpu/drm/nouveau/nvif/object.c +++ b/drivers/gpu/drm/nouveau/nvif/object.c @@ -44,8 +44,7 @@ nvif_object_ioctl(struct nvif_object *object, void *data, u32 size, void **hack) } else return -ENOSYS; - return client->driver->ioctl(client->object.priv, client->super, - data, size, hack); + return client->driver->ioctl(client->object.priv, data, size, hack); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index d777df5a64e6..735cb6816f10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -426,8 +426,7 @@ nvkm_ioctl_path(struct nvkm_client *client, u64 handle, u32 type, } int -nvkm_ioctl(struct nvkm_client *client, bool supervisor, - void *data, u32 size, void **hack) +nvkm_ioctl(struct nvkm_client *client, void *data, u32 size, void **hack) { struct nvkm_object *object = &client->object; union { @@ -435,7 +434,6 @@ nvkm_ioctl(struct nvkm_client *client, bool supervisor, } *args = data; int ret = -ENOSYS; - client->super = supervisor; nvif_ioctl(object, "size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index b930f539feec..93ddf63d1114 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2624,6 +2624,26 @@ nv174_chipset = { .dma = { 0x00000001, gv100_dma_new }, }; +static const struct nvkm_device_chip +nv177_chipset = { + .name = "GA107", + .bar = { 0x00000001, tu102_bar_new }, + .bios = { 0x00000001, nvkm_bios_new }, + .devinit = { 0x00000001, ga100_devinit_new }, + .fb = { 0x00000001, ga102_fb_new }, + .gpio = { 0x00000001, ga102_gpio_new }, + .i2c = { 0x00000001, gm200_i2c_new }, + .imem = { 0x00000001, nv50_instmem_new }, + .mc = { 0x00000001, ga100_mc_new }, + .mmu = { 0x00000001, tu102_mmu_new }, + .pci = { 0x00000001, gp100_pci_new }, + .privring = { 0x00000001, gm200_privring_new }, + .timer = { 0x00000001, gk20a_timer_new }, + .top = { 0x00000001, ga100_top_new }, + .disp = { 0x00000001, ga102_disp_new }, + .dma = { 0x00000001, gv100_dma_new }, +}; + static int nvkm_device_event_ctor(struct nvkm_object *object, void *data, u32 size, struct nvkm_notify *notify) @@ -3049,6 +3069,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, case 0x168: device->chip = &nv168_chipset; break; case 0x172: device->chip = &nv172_chipset; break; case 0x174: device->chip = &nv174_chipset; break; + case 0x177: device->chip = &nv177_chipset; break; default: if (nvkm_boolopt(device->cfgopt, "NvEnableUnsupportedChipsets", false)) { switch (device->chipset) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index fea9d8f2b10c..f28894fdede9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -397,7 +397,7 @@ nvkm_udevice_new(const struct nvkm_oclass *oclass, void *data, u32 size, return ret; /* give priviledged clients register access */ - if (client->super) + if (args->v0.priv) func = &nvkm_udevice_super; else func = &nvkm_udevice; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c index 55fbfe28c6dc..9669472a2749 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c @@ -440,7 +440,7 @@ nvkm_dp_train(struct nvkm_dp *dp, u32 dataKBps) return ret; } -static void +void nvkm_dp_disable(struct nvkm_outp *outp, struct nvkm_ior *ior) { struct nvkm_dp *dp = nvkm_dp(outp); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h index 428b3f488f03..e484d0c3b0d4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h @@ -32,6 +32,7 @@ struct nvkm_dp { int nvkm_dp_new(struct nvkm_disp *, int index, struct dcb_output *, struct nvkm_outp **); +void nvkm_dp_disable(struct nvkm_outp *, struct nvkm_ior *); /* DPCD Receiver Capabilities */ #define DPCD_RC00_DPCD_REV 0x00000 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c index dffcac249211..129982fef7ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "outp.h" +#include "dp.h" #include "ior.h" #include <subdev/bios.h> @@ -257,6 +258,14 @@ nvkm_outp_init_route(struct nvkm_outp *outp) if (!ior->arm.head || ior->arm.proto != proto) { OUTP_DBG(outp, "no heads (%x %d %d)", ior->arm.head, ior->arm.proto, proto); + + /* The EFI GOP driver on Ampere can leave unused DP links routed, + * which we don't expect. The DisableLT IED script *should* get + * us back to where we need to be. + */ + if (ior->func->route.get && !ior->arm.head && outp->info.type == DCB_OUTPUT_DP) + nvkm_dp_disable(outp, ior); + return; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c index d20cc0681a88..797131ed7d67 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.c @@ -26,7 +26,6 @@ #include <core/client.h> #include <core/gpuobj.h> #include <subdev/fb.h> -#include <subdev/instmem.h> #include <nvif/cl0002.h> #include <nvif/unpack.h> @@ -72,11 +71,7 @@ nvkm_dmaobj_ctor(const struct nvkm_dmaobj_func *func, struct nvkm_dma *dma, union { struct nv_dma_v0 v0; } *args = *pdata; - struct nvkm_device *device = dma->engine.subdev.device; - struct nvkm_client *client = oclass->client; struct nvkm_object *parent = oclass->parent; - struct nvkm_instmem *instmem = device->imem; - struct nvkm_fb *fb = device->fb; void *data = *pdata; u32 size = *psize; int ret = -ENOSYS; @@ -109,23 +104,13 @@ nvkm_dmaobj_ctor(const struct nvkm_dmaobj_func *func, struct nvkm_dma *dma, dmaobj->target = NV_MEM_TARGET_VM; break; case NV_DMA_V0_TARGET_VRAM: - if (!client->super) { - if (dmaobj->limit >= fb->ram->size - instmem->reserved) - return -EACCES; - if (device->card_type >= NV_50) - return -EACCES; - } dmaobj->target = NV_MEM_TARGET_VRAM; break; case NV_DMA_V0_TARGET_PCI: - if (!client->super) - return -EACCES; dmaobj->target = NV_MEM_TARGET_PCI; break; case NV_DMA_V0_TARGET_PCI_US: case NV_DMA_V0_TARGET_AGP: - if (!client->super) - return -EACCES; dmaobj->target = NV_MEM_TARGET_PCI_NOSNOOP; break; default: diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 90e9a0972a44..3209eb7af65f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -27,8 +27,6 @@ nvkm-y += nvkm/engine/fifo/dmanv04.o nvkm-y += nvkm/engine/fifo/dmanv10.o nvkm-y += nvkm/engine/fifo/dmanv17.o nvkm-y += nvkm/engine/fifo/dmanv40.o -nvkm-y += nvkm/engine/fifo/dmanv50.o -nvkm-y += nvkm/engine/fifo/dmag84.o nvkm-y += nvkm/engine/fifo/gpfifonv50.o nvkm-y += nvkm/engine/fifo/gpfifog84.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index af8bdf275552..3a95730d7ff5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -48,8 +48,6 @@ void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int); int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); -extern const struct nvkm_fifo_chan_oclass nv50_fifo_dma_oclass; extern const struct nvkm_fifo_chan_oclass nv50_fifo_gpfifo_oclass; -extern const struct nvkm_fifo_chan_oclass g84_fifo_dma_oclass; extern const struct nvkm_fifo_chan_oclass g84_fifo_gpfifo_oclass; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c deleted file mode 100644 index fc34cddcd2f5..000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include <core/client.h> -#include <core/ramht.h> - -#include <nvif/class.h> -#include <nvif/cl826e.h> -#include <nvif/unpack.h> - -static int -g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct g82_channel_dma_v0 v0; - } *args = data; - struct nv50_fifo *fifo = nv50_fifo(base); - struct nv50_fifo_chan *chan; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d vmm %llx " - "pushbuf %llx offset %016llx\n", - args->v0.version, args->v0.vmm, args->v0.pushbuf, - args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, - oclass, chan); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - nvkm_kmap(chan->ramfc); - nvkm_wo32(chan->ramfc, 0x08, lower_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x0c, upper_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x10, lower_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x14, upper_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x3c, 0x003f6078); - nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); - nvkm_wo32(chan->ramfc, 0x48, chan->base.push->node->offset >> 4); - nvkm_wo32(chan->ramfc, 0x4c, 0xffffffff); - nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); - nvkm_wo32(chan->ramfc, 0x78, 0x00000000); - nvkm_wo32(chan->ramfc, 0x7c, 0x30000001); - nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | - (4 << 24) /* SEARCH_FULL */ | - (chan->ramht->gpuobj->node->offset >> 4)); - nvkm_wo32(chan->ramfc, 0x88, chan->cache->addr >> 10); - nvkm_wo32(chan->ramfc, 0x98, chan->base.inst->addr >> 12); - nvkm_done(chan->ramfc); - return 0; -} - -const struct nvkm_fifo_chan_oclass -g84_fifo_dma_oclass = { - .base.oclass = G82_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, - .ctor = g84_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c deleted file mode 100644 index 8043718ad150..000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include <core/client.h> -#include <core/ramht.h> - -#include <nvif/class.h> -#include <nvif/cl506e.h> -#include <nvif/unpack.h> - -static int -nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv50_channel_dma_v0 v0; - } *args = data; - struct nv50_fifo *fifo = nv50_fifo(base); - struct nv50_fifo_chan *chan; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d vmm %llx " - "pushbuf %llx offset %016llx\n", - args->v0.version, args->v0.vmm, args->v0.pushbuf, - args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, - oclass, chan); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - nvkm_kmap(chan->ramfc); - nvkm_wo32(chan->ramfc, 0x08, lower_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x0c, upper_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x10, lower_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x14, upper_32_bits(args->v0.offset)); - nvkm_wo32(chan->ramfc, 0x3c, 0x003f6078); - nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); - nvkm_wo32(chan->ramfc, 0x48, chan->base.push->node->offset >> 4); - nvkm_wo32(chan->ramfc, 0x4c, 0xffffffff); - nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); - nvkm_wo32(chan->ramfc, 0x78, 0x00000000); - nvkm_wo32(chan->ramfc, 0x7c, 0x30000001); - nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | - (4 << 24) /* SEARCH_FULL */ | - (chan->ramht->gpuobj->node->offset >> 4)); - nvkm_done(chan->ramfc); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv50_fifo_dma_oclass = { - .base.oclass = NV50_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, - .ctor = nv50_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index c0a7d0f21dac..3885c3830b94 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -119,7 +119,6 @@ g84_fifo = { .uevent_init = g84_fifo_uevent_init, .uevent_fini = g84_fifo_uevent_fini, .chan = { - &g84_fifo_dma_oclass, &g84_fifo_gpfifo_oclass, NULL }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index b6900a52bcce..ae6c4d846eb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -341,8 +341,6 @@ gk104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, "runlist %016llx priv %d\n", args->v0.version, args->v0.vmm, args->v0.ioffset, args->v0.ilength, args->v0.runlist, args->v0.priv); - if (args->v0.priv && !oclass->client->super) - return -EINVAL; return gk104_fifo_gpfifo_new_(fifo, &args->v0.runlist, &args->v0.chid, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index ee4967b706a7..743791c514fe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -226,8 +226,6 @@ gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, "runlist %016llx priv %d\n", args->v0.version, args->v0.vmm, args->v0.ioffset, args->v0.ilength, args->v0.runlist, args->v0.priv); - if (args->v0.priv && !oclass->client->super) - return -EINVAL; return gv100_fifo_gpfifo_new_(&gv100_fifo_gpfifo, fifo, &args->v0.runlist, &args->v0.chid, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index abef7fb6e2d3..99aafa103a31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -65,8 +65,6 @@ tu102_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, "runlist %016llx priv %d\n", args->v0.version, args->v0.vmm, args->v0.ioffset, args->v0.ilength, args->v0.runlist, args->v0.priv); - if (args->v0.priv && !oclass->client->super) - return -EINVAL; return gv100_fifo_gpfifo_new_(&tu102_fifo_gpfifo, fifo, &args->v0.runlist, &args->v0.chid, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index be94156ea248..a08742cf425a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -136,7 +136,6 @@ nv50_fifo = { .pause = nv04_fifo_pause, .start = nv04_fifo_start, .chan = { - &nv50_fifo_dma_oclass, &nv50_fifo_gpfifo_oclass, NULL }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c index fac2f9a45ea6..e530bb8b3b17 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c @@ -41,7 +41,7 @@ nvkm_umem_search(struct nvkm_client *client, u64 handle) object = nvkm_object_search(client, handle, &nvkm_umem); if (IS_ERR(object)) { - if (client->super && client != master) { + if (client != master) { spin_lock(&master->lock); list_for_each_entry(umem, &master->umem, head) { if (umem->object.object == handle) { @@ -53,8 +53,7 @@ nvkm_umem_search(struct nvkm_client *client, u64 handle) } } else { umem = nvkm_umem(object); - if (!umem->priv || client->super) - memory = nvkm_memory_ref(umem->memory); + memory = nvkm_memory_ref(umem->memory); } return memory ? memory : ERR_PTR(-ENOENT); @@ -167,7 +166,6 @@ nvkm_umem_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, nvkm_object_ctor(&nvkm_umem, oclass, &umem->object); umem->mmu = mmu; umem->type = mmu->type[type].type; - umem->priv = oclass->client->super; INIT_LIST_HEAD(&umem->head); *pobject = &umem->object; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h index 85cf692d620a..d56a594016cc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h @@ -8,7 +8,6 @@ struct nvkm_umem { struct nvkm_object object; struct nvkm_mmu *mmu; u8 type:8; - bool priv:1; bool mappable:1; bool io:1; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c index 0e4b8941da37..6870fda4b188 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c @@ -34,7 +34,7 @@ nvkm_ummu_sclass(struct nvkm_object *object, int index, { struct nvkm_mmu *mmu = nvkm_ummu(object)->mmu; - if (mmu->func->mem.user.oclass && oclass->client->super) { + if (mmu->func->mem.user.oclass) { if (index-- == 0) { oclass->base = mmu->func->mem.user; oclass->ctor = nvkm_umem_new; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c index c43b8248c682..d6a1f8d04c09 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -45,7 +45,6 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle) static int nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { - struct nvkm_client *client = uvmm->object.client; union { struct nvif_vmm_pfnclr_v0 v0; } *args = argv; @@ -59,9 +58,6 @@ nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) } else return ret; - if (!client->super) - return -ENOENT; - if (size) { mutex_lock(&vmm->mutex); ret = nvkm_vmm_pfn_unmap(vmm, addr, size); @@ -74,7 +70,6 @@ nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) static int nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { - struct nvkm_client *client = uvmm->object.client; union { struct nvif_vmm_pfnmap_v0 v0; } *args = argv; @@ -93,9 +88,6 @@ nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) } else return ret; - if (!client->super) - return -ENOENT; - if (size) { mutex_lock(&vmm->mutex); ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); @@ -108,7 +100,6 @@ nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) static int nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { - struct nvkm_client *client = uvmm->object.client; union { struct nvif_vmm_unmap_v0 v0; } *args = argv; @@ -130,9 +121,8 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto done; } - if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { - VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, - vma->user, !client->super, vma->busy); + if (ret = -ENOENT, vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy); goto done; } @@ -181,9 +171,8 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } - if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { - VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, - vma->user, !client->super, vma->busy); + if (ret = -ENOENT, vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy); goto fail; } @@ -230,7 +219,6 @@ fail: static int nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { - struct nvkm_client *client = uvmm->object.client; union { struct nvif_vmm_put_v0 v0; } *args = argv; @@ -252,9 +240,8 @@ nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto done; } - if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { - VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, - vma->user, !client->super, vma->busy); + if (ret = -ENOENT, vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d", addr, vma->busy); goto done; } @@ -268,7 +255,6 @@ done: static int nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { - struct nvkm_client *client = uvmm->object.client; union { struct nvif_vmm_get_v0 v0; } *args = argv; @@ -297,7 +283,6 @@ nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc) return ret; args->v0.addr = vma->addr; - vma->user = !client->super; return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 710f3f8dc7c9..8bf00b396ec1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -774,7 +774,6 @@ nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) new->refd = vma->refd; new->used = vma->used; new->part = vma->part; - new->user = vma->user; new->busy = vma->busy; new->mapped = vma->mapped; list_add(&new->head, &vma->head); @@ -951,7 +950,7 @@ nvkm_vmm_node_split(struct nvkm_vmm *vmm, static void nvkm_vma_dump(struct nvkm_vma *vma) { - printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n", + printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n", vma->addr, (u64)vma->size, vma->used ? '-' : 'F', vma->mapref ? 'R' : '-', @@ -959,7 +958,6 @@ nvkm_vma_dump(struct nvkm_vma *vma) vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-', vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-', vma->part ? 'P' : '-', - vma->user ? 'U' : '-', vma->busy ? 'B' : '-', vma->mapped ? 'M' : '-', vma->memory); @@ -1024,7 +1022,6 @@ nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size) vma->mapref = true; vma->sparse = false; vma->used = true; - vma->user = true; nvkm_vmm_node_insert(vmm, vma); list_add_tail(&vma->head, &vmm->list); return 0; @@ -1615,7 +1612,6 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) vma->page = NVKM_VMA_PAGE_NONE; vma->refd = NVKM_VMA_PAGE_NONE; vma->used = false; - vma->user = false; nvkm_vmm_put_region(vmm, vma); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c index f02abd9cb4dd..b5e733783b5b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -534,15 +534,13 @@ int gp100_vmm_mthd(struct nvkm_vmm *vmm, struct nvkm_client *client, u32 mthd, void *argv, u32 argc) { - if (client->super) { - switch (mthd) { - case GP100_VMM_VN_FAULT_REPLAY: - return gp100_vmm_fault_replay(vmm, argv, argc); - case GP100_VMM_VN_FAULT_CANCEL: - return gp100_vmm_fault_cancel(vmm, argv, argc); - default: - break; - } + switch (mthd) { + case GP100_VMM_VN_FAULT_REPLAY: + return gp100_vmm_fault_replay(vmm, argv, argc); + case GP100_VMM_VN_FAULT_CANCEL: + return gp100_vmm_fault_cancel(vmm, argv, argc); + default: + break; } return -EINVAL; } diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 4ec310a650cd..925079209086 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -4428,7 +4428,7 @@ static const struct drm_display_mode yes_optoelectronics_ytc700tlag_05_201c_mode static const struct panel_desc yes_optoelectronics_ytc700tlag_05_201c = { .modes = &yes_optoelectronics_ytc700tlag_05_201c_mode, .num_modes = 1, - .bpc = 6, + .bpc = 8, .size = { .width = 154, .height = 90, diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index c3fbe0ad9090..f7f83cf63f42 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -58,25 +58,16 @@ static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd) } static void lock_region(struct panfrost_device *pfdev, u32 as_nr, - u64 iova, size_t size) + u64 iova, u64 size) { u8 region_width; u64 region = iova & PAGE_MASK; - /* - * fls returns: - * 1 .. 32 - * - * 10 + fls(num_pages) - * results in the range (11 .. 42) - */ - - size = round_up(size, PAGE_SIZE); - region_width = 10 + fls(size >> PAGE_SHIFT); - if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) { - /* not pow2, so must go up to the next pow2 */ - region_width += 1; - } + /* The size is encoded as ceil(log2) minus(1), which may be calculated + * with fls. The size must be clamped to hardware bounds. + */ + size = max_t(u64, size, AS_LOCK_REGION_MIN_SIZE); + region_width = fls64(size - 1) - 1; region |= region_width; /* Lock the region that needs to be updated */ @@ -87,7 +78,7 @@ static void lock_region(struct panfrost_device *pfdev, u32 as_nr, static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr, - u64 iova, size_t size, u32 op) + u64 iova, u64 size, u32 op) { if (as_nr < 0) return 0; @@ -104,7 +95,7 @@ static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr, static int mmu_hw_do_operation(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, - u64 iova, size_t size, u32 op) + u64 iova, u64 size, u32 op) { int ret; @@ -121,7 +112,7 @@ static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_m u64 transtab = cfg->arm_mali_lpae_cfg.transtab; u64 memattr = cfg->arm_mali_lpae_cfg.memattr; - mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM); mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab)); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab)); @@ -137,7 +128,7 @@ static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_m static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr) { - mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM); mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0); @@ -251,7 +242,7 @@ static size_t get_pgsize(u64 addr, size_t size) static void panfrost_mmu_flush_range(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, - u64 iova, size_t size) + u64 iova, u64 size) { if (mmu->as < 0) return; diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h index 1940ff86e49a..6c5a11ef1ee8 100644 --- a/drivers/gpu/drm/panfrost/panfrost_regs.h +++ b/drivers/gpu/drm/panfrost/panfrost_regs.h @@ -316,6 +316,8 @@ #define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2 << 8) #define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3 << 8) +#define AS_LOCK_REGION_MIN_SIZE (1ULL << 15) + #define gpu_write(dev, reg, data) writel(data, dev->iomem + reg) #define gpu_read(dev, reg) readl(dev->iomem + reg) diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h index 4b86e8b45009..83e8b8547f9b 100644 --- a/drivers/gpu/drm/radeon/atombios.h +++ b/drivers/gpu/drm/radeon/atombios.h @@ -2802,8 +2802,8 @@ ucMaxNBVoltageHigh: Voltage regulator dependent PWM value. High 8 bits of t ucMinNBVoltageHigh: Voltage regulator dependent PWM value. High 8 bits of the value for the min voltage.Set this one to 0x00 if VC without PWM or no VC at all. -usInterNBVoltageLow: Voltage regulator dependent PWM value. The value makes the the voltage >=Min NB voltage but <=InterNBVoltageHigh. Set this to 0x0000 if VC without PWM or no VC at all. -usInterNBVoltageHigh: Voltage regulator dependent PWM value. The value makes the the voltage >=InterNBVoltageLow but <=Max NB voltage.Set this to 0x0000 if VC without PWM or no VC at all. +usInterNBVoltageLow: Voltage regulator dependent PWM value. The value makes the voltage >=Min NB voltage but <=InterNBVoltageHigh. Set this to 0x0000 if VC without PWM or no VC at all. +usInterNBVoltageHigh: Voltage regulator dependent PWM value. The value makes the voltage >=InterNBVoltageLow but <=Max NB voltage.Set this to 0x0000 if VC without PWM or no VC at all. */ diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 36a888e1b179..eeb590d2dec2 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -28,6 +28,7 @@ #include <drm/drm_vblank.h> #include <drm/radeon_drm.h> +#include <drm/drm_fourcc.h> #include "atom.h" #include "avivod.h" @@ -1414,10 +1415,15 @@ void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + struct drm_framebuffer *fb = radeon_crtc->base.primary->fb; - /* update the scanout addresses */ + /* flip at hsync for async, default is vsync */ WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); + /* update pitch */ + WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset, + fb->pitches[0] / fb->format->cpp[0]); + /* update the scanout addresses */ WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, upper_32_bits(crtc_base)); WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index ba724198b72e..2dd85ba1faa2 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -162,6 +162,8 @@ void r100_wait_for_vblank(struct radeon_device *rdev, int crtc) void r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + uint32_t crtc_pitch, pitch_pixels; + struct drm_framebuffer *fb = radeon_crtc->base.primary->fb; u32 tmp = ((u32)crtc_base) | RADEON_CRTC_OFFSET__OFFSET_LOCK; int i; @@ -169,6 +171,13 @@ void r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool /* update the scanout addresses */ WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp); + /* update pitch */ + pitch_pixels = fb->pitches[0] / fb->format->cpp[0]; + crtc_pitch = DIV_ROUND_UP(pitch_pixels * fb->format->cpp[0] * 8, + fb->format->cpp[0] * 8 * 8); + crtc_pitch |= crtc_pitch << 16; + WREG32(RADEON_CRTC_PITCH + radeon_crtc->crtc_offset, crtc_pitch); + /* Wait for update_pending to go high. */ for (i = 0; i < rdev->usec_timeout; i++) { if (RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET) diff --git a/drivers/gpu/drm/radeon/r300_reg.h b/drivers/gpu/drm/radeon/r300_reg.h index 00c0d2ba22d3..60d5413bafa1 100644 --- a/drivers/gpu/drm/radeon/r300_reg.h +++ b/drivers/gpu/drm/radeon/r300_reg.h @@ -353,7 +353,7 @@ # define R300_PVS_CNTL_1_PROGRAM_START_SHIFT 0 # define R300_PVS_CNTL_1_POS_END_SHIFT 10 # define R300_PVS_CNTL_1_PROGRAM_END_SHIFT 20 -/* Addresses are relative the the vertex program parameters area. */ +/* Addresses are relative the vertex program parameters area. */ #define R300_VAP_PVS_CNTL_2 0x22D4 # define R300_PVS_CNTL_2_PARAM_OFFSET_SHIFT 0 # define R300_PVS_CNTL_2_PARAM_COUNT_SHIFT 16 diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index cec03238e14d..4f0fbf667431 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -406,7 +406,7 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) /* * radeon_wb_*() - * Writeback is the the method by which the the GPU updates special pages + * Writeback is the method by which the GPU updates special pages * in memory with the status of certain GPU events (fences, ring pointers, * etc.). */ @@ -785,7 +785,7 @@ int radeon_dummy_page_init(struct radeon_device *rdev) if (rdev->dummy_page.page == NULL) return -ENOMEM; rdev->dummy_page.addr = dma_map_page(&rdev->pdev->dev, rdev->dummy_page.page, - 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + 0, PAGE_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(&rdev->pdev->dev, rdev->dummy_page.addr)) { dev_err(&rdev->pdev->dev, "Failed to DMA MAP the dummy page\n"); __free_page(rdev->dummy_page.page); @@ -808,8 +808,8 @@ void radeon_dummy_page_fini(struct radeon_device *rdev) { if (rdev->dummy_page.page == NULL) return; - pci_unmap_page(rdev->pdev, rdev->dummy_page.addr, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(&rdev->pdev->dev, rdev->dummy_page.addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); __free_page(rdev->dummy_page.page); rdev->dummy_page.page = NULL; } diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c index 0b206b052972..ca382fbf7a86 100644 --- a/drivers/gpu/drm/radeon/radeon_fb.c +++ b/drivers/gpu/drm/radeon/radeon_fb.c @@ -54,6 +54,7 @@ radeonfb_open(struct fb_info *info, int user) struct radeon_fbdev *rfbdev = info->par; struct radeon_device *rdev = rfbdev->rdev; int ret = pm_runtime_get_sync(rdev->ddev->dev); + if (ret < 0 && ret != -EACCES) { pm_runtime_mark_last_busy(rdev->ddev->dev); pm_runtime_put_autosuspend(rdev->ddev->dev); @@ -167,6 +168,7 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev, break; case 2: tiling_flags |= RADEON_TILING_SWAP_16BIT; + break; default: break; } @@ -196,9 +198,8 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev, radeon_bo_check_tiling(rbo, 0, 0); ret = radeon_bo_kmap(rbo, NULL); radeon_bo_unreserve(rbo); - if (ret) { + if (ret) goto out_unref; - } *gobj_p = gobj; return 0; @@ -294,9 +295,6 @@ static int radeonfb_create(struct drm_fb_helper *helper, return 0; out: - if (rbo) { - - } if (fb && ret) { drm_gem_object_put(gobj); drm_framebuffer_unregister_private(fb); diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 3f351d222cbb..73e3117420bf 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -50,7 +50,7 @@ * for GPU/CPU synchronization. When the fence is written, * it is expected that all buffers associated with that fence * are no longer in use by the associated ring on the GPU and - * that the the relevant GPU caches have been flushed. Whether + * that the relevant GPU caches have been flushed. Whether * we use a scratch register or memory location depends on the asic * and whether writeback is enabled. */ diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 36a38adaaea9..bb53016f3138 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -41,7 +41,7 @@ * (uncached system pages). * Each VM has an ID associated with it and there is a page table * associated with each VMID. When execting a command buffer, - * the kernel tells the the ring what VMID to use for that command + * the kernel tells the ring what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c index b2d22e25eee1..b87dd551e939 100644 --- a/drivers/gpu/drm/radeon/rs600.c +++ b/drivers/gpu/drm/radeon/rs600.c @@ -41,6 +41,7 @@ #include <drm/drm_device.h> #include <drm/drm_vblank.h> +#include <drm/drm_fourcc.h> #include "atom.h" #include "radeon.h" @@ -118,6 +119,7 @@ void avivo_wait_for_vblank(struct radeon_device *rdev, int crtc) void rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + struct drm_framebuffer *fb = radeon_crtc->base.primary->fb; u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset); int i; @@ -125,9 +127,13 @@ void rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, boo tmp |= AVIVO_D1GRPH_UPDATE_LOCK; WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); - /* update the scanout addresses */ + /* flip at hsync for async, default is vsync */ WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); + /* update pitch */ + WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset, + fb->pitches[0] / fb->format->cpp[0]); + /* update the scanout addresses */ WREG32(AVIVO_D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, (u32)crtc_base); WREG32(AVIVO_D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c index 74499307285b..e592e57be1bb 100644 --- a/drivers/gpu/drm/radeon/rv770.c +++ b/drivers/gpu/drm/radeon/rv770.c @@ -32,6 +32,7 @@ #include <drm/drm_device.h> #include <drm/radeon_drm.h> +#include <drm/drm_fourcc.h> #include "atom.h" #include "avivod.h" @@ -809,6 +810,7 @@ u32 rv770_get_xclk(struct radeon_device *rdev) void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + struct drm_framebuffer *fb = radeon_crtc->base.primary->fb; u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset); int i; @@ -816,9 +818,13 @@ void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, boo tmp |= AVIVO_D1GRPH_UPDATE_LOCK; WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); - /* update the scanout addresses */ + /* flip at hsync for async, default is vsync */ WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); + /* update pitch */ + WREG32(AVIVO_D1GRPH_PITCH + radeon_crtc->crtc_offset, + fb->pitches[0] / fb->format->cpp[0]); + /* update the scanout addresses */ if (radeon_crtc->crtc_id) { WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base)); WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base)); diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig index 5043dcaf1cf9..1650a448eabd 100644 --- a/drivers/gpu/drm/tegra/Kconfig +++ b/drivers/gpu/drm/tegra/Kconfig @@ -9,6 +9,7 @@ config DRM_TEGRA select DRM_MIPI_DSI select DRM_PANEL select TEGRA_HOST1X + select INTERCONNECT select IOMMU_IOVA select CEC_CORE if CEC_NOTIFIER help diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile index d6cf202414f0..d801909182cf 100644 --- a/drivers/gpu/drm/tegra/Makefile +++ b/drivers/gpu/drm/tegra/Makefile @@ -3,6 +3,9 @@ ccflags-$(CONFIG_DRM_TEGRA_DEBUG) += -DDEBUG tegra-drm-y := \ drm.o \ + uapi.o \ + submit.o \ + firewall.o \ gem.o \ fb.o \ dp.o \ diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c index 51bbbc42a144..16c7aabb94d3 100644 --- a/drivers/gpu/drm/tegra/dc.c +++ b/drivers/gpu/drm/tegra/dc.c @@ -8,6 +8,7 @@ #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/iommu.h> +#include <linux/interconnect.h> #include <linux/module.h> #include <linux/of_device.h> #include <linux/pm_runtime.h> @@ -618,9 +619,14 @@ static int tegra_plane_atomic_check(struct drm_plane *plane, struct tegra_dc *dc = to_tegra_dc(new_plane_state->crtc); int err; + plane_state->peak_memory_bandwidth = 0; + plane_state->avg_memory_bandwidth = 0; + /* no need for further checks if the plane is being disabled */ - if (!new_plane_state->crtc) + if (!new_plane_state->crtc) { + plane_state->total_peak_memory_bandwidth = 0; return 0; + } err = tegra_plane_format(new_plane_state->fb->format->format, &plane_state->format, @@ -808,6 +814,12 @@ static struct drm_plane *tegra_primary_plane_create(struct drm_device *drm, formats = dc->soc->primary_formats; modifiers = dc->soc->modifiers; + err = tegra_plane_interconnect_init(plane); + if (err) { + kfree(plane); + return ERR_PTR(err); + } + err = drm_universal_plane_init(drm, &plane->base, possible_crtcs, &tegra_plane_funcs, formats, num_formats, modifiers, type, NULL); @@ -845,12 +857,18 @@ static int tegra_cursor_atomic_check(struct drm_plane *plane, { struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane); + struct tegra_plane_state *plane_state = to_tegra_plane_state(new_plane_state); struct tegra_plane *tegra = to_tegra_plane(plane); int err; + plane_state->peak_memory_bandwidth = 0; + plane_state->avg_memory_bandwidth = 0; + /* no need for further checks if the plane is being disabled */ - if (!new_plane_state->crtc) + if (!new_plane_state->crtc) { + plane_state->total_peak_memory_bandwidth = 0; return 0; + } /* scaling not supported for cursor */ if ((new_plane_state->src_w >> 16 != new_plane_state->crtc_w) || @@ -1030,6 +1048,12 @@ static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm, if (!dc->soc->has_nvdisplay) { num_formats = ARRAY_SIZE(tegra_legacy_cursor_plane_formats); formats = tegra_legacy_cursor_plane_formats; + + err = tegra_plane_interconnect_init(plane); + if (err) { + kfree(plane); + return ERR_PTR(err); + } } else { num_formats = ARRAY_SIZE(tegra_cursor_plane_formats); formats = tegra_cursor_plane_formats; @@ -1149,6 +1173,12 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm, num_formats = dc->soc->num_overlay_formats; formats = dc->soc->overlay_formats; + err = tegra_plane_interconnect_init(plane); + if (err) { + kfree(plane); + return ERR_PTR(err); + } + if (!cursor) type = DRM_PLANE_TYPE_OVERLAY; else @@ -1572,6 +1602,11 @@ static int tegra_dc_show_stats(struct seq_file *s, void *data) seq_printf(s, "underflow: %lu\n", dc->stats.underflow); seq_printf(s, "overflow: %lu\n", dc->stats.overflow); + seq_printf(s, "frames total: %lu\n", dc->stats.frames_total); + seq_printf(s, "vblank total: %lu\n", dc->stats.vblank_total); + seq_printf(s, "underflow total: %lu\n", dc->stats.underflow_total); + seq_printf(s, "overflow total: %lu\n", dc->stats.overflow_total); + return 0; } @@ -1804,6 +1839,106 @@ static int tegra_dc_wait_idle(struct tegra_dc *dc, unsigned long timeout) return -ETIMEDOUT; } +static void +tegra_crtc_update_memory_bandwidth(struct drm_crtc *crtc, + struct drm_atomic_state *state, + bool prepare_bandwidth_transition) +{ + const struct tegra_plane_state *old_tegra_state, *new_tegra_state; + const struct tegra_dc_state *old_dc_state, *new_dc_state; + u32 i, new_avg_bw, old_avg_bw, new_peak_bw, old_peak_bw; + const struct drm_plane_state *old_plane_state; + const struct drm_crtc_state *old_crtc_state; + struct tegra_dc_window window, old_window; + struct tegra_dc *dc = to_tegra_dc(crtc); + struct tegra_plane *tegra; + struct drm_plane *plane; + + if (dc->soc->has_nvdisplay) + return; + + old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc); + old_dc_state = to_const_dc_state(old_crtc_state); + new_dc_state = to_const_dc_state(crtc->state); + + if (!crtc->state->active) { + if (!old_crtc_state->active) + return; + + /* + * When CRTC is disabled on DPMS, the state of attached planes + * is kept unchanged. Hence we need to enforce removal of the + * bandwidths from the ICC paths. + */ + drm_atomic_crtc_for_each_plane(plane, crtc) { + tegra = to_tegra_plane(plane); + + icc_set_bw(tegra->icc_mem, 0, 0); + icc_set_bw(tegra->icc_mem_vfilter, 0, 0); + } + + return; + } + + for_each_old_plane_in_state(old_crtc_state->state, plane, + old_plane_state, i) { + old_tegra_state = to_const_tegra_plane_state(old_plane_state); + new_tegra_state = to_const_tegra_plane_state(plane->state); + tegra = to_tegra_plane(plane); + + /* + * We're iterating over the global atomic state and it contains + * planes from another CRTC, hence we need to filter out the + * planes unrelated to this CRTC. + */ + if (tegra->dc != dc) + continue; + + new_avg_bw = new_tegra_state->avg_memory_bandwidth; + old_avg_bw = old_tegra_state->avg_memory_bandwidth; + + new_peak_bw = new_tegra_state->total_peak_memory_bandwidth; + old_peak_bw = old_tegra_state->total_peak_memory_bandwidth; + + /* + * See the comment related to !crtc->state->active above, + * which explains why bandwidths need to be updated when + * CRTC is turning ON. + */ + if (new_avg_bw == old_avg_bw && new_peak_bw == old_peak_bw && + old_crtc_state->active) + continue; + + window.src.h = drm_rect_height(&plane->state->src) >> 16; + window.dst.h = drm_rect_height(&plane->state->dst); + + old_window.src.h = drm_rect_height(&old_plane_state->src) >> 16; + old_window.dst.h = drm_rect_height(&old_plane_state->dst); + + /* + * During the preparation phase (atomic_begin), the memory + * freq should go high before the DC changes are committed + * if bandwidth requirement goes up, otherwise memory freq + * should to stay high if BW requirement goes down. The + * opposite applies to the completion phase (post_commit). + */ + if (prepare_bandwidth_transition) { + new_avg_bw = max(old_avg_bw, new_avg_bw); + new_peak_bw = max(old_peak_bw, new_peak_bw); + + if (tegra_plane_use_vertical_filtering(tegra, &old_window)) + window = old_window; + } + + icc_set_bw(tegra->icc_mem, new_avg_bw, new_peak_bw); + + if (tegra_plane_use_vertical_filtering(tegra, &window)) + icc_set_bw(tegra->icc_mem_vfilter, new_avg_bw, new_peak_bw); + else + icc_set_bw(tegra->icc_mem_vfilter, 0, 0); + } +} + static void tegra_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) { @@ -1985,6 +2120,8 @@ static void tegra_crtc_atomic_begin(struct drm_crtc *crtc, { unsigned long flags; + tegra_crtc_update_memory_bandwidth(crtc, state, true); + if (crtc->state->event) { spin_lock_irqsave(&crtc->dev->event_lock, flags); @@ -2017,7 +2154,207 @@ static void tegra_crtc_atomic_flush(struct drm_crtc *crtc, value = tegra_dc_readl(dc, DC_CMD_STATE_CONTROL); } +static bool tegra_plane_is_cursor(const struct drm_plane_state *state) +{ + const struct tegra_dc_soc_info *soc = to_tegra_dc(state->crtc)->soc; + const struct drm_format_info *fmt = state->fb->format; + unsigned int src_w = drm_rect_width(&state->src) >> 16; + unsigned int dst_w = drm_rect_width(&state->dst); + + if (state->plane->type != DRM_PLANE_TYPE_CURSOR) + return false; + + if (soc->supports_cursor) + return true; + + if (src_w != dst_w || fmt->num_planes != 1 || src_w * fmt->cpp[0] > 256) + return false; + + return true; +} + +static unsigned long +tegra_plane_overlap_mask(struct drm_crtc_state *state, + const struct drm_plane_state *plane_state) +{ + const struct drm_plane_state *other_state; + const struct tegra_plane *tegra; + unsigned long overlap_mask = 0; + struct drm_plane *plane; + struct drm_rect rect; + + if (!plane_state->visible || !plane_state->fb) + return 0; + + /* + * Data-prefetch FIFO will easily help to overcome temporal memory + * pressure if other plane overlaps with the cursor plane. + */ + if (tegra_plane_is_cursor(plane_state)) + return 0; + + drm_atomic_crtc_state_for_each_plane_state(plane, other_state, state) { + rect = plane_state->dst; + + tegra = to_tegra_plane(other_state->plane); + + if (!other_state->visible || !other_state->fb) + continue; + + /* + * Ignore cursor plane overlaps because it's not practical to + * assume that it contributes to the bandwidth in overlapping + * area if window width is small. + */ + if (tegra_plane_is_cursor(other_state)) + continue; + + if (drm_rect_intersect(&rect, &other_state->dst)) + overlap_mask |= BIT(tegra->index); + } + + return overlap_mask; +} + +static int tegra_crtc_calculate_memory_bandwidth(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + ulong overlap_mask[TEGRA_DC_LEGACY_PLANES_NUM] = {}, mask; + u32 plane_peak_bw[TEGRA_DC_LEGACY_PLANES_NUM] = {}; + bool all_planes_overlap_simultaneously = true; + const struct tegra_plane_state *tegra_state; + const struct drm_plane_state *plane_state; + struct tegra_dc *dc = to_tegra_dc(crtc); + const struct drm_crtc_state *old_state; + struct drm_crtc_state *new_state; + struct tegra_plane *tegra; + struct drm_plane *plane; + + /* + * The nv-display uses shared planes. The algorithm below assumes + * maximum 3 planes per-CRTC, this assumption isn't applicable to + * the nv-display. Note that T124 support has additional windows, + * but currently they aren't supported by the driver. + */ + if (dc->soc->has_nvdisplay) + return 0; + + new_state = drm_atomic_get_new_crtc_state(state, crtc); + old_state = drm_atomic_get_old_crtc_state(state, crtc); + + /* + * For overlapping planes pixel's data is fetched for each plane at + * the same time, hence bandwidths are accumulated in this case. + * This needs to be taken into account for calculating total bandwidth + * consumed by all planes. + * + * Here we get the overlapping state of each plane, which is a + * bitmask of plane indices telling with what planes there is an + * overlap. Note that bitmask[plane] includes BIT(plane) in order + * to make further code nicer and simpler. + */ + drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, new_state) { + tegra_state = to_const_tegra_plane_state(plane_state); + tegra = to_tegra_plane(plane); + + if (WARN_ON_ONCE(tegra->index >= TEGRA_DC_LEGACY_PLANES_NUM)) + return -EINVAL; + + plane_peak_bw[tegra->index] = tegra_state->peak_memory_bandwidth; + mask = tegra_plane_overlap_mask(new_state, plane_state); + overlap_mask[tegra->index] = mask; + + if (hweight_long(mask) != 3) + all_planes_overlap_simultaneously = false; + } + + /* + * Then we calculate maximum bandwidth of each plane state. + * The bandwidth includes the plane BW + BW of the "simultaneously" + * overlapping planes, where "simultaneously" means areas where DC + * fetches from the planes simultaneously during of scan-out process. + * + * For example, if plane A overlaps with planes B and C, but B and C + * don't overlap, then the peak bandwidth will be either in area where + * A-and-B or A-and-C planes overlap. + * + * The plane_peak_bw[] contains peak memory bandwidth values of + * each plane, this information is needed by interconnect provider + * in order to set up latency allowance based on the peak BW, see + * tegra_crtc_update_memory_bandwidth(). + */ + drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, new_state) { + u32 i, old_peak_bw, new_peak_bw, overlap_bw = 0; + + /* + * Note that plane's atomic check doesn't touch the + * total_peak_memory_bandwidth of enabled plane, hence the + * current state contains the old bandwidth state from the + * previous CRTC commit. + */ + tegra_state = to_const_tegra_plane_state(plane_state); + tegra = to_tegra_plane(plane); + + for_each_set_bit(i, &overlap_mask[tegra->index], 3) { + if (i == tegra->index) + continue; + + if (all_planes_overlap_simultaneously) + overlap_bw += plane_peak_bw[i]; + else + overlap_bw = max(overlap_bw, plane_peak_bw[i]); + } + + new_peak_bw = plane_peak_bw[tegra->index] + overlap_bw; + old_peak_bw = tegra_state->total_peak_memory_bandwidth; + + /* + * If plane's peak bandwidth changed (for example plane isn't + * overlapped anymore) and plane isn't in the atomic state, + * then add plane to the state in order to have the bandwidth + * updated. + */ + if (old_peak_bw != new_peak_bw) { + struct tegra_plane_state *new_tegra_state; + struct drm_plane_state *new_plane_state; + + new_plane_state = drm_atomic_get_plane_state(state, plane); + if (IS_ERR(new_plane_state)) + return PTR_ERR(new_plane_state); + + new_tegra_state = to_tegra_plane_state(new_plane_state); + new_tegra_state->total_peak_memory_bandwidth = new_peak_bw; + } + } + + return 0; +} + +static int tegra_crtc_atomic_check(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + int err; + + err = tegra_crtc_calculate_memory_bandwidth(crtc, state); + if (err) + return err; + + return 0; +} + +void tegra_crtc_atomic_post_commit(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + /* + * Display bandwidth is allowed to go down only once hardware state + * is known to be armed, i.e. state was committed and VBLANK event + * received. + */ + tegra_crtc_update_memory_bandwidth(crtc, state, false); +} + static const struct drm_crtc_helper_funcs tegra_crtc_helper_funcs = { + .atomic_check = tegra_crtc_atomic_check, .atomic_begin = tegra_crtc_atomic_begin, .atomic_flush = tegra_crtc_atomic_flush, .atomic_enable = tegra_crtc_atomic_enable, @@ -2036,6 +2373,7 @@ static irqreturn_t tegra_dc_irq(int irq, void *data) /* dev_dbg(dc->dev, "%s(): frame end\n", __func__); */ + dc->stats.frames_total++; dc->stats.frames++; } @@ -2044,6 +2382,7 @@ static irqreturn_t tegra_dc_irq(int irq, void *data) dev_dbg(dc->dev, "%s(): vertical blank\n", __func__); */ drm_crtc_handle_vblank(&dc->base); + dc->stats.vblank_total++; dc->stats.vblank++; } @@ -2051,6 +2390,7 @@ static irqreturn_t tegra_dc_irq(int irq, void *data) /* dev_dbg(dc->dev, "%s(): underflow\n", __func__); */ + dc->stats.underflow_total++; dc->stats.underflow++; } @@ -2058,11 +2398,13 @@ static irqreturn_t tegra_dc_irq(int irq, void *data) /* dev_dbg(dc->dev, "%s(): overflow\n", __func__); */ + dc->stats.overflow_total++; dc->stats.overflow++; } if (status & HEAD_UF_INT) { dev_dbg_ratelimited(dc->dev, "%s(): head underflow\n", __func__); + dc->stats.underflow_total++; dc->stats.underflow++; } @@ -2343,7 +2685,9 @@ static const struct tegra_dc_soc_info tegra20_dc_soc_info = { .overlay_formats = tegra20_overlay_formats, .modifiers = tegra20_modifiers, .has_win_a_without_filters = true, + .has_win_b_vfilter_mem_client = true, .has_win_c_without_vert_filter = true, + .plane_tiled_memory_bandwidth_x2 = false, }; static const struct tegra_dc_soc_info tegra30_dc_soc_info = { @@ -2363,7 +2707,9 @@ static const struct tegra_dc_soc_info tegra30_dc_soc_info = { .overlay_formats = tegra20_overlay_formats, .modifiers = tegra20_modifiers, .has_win_a_without_filters = false, + .has_win_b_vfilter_mem_client = true, .has_win_c_without_vert_filter = false, + .plane_tiled_memory_bandwidth_x2 = true, }; static const struct tegra_dc_soc_info tegra114_dc_soc_info = { @@ -2383,7 +2729,9 @@ static const struct tegra_dc_soc_info tegra114_dc_soc_info = { .overlay_formats = tegra114_overlay_formats, .modifiers = tegra20_modifiers, .has_win_a_without_filters = false, + .has_win_b_vfilter_mem_client = false, .has_win_c_without_vert_filter = false, + .plane_tiled_memory_bandwidth_x2 = true, }; static const struct tegra_dc_soc_info tegra124_dc_soc_info = { @@ -2403,7 +2751,9 @@ static const struct tegra_dc_soc_info tegra124_dc_soc_info = { .overlay_formats = tegra124_overlay_formats, .modifiers = tegra124_modifiers, .has_win_a_without_filters = false, + .has_win_b_vfilter_mem_client = false, .has_win_c_without_vert_filter = false, + .plane_tiled_memory_bandwidth_x2 = false, }; static const struct tegra_dc_soc_info tegra210_dc_soc_info = { @@ -2423,7 +2773,9 @@ static const struct tegra_dc_soc_info tegra210_dc_soc_info = { .overlay_formats = tegra114_overlay_formats, .modifiers = tegra124_modifiers, .has_win_a_without_filters = false, + .has_win_b_vfilter_mem_client = false, .has_win_c_without_vert_filter = false, + .plane_tiled_memory_bandwidth_x2 = false, }; static const struct tegra_windowgroup_soc tegra186_dc_wgrps[] = { @@ -2473,6 +2825,7 @@ static const struct tegra_dc_soc_info tegra186_dc_soc_info = { .has_nvdisplay = true, .wgrps = tegra186_dc_wgrps, .num_wgrps = ARRAY_SIZE(tegra186_dc_wgrps), + .plane_tiled_memory_bandwidth_x2 = false, }; static const struct tegra_windowgroup_soc tegra194_dc_wgrps[] = { @@ -2522,6 +2875,7 @@ static const struct tegra_dc_soc_info tegra194_dc_soc_info = { .has_nvdisplay = true, .wgrps = tegra194_dc_wgrps, .num_wgrps = ARRAY_SIZE(tegra194_dc_wgrps), + .plane_tiled_memory_bandwidth_x2 = false, }; static const struct of_device_id tegra_dc_of_match[] = { diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h index 5e13f1cfd749..f0cb691852a1 100644 --- a/drivers/gpu/drm/tegra/dc.h +++ b/drivers/gpu/drm/tegra/dc.h @@ -15,6 +15,8 @@ struct tegra_output; +#define TEGRA_DC_LEGACY_PLANES_NUM 7 + struct tegra_dc_state { struct drm_crtc_state base; @@ -33,11 +35,22 @@ static inline struct tegra_dc_state *to_dc_state(struct drm_crtc_state *state) return NULL; } +static inline const struct tegra_dc_state * +to_const_dc_state(const struct drm_crtc_state *state) +{ + return to_dc_state((struct drm_crtc_state *)state); +} + struct tegra_dc_stats { unsigned long frames; unsigned long vblank; unsigned long underflow; unsigned long overflow; + + unsigned long frames_total; + unsigned long vblank_total; + unsigned long underflow_total; + unsigned long overflow_total; }; struct tegra_windowgroup_soc { @@ -66,7 +79,9 @@ struct tegra_dc_soc_info { unsigned int num_overlay_formats; const u64 *modifiers; bool has_win_a_without_filters; + bool has_win_b_vfilter_mem_client; bool has_win_c_without_vert_filter; + bool plane_tiled_memory_bandwidth_x2; }; struct tegra_dc { @@ -152,6 +167,8 @@ int tegra_dc_state_setup_clock(struct tegra_dc *dc, struct drm_crtc_state *crtc_state, struct clk *clk, unsigned long pclk, unsigned int div); +void tegra_crtc_atomic_post_commit(struct drm_crtc *crtc, + struct drm_atomic_state *state); /* from rgb.c */ int tegra_dc_rgb_probe(struct tegra_dc *dc); diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c index 8c6069b33160..8d37d6b00562 100644 --- a/drivers/gpu/drm/tegra/drm.c +++ b/drivers/gpu/drm/tegra/drm.c @@ -21,24 +21,21 @@ #include <drm/drm_prime.h> #include <drm/drm_vblank.h> +#include "dc.h" #include "drm.h" #include "gem.h" +#include "uapi.h" #define DRIVER_NAME "tegra" #define DRIVER_DESC "NVIDIA Tegra graphics" #define DRIVER_DATE "20120330" -#define DRIVER_MAJOR 0 +#define DRIVER_MAJOR 1 #define DRIVER_MINOR 0 #define DRIVER_PATCHLEVEL 0 #define CARVEOUT_SZ SZ_64M #define CDMA_GATHER_FETCHES_MAX_NB 16383 -struct tegra_drm_file { - struct idr contexts; - struct mutex lock; -}; - static int tegra_atomic_check(struct drm_device *drm, struct drm_atomic_state *state) { @@ -60,6 +57,17 @@ static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = { .atomic_commit = drm_atomic_helper_commit, }; +static void tegra_atomic_post_commit(struct drm_device *drm, + struct drm_atomic_state *old_state) +{ + struct drm_crtc_state *old_crtc_state __maybe_unused; + struct drm_crtc *crtc; + unsigned int i; + + for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i) + tegra_crtc_atomic_post_commit(crtc, old_state); +} + static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state) { struct drm_device *drm = old_state->dev; @@ -79,6 +87,8 @@ static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state) } else { drm_atomic_helper_commit_tail_rpm(old_state); } + + tegra_atomic_post_commit(drm, old_state); } static const struct drm_mode_config_helper_funcs @@ -94,7 +104,9 @@ static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp) if (!fpriv) return -ENOMEM; - idr_init_base(&fpriv->contexts, 1); + idr_init_base(&fpriv->legacy_contexts, 1); + xa_init_flags(&fpriv->contexts, XA_FLAGS_ALLOC1); + xa_init(&fpriv->syncpoints); mutex_init(&fpriv->lock); filp->driver_priv = fpriv; @@ -107,20 +119,6 @@ static void tegra_drm_context_free(struct tegra_drm_context *context) kfree(context); } -static struct host1x_bo * -host1x_bo_lookup(struct drm_file *file, u32 handle) -{ - struct drm_gem_object *gem; - struct tegra_bo *bo; - - gem = drm_gem_object_lookup(file, handle); - if (!gem) - return NULL; - - bo = to_tegra_bo(gem); - return &bo->base; -} - static int host1x_reloc_copy_from_user(struct host1x_reloc *dest, struct drm_tegra_reloc __user *src, struct drm_device *drm, @@ -151,11 +149,11 @@ static int host1x_reloc_copy_from_user(struct host1x_reloc *dest, dest->flags = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE; - dest->cmdbuf.bo = host1x_bo_lookup(file, cmdbuf); + dest->cmdbuf.bo = tegra_gem_lookup(file, cmdbuf); if (!dest->cmdbuf.bo) return -ENOENT; - dest->target.bo = host1x_bo_lookup(file, target); + dest->target.bo = tegra_gem_lookup(file, target); if (!dest->target.bo) return -ENOENT; @@ -193,7 +191,7 @@ int tegra_drm_submit(struct tegra_drm_context *context, return -EINVAL; job = host1x_job_alloc(context->channel, args->num_cmdbufs, - args->num_relocs); + args->num_relocs, false); if (!job) return -ENOMEM; @@ -201,6 +199,7 @@ int tegra_drm_submit(struct tegra_drm_context *context, job->client = client; job->class = client->class; job->serialize = true; + job->syncpt_recovery = true; /* * Track referenced BOs so that they can be unreferenced after the @@ -237,7 +236,7 @@ int tegra_drm_submit(struct tegra_drm_context *context, goto fail; } - bo = host1x_bo_lookup(file, cmdbuf.handle); + bo = tegra_gem_lookup(file, cmdbuf.handle); if (!bo) { err = -ENOENT; goto fail; @@ -432,7 +431,7 @@ static int tegra_client_open(struct tegra_drm_file *fpriv, if (err < 0) return err; - err = idr_alloc(&fpriv->contexts, context, 1, 0, GFP_KERNEL); + err = idr_alloc(&fpriv->legacy_contexts, context, 1, 0, GFP_KERNEL); if (err < 0) { client->ops->close_channel(context); return err; @@ -487,13 +486,13 @@ static int tegra_close_channel(struct drm_device *drm, void *data, mutex_lock(&fpriv->lock); - context = idr_find(&fpriv->contexts, args->context); + context = idr_find(&fpriv->legacy_contexts, args->context); if (!context) { err = -EINVAL; goto unlock; } - idr_remove(&fpriv->contexts, context->id); + idr_remove(&fpriv->legacy_contexts, context->id); tegra_drm_context_free(context); unlock: @@ -512,7 +511,7 @@ static int tegra_get_syncpt(struct drm_device *drm, void *data, mutex_lock(&fpriv->lock); - context = idr_find(&fpriv->contexts, args->context); + context = idr_find(&fpriv->legacy_contexts, args->context); if (!context) { err = -ENODEV; goto unlock; @@ -541,7 +540,7 @@ static int tegra_submit(struct drm_device *drm, void *data, mutex_lock(&fpriv->lock); - context = idr_find(&fpriv->contexts, args->context); + context = idr_find(&fpriv->legacy_contexts, args->context); if (!context) { err = -ENODEV; goto unlock; @@ -566,7 +565,7 @@ static int tegra_get_syncpt_base(struct drm_device *drm, void *data, mutex_lock(&fpriv->lock); - context = idr_find(&fpriv->contexts, args->context); + context = idr_find(&fpriv->legacy_contexts, args->context); if (!context) { err = -ENODEV; goto unlock; @@ -735,10 +734,25 @@ static int tegra_gem_get_flags(struct drm_device *drm, void *data, static const struct drm_ioctl_desc tegra_drm_ioctls[] = { #ifdef CONFIG_DRM_TEGRA_STAGING - DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, + DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_OPEN, tegra_drm_ioctl_channel_open, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_CLOSE, tegra_drm_ioctl_channel_close, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_MAP, tegra_drm_ioctl_channel_map, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_UNMAP, tegra_drm_ioctl_channel_unmap, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_SUBMIT, tegra_drm_ioctl_channel_submit, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_ALLOCATE, tegra_drm_ioctl_syncpoint_allocate, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, + DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_FREE, tegra_drm_ioctl_syncpoint_free, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_WAIT, tegra_drm_ioctl_syncpoint_wait, + DRM_RENDER_ALLOW), + + DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr, @@ -792,10 +806,11 @@ static void tegra_drm_postclose(struct drm_device *drm, struct drm_file *file) struct tegra_drm_file *fpriv = file->driver_priv; mutex_lock(&fpriv->lock); - idr_for_each(&fpriv->contexts, tegra_drm_context_cleanup, NULL); + idr_for_each(&fpriv->legacy_contexts, tegra_drm_context_cleanup, NULL); + tegra_drm_uapi_close_file(fpriv); mutex_unlock(&fpriv->lock); - idr_destroy(&fpriv->contexts); + idr_destroy(&fpriv->legacy_contexts); mutex_destroy(&fpriv->lock); kfree(fpriv); } @@ -853,7 +868,7 @@ static void tegra_debugfs_init(struct drm_minor *minor) static const struct drm_driver tegra_drm_driver = { .driver_features = DRIVER_MODESET | DRIVER_GEM | - DRIVER_ATOMIC | DRIVER_RENDER, + DRIVER_ATOMIC | DRIVER_RENDER | DRIVER_SYNCOBJ, .open = tegra_drm_open, .postclose = tegra_drm_postclose, .lastclose = drm_fb_helper_lastclose, @@ -883,6 +898,14 @@ static const struct drm_driver tegra_drm_driver = { int tegra_drm_register_client(struct tegra_drm *tegra, struct tegra_drm_client *client) { + /* + * When MLOCKs are implemented, change to allocate a shared channel + * only when MLOCKs are disabled. + */ + client->shared_channel = host1x_channel_request(&client->base); + if (!client->shared_channel) + return -EBUSY; + mutex_lock(&tegra->clients_lock); list_add_tail(&client->list, &tegra->clients); client->drm = tegra; @@ -899,6 +922,9 @@ int tegra_drm_unregister_client(struct tegra_drm *tegra, client->drm = NULL; mutex_unlock(&tegra->clients_lock); + if (client->shared_channel) + host1x_channel_put(client->shared_channel); + return 0; } diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h index 0cb868065348..8b28327c931c 100644 --- a/drivers/gpu/drm/tegra/drm.h +++ b/drivers/gpu/drm/tegra/drm.h @@ -64,12 +64,22 @@ struct tegra_drm { struct tegra_display_hub *hub; }; +static inline struct host1x *tegra_drm_to_host1x(struct tegra_drm *tegra) +{ + return dev_get_drvdata(tegra->drm->dev->parent); +} + struct tegra_drm_client; struct tegra_drm_context { struct tegra_drm_client *client; struct host1x_channel *channel; + + /* Only used by legacy UAPI. */ unsigned int id; + + /* Only used by new UAPI. */ + struct xarray mappings; }; struct tegra_drm_client_ops { @@ -91,7 +101,9 @@ struct tegra_drm_client { struct host1x_client base; struct list_head list; struct tegra_drm *drm; + struct host1x_channel *shared_channel; + /* Set by driver */ unsigned int version; const struct tegra_drm_client_ops *ops; }; diff --git a/drivers/gpu/drm/tegra/firewall.c b/drivers/gpu/drm/tegra/firewall.c new file mode 100644 index 000000000000..1824d2db0e2c --- /dev/null +++ b/drivers/gpu/drm/tegra/firewall.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2010-2020 NVIDIA Corporation */ + +#include "drm.h" +#include "submit.h" +#include "uapi.h" + +struct tegra_drm_firewall { + struct tegra_drm_submit_data *submit; + struct tegra_drm_client *client; + u32 *data; + u32 pos; + u32 end; + u32 class; +}; + +static int fw_next(struct tegra_drm_firewall *fw, u32 *word) +{ + if (fw->pos == fw->end) + return -EINVAL; + + *word = fw->data[fw->pos++]; + + return 0; +} + +static bool fw_check_addr_valid(struct tegra_drm_firewall *fw, u32 offset) +{ + u32 i; + + for (i = 0; i < fw->submit->num_used_mappings; i++) { + struct tegra_drm_mapping *m = fw->submit->used_mappings[i].mapping; + + if (offset >= m->iova && offset <= m->iova_end) + return true; + } + + return false; +} + +static int fw_check_reg(struct tegra_drm_firewall *fw, u32 offset) +{ + bool is_addr; + u32 word; + int err; + + err = fw_next(fw, &word); + if (err) + return err; + + if (!fw->client->ops->is_addr_reg) + return 0; + + is_addr = fw->client->ops->is_addr_reg(fw->client->base.dev, fw->class, + offset); + + if (!is_addr) + return 0; + + if (!fw_check_addr_valid(fw, word)) + return -EINVAL; + + return 0; +} + +static int fw_check_regs_seq(struct tegra_drm_firewall *fw, u32 offset, + u32 count, bool incr) +{ + u32 i; + + for (i = 0; i < count; i++) { + if (fw_check_reg(fw, offset)) + return -EINVAL; + + if (incr) + offset++; + } + + return 0; +} + +static int fw_check_regs_mask(struct tegra_drm_firewall *fw, u32 offset, + u16 mask) +{ + unsigned long bmask = mask; + unsigned int bit; + + for_each_set_bit(bit, &bmask, 16) { + if (fw_check_reg(fw, offset+bit)) + return -EINVAL; + } + + return 0; +} + +static int fw_check_regs_imm(struct tegra_drm_firewall *fw, u32 offset) +{ + bool is_addr; + + is_addr = fw->client->ops->is_addr_reg(fw->client->base.dev, fw->class, + offset); + if (is_addr) + return -EINVAL; + + return 0; +} + +static int fw_check_class(struct tegra_drm_firewall *fw, u32 class) +{ + if (!fw->client->ops->is_valid_class) { + if (class == fw->client->base.class) + return 0; + else + return -EINVAL; + } + + if (!fw->client->ops->is_valid_class(class)) + return -EINVAL; + + return 0; +} + +enum { + HOST1X_OPCODE_SETCLASS = 0x00, + HOST1X_OPCODE_INCR = 0x01, + HOST1X_OPCODE_NONINCR = 0x02, + HOST1X_OPCODE_MASK = 0x03, + HOST1X_OPCODE_IMM = 0x04, + HOST1X_OPCODE_RESTART = 0x05, + HOST1X_OPCODE_GATHER = 0x06, + HOST1X_OPCODE_SETSTRMID = 0x07, + HOST1X_OPCODE_SETAPPID = 0x08, + HOST1X_OPCODE_SETPYLD = 0x09, + HOST1X_OPCODE_INCR_W = 0x0a, + HOST1X_OPCODE_NONINCR_W = 0x0b, + HOST1X_OPCODE_GATHER_W = 0x0c, + HOST1X_OPCODE_RESTART_W = 0x0d, + HOST1X_OPCODE_EXTEND = 0x0e, +}; + +int tegra_drm_fw_validate(struct tegra_drm_client *client, u32 *data, u32 start, + u32 words, struct tegra_drm_submit_data *submit, + u32 *job_class) +{ + struct tegra_drm_firewall fw = { + .submit = submit, + .client = client, + .data = data, + .pos = start, + .end = start+words, + .class = *job_class, + }; + bool payload_valid = false; + u32 payload; + int err; + + while (fw.pos != fw.end) { + u32 word, opcode, offset, count, mask, class; + + err = fw_next(&fw, &word); + if (err) + return err; + + opcode = (word & 0xf0000000) >> 28; + + switch (opcode) { + case HOST1X_OPCODE_SETCLASS: + offset = word >> 16 & 0xfff; + mask = word & 0x3f; + class = (word >> 6) & 0x3ff; + err = fw_check_class(&fw, class); + fw.class = class; + *job_class = class; + if (!err) + err = fw_check_regs_mask(&fw, offset, mask); + if (err) + dev_warn(client->base.dev, + "illegal SETCLASS(offset=0x%x, mask=0x%x, class=0x%x) at word %u", + offset, mask, class, fw.pos-1); + break; + case HOST1X_OPCODE_INCR: + offset = (word >> 16) & 0xfff; + count = word & 0xffff; + err = fw_check_regs_seq(&fw, offset, count, true); + if (err) + dev_warn(client->base.dev, + "illegal INCR(offset=0x%x, count=%u) in class 0x%x at word %u", + offset, count, fw.class, fw.pos-1); + break; + case HOST1X_OPCODE_NONINCR: + offset = (word >> 16) & 0xfff; + count = word & 0xffff; + err = fw_check_regs_seq(&fw, offset, count, false); + if (err) + dev_warn(client->base.dev, + "illegal NONINCR(offset=0x%x, count=%u) in class 0x%x at word %u", + offset, count, fw.class, fw.pos-1); + break; + case HOST1X_OPCODE_MASK: + offset = (word >> 16) & 0xfff; + mask = word & 0xffff; + err = fw_check_regs_mask(&fw, offset, mask); + if (err) + dev_warn(client->base.dev, + "illegal MASK(offset=0x%x, mask=0x%x) in class 0x%x at word %u", + offset, mask, fw.class, fw.pos-1); + break; + case HOST1X_OPCODE_IMM: + /* IMM cannot reasonably be used to write a pointer */ + offset = (word >> 16) & 0xfff; + err = fw_check_regs_imm(&fw, offset); + if (err) + dev_warn(client->base.dev, + "illegal IMM(offset=0x%x) in class 0x%x at word %u", + offset, fw.class, fw.pos-1); + break; + case HOST1X_OPCODE_SETPYLD: + payload = word & 0xffff; + payload_valid = true; + break; + case HOST1X_OPCODE_INCR_W: + if (!payload_valid) + return -EINVAL; + + offset = word & 0x3fffff; + err = fw_check_regs_seq(&fw, offset, payload, true); + if (err) + dev_warn(client->base.dev, + "illegal INCR_W(offset=0x%x) in class 0x%x at word %u", + offset, fw.class, fw.pos-1); + break; + case HOST1X_OPCODE_NONINCR_W: + if (!payload_valid) + return -EINVAL; + + offset = word & 0x3fffff; + err = fw_check_regs_seq(&fw, offset, payload, false); + if (err) + dev_warn(client->base.dev, + "illegal NONINCR(offset=0x%x) in class 0x%x at word %u", + offset, fw.class, fw.pos-1); + break; + default: + dev_warn(client->base.dev, "illegal opcode at word %u", + fw.pos-1); + return -EINVAL; + } + + if (err) + return err; + } + + return 0; +} diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 26af8daa9a16..6ec598f5d5b3 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -707,3 +707,16 @@ struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm, return &bo->gem; } + +struct host1x_bo *tegra_gem_lookup(struct drm_file *file, u32 handle) +{ + struct drm_gem_object *gem; + struct tegra_bo *bo; + + gem = drm_gem_object_lookup(file, handle); + if (!gem) + return NULL; + + bo = to_tegra_bo(gem); + return &bo->base; +} diff --git a/drivers/gpu/drm/tegra/gem.h b/drivers/gpu/drm/tegra/gem.h index c15fd99d6cb2..cb5146a67668 100644 --- a/drivers/gpu/drm/tegra/gem.h +++ b/drivers/gpu/drm/tegra/gem.h @@ -80,4 +80,6 @@ struct dma_buf *tegra_gem_prime_export(struct drm_gem_object *gem, struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm, struct dma_buf *buf); +struct host1x_bo *tegra_gem_lookup(struct drm_file *file, u32 handle); + #endif diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c index f7496425fa83..16a1cdc28657 100644 --- a/drivers/gpu/drm/tegra/plane.c +++ b/drivers/gpu/drm/tegra/plane.c @@ -4,6 +4,7 @@ */ #include <linux/iommu.h> +#include <linux/interconnect.h> #include <drm/drm_atomic.h> #include <drm/drm_atomic_helper.h> @@ -64,6 +65,9 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane) copy->reflect_x = state->reflect_x; copy->reflect_y = state->reflect_y; copy->opaque = state->opaque; + copy->total_peak_memory_bandwidth = state->total_peak_memory_bandwidth; + copy->peak_memory_bandwidth = state->peak_memory_bandwidth; + copy->avg_memory_bandwidth = state->avg_memory_bandwidth; for (i = 0; i < 2; i++) copy->blending[i] = state->blending[i]; @@ -244,6 +248,78 @@ void tegra_plane_cleanup_fb(struct drm_plane *plane, tegra_dc_unpin(dc, to_tegra_plane_state(state)); } +static int tegra_plane_calculate_memory_bandwidth(struct drm_plane_state *state) +{ + struct tegra_plane_state *tegra_state = to_tegra_plane_state(state); + unsigned int i, bpp, dst_w, dst_h, src_w, src_h, mul; + const struct tegra_dc_soc_info *soc; + const struct drm_format_info *fmt; + struct drm_crtc_state *crtc_state; + u64 avg_bandwidth, peak_bandwidth; + + if (!state->visible) + return 0; + + crtc_state = drm_atomic_get_new_crtc_state(state->state, state->crtc); + if (!crtc_state) + return -EINVAL; + + src_w = drm_rect_width(&state->src) >> 16; + src_h = drm_rect_height(&state->src) >> 16; + dst_w = drm_rect_width(&state->dst); + dst_h = drm_rect_height(&state->dst); + + fmt = state->fb->format; + soc = to_tegra_dc(state->crtc)->soc; + + /* + * Note that real memory bandwidth vary depending on format and + * memory layout, we are not taking that into account because small + * estimation error isn't important since bandwidth is rounded up + * anyway. + */ + for (i = 0, bpp = 0; i < fmt->num_planes; i++) { + unsigned int bpp_plane = fmt->cpp[i] * 8; + + /* + * Sub-sampling is relevant for chroma planes only and vertical + * readouts are not cached, hence only horizontal sub-sampling + * matters. + */ + if (i > 0) + bpp_plane /= fmt->hsub; + + bpp += bpp_plane; + } + + /* average bandwidth in kbytes/sec */ + avg_bandwidth = min(src_w, dst_w) * min(src_h, dst_h); + avg_bandwidth *= drm_mode_vrefresh(&crtc_state->adjusted_mode); + avg_bandwidth = DIV_ROUND_UP(avg_bandwidth * bpp, 8) + 999; + do_div(avg_bandwidth, 1000); + + /* mode.clock in kHz, peak bandwidth in kbytes/sec */ + peak_bandwidth = DIV_ROUND_UP(crtc_state->adjusted_mode.clock * bpp, 8); + + /* + * Tegra30/114 Memory Controller can't interleave DC memory requests + * for the tiled windows because DC uses 16-bytes atom, while DDR3 + * uses 32-bytes atom. Hence there is x2 memory overfetch for tiled + * framebuffer and DDR3 on these SoCs. + */ + if (soc->plane_tiled_memory_bandwidth_x2 && + tegra_state->tiling.mode == TEGRA_BO_TILING_MODE_TILED) + mul = 2; + else + mul = 1; + + /* ICC bandwidth in kbytes/sec */ + tegra_state->peak_memory_bandwidth = kBps_to_icc(peak_bandwidth) * mul; + tegra_state->avg_memory_bandwidth = kBps_to_icc(avg_bandwidth) * mul; + + return 0; +} + int tegra_plane_state_add(struct tegra_plane *plane, struct drm_plane_state *state) { @@ -262,6 +338,10 @@ int tegra_plane_state_add(struct tegra_plane *plane, if (err < 0) return err; + err = tegra_plane_calculate_memory_bandwidth(state); + if (err < 0) + return err; + tegra = to_dc_state(crtc_state); tegra->planes |= WIN_A_ACT_REQ << plane->index; @@ -646,3 +726,40 @@ int tegra_plane_setup_legacy_state(struct tegra_plane *tegra, return 0; } + +static const char * const tegra_plane_icc_names[TEGRA_DC_LEGACY_PLANES_NUM] = { + "wina", "winb", "winc", NULL, NULL, NULL, "cursor", +}; + +int tegra_plane_interconnect_init(struct tegra_plane *plane) +{ + const char *icc_name = tegra_plane_icc_names[plane->index]; + struct device *dev = plane->dc->dev; + struct tegra_dc *dc = plane->dc; + int err; + + if (WARN_ON(plane->index >= TEGRA_DC_LEGACY_PLANES_NUM) || + WARN_ON(!tegra_plane_icc_names[plane->index])) + return -EINVAL; + + plane->icc_mem = devm_of_icc_get(dev, icc_name); + err = PTR_ERR_OR_ZERO(plane->icc_mem); + if (err) { + dev_err_probe(dev, err, "failed to get %s interconnect\n", + icc_name); + return err; + } + + /* plane B on T20/30 has a dedicated memory client for a 6-tap vertical filter */ + if (plane->index == 1 && dc->soc->has_win_b_vfilter_mem_client) { + plane->icc_mem_vfilter = devm_of_icc_get(dev, "winb-vfilter"); + err = PTR_ERR_OR_ZERO(plane->icc_mem_vfilter); + if (err) { + dev_err_probe(dev, err, "failed to get %s interconnect\n", + "winb-vfilter"); + return err; + } + } + + return 0; +} diff --git a/drivers/gpu/drm/tegra/plane.h b/drivers/gpu/drm/tegra/plane.h index 1785c1559c0c..d9470780c803 100644 --- a/drivers/gpu/drm/tegra/plane.h +++ b/drivers/gpu/drm/tegra/plane.h @@ -8,6 +8,7 @@ #include <drm/drm_plane.h> +struct icc_path; struct tegra_bo; struct tegra_dc; @@ -16,6 +17,9 @@ struct tegra_plane { struct tegra_dc *dc; unsigned int offset; unsigned int index; + + struct icc_path *icc_mem; + struct icc_path *icc_mem_vfilter; }; struct tegra_cursor { @@ -52,6 +56,11 @@ struct tegra_plane_state { /* used for legacy blending support only */ struct tegra_plane_legacy_blending_state blending[2]; bool opaque; + + /* bandwidths are in ICC units, i.e. kbytes/sec */ + u32 total_peak_memory_bandwidth; + u32 peak_memory_bandwidth; + u32 avg_memory_bandwidth; }; static inline struct tegra_plane_state * @@ -63,6 +72,12 @@ to_tegra_plane_state(struct drm_plane_state *state) return NULL; } +static inline const struct tegra_plane_state * +to_const_tegra_plane_state(const struct drm_plane_state *state) +{ + return to_tegra_plane_state((struct drm_plane_state *)state); +} + extern const struct drm_plane_funcs tegra_plane_funcs; int tegra_plane_prepare_fb(struct drm_plane *plane, @@ -78,5 +93,6 @@ bool tegra_plane_format_is_indexed(unsigned int format); bool tegra_plane_format_is_yuv(unsigned int format, bool *planar, unsigned int *bpc); int tegra_plane_setup_legacy_state(struct tegra_plane *tegra, struct tegra_plane_state *state); +int tegra_plane_interconnect_init(struct tegra_plane *plane); #endif /* TEGRA_PLANE_H */ diff --git a/drivers/gpu/drm/tegra/submit.c b/drivers/gpu/drm/tegra/submit.c new file mode 100644 index 000000000000..776f825df52f --- /dev/null +++ b/drivers/gpu/drm/tegra/submit.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 NVIDIA Corporation */ + +#include <linux/dma-fence-array.h> +#include <linux/dma-mapping.h> +#include <linux/file.h> +#include <linux/host1x.h> +#include <linux/iommu.h> +#include <linux/kref.h> +#include <linux/list.h> +#include <linux/nospec.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/sync_file.h> + +#include <drm/drm_drv.h> +#include <drm/drm_file.h> +#include <drm/drm_syncobj.h> + +#include "drm.h" +#include "gem.h" +#include "submit.h" +#include "uapi.h" + +#define SUBMIT_ERR(context, fmt, ...) \ + dev_err_ratelimited(context->client->base.dev, \ + "%s: job submission failed: " fmt "\n", \ + current->comm, ##__VA_ARGS__) + +struct gather_bo { + struct host1x_bo base; + + struct kref ref; + + struct device *dev; + u32 *gather_data; + dma_addr_t gather_data_dma; + size_t gather_data_words; +}; + +static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) +{ + struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); + + kref_get(&bo->ref); + + return host_bo; +} + +static void gather_bo_release(struct kref *ref) +{ + struct gather_bo *bo = container_of(ref, struct gather_bo, ref); + + dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, + 0); + kfree(bo); +} + +static void gather_bo_put(struct host1x_bo *host_bo) +{ + struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); + + kref_put(&bo->ref, gather_bo_release); +} + +static struct sg_table * +gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys) +{ + struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); + struct sg_table *sgt; + int err; + + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) + return ERR_PTR(-ENOMEM); + + err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma, + bo->gather_data_words * 4); + if (err) { + kfree(sgt); + return ERR_PTR(err); + } + + return sgt; +} + +static void gather_bo_unpin(struct device *dev, struct sg_table *sgt) +{ + if (sgt) { + sg_free_table(sgt); + kfree(sgt); + } +} + +static void *gather_bo_mmap(struct host1x_bo *host_bo) +{ + struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); + + return bo->gather_data; +} + +static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) +{ +} + +const struct host1x_bo_ops gather_bo_ops = { + .get = gather_bo_get, + .put = gather_bo_put, + .pin = gather_bo_pin, + .unpin = gather_bo_unpin, + .mmap = gather_bo_mmap, + .munmap = gather_bo_munmap, +}; + +static struct tegra_drm_mapping * +tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) +{ + struct tegra_drm_mapping *mapping; + + xa_lock(&context->mappings); + + mapping = xa_load(&context->mappings, id); + if (mapping) + kref_get(&mapping->ref); + + xa_unlock(&context->mappings); + + return mapping; +} + +static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) +{ + size_t copy_len; + void *data; + + if (check_mul_overflow(count, size, ©_len)) + return ERR_PTR(-EINVAL); + + if (copy_len > 0x4000) + return ERR_PTR(-E2BIG); + + data = kvmalloc(copy_len, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + + if (copy_from_user(data, from, copy_len)) { + kvfree(data); + return ERR_PTR(-EFAULT); + } + + return data; +} + +static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, + struct tegra_drm_context *context, + struct drm_tegra_channel_submit *args) +{ + struct gather_bo *bo; + size_t copy_len; + + if (args->gather_data_words == 0) { + SUBMIT_ERR(context, "gather_data_words cannot be zero"); + return -EINVAL; + } + + if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { + SUBMIT_ERR(context, "gather_data_words is too large"); + return -EINVAL; + } + + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (!bo) { + SUBMIT_ERR(context, "failed to allocate memory for bo info"); + return -ENOMEM; + } + + host1x_bo_init(&bo->base, &gather_bo_ops); + kref_init(&bo->ref); + bo->dev = dev; + + bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, + GFP_KERNEL | __GFP_NOWARN, 0); + if (!bo->gather_data) { + SUBMIT_ERR(context, "failed to allocate memory for gather data"); + kfree(bo); + return -ENOMEM; + } + + if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { + SUBMIT_ERR(context, "failed to copy gather data from userspace"); + dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); + kfree(bo); + return -EFAULT; + } + + bo->gather_data_words = args->gather_data_words; + + *pbo = bo; + + return 0; +} + +static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, + struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) +{ + /* TODO check that target_offset is within bounds */ + dma_addr_t iova = mapping->iova + buf->reloc.target_offset; + u32 written_ptr; + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) + iova |= BIT_ULL(39); +#endif + + written_ptr = iova >> buf->reloc.shift; + + if (buf->reloc.gather_offset_words >= bo->gather_data_words) { + SUBMIT_ERR(context, + "relocation has too large gather offset (%u vs gather length %zu)", + buf->reloc.gather_offset_words, bo->gather_data_words); + return -EINVAL; + } + + buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, + bo->gather_data_words); + + bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; + + return 0; +} + +static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, + struct drm_tegra_channel_submit *args, + struct tegra_drm_submit_data *job_data) +{ + struct tegra_drm_used_mapping *mappings; + struct drm_tegra_submit_buf *bufs; + int err; + u32 i; + + bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, + sizeof(*bufs)); + if (IS_ERR(bufs)) { + SUBMIT_ERR(context, "failed to copy bufs array from userspace"); + return PTR_ERR(bufs); + } + + mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL); + if (!mappings) { + SUBMIT_ERR(context, "failed to allocate memory for mapping info"); + err = -ENOMEM; + goto done; + } + + for (i = 0; i < args->num_bufs; i++) { + struct drm_tegra_submit_buf *buf = &bufs[i]; + struct tegra_drm_mapping *mapping; + + if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { + SUBMIT_ERR(context, "invalid flag specified for buffer"); + err = -EINVAL; + goto drop_refs; + } + + mapping = tegra_drm_mapping_get(context, buf->mapping); + if (!mapping) { + SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); + err = -EINVAL; + goto drop_refs; + } + + err = submit_write_reloc(context, bo, buf, mapping); + if (err) { + tegra_drm_mapping_put(mapping); + goto drop_refs; + } + + mappings[i].mapping = mapping; + mappings[i].flags = buf->flags; + } + + job_data->used_mappings = mappings; + job_data->num_used_mappings = i; + + err = 0; + + goto done; + +drop_refs: + while (i--) + tegra_drm_mapping_put(mappings[i].mapping); + + kfree(mappings); + job_data->used_mappings = NULL; + +done: + kvfree(bufs); + + return err; +} + +static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, + struct xarray *syncpoints, struct drm_tegra_channel_submit *args) +{ + struct host1x_syncpt *sp; + + if (args->syncpt.flags) { + SUBMIT_ERR(context, "invalid flag specified for syncpt"); + return -EINVAL; + } + + /* Syncpt ref will be dropped on job release */ + sp = xa_load(syncpoints, args->syncpt.id); + if (!sp) { + SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); + return -EINVAL; + } + + job->syncpt = host1x_syncpt_get(sp); + job->syncpt_incrs = args->syncpt.increments; + + return 0; +} + +static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, + struct drm_tegra_submit_cmd_gather_uptr *cmd, + struct gather_bo *bo, u32 *offset, + struct tegra_drm_submit_data *job_data, + u32 *class) +{ + u32 next_offset; + + if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { + SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); + return -EINVAL; + } + + /* Check for maximum gather size */ + if (cmd->words > 16383) { + SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); + return -EINVAL; + } + + if (check_add_overflow(*offset, cmd->words, &next_offset)) { + SUBMIT_ERR(context, "too many total words in job"); + return -EINVAL; + } + + if (next_offset > bo->gather_data_words) { + SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); + return -EINVAL; + } + + if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, + cmd->words, job_data, class)) { + SUBMIT_ERR(context, "job was rejected by firewall"); + return -EINVAL; + } + + host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); + + *offset = next_offset; + + return 0; +} + +static struct host1x_job * +submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, + struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, + struct xarray *syncpoints) +{ + struct drm_tegra_submit_cmd *cmds; + u32 i, gather_offset = 0, class; + struct host1x_job *job; + int err; + + /* Set initial class for firewall. */ + class = context->client->base.class; + + cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, + sizeof(*cmds)); + if (IS_ERR(cmds)) { + SUBMIT_ERR(context, "failed to copy cmds array from userspace"); + return ERR_CAST(cmds); + } + + job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); + if (!job) { + SUBMIT_ERR(context, "failed to allocate memory for job"); + job = ERR_PTR(-ENOMEM); + goto done; + } + + err = submit_get_syncpt(context, job, syncpoints, args); + if (err < 0) + goto free_job; + + job->client = &context->client->base; + job->class = context->client->base.class; + job->serialize = true; + + for (i = 0; i < args->num_cmds; i++) { + struct drm_tegra_submit_cmd *cmd = &cmds[i]; + + if (cmd->flags) { + SUBMIT_ERR(context, "unknown flags given for cmd"); + err = -EINVAL; + goto free_job; + } + + if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { + err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, + &gather_offset, job_data, &class); + if (err) + goto free_job; + } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { + if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { + SUBMIT_ERR(context, "non-zero reserved value"); + err = -EINVAL; + goto free_job; + } + + host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, + false, class); + } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { + if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { + SUBMIT_ERR(context, "non-zero reserved value"); + err = -EINVAL; + goto free_job; + } + + if (cmd->wait_syncpt.id != args->syncpt.id) { + SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); + err = -EINVAL; + goto free_job; + } + + host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, + true, class); + } else { + SUBMIT_ERR(context, "unknown cmd type"); + err = -EINVAL; + goto free_job; + } + } + + if (gather_offset == 0) { + SUBMIT_ERR(context, "job must have at least one gather"); + err = -EINVAL; + goto free_job; + } + + goto done; + +free_job: + host1x_job_put(job); + job = ERR_PTR(err); + +done: + kvfree(cmds); + + return job; +} + +static void release_job(struct host1x_job *job) +{ + struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); + struct tegra_drm_submit_data *job_data = job->user_data; + u32 i; + + for (i = 0; i < job_data->num_used_mappings; i++) + tegra_drm_mapping_put(job_data->used_mappings[i].mapping); + + kfree(job_data->used_mappings); + kfree(job_data); + + if (pm_runtime_enabled(client->base.dev)) + pm_runtime_put_autosuspend(client->base.dev); +} + +int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_channel_submit *args = data; + struct tegra_drm_submit_data *job_data; + struct drm_syncobj *syncobj = NULL; + struct tegra_drm_context *context; + struct host1x_job *job; + struct gather_bo *bo; + u32 i; + int err; + + mutex_lock(&fpriv->lock); + + context = xa_load(&fpriv->contexts, args->context); + if (!context) { + mutex_unlock(&fpriv->lock); + pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, + current->comm, args->context); + return -EINVAL; + } + + if (args->syncobj_in) { + struct dma_fence *fence; + + err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); + if (err) { + SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); + goto unlock; + } + + err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000)); + dma_fence_put(fence); + if (err) { + SUBMIT_ERR(context, "wait for syncobj_in timed out"); + goto unlock; + } + } + + if (args->syncobj_out) { + syncobj = drm_syncobj_find(file, args->syncobj_out); + if (!syncobj) { + SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); + err = -ENOENT; + goto unlock; + } + } + + /* Allocate gather BO and copy gather words in. */ + err = submit_copy_gather_data(&bo, drm->dev, context, args); + if (err) + goto unlock; + + job_data = kzalloc(sizeof(*job_data), GFP_KERNEL); + if (!job_data) { + SUBMIT_ERR(context, "failed to allocate memory for job data"); + err = -ENOMEM; + goto put_bo; + } + + /* Get data buffer mappings and do relocation patching. */ + err = submit_process_bufs(context, bo, args, job_data); + if (err) + goto free_job_data; + + /* Allocate host1x_job and add gathers and waits to it. */ + job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto free_job_data; + } + + /* Map gather data for Host1x. */ + err = host1x_job_pin(job, context->client->base.dev); + if (err) { + SUBMIT_ERR(context, "failed to pin job: %d", err); + goto put_job; + } + + /* Boot engine. */ + if (pm_runtime_enabled(context->client->base.dev)) { + err = pm_runtime_resume_and_get(context->client->base.dev); + if (err < 0) { + SUBMIT_ERR(context, "could not power up engine: %d", err); + goto unpin_job; + } + } + + job->user_data = job_data; + job->release = release_job; + job->timeout = 10000; + + /* + * job_data is now part of job reference counting, so don't release + * it from here. + */ + job_data = NULL; + + /* Submit job to hardware. */ + err = host1x_job_submit(job); + if (err) { + SUBMIT_ERR(context, "host1x job submission failed: %d", err); + goto unpin_job; + } + + /* Return postfences to userspace and add fences to DMA reservations. */ + args->syncpt.value = job->syncpt_end; + + if (syncobj) { + struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + SUBMIT_ERR(context, "failed to create postfence: %d", err); + } + + drm_syncobj_replace_fence(syncobj, fence); + } + + goto put_job; + +unpin_job: + host1x_job_unpin(job); +put_job: + host1x_job_put(job); +free_job_data: + if (job_data && job_data->used_mappings) { + for (i = 0; i < job_data->num_used_mappings; i++) + tegra_drm_mapping_put(job_data->used_mappings[i].mapping); + + kfree(job_data->used_mappings); + } + + if (job_data) + kfree(job_data); +put_bo: + gather_bo_put(&bo->base); +unlock: + if (syncobj) + drm_syncobj_put(syncobj); + + mutex_unlock(&fpriv->lock); + return err; +} diff --git a/drivers/gpu/drm/tegra/submit.h b/drivers/gpu/drm/tegra/submit.h new file mode 100644 index 000000000000..cf6a2f0a29fc --- /dev/null +++ b/drivers/gpu/drm/tegra/submit.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 NVIDIA Corporation */ + +#ifndef _TEGRA_DRM_UAPI_SUBMIT_H +#define _TEGRA_DRM_UAPI_SUBMIT_H + +struct tegra_drm_used_mapping { + struct tegra_drm_mapping *mapping; + u32 flags; +}; + +struct tegra_drm_submit_data { + struct tegra_drm_used_mapping *used_mappings; + u32 num_used_mappings; +}; + +int tegra_drm_fw_validate(struct tegra_drm_client *client, u32 *data, u32 start, + u32 words, struct tegra_drm_submit_data *submit, + u32 *job_class); + +#endif diff --git a/drivers/gpu/drm/tegra/uapi.c b/drivers/gpu/drm/tegra/uapi.c new file mode 100644 index 000000000000..dc16a24f4dbe --- /dev/null +++ b/drivers/gpu/drm/tegra/uapi.c @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 NVIDIA Corporation */ + +#include <linux/host1x.h> +#include <linux/iommu.h> +#include <linux/list.h> + +#include <drm/drm_drv.h> +#include <drm/drm_file.h> +#include <drm/drm_utils.h> + +#include "drm.h" +#include "uapi.h" + +static void tegra_drm_mapping_release(struct kref *ref) +{ + struct tegra_drm_mapping *mapping = + container_of(ref, struct tegra_drm_mapping, ref); + + if (mapping->sgt) + dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, + DMA_ATTR_SKIP_CPU_SYNC); + + host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); + host1x_bo_put(mapping->bo); + + kfree(mapping); +} + +void tegra_drm_mapping_put(struct tegra_drm_mapping *mapping) +{ + kref_put(&mapping->ref, tegra_drm_mapping_release); +} + +static void tegra_drm_channel_context_close(struct tegra_drm_context *context) +{ + struct tegra_drm_mapping *mapping; + unsigned long id; + + xa_for_each(&context->mappings, id, mapping) + tegra_drm_mapping_put(mapping); + + xa_destroy(&context->mappings); + + host1x_channel_put(context->channel); + + kfree(context); +} + +void tegra_drm_uapi_close_file(struct tegra_drm_file *file) +{ + struct tegra_drm_context *context; + struct host1x_syncpt *sp; + unsigned long id; + + xa_for_each(&file->contexts, id, context) + tegra_drm_channel_context_close(context); + + xa_for_each(&file->syncpoints, id, sp) + host1x_syncpt_put(sp); + + xa_destroy(&file->contexts); + xa_destroy(&file->syncpoints); +} + +static struct tegra_drm_client *tegra_drm_find_client(struct tegra_drm *tegra, u32 class) +{ + struct tegra_drm_client *client; + + list_for_each_entry(client, &tegra->clients, list) + if (client->base.class == class) + return client; + + return NULL; +} + +int tegra_drm_ioctl_channel_open(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct tegra_drm *tegra = drm->dev_private; + struct drm_tegra_channel_open *args = data; + struct tegra_drm_client *client = NULL; + struct tegra_drm_context *context; + int err; + + if (args->flags) + return -EINVAL; + + context = kzalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + client = tegra_drm_find_client(tegra, args->host1x_class); + if (!client) { + err = -ENODEV; + goto free; + } + + if (client->shared_channel) { + context->channel = host1x_channel_get(client->shared_channel); + } else { + context->channel = host1x_channel_request(&client->base); + if (!context->channel) { + err = -EBUSY; + goto free; + } + } + + err = xa_alloc(&fpriv->contexts, &args->context, context, XA_LIMIT(1, U32_MAX), + GFP_KERNEL); + if (err < 0) + goto put_channel; + + context->client = client; + xa_init_flags(&context->mappings, XA_FLAGS_ALLOC1); + + args->version = client->version; + args->capabilities = 0; + + if (device_get_dma_attr(client->base.dev) == DEV_DMA_COHERENT) + args->capabilities |= DRM_TEGRA_CHANNEL_CAP_CACHE_COHERENT; + + return 0; + +put_channel: + host1x_channel_put(context->channel); +free: + kfree(context); + + return err; +} + +int tegra_drm_ioctl_channel_close(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_channel_close *args = data; + struct tegra_drm_context *context; + + mutex_lock(&fpriv->lock); + + context = xa_load(&fpriv->contexts, args->context); + if (!context) { + mutex_unlock(&fpriv->lock); + return -EINVAL; + } + + xa_erase(&fpriv->contexts, args->context); + + mutex_unlock(&fpriv->lock); + + tegra_drm_channel_context_close(context); + + return 0; +} + +int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_channel_map *args = data; + struct tegra_drm_mapping *mapping; + struct tegra_drm_context *context; + int err = 0; + + if (args->flags & ~DRM_TEGRA_CHANNEL_MAP_READ_WRITE) + return -EINVAL; + + mutex_lock(&fpriv->lock); + + context = xa_load(&fpriv->contexts, args->context); + if (!context) { + mutex_unlock(&fpriv->lock); + return -EINVAL; + } + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + err = -ENOMEM; + goto unlock; + } + + kref_init(&mapping->ref); + + mapping->dev = context->client->base.dev; + mapping->bo = tegra_gem_lookup(file, args->handle); + if (!mapping->bo) { + err = -EINVAL; + goto unlock; + } + + if (context->client->base.group) { + /* IOMMU domain managed directly using IOMMU API */ + host1x_bo_pin(mapping->dev, mapping->bo, &mapping->iova); + } else { + switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { + case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: + mapping->direction = DMA_BIDIRECTIONAL; + break; + + case DRM_TEGRA_CHANNEL_MAP_WRITE: + mapping->direction = DMA_FROM_DEVICE; + break; + + case DRM_TEGRA_CHANNEL_MAP_READ: + mapping->direction = DMA_TO_DEVICE; + break; + + default: + return -EINVAL; + } + + mapping->sgt = host1x_bo_pin(mapping->dev, mapping->bo, NULL); + if (IS_ERR(mapping->sgt)) { + err = PTR_ERR(mapping->sgt); + goto put_gem; + } + + err = dma_map_sgtable(mapping->dev, mapping->sgt, mapping->direction, + DMA_ATTR_SKIP_CPU_SYNC); + if (err) + goto unpin; + + mapping->iova = sg_dma_address(mapping->sgt->sgl); + } + + mapping->iova_end = mapping->iova + host1x_to_tegra_bo(mapping->bo)->size; + + err = xa_alloc(&context->mappings, &args->mapping, mapping, XA_LIMIT(1, U32_MAX), + GFP_KERNEL); + if (err < 0) + goto unmap; + + mutex_unlock(&fpriv->lock); + + return 0; + +unmap: + if (mapping->sgt) { + dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, + DMA_ATTR_SKIP_CPU_SYNC); + } +unpin: + host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); +put_gem: + host1x_bo_put(mapping->bo); + kfree(mapping); +unlock: + mutex_unlock(&fpriv->lock); + return err; +} + +int tegra_drm_ioctl_channel_unmap(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_channel_unmap *args = data; + struct tegra_drm_mapping *mapping; + struct tegra_drm_context *context; + + mutex_lock(&fpriv->lock); + + context = xa_load(&fpriv->contexts, args->context); + if (!context) { + mutex_unlock(&fpriv->lock); + return -EINVAL; + } + + mapping = xa_erase(&context->mappings, args->mapping); + + mutex_unlock(&fpriv->lock); + + if (!mapping) + return -EINVAL; + + tegra_drm_mapping_put(mapping); + return 0; +} + +int tegra_drm_ioctl_syncpoint_allocate(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct host1x *host1x = tegra_drm_to_host1x(drm->dev_private); + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_syncpoint_allocate *args = data; + struct host1x_syncpt *sp; + int err; + + if (args->id) + return -EINVAL; + + sp = host1x_syncpt_alloc(host1x, HOST1X_SYNCPT_CLIENT_MANAGED, current->comm); + if (!sp) + return -EBUSY; + + args->id = host1x_syncpt_id(sp); + + err = xa_insert(&fpriv->syncpoints, args->id, sp, GFP_KERNEL); + if (err) { + host1x_syncpt_put(sp); + return err; + } + + return 0; +} + +int tegra_drm_ioctl_syncpoint_free(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct tegra_drm_file *fpriv = file->driver_priv; + struct drm_tegra_syncpoint_allocate *args = data; + struct host1x_syncpt *sp; + + mutex_lock(&fpriv->lock); + sp = xa_erase(&fpriv->syncpoints, args->id); + mutex_unlock(&fpriv->lock); + + if (!sp) + return -EINVAL; + + host1x_syncpt_put(sp); + + return 0; +} + +int tegra_drm_ioctl_syncpoint_wait(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct host1x *host1x = tegra_drm_to_host1x(drm->dev_private); + struct drm_tegra_syncpoint_wait *args = data; + signed long timeout_jiffies; + struct host1x_syncpt *sp; + + if (args->padding != 0) + return -EINVAL; + + sp = host1x_syncpt_get_by_id_noref(host1x, args->id); + if (!sp) + return -EINVAL; + + timeout_jiffies = drm_timeout_abs_to_jiffies(args->timeout_ns); + + return host1x_syncpt_wait(sp, args->threshold, timeout_jiffies, &args->value); +} diff --git a/drivers/gpu/drm/tegra/uapi.h b/drivers/gpu/drm/tegra/uapi.h new file mode 100644 index 000000000000..12adad770ad3 --- /dev/null +++ b/drivers/gpu/drm/tegra/uapi.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 NVIDIA Corporation */ + +#ifndef _TEGRA_DRM_UAPI_H +#define _TEGRA_DRM_UAPI_H + +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/kref.h> +#include <linux/xarray.h> + +#include <drm/drm.h> + +struct drm_file; +struct drm_device; + +struct tegra_drm_file { + /* Legacy UAPI state */ + struct idr legacy_contexts; + struct mutex lock; + + /* New UAPI state */ + struct xarray contexts; + struct xarray syncpoints; +}; + +struct tegra_drm_mapping { + struct kref ref; + + struct device *dev; + struct host1x_bo *bo; + struct sg_table *sgt; + enum dma_data_direction direction; + dma_addr_t iova; + dma_addr_t iova_end; +}; + +int tegra_drm_ioctl_channel_open(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_channel_close(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_channel_unmap(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_syncpoint_allocate(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_syncpoint_free(struct drm_device *drm, void *data, + struct drm_file *file); +int tegra_drm_ioctl_syncpoint_wait(struct drm_device *drm, void *data, + struct drm_file *file); + +void tegra_drm_uapi_close_file(struct tegra_drm_file *file); +void tegra_drm_mapping_put(struct tegra_drm_mapping *mapping); + +#endif diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c index c9d55a9a3180..c02010ff2b7f 100644 --- a/drivers/gpu/drm/tegra/vic.c +++ b/drivers/gpu/drm/tegra/vic.c @@ -29,7 +29,6 @@ struct vic_config { struct vic { struct falcon falcon; - bool booted; void __iomem *regs; struct tegra_drm_client client; @@ -52,48 +51,6 @@ static void vic_writel(struct vic *vic, u32 value, unsigned int offset) writel(value, vic->regs + offset); } -static int vic_runtime_resume(struct device *dev) -{ - struct vic *vic = dev_get_drvdata(dev); - int err; - - err = clk_prepare_enable(vic->clk); - if (err < 0) - return err; - - usleep_range(10, 20); - - err = reset_control_deassert(vic->rst); - if (err < 0) - goto disable; - - usleep_range(10, 20); - - return 0; - -disable: - clk_disable_unprepare(vic->clk); - return err; -} - -static int vic_runtime_suspend(struct device *dev) -{ - struct vic *vic = dev_get_drvdata(dev); - int err; - - err = reset_control_assert(vic->rst); - if (err < 0) - return err; - - usleep_range(2000, 4000); - - clk_disable_unprepare(vic->clk); - - vic->booted = false; - - return 0; -} - static int vic_boot(struct vic *vic) { #ifdef CONFIG_IOMMU_API @@ -103,9 +60,6 @@ static int vic_boot(struct vic *vic) void *hdr; int err = 0; - if (vic->booted) - return 0; - #ifdef CONFIG_IOMMU_API if (vic->config->supports_sid && spec) { u32 value; @@ -168,8 +122,6 @@ static int vic_boot(struct vic *vic) return err; } - vic->booted = true; - return 0; } @@ -323,35 +275,74 @@ cleanup: return err; } -static int vic_open_channel(struct tegra_drm_client *client, - struct tegra_drm_context *context) + +static int vic_runtime_resume(struct device *dev) { - struct vic *vic = to_vic(client); + struct vic *vic = dev_get_drvdata(dev); int err; - err = pm_runtime_resume_and_get(vic->dev); + err = clk_prepare_enable(vic->clk); if (err < 0) return err; + usleep_range(10, 20); + + err = reset_control_deassert(vic->rst); + if (err < 0) + goto disable; + + usleep_range(10, 20); + err = vic_load_firmware(vic); if (err < 0) - goto rpm_put; + goto assert; err = vic_boot(vic); if (err < 0) - goto rpm_put; + goto assert; + + return 0; + +assert: + reset_control_assert(vic->rst); +disable: + clk_disable_unprepare(vic->clk); + return err; +} + +static int vic_runtime_suspend(struct device *dev) +{ + struct vic *vic = dev_get_drvdata(dev); + int err; + + err = reset_control_assert(vic->rst); + if (err < 0) + return err; + + usleep_range(2000, 4000); + + clk_disable_unprepare(vic->clk); + + return 0; +} + +static int vic_open_channel(struct tegra_drm_client *client, + struct tegra_drm_context *context) +{ + struct vic *vic = to_vic(client); + int err; + + err = pm_runtime_resume_and_get(vic->dev); + if (err < 0) + return err; context->channel = host1x_channel_get(vic->channel); if (!context->channel) { - err = -ENOMEM; - goto rpm_put; + pm_runtime_put(vic->dev); + return -ENOMEM; } return 0; - -rpm_put: - pm_runtime_put(vic->dev); - return err; } static void vic_close_channel(struct tegra_drm_context *context) @@ -359,7 +350,6 @@ static void vic_close_channel(struct tegra_drm_context *context) struct vic *vic = to_vic(context->client); host1x_channel_put(context->channel); - pm_runtime_put(vic->dev); } diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 0a3127436f61..3b22c0013dbf 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1124,9 +1124,9 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, } if (bo->deleted) { - ttm_bo_cleanup_refs(bo, false, false, locked); + ret = ttm_bo_cleanup_refs(bo, false, false, locked); ttm_bo_put(bo); - return 0; + return ret == -EBUSY ? -ENOSPC : ret; } ttm_bo_move_to_pinned(bo); @@ -1180,7 +1180,7 @@ out: if (locked) dma_resv_unlock(bo->base.resv); ttm_bo_put(bo); - return ret; + return ret == -EBUSY ? -ENOSPC : ret; } void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 763fa6f4e07d..1c5ffe2935af 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -143,7 +143,6 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, struct ttm_resource *src_mem = bo->resource; struct ttm_resource_manager *src_man = ttm_manager_type(bdev, src_mem->mem_type); - struct ttm_resource src_copy = *src_mem; union { struct ttm_kmap_iter_tt tt; struct ttm_kmap_iter_linear_io io; @@ -173,11 +172,11 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, } ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter); - src_copy = *src_mem; - ttm_bo_move_sync_cleanup(bo, dst_mem); if (!src_iter->ops->maps_tt) - ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, &src_copy); + ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem); + ttm_bo_move_sync_cleanup(bo, dst_mem); + out_src_iter: if (!dst_iter->ops->maps_tt) ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem); diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index a021d029b730..be24bb6cefd0 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -44,6 +44,8 @@ static unsigned ttm_glob_use_count; struct ttm_global ttm_glob; EXPORT_SYMBOL(ttm_glob); +struct dentry *ttm_debugfs_root; + static void ttm_global_release(void) { struct ttm_global *glob = &ttm_glob; @@ -53,6 +55,7 @@ static void ttm_global_release(void) goto out; ttm_pool_mgr_fini(); + debugfs_remove(ttm_debugfs_root); __free_page(glob->dummy_read_page); memset(glob, 0, sizeof(*glob)); @@ -73,6 +76,11 @@ static int ttm_global_init(void) si_meminfo(&si); + ttm_debugfs_root = debugfs_create_dir("ttm", NULL); + if (IS_ERR(ttm_debugfs_root)) { + ttm_debugfs_root = NULL; + } + /* Limit the number of pages in the pool to about 50% of the total * system memory. */ @@ -100,6 +108,8 @@ static int ttm_global_init(void) debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root, &glob->bo_count); out: + if (ret && ttm_debugfs_root) + debugfs_remove(ttm_debugfs_root); if (ret) --ttm_glob_use_count; mutex_unlock(&ttm_global_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 6c19290f7ea9..0037eefe3239 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -84,22 +84,6 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp) return tmp; } -struct dentry *ttm_debugfs_root; - -static int __init ttm_init(void) -{ - ttm_debugfs_root = debugfs_create_dir("ttm", NULL); - return 0; -} - -static void __exit ttm_exit(void) -{ - debugfs_remove(ttm_debugfs_root); -} - -module_init(ttm_init); -module_exit(ttm_exit); - MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse"); MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)"); MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 506b3c926a68..dae52433beeb 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -32,7 +32,6 @@ #define pr_fmt(fmt) "[TTM] " fmt #include <linux/sched.h> -#include <linux/pagemap.h> #include <linux/shmem_fs.h> #include <linux/file.h> #include <drm/drm_cache.h> diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index b7dc32a0c9bb..4a1115043114 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -1462,7 +1462,7 @@ static const struct hdmi_codec_ops vc4_hdmi_codec_ops = { .audio_startup = vc4_hdmi_audio_startup, }; -struct hdmi_codec_pdata vc4_hdmi_codec_pdata = { +static struct hdmi_codec_pdata vc4_hdmi_codec_pdata = { .ops = &vc4_hdmi_codec_ops, .max_i2s_channels = 8, .i2s = 1, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c index f35bdc1cb197..e899a936a42a 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c @@ -328,7 +328,6 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) int ret = 0; static size_t sgl_size; static size_t sgt_size; - struct scatterlist *sg; if (vmw_tt->mapped) return 0; @@ -351,15 +350,12 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) if (unlikely(ret != 0)) return ret; - sg = __sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages, - vsgt->num_pages, 0, - (unsigned long) vsgt->num_pages << PAGE_SHIFT, - dma_get_max_seg_size(dev_priv->drm.dev), - NULL, 0, GFP_KERNEL); - if (IS_ERR(sg)) { - ret = PTR_ERR(sg); + ret = sg_alloc_table_from_pages_segment( + &vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, + (unsigned long)vsgt->num_pages << PAGE_SHIFT, + dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL); + if (ret) goto out_sg_alloc_fail; - } if (vsgt->num_pages > vmw_tt->sgt.orig_nents) { uint64_t over_alloc = diff --git a/drivers/gpu/drm/xlnx/Kconfig b/drivers/gpu/drm/xlnx/Kconfig index b52c6cdfc0b8..c3d08269faa9 100644 --- a/drivers/gpu/drm/xlnx/Kconfig +++ b/drivers/gpu/drm/xlnx/Kconfig @@ -3,6 +3,8 @@ config DRM_ZYNQMP_DPSUB depends on ARCH_ZYNQMP || COMPILE_TEST depends on COMMON_CLK && DRM && OF depends on DMADEVICES + depends on PHY_XILINX_ZYNQMP + depends on XILINX_ZYNQMP_DPDMA select DMA_ENGINE select DRM_GEM_CMA_HELPER select DRM_KMS_CMA_HELPER diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.c b/drivers/gpu/drm/xlnx/zynqmp_disp.c index 109d627968ac..ff2b308d8651 100644 --- a/drivers/gpu/drm/xlnx/zynqmp_disp.c +++ b/drivers/gpu/drm/xlnx/zynqmp_disp.c @@ -91,7 +91,7 @@ struct zynqmp_disp_format { }; /** - * enum zynqmp_disp_id - Layer identifier + * enum zynqmp_disp_layer_id - Layer identifier * @ZYNQMP_DISP_LAYER_VID: Video layer * @ZYNQMP_DISP_LAYER_GFX: Graphics layer */ @@ -159,42 +159,16 @@ struct zynqmp_disp_layer { }; /** - * struct zynqmp_disp_blend - Blender - * @base: Registers I/O base address - */ -struct zynqmp_disp_blend { - void __iomem *base; -}; - -/** - * struct zynqmp_disp_avbuf - Audio/video buffer manager - * @base: Registers I/O base address - */ -struct zynqmp_disp_avbuf { - void __iomem *base; -}; - -/** - * struct zynqmp_disp_audio - Audio mixer - * @base: Registers I/O base address - * @clk: Audio clock - * @clk_from_ps: True of the audio clock comes from PS, false from PL - */ -struct zynqmp_disp_audio { - void __iomem *base; - struct clk *clk; - bool clk_from_ps; -}; - -/** * struct zynqmp_disp - Display controller * @dev: Device structure * @drm: DRM core * @dpsub: Display subsystem * @crtc: DRM CRTC - * @blend: Blender (video rendering pipeline) - * @avbuf: Audio/video buffer manager - * @audio: Audio mixer + * @blend.base: Register I/O base address for the blender + * @avbuf.base: Register I/O base address for the audio/video buffer manager + * @audio.base: Registers I/O base address for the audio mixer + * @audio.clk: Audio clock + * @audio.clk_from_ps: True of the audio clock comes from PS, false from PL * @layers: Layers (planes) * @event: Pending vblank event request * @pclk: Pixel clock @@ -207,9 +181,17 @@ struct zynqmp_disp { struct drm_crtc crtc; - struct zynqmp_disp_blend blend; - struct zynqmp_disp_avbuf avbuf; - struct zynqmp_disp_audio audio; + struct { + void __iomem *base; + } blend; + struct { + void __iomem *base; + } avbuf; + struct { + void __iomem *base; + struct clk *clk; + bool clk_from_ps; + } audio; struct zynqmp_disp_layer layers[ZYNQMP_DISP_NUM_LAYERS]; @@ -423,51 +405,60 @@ static const struct zynqmp_disp_format avbuf_gfx_fmts[] = { }, }; -static u32 zynqmp_disp_avbuf_read(struct zynqmp_disp_avbuf *avbuf, int reg) +static u32 zynqmp_disp_avbuf_read(struct zynqmp_disp *disp, int reg) { - return readl(avbuf->base + reg); + return readl(disp->avbuf.base + reg); } -static void zynqmp_disp_avbuf_write(struct zynqmp_disp_avbuf *avbuf, - int reg, u32 val) +static void zynqmp_disp_avbuf_write(struct zynqmp_disp *disp, int reg, u32 val) { - writel(val, avbuf->base + reg); + writel(val, disp->avbuf.base + reg); +} + +static bool zynqmp_disp_layer_is_gfx(const struct zynqmp_disp_layer *layer) +{ + return layer->id == ZYNQMP_DISP_LAYER_GFX; +} + +static bool zynqmp_disp_layer_is_video(const struct zynqmp_disp_layer *layer) +{ + return layer->id == ZYNQMP_DISP_LAYER_VID; } /** * zynqmp_disp_avbuf_set_format - Set the input format for a layer - * @avbuf: Audio/video buffer manager - * @layer: The layer ID + * @disp: Display controller + * @layer: The layer * @fmt: The format information * * Set the video buffer manager format for @layer to @fmt. */ -static void zynqmp_disp_avbuf_set_format(struct zynqmp_disp_avbuf *avbuf, - enum zynqmp_disp_layer_id layer, +static void zynqmp_disp_avbuf_set_format(struct zynqmp_disp *disp, + struct zynqmp_disp_layer *layer, const struct zynqmp_disp_format *fmt) { unsigned int i; u32 val; - val = zynqmp_disp_avbuf_read(avbuf, ZYNQMP_DISP_AV_BUF_FMT); - val &= layer == ZYNQMP_DISP_LAYER_VID + val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_FMT); + val &= zynqmp_disp_layer_is_video(layer) ? ~ZYNQMP_DISP_AV_BUF_FMT_NL_VID_MASK : ~ZYNQMP_DISP_AV_BUF_FMT_NL_GFX_MASK; val |= fmt->buf_fmt; - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_FMT, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_FMT, val); for (i = 0; i < ZYNQMP_DISP_AV_BUF_NUM_SF; i++) { - unsigned int reg = layer == ZYNQMP_DISP_LAYER_VID + unsigned int reg = zynqmp_disp_layer_is_video(layer) ? ZYNQMP_DISP_AV_BUF_VID_COMP_SF(i) : ZYNQMP_DISP_AV_BUF_GFX_COMP_SF(i); - zynqmp_disp_avbuf_write(avbuf, reg, fmt->sf[i]); + zynqmp_disp_avbuf_write(disp, reg, fmt->sf[i]); } } /** * zynqmp_disp_avbuf_set_clocks_sources - Set the clocks sources - * @avbuf: Audio/video buffer manager + * @disp: Display controller * @video_from_ps: True if the video clock originates from the PS * @audio_from_ps: True if the audio clock originates from the PS * @timings_internal: True if video timings are generated internally @@ -477,7 +468,7 @@ static void zynqmp_disp_avbuf_set_format(struct zynqmp_disp_avbuf *avbuf, * generated internally or externally. */ static void -zynqmp_disp_avbuf_set_clocks_sources(struct zynqmp_disp_avbuf *avbuf, +zynqmp_disp_avbuf_set_clocks_sources(struct zynqmp_disp *disp, bool video_from_ps, bool audio_from_ps, bool timings_internal) { @@ -490,16 +481,16 @@ zynqmp_disp_avbuf_set_clocks_sources(struct zynqmp_disp_avbuf *avbuf, if (timings_internal) val |= ZYNQMP_DISP_AV_BUF_CLK_SRC_VID_INTERNAL_TIMING; - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_CLK_SRC, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_CLK_SRC, val); } /** * zynqmp_disp_avbuf_enable_channels - Enable buffer channels - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * Enable all (video and audio) buffer channels. */ -static void zynqmp_disp_avbuf_enable_channels(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_enable_channels(struct zynqmp_disp *disp) { unsigned int i; u32 val; @@ -509,7 +500,7 @@ static void zynqmp_disp_avbuf_enable_channels(struct zynqmp_disp_avbuf *avbuf) ZYNQMP_DISP_AV_BUF_CHBUF_BURST_LEN_SHIFT); for (i = 0; i < ZYNQMP_DISP_AV_BUF_NUM_VID_GFX_BUFFERS; i++) - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_CHBUF(i), + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_CHBUF(i), val); val = ZYNQMP_DISP_AV_BUF_CHBUF_EN | @@ -517,75 +508,75 @@ static void zynqmp_disp_avbuf_enable_channels(struct zynqmp_disp_avbuf *avbuf) ZYNQMP_DISP_AV_BUF_CHBUF_BURST_LEN_SHIFT); for (; i < ZYNQMP_DISP_AV_BUF_NUM_BUFFERS; i++) - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_CHBUF(i), + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_CHBUF(i), val); } /** * zynqmp_disp_avbuf_disable_channels - Disable buffer channels - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * Disable all (video and audio) buffer channels. */ -static void zynqmp_disp_avbuf_disable_channels(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_disable_channels(struct zynqmp_disp *disp) { unsigned int i; for (i = 0; i < ZYNQMP_DISP_AV_BUF_NUM_BUFFERS; i++) - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_CHBUF(i), + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_CHBUF(i), ZYNQMP_DISP_AV_BUF_CHBUF_FLUSH); } /** * zynqmp_disp_avbuf_enable_audio - Enable audio - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * Enable all audio buffers with a non-live (memory) source. */ -static void zynqmp_disp_avbuf_enable_audio(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_enable_audio(struct zynqmp_disp *disp) { u32 val; - val = zynqmp_disp_avbuf_read(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT); + val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_OUTPUT); val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_AUD1_MASK; val |= ZYNQMP_DISP_AV_BUF_OUTPUT_AUD1_MEM; val |= ZYNQMP_DISP_AV_BUF_OUTPUT_AUD2_EN; - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_OUTPUT, val); } /** * zynqmp_disp_avbuf_disable_audio - Disable audio - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * Disable all audio buffers. */ -static void zynqmp_disp_avbuf_disable_audio(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_disable_audio(struct zynqmp_disp *disp) { u32 val; - val = zynqmp_disp_avbuf_read(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT); + val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_OUTPUT); val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_AUD1_MASK; val |= ZYNQMP_DISP_AV_BUF_OUTPUT_AUD1_DISABLE; val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_AUD2_EN; - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_OUTPUT, val); } /** * zynqmp_disp_avbuf_enable_video - Enable a video layer - * @avbuf: Audio/video buffer manager - * @layer: The layer ID + * @disp: Display controller + * @layer: The layer * @mode: Operating mode of layer * * Enable the video/graphics buffer for @layer. */ -static void zynqmp_disp_avbuf_enable_video(struct zynqmp_disp_avbuf *avbuf, - enum zynqmp_disp_layer_id layer, +static void zynqmp_disp_avbuf_enable_video(struct zynqmp_disp *disp, + struct zynqmp_disp_layer *layer, enum zynqmp_disp_layer_mode mode) { u32 val; - val = zynqmp_disp_avbuf_read(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT); - if (layer == ZYNQMP_DISP_LAYER_VID) { + val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_OUTPUT); + if (zynqmp_disp_layer_is_video(layer)) { val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_VID1_MASK; if (mode == ZYNQMP_DISP_LAYER_NONLIVE) val |= ZYNQMP_DISP_AV_BUF_OUTPUT_VID1_MEM; @@ -599,52 +590,52 @@ static void zynqmp_disp_avbuf_enable_video(struct zynqmp_disp_avbuf *avbuf, else val |= ZYNQMP_DISP_AV_BUF_OUTPUT_VID2_LIVE; } - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_OUTPUT, val); } /** * zynqmp_disp_avbuf_disable_video - Disable a video layer - * @avbuf: Audio/video buffer manager - * @layer: The layer ID + * @disp: Display controller + * @layer: The layer * * Disable the video/graphics buffer for @layer. */ -static void zynqmp_disp_avbuf_disable_video(struct zynqmp_disp_avbuf *avbuf, - enum zynqmp_disp_layer_id layer) +static void zynqmp_disp_avbuf_disable_video(struct zynqmp_disp *disp, + struct zynqmp_disp_layer *layer) { u32 val; - val = zynqmp_disp_avbuf_read(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT); - if (layer == ZYNQMP_DISP_LAYER_VID) { + val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_OUTPUT); + if (zynqmp_disp_layer_is_video(layer)) { val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_VID1_MASK; val |= ZYNQMP_DISP_AV_BUF_OUTPUT_VID1_NONE; } else { val &= ~ZYNQMP_DISP_AV_BUF_OUTPUT_VID2_MASK; val |= ZYNQMP_DISP_AV_BUF_OUTPUT_VID2_DISABLE; } - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_OUTPUT, val); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_OUTPUT, val); } /** * zynqmp_disp_avbuf_enable - Enable the video pipe - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * De-assert the video pipe reset. */ -static void zynqmp_disp_avbuf_enable(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_enable(struct zynqmp_disp *disp) { - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_SRST_REG, 0); + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_SRST_REG, 0); } /** * zynqmp_disp_avbuf_disable - Disable the video pipe - * @avbuf: Audio/video buffer manager + * @disp: Display controller * * Assert the video pipe reset. */ -static void zynqmp_disp_avbuf_disable(struct zynqmp_disp_avbuf *avbuf) +static void zynqmp_disp_avbuf_disable(struct zynqmp_disp *disp) { - zynqmp_disp_avbuf_write(avbuf, ZYNQMP_DISP_AV_BUF_SRST_REG, + zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_SRST_REG, ZYNQMP_DISP_AV_BUF_SRST_REG_VID_RST); } @@ -652,10 +643,9 @@ static void zynqmp_disp_avbuf_disable(struct zynqmp_disp_avbuf *avbuf) * Blender (Video Pipeline) */ -static void zynqmp_disp_blend_write(struct zynqmp_disp_blend *blend, - int reg, u32 val) +static void zynqmp_disp_blend_write(struct zynqmp_disp *disp, int reg, u32 val) { - writel(val, blend->base + reg); + writel(val, disp->blend.base + reg); } /* @@ -701,12 +691,12 @@ static const u32 csc_sdtv_to_rgb_offsets[] = { /** * zynqmp_disp_blend_set_output_format - Set the output format of the blender - * @blend: Blender object + * @disp: Display controller * @format: Output format * * Set the output format of the blender to @format. */ -static void zynqmp_disp_blend_set_output_format(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_set_output_format(struct zynqmp_disp *disp, enum zynqmp_dpsub_format format) { static const unsigned int blend_output_fmts[] = { @@ -722,7 +712,7 @@ static void zynqmp_disp_blend_set_output_format(struct zynqmp_disp_blend *blend, const u32 *offsets; unsigned int i; - zynqmp_disp_blend_write(blend, ZYNQMP_DISP_V_BLEND_OUTPUT_VID_FMT, fmt); + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_OUTPUT_VID_FMT, fmt); if (fmt == ZYNQMP_DISP_V_BLEND_OUTPUT_VID_FMT_RGB) { coeffs = csc_identity_matrix; offsets = csc_zero_offsets; @@ -732,19 +722,19 @@ static void zynqmp_disp_blend_set_output_format(struct zynqmp_disp_blend *blend, } for (i = 0; i < ZYNQMP_DISP_V_BLEND_NUM_COEFF; i++) - zynqmp_disp_blend_write(blend, + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_RGB2YCBCR_COEFF(i), coeffs[i]); for (i = 0; i < ZYNQMP_DISP_V_BLEND_NUM_OFFSET; i++) - zynqmp_disp_blend_write(blend, + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_OUTCSC_OFFSET(i), offsets[i]); } /** * zynqmp_disp_blend_set_bg_color - Set the background color - * @blend: Blender object + * @disp: Display controller * @rcr: Red/Cr color component * @gy: Green/Y color component * @bcb: Blue/Cb color component @@ -753,31 +743,31 @@ static void zynqmp_disp_blend_set_output_format(struct zynqmp_disp_blend *blend, * B or Cr, Y and Cb components respectively depending on the selected output * format. */ -static void zynqmp_disp_blend_set_bg_color(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_set_bg_color(struct zynqmp_disp *disp, u32 rcr, u32 gy, u32 bcb) { - zynqmp_disp_blend_write(blend, ZYNQMP_DISP_V_BLEND_BG_CLR_0, rcr); - zynqmp_disp_blend_write(blend, ZYNQMP_DISP_V_BLEND_BG_CLR_1, gy); - zynqmp_disp_blend_write(blend, ZYNQMP_DISP_V_BLEND_BG_CLR_2, bcb); + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_BG_CLR_0, rcr); + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_BG_CLR_1, gy); + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_BG_CLR_2, bcb); } /** * zynqmp_disp_blend_set_global_alpha - Configure global alpha blending - * @blend: Blender object + * @disp: Display controller * @enable: True to enable global alpha blending * @alpha: Global alpha value (ignored if @enabled is false) */ -static void zynqmp_disp_blend_set_global_alpha(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_set_global_alpha(struct zynqmp_disp *disp, bool enable, u32 alpha) { - zynqmp_disp_blend_write(blend, ZYNQMP_DISP_V_BLEND_SET_GLOBAL_ALPHA, + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_SET_GLOBAL_ALPHA, ZYNQMP_DISP_V_BLEND_SET_GLOBAL_ALPHA_VALUE(alpha) | (enable ? ZYNQMP_DISP_V_BLEND_SET_GLOBAL_ALPHA_EN : 0)); } /** * zynqmp_disp_blend_layer_set_csc - Configure colorspace conversion for layer - * @blend: Blender object + * @disp: Display controller * @layer: The layer * @coeffs: Colorspace conversion matrix * @offsets: Colorspace conversion offsets @@ -786,7 +776,7 @@ static void zynqmp_disp_blend_set_global_alpha(struct zynqmp_disp_blend *blend, * Columns of the matrix are automatically swapped based on the input format to * handle RGB and YCrCb components permutations. */ -static void zynqmp_disp_blend_layer_set_csc(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_layer_set_csc(struct zynqmp_disp *disp, struct zynqmp_disp_layer *layer, const u16 *coeffs, const u32 *offsets) @@ -807,32 +797,32 @@ static void zynqmp_disp_blend_layer_set_csc(struct zynqmp_disp_blend *blend, } } - if (layer->id == ZYNQMP_DISP_LAYER_VID) + if (zynqmp_disp_layer_is_video(layer)) reg = ZYNQMP_DISP_V_BLEND_IN1CSC_COEFF(0); else reg = ZYNQMP_DISP_V_BLEND_IN2CSC_COEFF(0); for (i = 0; i < ZYNQMP_DISP_V_BLEND_NUM_COEFF; i += 3, reg += 12) { - zynqmp_disp_blend_write(blend, reg + 0, coeffs[i + swap[0]]); - zynqmp_disp_blend_write(blend, reg + 4, coeffs[i + swap[1]]); - zynqmp_disp_blend_write(blend, reg + 8, coeffs[i + swap[2]]); + zynqmp_disp_blend_write(disp, reg + 0, coeffs[i + swap[0]]); + zynqmp_disp_blend_write(disp, reg + 4, coeffs[i + swap[1]]); + zynqmp_disp_blend_write(disp, reg + 8, coeffs[i + swap[2]]); } - if (layer->id == ZYNQMP_DISP_LAYER_VID) + if (zynqmp_disp_layer_is_video(layer)) reg = ZYNQMP_DISP_V_BLEND_IN1CSC_OFFSET(0); else reg = ZYNQMP_DISP_V_BLEND_IN2CSC_OFFSET(0); for (i = 0; i < ZYNQMP_DISP_V_BLEND_NUM_OFFSET; i++) - zynqmp_disp_blend_write(blend, reg + i * 4, offsets[i]); + zynqmp_disp_blend_write(disp, reg + i * 4, offsets[i]); } /** * zynqmp_disp_blend_layer_enable - Enable a layer - * @blend: Blender object + * @disp: Display controller * @layer: The layer */ -static void zynqmp_disp_blend_layer_enable(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_layer_enable(struct zynqmp_disp *disp, struct zynqmp_disp_layer *layer) { const u16 *coeffs; @@ -844,7 +834,7 @@ static void zynqmp_disp_blend_layer_enable(struct zynqmp_disp_blend *blend, (layer->drm_fmt->hsub > 1 ? ZYNQMP_DISP_V_BLEND_LAYER_CONTROL_EN_US : 0); - zynqmp_disp_blend_write(blend, + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_LAYER_CONTROL(layer->id), val); @@ -856,22 +846,22 @@ static void zynqmp_disp_blend_layer_enable(struct zynqmp_disp_blend *blend, offsets = csc_zero_offsets; } - zynqmp_disp_blend_layer_set_csc(blend, layer, coeffs, offsets); + zynqmp_disp_blend_layer_set_csc(disp, layer, coeffs, offsets); } /** * zynqmp_disp_blend_layer_disable - Disable a layer - * @blend: Blender object + * @disp: Display controller * @layer: The layer */ -static void zynqmp_disp_blend_layer_disable(struct zynqmp_disp_blend *blend, +static void zynqmp_disp_blend_layer_disable(struct zynqmp_disp *disp, struct zynqmp_disp_layer *layer) { - zynqmp_disp_blend_write(blend, + zynqmp_disp_blend_write(disp, ZYNQMP_DISP_V_BLEND_LAYER_CONTROL(layer->id), 0); - zynqmp_disp_blend_layer_set_csc(blend, layer, csc_zero_matrix, + zynqmp_disp_blend_layer_set_csc(disp, layer, csc_zero_matrix, csc_zero_offsets); } @@ -879,57 +869,55 @@ static void zynqmp_disp_blend_layer_disable(struct zynqmp_disp_blend *blend, * Audio Mixer */ -static void zynqmp_disp_audio_write(struct zynqmp_disp_audio *audio, - int reg, u32 val) +static void zynqmp_disp_audio_write(struct zynqmp_disp *disp, int reg, u32 val) { - writel(val, audio->base + reg); + writel(val, disp->audio.base + reg); } /** * zynqmp_disp_audio_enable - Enable the audio mixer - * @audio: Audio mixer + * @disp: Display controller * * Enable the audio mixer by de-asserting the soft reset. The audio state is set to * default values by the reset, set the default mixer volume explicitly. */ -static void zynqmp_disp_audio_enable(struct zynqmp_disp_audio *audio) +static void zynqmp_disp_audio_enable(struct zynqmp_disp *disp) { /* Clear the audio soft reset register as it's an non-reset flop. */ - zynqmp_disp_audio_write(audio, ZYNQMP_DISP_AUD_SOFT_RESET, 0); - zynqmp_disp_audio_write(audio, ZYNQMP_DISP_AUD_MIXER_VOLUME, + zynqmp_disp_audio_write(disp, ZYNQMP_DISP_AUD_SOFT_RESET, 0); + zynqmp_disp_audio_write(disp, ZYNQMP_DISP_AUD_MIXER_VOLUME, ZYNQMP_DISP_AUD_MIXER_VOLUME_NO_SCALE); } /** * zynqmp_disp_audio_disable - Disable the audio mixer - * @audio: Audio mixer + * @disp: Display controller * * Disable the audio mixer by asserting its soft reset. */ -static void zynqmp_disp_audio_disable(struct zynqmp_disp_audio *audio) +static void zynqmp_disp_audio_disable(struct zynqmp_disp *disp) { - zynqmp_disp_audio_write(audio, ZYNQMP_DISP_AUD_SOFT_RESET, + zynqmp_disp_audio_write(disp, ZYNQMP_DISP_AUD_SOFT_RESET, ZYNQMP_DISP_AUD_SOFT_RESET_AUD_SRST); } -static void zynqmp_disp_audio_init(struct device *dev, - struct zynqmp_disp_audio *audio) +static void zynqmp_disp_audio_init(struct zynqmp_disp *disp) { /* Try the live PL audio clock. */ - audio->clk = devm_clk_get(dev, "dp_live_audio_aclk"); - if (!IS_ERR(audio->clk)) { - audio->clk_from_ps = false; + disp->audio.clk = devm_clk_get(disp->dev, "dp_live_audio_aclk"); + if (!IS_ERR(disp->audio.clk)) { + disp->audio.clk_from_ps = false; return; } /* If the live PL audio clock is not valid, fall back to PS clock. */ - audio->clk = devm_clk_get(dev, "dp_aud_clk"); - if (!IS_ERR(audio->clk)) { - audio->clk_from_ps = true; + disp->audio.clk = devm_clk_get(disp->dev, "dp_aud_clk"); + if (!IS_ERR(disp->audio.clk)) { + disp->audio.clk_from_ps = true; return; } - dev_err(dev, "audio disabled due to missing clock\n"); + dev_err(disp->dev, "audio disabled due to missing clock\n"); } /* ----------------------------------------------------------------------------- @@ -1025,9 +1013,9 @@ zynqmp_disp_layer_find_format(struct zynqmp_disp_layer *layer, */ static void zynqmp_disp_layer_enable(struct zynqmp_disp_layer *layer) { - zynqmp_disp_avbuf_enable_video(&layer->disp->avbuf, layer->id, + zynqmp_disp_avbuf_enable_video(layer->disp, layer, ZYNQMP_DISP_LAYER_NONLIVE); - zynqmp_disp_blend_layer_enable(&layer->disp->blend, layer); + zynqmp_disp_blend_layer_enable(layer->disp, layer); layer->mode = ZYNQMP_DISP_LAYER_NONLIVE; } @@ -1046,8 +1034,8 @@ static void zynqmp_disp_layer_disable(struct zynqmp_disp_layer *layer) for (i = 0; i < layer->drm_fmt->num_planes; i++) dmaengine_terminate_sync(layer->dmas[i].chan); - zynqmp_disp_avbuf_disable_video(&layer->disp->avbuf, layer->id); - zynqmp_disp_blend_layer_disable(&layer->disp->blend, layer); + zynqmp_disp_avbuf_disable_video(layer->disp, layer); + zynqmp_disp_blend_layer_disable(layer->disp, layer); } /** @@ -1067,8 +1055,7 @@ static void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer, layer->disp_fmt = zynqmp_disp_layer_find_format(layer, info->format); layer->drm_fmt = info; - zynqmp_disp_avbuf_set_format(&layer->disp->avbuf, layer->id, - layer->disp_fmt); + zynqmp_disp_avbuf_set_format(layer->disp, layer, layer->disp_fmt); /* * Set slave_id for each DMA channel to indicate they're part of a @@ -1175,6 +1162,10 @@ zynqmp_disp_plane_atomic_disable(struct drm_plane *plane, return; zynqmp_disp_layer_disable(layer); + + if (zynqmp_disp_layer_is_gfx(layer)) + zynqmp_disp_blend_set_global_alpha(layer->disp, false, + plane->state->alpha >> 8); } static void @@ -1204,6 +1195,10 @@ zynqmp_disp_plane_atomic_update(struct drm_plane *plane, zynqmp_disp_layer_update(layer, new_state); + if (zynqmp_disp_layer_is_gfx(layer)) + zynqmp_disp_blend_set_global_alpha(layer->disp, true, + plane->state->alpha >> 8); + /* Enable or re-enable the plane is the format has changed. */ if (format_changed) zynqmp_disp_layer_enable(layer); @@ -1244,8 +1239,8 @@ static int zynqmp_disp_create_planes(struct zynqmp_disp *disp) drm_formats[j] = layer->info->formats[j].drm_fmt; /* Graphics layer is primary, and video layer is overlay. */ - type = i == ZYNQMP_DISP_LAYER_GFX - ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; + type = zynqmp_disp_layer_is_video(layer) + ? DRM_PLANE_TYPE_OVERLAY : DRM_PLANE_TYPE_PRIMARY; ret = drm_universal_plane_init(disp->drm, &layer->plane, 0, &zynqmp_disp_plane_funcs, drm_formats, @@ -1256,6 +1251,10 @@ static int zynqmp_disp_create_planes(struct zynqmp_disp *disp) drm_plane_helper_add(&layer->plane, &zynqmp_disp_plane_helper_funcs); + + drm_plane_create_zpos_immutable_property(&layer->plane, i); + if (zynqmp_disp_layer_is_gfx(layer)) + drm_plane_create_alpha_property(&layer->plane); } return 0; @@ -1387,14 +1386,14 @@ err: */ static void zynqmp_disp_enable(struct zynqmp_disp *disp) { - zynqmp_disp_avbuf_enable(&disp->avbuf); + zynqmp_disp_avbuf_enable(disp); /* Choose clock source based on the DT clock handle. */ - zynqmp_disp_avbuf_set_clocks_sources(&disp->avbuf, disp->pclk_from_ps, + zynqmp_disp_avbuf_set_clocks_sources(disp, disp->pclk_from_ps, disp->audio.clk_from_ps, true); - zynqmp_disp_avbuf_enable_channels(&disp->avbuf); - zynqmp_disp_avbuf_enable_audio(&disp->avbuf); + zynqmp_disp_avbuf_enable_channels(disp); + zynqmp_disp_avbuf_enable_audio(disp); - zynqmp_disp_audio_enable(&disp->audio); + zynqmp_disp_audio_enable(disp); } /** @@ -1403,11 +1402,11 @@ static void zynqmp_disp_enable(struct zynqmp_disp *disp) */ static void zynqmp_disp_disable(struct zynqmp_disp *disp) { - zynqmp_disp_audio_disable(&disp->audio); + zynqmp_disp_audio_disable(disp); - zynqmp_disp_avbuf_disable_audio(&disp->avbuf); - zynqmp_disp_avbuf_disable_channels(&disp->avbuf); - zynqmp_disp_avbuf_disable(&disp->avbuf); + zynqmp_disp_avbuf_disable_audio(disp); + zynqmp_disp_avbuf_disable_channels(disp); + zynqmp_disp_avbuf_disable(disp); } static inline struct zynqmp_disp *crtc_to_disp(struct drm_crtc *crtc) @@ -1452,9 +1451,10 @@ zynqmp_disp_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode; int ret, vrefresh; + pm_runtime_get_sync(disp->dev); + zynqmp_disp_crtc_setup_clock(crtc, adjusted_mode); - pm_runtime_get_sync(disp->dev); ret = clk_prepare_enable(disp->pclk); if (ret) { dev_err(disp->dev, "failed to enable a pixel clock\n"); @@ -1462,10 +1462,8 @@ zynqmp_disp_crtc_atomic_enable(struct drm_crtc *crtc, return; } - zynqmp_disp_blend_set_output_format(&disp->blend, - ZYNQMP_DPSUB_FORMAT_RGB); - zynqmp_disp_blend_set_bg_color(&disp->blend, 0, 0, 0); - zynqmp_disp_blend_set_global_alpha(&disp->blend, false, 0); + zynqmp_disp_blend_set_output_format(disp, ZYNQMP_DPSUB_FORMAT_RGB); + zynqmp_disp_blend_set_bg_color(disp, 0, 0, 0); zynqmp_disp_enable(disp); @@ -1674,7 +1672,7 @@ int zynqmp_disp_probe(struct zynqmp_dpsub *dpsub, struct drm_device *drm) disp->pclk_from_ps = true; } - zynqmp_disp_audio_init(disp->dev, &disp->audio); + zynqmp_disp_audio_init(disp); ret = zynqmp_disp_create_layers(disp); if (ret) diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c index 82430ca9b913..6f588dc09ba6 100644 --- a/drivers/gpu/drm/xlnx/zynqmp_dp.c +++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c @@ -402,10 +402,6 @@ static int zynqmp_dp_phy_init(struct zynqmp_dp *dp) } } - ret = zynqmp_dp_reset(dp, false); - if (ret < 0) - return ret; - zynqmp_dp_clr(dp, ZYNQMP_DP_PHY_RESET, ZYNQMP_DP_PHY_RESET_ALL_RESET); /* @@ -441,8 +437,6 @@ static void zynqmp_dp_phy_exit(struct zynqmp_dp *dp) ret); } - zynqmp_dp_reset(dp, true); - for (i = 0; i < dp->num_lanes; i++) { ret = phy_exit(dp->phy[i]); if (ret) @@ -1683,9 +1677,13 @@ int zynqmp_dp_probe(struct zynqmp_dpsub *dpsub, struct drm_device *drm) return PTR_ERR(dp->reset); } + ret = zynqmp_dp_reset(dp, false); + if (ret < 0) + return ret; + ret = zynqmp_dp_phy_probe(dp); if (ret) - return ret; + goto err_reset; /* Initialize the hardware. */ zynqmp_dp_write(dp, ZYNQMP_DP_TX_PHY_POWER_DOWN, @@ -1697,7 +1695,7 @@ int zynqmp_dp_probe(struct zynqmp_dpsub *dpsub, struct drm_device *drm) ret = zynqmp_dp_phy_init(dp); if (ret) - return ret; + goto err_reset; zynqmp_dp_write(dp, ZYNQMP_DP_TRANSMITTER_ENABLE, 1); @@ -1709,15 +1707,18 @@ int zynqmp_dp_probe(struct zynqmp_dpsub *dpsub, struct drm_device *drm) zynqmp_dp_irq_handler, IRQF_ONESHOT, dev_name(dp->dev), dp); if (ret < 0) - goto error; + goto err_phy_exit; dev_dbg(dp->dev, "ZynqMP DisplayPort Tx probed with %u lanes\n", dp->num_lanes); return 0; -error: +err_phy_exit: zynqmp_dp_phy_exit(dp); +err_reset: + zynqmp_dp_reset(dp, true); + return ret; } @@ -1735,4 +1736,5 @@ void zynqmp_dp_remove(struct zynqmp_dpsub *dpsub) zynqmp_dp_write(dp, ZYNQMP_DP_INT_DS, 0xffffffff); zynqmp_dp_phy_exit(dp); + zynqmp_dp_reset(dp, true); } |