diff options
Diffstat (limited to 'drivers/gpu')
151 files changed, 3510 insertions, 1526 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 3f2b695cf19e..ffe149aafc39 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -27,6 +27,7 @@ #include <linux/types.h> #include <linux/mm.h> +#include <linux/kthread.h> #include <linux/workqueue.h> #include <kgd_kfd_interface.h> #include <drm/ttm/ttm_execbuf_util.h> @@ -186,7 +187,7 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s * disabled. The memory must be pinned and mapped to the hardware when * this is called in hqd_load functions, so it should never fault in * the first place. This resolves a circular lock dependency involving - * four locks, including the DQM lock and mmap_sem. + * four locks, including the DQM lock and mmap_lock. */ #define read_user_wptr(mmptr, wptr, dst) \ ({ \ @@ -195,10 +196,10 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s pagefault_disable(); \ if ((mmptr) == current->mm) { \ valid = !get_user((dst), (wptr)); \ - } else if (current->mm == NULL) { \ - use_mm(mmptr); \ + } else if (current->flags & PF_KTHREAD) { \ + kthread_use_mm(mmptr); \ valid = !get_user((dst), (wptr)); \ - unuse_mm(mmptr); \ + kthread_unuse_mm(mmptr); \ } \ pagefault_enable(); \ } \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c index 6529caca88fe..35d4a5ab0228 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c @@ -22,7 +22,6 @@ #include <linux/module.h> #include <linux/fdtable.h> #include <linux/uaccess.h> -#include <linux/mmu_context.h> #include <linux/firmware.h> #include "amdgpu.h" #include "amdgpu_amdkfd.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 691c89705bcd..bf927f432506 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -19,7 +19,6 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gc/gc_10_1_0_offset.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 0b7e78748540..744366c7ee85 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -20,8 +20,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "cikd.h" @@ -237,7 +235,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); - /* read_user_ptr may take the mm->mmap_sem. + /* read_user_ptr may take the mm->mmap_lock. * release srbm_mutex to avoid circular dependency between * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index ccd635b812b5..feab4cc6e836 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -20,8 +20,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gfx_v8_0.h" @@ -224,7 +222,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); - /* read_user_ptr may take the mm->mmap_sem. + /* read_user_ptr may take the mm->mmap_lock. * release srbm_mutex to avoid circular dependency between * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index df841c2ac5e7..c7fd0c47b254 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -19,8 +19,6 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gc/gc_9_0_offset.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 68e6e1bc8f3a..b91b5171270f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1393,9 +1393,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( * concurrently and the queues are actually stopped */ if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); is_invalid_userptr = atomic_read(&mem->invalid); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } mutex_lock(&mem->lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c index d7646cbce346..16596a9ccabe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c @@ -163,6 +163,9 @@ static ssize_t amdgpu_get_power_dpm_state(struct device *dev, enum amd_pm_state_type pm; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -196,6 +199,9 @@ static ssize_t amdgpu_set_power_dpm_state(struct device *dev, enum amd_pm_state_type state; int ret; + if (adev->in_gpu_reset) + return -EPERM; + if (strncmp("battery", buf, strlen("battery")) == 0) state = POWER_STATE_TYPE_BATTERY; else if (strncmp("balanced", buf, strlen("balanced")) == 0) @@ -297,6 +303,9 @@ static ssize_t amdgpu_get_power_dpm_force_performance_level(struct device *dev, enum amd_dpm_forced_level level = 0xff; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -334,6 +343,9 @@ static ssize_t amdgpu_set_power_dpm_force_performance_level(struct device *dev, enum amd_dpm_forced_level current_level = 0xff; int ret = 0; + if (adev->in_gpu_reset) + return -EPERM; + if (strncmp("low", buf, strlen("low")) == 0) { level = AMD_DPM_FORCED_LEVEL_LOW; } else if (strncmp("high", buf, strlen("high")) == 0) { @@ -433,6 +445,9 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev, struct pp_states_info data; int i, buf_len, ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -472,6 +487,9 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev, enum amd_pm_state_type pm = 0; int i = 0, ret = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -508,6 +526,9 @@ static ssize_t amdgpu_get_pp_force_state(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = ddev->dev_private; + if (adev->in_gpu_reset) + return -EPERM; + if (adev->pp_force_state_enabled) return amdgpu_get_pp_cur_state(dev, attr, buf); else @@ -525,6 +546,9 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev, unsigned long idx; int ret; + if (adev->in_gpu_reset) + return -EPERM; + if (strlen(buf) == 1) adev->pp_force_state_enabled = false; else if (is_support_sw_smu(adev)) @@ -580,6 +604,9 @@ static ssize_t amdgpu_get_pp_table(struct device *dev, char *table = NULL; int size, ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -619,6 +646,9 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, struct amdgpu_device *adev = ddev->dev_private; int ret = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -666,7 +696,7 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * default power levels, write "r" (reset) to the file to reset them. * * - * < For Vega20 > + * < For Vega20 and newer ASICs > * * Reading the file will display: * @@ -721,6 +751,9 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, const char delimiter[3] = {' ', '\n', '\0'}; uint32_t type; + if (adev->in_gpu_reset) + return -EPERM; + if (count > 127) return -EINVAL; @@ -810,6 +843,9 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -859,6 +895,9 @@ static ssize_t amdgpu_set_pp_features(struct device *dev, uint64_t featuremask; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = kstrtou64(buf, 0, &featuremask); if (ret) return -EINVAL; @@ -899,6 +938,9 @@ static ssize_t amdgpu_get_pp_features(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -955,6 +997,9 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1018,6 +1063,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev, int ret; uint32_t mask = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1049,6 +1097,9 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1076,6 +1127,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, uint32_t mask = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1107,6 +1161,9 @@ static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1134,6 +1191,9 @@ static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev, int ret; uint32_t mask = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1167,6 +1227,9 @@ static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1194,6 +1257,9 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, int ret; uint32_t mask = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1227,6 +1293,9 @@ static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1254,6 +1323,9 @@ static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev, int ret; uint32_t mask = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1287,6 +1359,9 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1314,6 +1389,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev, int ret; uint32_t mask = 0; + if (adev->in_gpu_reset) + return -EPERM; + ret = amdgpu_read_mask(buf, count, &mask); if (ret) return ret; @@ -1347,6 +1425,9 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev, uint32_t value = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1372,6 +1453,9 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev, int ret; long int value; + if (adev->in_gpu_reset) + return -EPERM; + ret = kstrtol(buf, 0, &value); if (ret) @@ -1410,6 +1494,9 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev, uint32_t value = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1435,6 +1522,9 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev, int ret; long int value; + if (adev->in_gpu_reset) + return -EPERM; + ret = kstrtol(buf, 0, &value); if (ret) @@ -1493,6 +1583,9 @@ static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev, ssize_t size; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(ddev->dev); if (ret < 0) return ret; @@ -1528,6 +1621,9 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, long int profile_mode = 0; const char delimiter[3] = {' ', '\n', '\0'}; + if (adev->in_gpu_reset) + return -EPERM; + tmp[0] = *(buf); tmp[1] = '\0'; ret = kstrtol(tmp, 0, &profile_mode); @@ -1572,7 +1668,7 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, } /** - * DOC: busy_percent + * DOC: gpu_busy_percent * * The amdgpu driver provides a sysfs API for reading how busy the GPU * is as a percentage. The file gpu_busy_percent is used for this. @@ -1587,6 +1683,9 @@ static ssize_t amdgpu_get_gpu_busy_percent(struct device *dev, struct amdgpu_device *adev = ddev->dev_private; int r, value, size = sizeof(value); + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(ddev->dev); if (r < 0) return r; @@ -1620,6 +1719,9 @@ static ssize_t amdgpu_get_mem_busy_percent(struct device *dev, struct amdgpu_device *adev = ddev->dev_private; int r, value, size = sizeof(value); + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(ddev->dev); if (r < 0) return r; @@ -1658,6 +1760,9 @@ static ssize_t amdgpu_get_pcie_bw(struct device *dev, uint64_t count0 = 0, count1 = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + if (adev->flags & AMD_IS_APU) return -ENODATA; @@ -1694,6 +1799,9 @@ static ssize_t amdgpu_get_unique_id(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = ddev->dev_private; + if (adev->in_gpu_reset) + return -EPERM; + if (adev->unique_id) return snprintf(buf, PAGE_SIZE, "%016llx\n", adev->unique_id); @@ -1888,6 +1996,9 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev, int channel = to_sensor_dev_attr(attr)->index; int r, temp = 0, size = sizeof(temp); + if (adev->in_gpu_reset) + return -EPERM; + if (channel >= PP_TEMP_MAX) return -EINVAL; @@ -2019,6 +2130,9 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev, u32 pwm_mode = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(adev->ddev->dev); if (ret < 0) return ret; @@ -2050,6 +2164,9 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, int err, ret; int value; + if (adev->in_gpu_reset) + return -EPERM; + err = kstrtoint(buf, 10, &value); if (err) return err; @@ -2099,6 +2216,9 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, u32 value; u32 pwm_mode; + if (adev->in_gpu_reset) + return -EPERM; + err = pm_runtime_get_sync(adev->ddev->dev); if (err < 0) return err; @@ -2148,6 +2268,9 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, int err; u32 speed = 0; + if (adev->in_gpu_reset) + return -EPERM; + err = pm_runtime_get_sync(adev->ddev->dev); if (err < 0) return err; @@ -2178,6 +2301,9 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, int err; u32 speed = 0; + if (adev->in_gpu_reset) + return -EPERM; + err = pm_runtime_get_sync(adev->ddev->dev); if (err < 0) return err; @@ -2207,6 +2333,9 @@ static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev, u32 size = sizeof(min_rpm); int r; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2232,6 +2361,9 @@ static ssize_t amdgpu_hwmon_get_fan1_max(struct device *dev, u32 size = sizeof(max_rpm); int r; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2256,6 +2388,9 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev, int err; u32 rpm = 0; + if (adev->in_gpu_reset) + return -EPERM; + err = pm_runtime_get_sync(adev->ddev->dev); if (err < 0) return err; @@ -2285,6 +2420,9 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, u32 value; u32 pwm_mode; + if (adev->in_gpu_reset) + return -EPERM; + err = pm_runtime_get_sync(adev->ddev->dev); if (err < 0) return err; @@ -2331,6 +2469,9 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev, u32 pwm_mode = 0; int ret; + if (adev->in_gpu_reset) + return -EPERM; + ret = pm_runtime_get_sync(adev->ddev->dev); if (ret < 0) return ret; @@ -2363,6 +2504,9 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, int value; u32 pwm_mode; + if (adev->in_gpu_reset) + return -EPERM; + err = kstrtoint(buf, 10, &value); if (err) return err; @@ -2403,6 +2547,9 @@ static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev, u32 vddgfx; int r, size = sizeof(vddgfx); + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2435,6 +2582,9 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev, u32 vddnb; int r, size = sizeof(vddnb); + if (adev->in_gpu_reset) + return -EPERM; + /* only APUs have vddnb */ if (!(adev->flags & AMD_IS_APU)) return -EINVAL; @@ -2472,6 +2622,9 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, int r, size = sizeof(u32); unsigned uw; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2508,6 +2661,9 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, ssize_t size; int r; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2537,6 +2693,9 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, ssize_t size; int r; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2567,6 +2726,9 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, int err; u32 value; + if (adev->in_gpu_reset) + return -EPERM; + if (amdgpu_sriov_vf(adev)) return -EINVAL; @@ -2605,6 +2767,9 @@ static ssize_t amdgpu_hwmon_show_sclk(struct device *dev, uint32_t sclk; int r, size = sizeof(sclk); + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -2637,6 +2802,9 @@ static ssize_t amdgpu_hwmon_show_mclk(struct device *dev, uint32_t mclk; int r, size = sizeof(mclk); + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(adev->ddev->dev); if (r < 0) return r; @@ -3497,6 +3665,9 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data) u32 flags = 0; int r; + if (adev->in_gpu_reset) + return -EPERM; + r = pm_runtime_get_sync(dev->dev); if (r < 0) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 9cbecd5ba814..e59c01a83dac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -910,7 +910,7 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) goto out_free_ranges; } - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, start); if (unlikely(!vma || start < vma->vm_start)) { r = -EFAULT; @@ -921,15 +921,15 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) r = -EPERM; goto out_unlock; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); retry: range->notifier_seq = mmu_interval_read_begin(&bo->notifier); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); r = hmm_range_fault(range); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (unlikely(r)) { /* * FIXME: This timeout should encompass the retry from @@ -954,7 +954,7 @@ retry: return 0; out_unlock: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_free_pfns: kvfree(range->hmm_pfns); out_free_ranges: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 15476fca8fa6..a9583b95fcc1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -901,7 +901,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, memset(&memory_exception_data, 0, sizeof(memory_exception_data)); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); memory_exception_data.gpu_id = dev->id; @@ -924,7 +924,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, memory_exception_data.failure.NoExecute = 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); pr_debug("notpresent %d, noexecute %d, readonly %d\n", diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index f0587d94294d..fee60921fccf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -40,6 +40,7 @@ #include <drm/drm_file.h> #include <drm/drm_drv.h> #include <drm/drm_device.h> +#include <drm/drm_ioctl.h> #include <kgd_kfd_interface.h> #include <linux/swap.h> @@ -1076,7 +1077,7 @@ static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd) #if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF) struct drm_device *ddev = kfd->ddev; - return devcgroup_check_permission(DEVCG_DEV_CHAR, ddev->driver->major, + return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR, ddev->render->index, DEVCG_ACC_WRITE | DEVCG_ACC_READ); #else diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index bdba0bfd6df1..7ced9f87be97 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1356,7 +1356,7 @@ static int dm_late_init(void *handle) unsigned int linear_lut[16]; int i; struct dmcu *dmcu = NULL; - bool ret = false; + bool ret; if (!adev->dm.fw_dmcu) return detect_mst_link_for_all_connectors(adev->ddev); @@ -1377,13 +1377,10 @@ static int dm_late_init(void *handle) */ params.min_abm_backlight = 0x28F; - /* todo will enable for navi10 */ - if (adev->asic_type <= CHIP_RAVEN) { - ret = dmcu_load_iram(dmcu, params); + ret = dmcu_load_iram(dmcu, params); - if (!ret) - return -EINVAL; - } + if (!ret) + return -EINVAL; return detect_mst_link_for_all_connectors(adev->ddev); } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 45cfb7c45566..6f93a6ca4cf0 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1016,9 +1016,17 @@ static void program_timing_sync( } } - /* set first pipe with plane as master */ + /* set first unblanked pipe as master */ for (j = 0; j < group_size; j++) { - if (pipe_set[j]->plane_state) { + bool is_blanked; + + if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked) + is_blanked = + pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp); + else + is_blanked = + pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg); + if (!is_blanked) { if (j == 0) break; @@ -1039,9 +1047,17 @@ static void program_timing_sync( status->timing_sync_info.master = false; } - /* remove any other pipes with plane as they have already been synced */ + /* remove any other unblanked pipes as they have already been synced */ for (j = j + 1; j < group_size; j++) { - if (pipe_set[j]->plane_state) { + bool is_blanked; + + if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked) + is_blanked = + pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp); + else + is_blanked = + pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg); + if (!is_blanked) { group_size--; pipe_set[j] = pipe_set[group_size]; j--; @@ -2522,6 +2538,12 @@ void dc_commit_updates_for_stream(struct dc *dc, copy_stream_update_to_stream(dc, context, stream, stream_update); + if (!dc->res_pool->funcs->validate_bandwidth(dc, context, false)) { + DC_ERROR("Mode validation failed for stream update!\n"); + dc_release_state(context); + return; + } + commit_planes_for_stream( dc, srf_updates, diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile index 3f66868df171..ea29cf95d470 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile @@ -28,8 +28,6 @@ endif endif CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_ccflags) -CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc_dpi.o := $(dsc_ccflags) -CFLAGS_$(AMDDALPATH)/dc/dsc/dc_dsc.o := $(dsc_ccflags) DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c index 0ea6662a1563..0c7f247bb7de 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c +++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c @@ -22,10 +22,12 @@ * Author: AMD */ +#include <drm/drm_dsc.h> #include "dc_hw_types.h" #include "dsc.h" #include <drm/drm_dp_helper.h> #include "dc.h" +#include "rc_calc.h" /* This module's internal functions */ @@ -304,22 +306,6 @@ static inline uint32_t dsc_div_by_10_round_up(uint32_t value) return (value + 9) / 10; } -static inline uint32_t calc_dsc_bpp_x16(uint32_t stream_bandwidth_kbps, uint32_t pix_clk_100hz, uint32_t bpp_increment_div) -{ - uint32_t dsc_target_bpp_x16; - float f_dsc_target_bpp; - float f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f; - uint32_t precision = bpp_increment_div; // bpp_increment_div is actually precision - - f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz; - - // Round down to the nearest precision stop to bring it into DSC spec range - dsc_target_bpp_x16 = (uint32_t)(f_dsc_target_bpp * precision); - dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision; - - return dsc_target_bpp_x16; -} - /* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock * and uncompressed bandwidth. */ diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c index 03ae15946c6d..667afbc260f9 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c +++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c @@ -23,6 +23,7 @@ * Authors: AMD * */ +#include <drm/drm_dsc.h> #include "os_types.h" #include "rc_calc.h" @@ -40,7 +41,8 @@ break -void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, enum max_min max_min, float bpp) +static void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, + enum max_min max_min, float bpp) { int mode = MODE_SELECT(444, 422, 420); int sel = table_hash(mode, bpc, max_min); @@ -85,7 +87,7 @@ void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, enum ma memcpy(qps, table[index].qps, sizeof(qp_set)); } -double dsc_roundf(double num) +static double dsc_roundf(double num) { if (num < 0.0) num = num - 0.5; @@ -95,7 +97,7 @@ double dsc_roundf(double num) return (int)(num); } -double dsc_ceil(double num) +static double dsc_ceil(double num) { double retval = (int)num; @@ -105,7 +107,7 @@ double dsc_ceil(double num) return (int)retval; } -void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp) +static void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp) { int *p = ofs; @@ -160,7 +162,7 @@ void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp) } } -int median3(int a, int b, int c) +static int median3(int a, int b, int c) { if (a > b) swap(a, b); @@ -172,13 +174,25 @@ int median3(int a, int b, int c) return b; } -void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version) +static void _do_calc_rc_params(struct rc_params *rc, enum colour_mode cm, + enum bits_per_comp bpc, u8 drm_bpp, + bool is_navite_422_or_420, + int slice_width, int slice_height, + int minor_version) { + float bpp; float bpp_group; float initial_xmit_delay_factor; int padding_pixels; int i; + bpp = ((float)drm_bpp / 16.0); + /* in native_422 or native_420 modes, the bits_per_pixel is double the + * target bpp (the latter is what calc_rc_params expects) + */ + if (is_navite_422_or_420) + bpp /= 2.0; + rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0); rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0); @@ -251,3 +265,128 @@ void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_com rc->rc_buf_thresh[13] = 8064; } +static u32 _do_bytes_per_pixel_calc(int slice_width, u8 drm_bpp, + bool is_navite_422_or_420) +{ + float bpp; + u32 bytes_per_pixel; + double d_bytes_per_pixel; + + bpp = ((float)drm_bpp / 16.0); + d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width; + // TODO: Make sure the formula for calculating this is precise (ceiling + // vs. floor, and at what point they should be applied) + if (is_navite_422_or_420) + d_bytes_per_pixel /= 2; + + bytes_per_pixel = (u32)dsc_ceil(d_bytes_per_pixel * 0x10000000); + + return bytes_per_pixel; +} + +static u32 _do_calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz, + u32 bpp_increment_div) +{ + u32 dsc_target_bpp_x16; + float f_dsc_target_bpp; + float f_stream_bandwidth_100bps; + // bpp_increment_div is actually precision + u32 precision = bpp_increment_div; + + f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f; + f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz; + + // Round down to the nearest precision stop to bring it into DSC spec + // range + dsc_target_bpp_x16 = (u32)(f_dsc_target_bpp * precision); + dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision; + + return dsc_target_bpp_x16; +} + +/** + * calc_rc_params - reads the user's cmdline mode + * @rc: DC internal DSC parameters + * @pps: DRM struct with all required DSC values + * + * This function expects a drm_dsc_config data struct with all the required DSC + * values previously filled out by our driver and based on this information it + * computes some of the DSC values. + * + * @note This calculation requires float point operation, most of it executes + * under kernel_fpu_{begin,end}. + */ +void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps) +{ + enum colour_mode mode; + enum bits_per_comp bpc; + bool is_navite_422_or_420; + u8 drm_bpp = pps->bits_per_pixel; + int slice_width = pps->slice_width; + int slice_height = pps->slice_height; + + mode = pps->convert_rgb ? CM_RGB : (pps->simple_422 ? CM_444 : + (pps->native_422 ? CM_422 : + pps->native_420 ? CM_420 : CM_444)); + bpc = (pps->bits_per_component == 8) ? BPC_8 : (pps->bits_per_component == 10) + ? BPC_10 : BPC_12; + + is_navite_422_or_420 = pps->native_422 || pps->native_420; + + DC_FP_START(); + _do_calc_rc_params(rc, mode, bpc, drm_bpp, is_navite_422_or_420, + slice_width, slice_height, + pps->dsc_version_minor); + DC_FP_END(); +} + +/** + * calc_dsc_bytes_per_pixel - calculate bytes per pixel + * @pps: DRM struct with all required DSC values + * + * Based on the information inside drm_dsc_config, this function calculates the + * total of bytes per pixel. + * + * @note This calculation requires float point operation, most of it executes + * under kernel_fpu_{begin,end}. + * + * Return: + * Return the number of bytes per pixel + */ +u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps) + +{ + u32 ret; + u8 drm_bpp = pps->bits_per_pixel; + int slice_width = pps->slice_width; + bool is_navite_422_or_420 = pps->native_422 || pps->native_420; + + DC_FP_START(); + ret = _do_bytes_per_pixel_calc(slice_width, drm_bpp, + is_navite_422_or_420); + DC_FP_END(); + return ret; +} + +/** + * calc_dsc_bpp_x16 - retrieve the dsc bits per pixel + * @stream_bandwidth_kbps: + * @pix_clk_100hz: + * @bpp_increment_div: + * + * Calculate the total of bits per pixel for DSC configuration. + * + * @note This calculation requires float point operation, most of it executes + * under kernel_fpu_{begin,end}. + */ +u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz, + u32 bpp_increment_div) +{ + u32 dsc_bpp; + + DC_FP_START(); + dsc_bpp = _do_calc_dsc_bpp_x16(stream_bandwidth_kbps, pix_clk_100hz, + bpp_increment_div); + DC_FP_END(); + return dsc_bpp; +} diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h index b6b1f09c2009..21723fa6561e 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h +++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h @@ -77,7 +77,10 @@ struct qp_entry { typedef struct qp_entry qp_table[]; -void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version); +void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps); +u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps); +u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz, + u32 bpp_increment_div); #endif diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c index 1f6e63b71456..ef830aded5b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c +++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c @@ -27,8 +27,6 @@ #include "dscc_types.h" #include "rc_calc.h" -double dsc_ceil(double num); - static void copy_pps_fields(struct drm_dsc_config *to, const struct drm_dsc_config *from) { to->line_buf_depth = from->line_buf_depth; @@ -100,34 +98,13 @@ static void copy_rc_to_cfg(struct drm_dsc_config *dsc_cfg, const struct rc_param int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps, struct dsc_parameters *dsc_params) { - enum colour_mode mode = pps->convert_rgb ? CM_RGB : - (pps->simple_422 ? CM_444 : - (pps->native_422 ? CM_422 : - pps->native_420 ? CM_420 : CM_444)); - enum bits_per_comp bpc = (pps->bits_per_component == 8) ? BPC_8 : - (pps->bits_per_component == 10) ? BPC_10 : BPC_12; - float bpp = ((float) pps->bits_per_pixel / 16.0); - int slice_width = pps->slice_width; - int slice_height = pps->slice_height; int ret; struct rc_params rc; struct drm_dsc_config dsc_cfg; - double d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width; - - // TODO: Make sure the formula for calculating this is precise (ceiling vs. floor, and at what point they should be applied) - if (pps->native_422 || pps->native_420) - d_bytes_per_pixel /= 2; - - dsc_params->bytes_per_pixel = (uint32_t)dsc_ceil(d_bytes_per_pixel * 0x10000000); - - /* in native_422 or native_420 modes, the bits_per_pixel is double the target bpp - * (the latter is what calc_rc_params expects) - */ - if (pps->native_422 || pps->native_420) - bpp /= 2.0; + dsc_params->bytes_per_pixel = calc_dsc_bytes_per_pixel(pps); - calc_rc_params(&rc, mode, bpc, bpp, slice_width, slice_height, pps->dsc_version_minor); + calc_rc_params(&rc, pps); dsc_params->pps = *pps; dsc_params->pps.initial_scale_value = 8 * rc.rc_model_size / (rc.rc_model_size - rc.initial_fullness_offset); diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c index 9431b48aecb4..bcfe34ef8c28 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c @@ -843,7 +843,7 @@ static bool build_regamma(struct pwl_float_data_ex *rgb_regamma, pow_buffer_ptr = -1; // reset back to no optimize ret = true; release: - kfree(coeff); + kvfree(coeff); return ret; } @@ -1777,7 +1777,7 @@ bool calculate_user_regamma_ramp(struct dc_transfer_func *output_tf, kfree(rgb_regamma); rgb_regamma_alloc_fail: - kvfree(rgb_user); + kfree(rgb_user); rgb_user_alloc_fail: return ret; } diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c index ae0361e225bb..aa76c2cea747 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c +++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c @@ -1561,6 +1561,7 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev, * events for SMCToHost interrupt. */ uint32_t ctxid = entry->src_data[0]; + uint32_t data; if (client_id == SOC15_IH_CLIENTID_THM) { switch (src_id) { @@ -1590,6 +1591,11 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev, orderly_poweroff(true); } else if (client_id == SOC15_IH_CLIENTID_MP1) { if (src_id == 0xfe) { + /* ACK SMUToHost interrupt */ + data = RREG32_SOC15(MP1, 0, mmMP1_SMN_IH_SW_INT_CTRL); + data = REG_SET_FIELD(data, MP1_SMN_IH_SW_INT_CTRL, INT_ACK, 1); + WREG32_SOC15(MP1, 0, mmMP1_SMN_IH_SW_INT_CTRL, data); + switch (ctxid) { case 0x3: dev_dbg(adev->dev, "Switched to AC mode!\n"); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c index 85e5b1ed22c2..56923a96b450 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c @@ -239,7 +239,7 @@ static void ci_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr) switch (dev_id) { case 0x67BA: - case 0x66B1: + case 0x67B1: smu_data->power_tune_defaults = &defaults_hawaii_pro; break; case 0x67B8: diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 7d39b858c9f1..3a3a511670c9 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -226,6 +226,7 @@ static void ast_set_vbios_color_reg(struct ast_private *ast, case 3: case 4: color_index = TrueCModeIndex; + break; default: return; } diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index 04f876e985de..43271c21d3fc 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -62,7 +62,7 @@ config DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW depends on OF select DRM_KMS_HELPER select DRM_PANEL - ---help--- + help This is a driver for the display bridges of GE B850v3 that convert dual channel LVDS to DP++. This is used with the i.MX6 imx-ldb @@ -89,7 +89,7 @@ config DRM_NXP_PTN3460 depends on OF select DRM_KMS_HELPER select DRM_PANEL - ---help--- + help NXP PTN3460 eDP-LVDS bridge chip driver. config DRM_PARADE_PS8622 @@ -98,7 +98,7 @@ config DRM_PARADE_PS8622 select DRM_PANEL select DRM_KMS_HELPER select BACKLIGHT_CLASS_DEVICE - ---help--- + help Parade eDP-LVDS bridge chip driver. config DRM_PARADE_PS8640 @@ -128,13 +128,13 @@ config DRM_SII902X select REGMAP_I2C select I2C_MUX select SND_SOC_HDMI_CODEC if SND_SOC - ---help--- + help Silicon Image sii902x bridge chip driver. config DRM_SII9234 tristate "Silicon Image SII9234 HDMI/MHL bridge" depends on OF - ---help--- + help Say Y here if you want support for the MHL interface. It is an I2C driver, that detects connection of MHL bridge and starts encapsulation of HDMI signal. @@ -150,7 +150,7 @@ config DRM_SIMPLE_BRIDGE config DRM_THINE_THC63LVD1024 tristate "Thine THC63LVD1024 LVDS decoder bridge" depends on OF - ---help--- + help Thine THC63LVD1024 LVDS/parallel converter driver. config DRM_TOSHIBA_TC358764 @@ -168,7 +168,7 @@ config DRM_TOSHIBA_TC358767 select DRM_KMS_HELPER select REGMAP_I2C select DRM_PANEL - ---help--- + help Toshiba TC358767 eDP bridge chip driver. config DRM_TOSHIBA_TC358768 @@ -185,7 +185,7 @@ config DRM_TI_TFP410 tristate "TI TFP410 DVI/HDMI bridge" depends on OF select DRM_KMS_HELPER - ---help--- + help Texas Instruments TFP410 DVI/HDMI Transmitter driver config DRM_TI_SN65DSI86 diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index b1099e1251a2..d877ddc6dc57 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -27,6 +27,7 @@ #include <drm/drm_print.h> #include <drm/drm_drv.h> #include <drm/drm_file.h> +#include <drm/drm_sysfs.h> #include <linux/uaccess.h> @@ -523,6 +524,10 @@ int drm_connector_register(struct drm_connector *connector) drm_mode_object_register(connector->dev, &connector->base); connector->registration_state = DRM_CONNECTOR_REGISTERED; + + /* Let userspace know we have a new connector */ + drm_sysfs_hotplug_event(connector->dev); + goto unlock; err_debugfs: diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 02fc24026872..170aa7689110 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -307,13 +307,13 @@ static void drm_fb_helper_sysrq(int dummy1) schedule_work(&drm_fb_helper_restore_work); } -static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { +static const struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { .handler = drm_fb_helper_sysrq, .help_msg = "force-fb(V)", .action_msg = "Restore framebuffer console", }; #else -static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { }; +static const struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { }; #endif static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode) diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c index 939f0032aab1..f0336c804639 100644 --- a/drivers/gpu/drm/drm_sysfs.c +++ b/drivers/gpu/drm/drm_sysfs.c @@ -291,9 +291,6 @@ int drm_sysfs_connector_add(struct drm_connector *connector) return PTR_ERR(connector->kdev); } - /* Let userspace know we have a new connector */ - drm_sysfs_hotplug_event(dev); - if (connector->ddc) return sysfs_create_link(&connector->kdev->kobj, &connector->ddc->dev.kobj, "ddc"); diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c index 56197ae0b2f9..4391e242356d 100644 --- a/drivers/gpu/drm/drm_vm.c +++ b/drivers/gpu/drm/drm_vm.c @@ -37,6 +37,7 @@ #include <linux/pci.h> #include <linux/seq_file.h> #include <linux/vmalloc.h> +#include <linux/pgtable.h> #if defined(__ia64__) #include <linux/efi.h> @@ -44,7 +45,6 @@ #endif #include <linux/mem_encrypt.h> -#include <asm/pgtable.h> #include <drm/drm_agpsupport.h> #include <drm/drm_device.h> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index dc9ef302f517..701f3995f621 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -661,7 +661,7 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr; int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT; - might_lock_read(¤t->mm->mmap_sem); + might_lock_read(¤t->mm->mmap_lock); if (userptr->mm != current->mm) return -EPERM; diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index aa22465bb56e..0575a1eea2a1 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -2579,14 +2579,14 @@ static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, static void tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, int link_clock, - u32 level) + u32 level, enum intel_output_type type) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port); const struct tgl_dkl_phy_ddi_buf_trans *ddi_translations; u32 n_entries, val, ln, dpcnt_mask, dpcnt_val; - if (encoder->type == INTEL_OUTPUT_HDMI) { + if (type == INTEL_OUTPUT_HDMI) { n_entries = ARRAY_SIZE(tgl_dkl_phy_hdmi_ddi_trans); ddi_translations = tgl_dkl_phy_hdmi_ddi_trans; } else { @@ -2638,7 +2638,7 @@ static void tgl_ddi_vswing_sequence(struct intel_encoder *encoder, if (intel_phy_is_combo(dev_priv, phy)) icl_combo_phy_ddi_vswing_sequence(encoder, level, type); else - tgl_dkl_phy_ddi_vswing_sequence(encoder, link_clock, level); + tgl_dkl_phy_ddi_vswing_sequence(encoder, link_clock, level, type); } static u32 translate_signal_level(struct intel_dp *intel_dp, int signal_levels) @@ -2987,7 +2987,7 @@ icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port, ln1 = intel_de_read(dev_priv, MG_DP_MODE(1, tc_port)); } - ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X1_MODE); + ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); ln1 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); /* DPPATC */ @@ -3472,7 +3472,9 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state, INTEL_OUTPUT_DP_MST); enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - intel_dp_set_infoframes(encoder, false, old_crtc_state, old_conn_state); + if (!is_mst) + intel_dp_set_infoframes(encoder, false, + old_crtc_state, old_conn_state); /* * Power down sink before disabling the port, otherwise we end diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 40d42dcff0b7..ed9e53c373a7 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -5206,6 +5206,9 @@ void intel_read_dp_sdp(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, unsigned int type) { + if (encoder->type != INTEL_OUTPUT_DDI) + return; + switch (type) { case DP_SDP_VSC: intel_read_dp_vsc_sdp(encoder, crtc_state, diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index d18b406f2a7d..f29e51ce489c 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -397,6 +397,14 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state, */ drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, false); + + /* + * BSpec 4287: disable DIP after the transcoder is disabled and before + * the transcoder clock select is set to none. + */ + if (last_mst_stream) + intel_dp_set_infoframes(&intel_dig_port->base, false, + old_crtc_state, NULL); /* * From TGL spec: "If multi-stream slave transcoder: Configure * Transcoder Clock Select to direct no clock to the transcoder" diff --git a/drivers/gpu/drm/i915/display/intel_global_state.c b/drivers/gpu/drm/i915/display/intel_global_state.c index 212d4ee68205..7a19215ad844 100644 --- a/drivers/gpu/drm/i915/display/intel_global_state.c +++ b/drivers/gpu/drm/i915/display/intel_global_state.c @@ -10,6 +10,28 @@ #include "intel_display_types.h" #include "intel_global_state.h" +static void __intel_atomic_global_state_free(struct kref *kref) +{ + struct intel_global_state *obj_state = + container_of(kref, struct intel_global_state, ref); + struct intel_global_obj *obj = obj_state->obj; + + obj->funcs->atomic_destroy_state(obj, obj_state); +} + +static void intel_atomic_global_state_put(struct intel_global_state *obj_state) +{ + kref_put(&obj_state->ref, __intel_atomic_global_state_free); +} + +static struct intel_global_state * +intel_atomic_global_state_get(struct intel_global_state *obj_state) +{ + kref_get(&obj_state->ref); + + return obj_state; +} + void intel_atomic_global_obj_init(struct drm_i915_private *dev_priv, struct intel_global_obj *obj, struct intel_global_state *state, @@ -17,6 +39,10 @@ void intel_atomic_global_obj_init(struct drm_i915_private *dev_priv, { memset(obj, 0, sizeof(*obj)); + state->obj = obj; + + kref_init(&state->ref); + obj->state = state; obj->funcs = funcs; list_add_tail(&obj->head, &dev_priv->global_obj_list); @@ -28,7 +54,9 @@ void intel_atomic_global_obj_cleanup(struct drm_i915_private *dev_priv) list_for_each_entry_safe(obj, next, &dev_priv->global_obj_list, head) { list_del(&obj->head); - obj->funcs->atomic_destroy_state(obj, obj->state); + + drm_WARN_ON(&dev_priv->drm, kref_read(&obj->state->ref) != 1); + intel_atomic_global_state_put(obj->state); } } @@ -97,10 +125,14 @@ intel_atomic_get_global_obj_state(struct intel_atomic_state *state, if (!obj_state) return ERR_PTR(-ENOMEM); + obj_state->obj = obj; obj_state->changed = false; + kref_init(&obj_state->ref); + state->global_objs[index].state = obj_state; - state->global_objs[index].old_state = obj->state; + state->global_objs[index].old_state = + intel_atomic_global_state_get(obj->state); state->global_objs[index].new_state = obj_state; state->global_objs[index].ptr = obj; obj_state->state = state; @@ -163,7 +195,9 @@ void intel_atomic_swap_global_state(struct intel_atomic_state *state) new_obj_state->state = NULL; state->global_objs[i].state = old_obj_state; - obj->state = new_obj_state; + + intel_atomic_global_state_put(obj->state); + obj->state = intel_atomic_global_state_get(new_obj_state); } } @@ -172,10 +206,9 @@ void intel_atomic_clear_global_state(struct intel_atomic_state *state) int i; for (i = 0; i < state->num_global_objs; i++) { - struct intel_global_obj *obj = state->global_objs[i].ptr; + intel_atomic_global_state_put(state->global_objs[i].old_state); + intel_atomic_global_state_put(state->global_objs[i].new_state); - obj->funcs->atomic_destroy_state(obj, - state->global_objs[i].state); state->global_objs[i].ptr = NULL; state->global_objs[i].state = NULL; state->global_objs[i].old_state = NULL; diff --git a/drivers/gpu/drm/i915/display/intel_global_state.h b/drivers/gpu/drm/i915/display/intel_global_state.h index e6163a469029..1f16fa3073c9 100644 --- a/drivers/gpu/drm/i915/display/intel_global_state.h +++ b/drivers/gpu/drm/i915/display/intel_global_state.h @@ -6,6 +6,7 @@ #ifndef __INTEL_GLOBAL_STATE_H__ #define __INTEL_GLOBAL_STATE_H__ +#include <linux/kref.h> #include <linux/list.h> struct drm_i915_private; @@ -54,7 +55,9 @@ struct intel_global_obj { for_each_if(obj) struct intel_global_state { + struct intel_global_obj *obj; struct intel_atomic_state *state; + struct kref ref; bool changed; }; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 900ea8b7fc8f..30c229fcb404 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -230,7 +230,7 @@ static void intel_context_set_gem(struct intel_context *ce, ce->timeline = intel_timeline_get(ctx->timeline); if (ctx->sched.priority >= I915_PRIORITY_NORMAL && - intel_engine_has_semaphores(ce->engine)) + intel_engine_has_timeslices(ce->engine)) __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); } @@ -1921,11 +1921,6 @@ get_engines(struct i915_gem_context *ctx, } user = u64_to_user_ptr(args->value); - if (!access_ok(user, size)) { - err = -EFAULT; - goto err_free; - } - if (put_user(0, &user->extensions)) { err = -EFAULT; goto err_free; @@ -1969,7 +1964,7 @@ static int __apply_priority(struct intel_context *ce, void *arg) { struct i915_gem_context *ctx = arg; - if (!intel_engine_has_semaphores(ce->engine)) + if (!intel_engine_has_timeslices(ce->engine)) return 0; if (ctx->sched.priority >= I915_PRIORITY_NORMAL) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 3ce185670ca4..db8eb1c6afe9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -1988,6 +1988,38 @@ static const struct dma_fence_work_ops eb_parse_ops = { .release = __eb_parse_release, }; +static inline int +__parser_mark_active(struct i915_vma *vma, + struct intel_timeline *tl, + struct dma_fence *fence) +{ + struct intel_gt_buffer_pool_node *node = vma->private; + + return i915_active_ref(&node->active, tl, fence); +} + +static int +parser_mark_active(struct eb_parse_work *pw, struct intel_timeline *tl) +{ + int err; + + mutex_lock(&tl->mutex); + + err = __parser_mark_active(pw->shadow, tl, &pw->base.dma); + if (err) + goto unlock; + + if (pw->trampoline) { + err = __parser_mark_active(pw->trampoline, tl, &pw->base.dma); + if (err) + goto unlock; + } + +unlock: + mutex_unlock(&tl->mutex); + return err; +} + static int eb_parse_pipeline(struct i915_execbuffer *eb, struct i915_vma *shadow, struct i915_vma *trampoline) @@ -2022,20 +2054,25 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, pw->shadow = shadow; pw->trampoline = trampoline; + /* Mark active refs early for this worker, in case we get interrupted */ + err = parser_mark_active(pw, eb->context->timeline); + if (err) + goto err_commit; + err = dma_resv_lock_interruptible(pw->batch->resv, NULL); if (err) - goto err_trampoline; + goto err_commit; err = dma_resv_reserve_shared(pw->batch->resv, 1); if (err) - goto err_batch_unlock; + goto err_commit_unlock; /* Wait for all writes (and relocs) into the batch to complete */ err = i915_sw_fence_await_reservation(&pw->base.chain, pw->batch->resv, NULL, false, 0, I915_FENCE_GFP); if (err < 0) - goto err_batch_unlock; + goto err_commit_unlock; /* Keep the batch alive and unwritten as we parse */ dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma); @@ -2050,11 +2087,13 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, dma_fence_work_commit_imm(&pw->base); return 0; -err_batch_unlock: +err_commit_unlock: dma_resv_unlock(pw->batch->resv); -err_trampoline: - if (trampoline) - i915_active_release(&trampoline->active); +err_commit: + i915_sw_fence_set_error_once(&pw->base.chain, err); + dma_fence_work_commit_imm(&pw->base); + return err; + err_shadow: i915_active_release(&shadow->active); err_batch: @@ -2100,6 +2139,7 @@ static int eb_parse(struct i915_execbuffer *eb) goto err; } i915_gem_object_set_readonly(shadow->obj); + shadow->private = pool; trampoline = NULL; if (CMDPARSER_USES_GGTT(eb->i915)) { @@ -2113,6 +2153,7 @@ static int eb_parse(struct i915_execbuffer *eb) shadow = trampoline; goto err_shadow; } + shadow->private = pool; eb->batch_flags |= I915_DISPATCH_SECURE; } @@ -2129,7 +2170,6 @@ static int eb_parse(struct i915_execbuffer *eb) eb->trampoline = trampoline; eb->batch_start_offset = 0; - shadow->private = pool; return 0; err_trampoline: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 70f5f82da288..fe45bd4d63a5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -93,7 +93,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { addr = -EINTR; goto err; } @@ -103,7 +103,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); else addr = -ENOMEM; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (IS_ERR_VALUE(addr)) goto err; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 5d5d7eef3f43..7aff3514d97a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -39,7 +39,6 @@ static int shmem_get_pages(struct drm_i915_gem_object *obj) unsigned long last_pfn = 0; /* suppress gcc warning */ unsigned int max_segment = i915_sg_segment_size(); unsigned int sg_page_sizes; - struct pagevec pvec; gfp_t noreclaim; int ret; @@ -192,13 +191,17 @@ err_sg: sg_mark_end(sg); err_pages: mapping_clear_unevictable(mapping); - pagevec_init(&pvec); - for_each_sgt_page(page, sgt_iter, st) { - if (!pagevec_add(&pvec, page)) + if (sg != st->sgl) { + struct pagevec pvec; + + pagevec_init(&pvec); + for_each_sgt_page(page, sgt_iter, st) { + if (!pagevec_add(&pvec, page)) + check_release_pagevec(&pvec); + } + if (pagevec_count(&pvec)) check_release_pagevec(&pvec); } - if (pagevec_count(&pvec)) - check_release_pagevec(&pvec); sg_free_table(st); kfree(st); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 33776b3f3fa5..c31a6744daee 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -200,10 +200,10 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) if (IS_ERR(mn)) err = PTR_ERR(mn); - down_write(&mm->mm->mmap_sem); + mmap_write_lock(mm->mm); mutex_lock(&mm->i915->mm_lock); if (mm->mn == NULL && !err) { - /* Protected by mmap_sem (write-lock) */ + /* Protected by mmap_lock (write-lock) */ err = __mmu_notifier_register(&mn->mn, mm->mm); if (!err) { /* Protected by mm_lock */ @@ -217,7 +217,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) err = 0; } mutex_unlock(&mm->i915->mm_lock); - up_write(&mm->mm->mmap_sem); + mmap_write_unlock(mm->mm); if (mn && !IS_ERR(mn)) kfree(mn); @@ -468,7 +468,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) if (mmget_not_zero(mm)) { while (pinned < npages) { if (!locked) { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); locked = 1; } ret = pin_user_pages_remote @@ -483,7 +483,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) pinned += ret; } if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } } @@ -522,8 +522,8 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj) /* Spawn a worker so that we can acquire the * user pages without holding our mutex. Access - * to the user pages requires mmap_sem, and we have - * a strict lock ordering of mmap_sem, struct_mutex - + * to the user pages requires mmap_lock, and we have + * a strict lock ordering of mmap_lock, struct_mutex - * we already hold struct_mutex here and so cannot * call gup without encountering a lock inversion. * diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 74ddb49b2941..e4aece20bc80 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -97,8 +97,6 @@ int __intel_context_do_pin(struct intel_context *ce) { int err; - GEM_BUG_ON(intel_context_is_closed(ce)); - if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { err = intel_context_alloc_state(ce); if (err) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index da5b61085257..8691eb61e185 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -646,7 +646,7 @@ static int engine_setup_common(struct intel_engine_cs *engine) struct measure_breadcrumb { struct i915_request rq; struct intel_ring ring; - u32 cs[1024]; + u32 cs[2048]; }; static int measure_breadcrumb_dw(struct intel_context *ce) @@ -668,6 +668,8 @@ static int measure_breadcrumb_dw(struct intel_context *ce) frame->ring.vaddr = frame->cs; frame->ring.size = sizeof(frame->cs); + frame->ring.wrap = + BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size); frame->ring.effective_size = frame->ring.size; intel_ring_update_space(&frame->ring); frame->rq.ring = &frame->ring; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 87e6c5bdd2dc..7c3d8ef4a47c 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -1134,6 +1134,13 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) list_move(&rq->sched.link, pl); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + /* Check in case we rollback so far we wrap [size/2] */ + if (intel_ring_direction(rq->ring, + intel_ring_wrap(rq->ring, + rq->tail), + rq->ring->tail) > 0) + rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE; + active = rq; } else { struct intel_engine_cs *owner = rq->context->engine; @@ -1498,8 +1505,9 @@ static u64 execlists_update_context(struct i915_request *rq) * HW has a tendency to ignore us rewinding the TAIL to the end of * an earlier request. */ + GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail); + prev = rq->ring->tail; tail = intel_ring_set_tail(rq->ring, rq->tail); - prev = ce->lrc_reg_state[CTX_RING_TAIL]; if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0)) desc |= CTX_DESC_FORCE_RESTORE; ce->lrc_reg_state[CTX_RING_TAIL] = tail; @@ -1895,7 +1903,8 @@ static void defer_active(struct intel_engine_cs *engine) static bool need_timeslice(const struct intel_engine_cs *engine, - const struct i915_request *rq) + const struct i915_request *rq, + const struct rb_node *rb) { int hint; @@ -1903,9 +1912,28 @@ need_timeslice(const struct intel_engine_cs *engine, return false; hint = engine->execlists.queue_priority_hint; + + if (rb) { + const struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + const struct intel_engine_cs *inflight = + intel_context_inflight(&ve->context); + + if (!inflight || inflight == engine) { + struct i915_request *next; + + rcu_read_lock(); + next = READ_ONCE(ve->request); + if (next) + hint = max(hint, rq_prio(next)); + rcu_read_unlock(); + } + } + if (!list_is_last(&rq->sched.link, &engine->active.requests)) hint = max(hint, rq_prio(list_next_entry(rq, sched.link))); + GEM_BUG_ON(hint >= I915_PRIORITY_UNPREEMPTABLE); return hint >= effective_prio(rq); } @@ -1977,10 +2005,9 @@ static void set_timeslice(struct intel_engine_cs *engine) set_timer_ms(&engine->execlists.timer, duration); } -static void start_timeslice(struct intel_engine_cs *engine) +static void start_timeslice(struct intel_engine_cs *engine, int prio) { struct intel_engine_execlists *execlists = &engine->execlists; - const int prio = queue_prio(execlists); unsigned long duration; if (!intel_engine_has_timeslices(engine)) @@ -2140,7 +2167,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) __unwind_incomplete_requests(engine); last = NULL; - } else if (need_timeslice(engine, last) && + } else if (need_timeslice(engine, last, rb) && timeslice_expired(execlists, last)) { if (i915_request_completed(last)) { tasklet_hi_schedule(&execlists->tasklet); @@ -2188,7 +2215,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * Even if ELSP[1] is occupied and not worthy * of timeslices, our queue might be. */ - start_timeslice(engine); + start_timeslice(engine, queue_prio(execlists)); return; } } @@ -2223,7 +2250,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (last && !can_merge_rq(last, rq)) { spin_unlock(&ve->base.active.lock); - start_timeslice(engine); + start_timeslice(engine, rq_prio(rq)); return; /* leave this for another sibling */ } @@ -4739,6 +4766,14 @@ static int gen12_emit_flush(struct i915_request *request, u32 mode) return 0; } +static void assert_request_valid(struct i915_request *rq) +{ + struct intel_ring *ring __maybe_unused = rq->ring; + + /* Can we unwind this request without appearing to go forwards? */ + GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0); +} + /* * Reserve space for 2 NOOPs at the end of each request to be * used as a workaround for not being allowed to do lite @@ -4751,6 +4786,9 @@ static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs) *cs++ = MI_NOOP; request->wa_tail = intel_ring_offset(request, cs); + /* Check that entire request is less than half the ring */ + assert_request_valid(request); + return cs; } diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index 8cda1b7e17ba..bdb324167ef3 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -315,3 +315,7 @@ int intel_ring_cacheline_align(struct i915_request *rq) GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1)); return 0; } + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftest_ring.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 90a2b9e399b0..85d2bef51524 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -179,6 +179,12 @@ wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) } static void +wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) +{ + wa_write_masked_or(wal, reg, clr, 0); +} + +static void wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) { wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val); @@ -687,6 +693,227 @@ int intel_engine_emit_ctx_wa(struct i915_request *rq) } static void +gen4_gt_workarounds_init(struct drm_i915_private *i915, + struct i915_wa_list *wal) +{ + /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */ + wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); +} + +static void +g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + gen4_gt_workarounds_init(i915, wal); + + /* WaDisableRenderCachePipelinedFlush:g4x,ilk */ + wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); +} + +static void +ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + g4x_gt_workarounds_init(i915, wal); + + wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); +} + +static void +snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ + wa_masked_en(wal, + _3D_CHICKEN, + _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB); + + /* WaDisable_RenderCache_OperationalFlush:snb */ + wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + wa_add(wal, + GEN6_GT_MODE, 0, + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), + GEN6_WIZ_HASHING_16x4); + + wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB); + + wa_masked_en(wal, + _3D_CHICKEN3, + /* WaStripsFansDisableFastClipPerformanceFix:snb */ + _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL | + /* + * Bspec says: + * "This bit must be set if 3DSTATE_CLIP clip mode is set + * to normal and 3DSTATE_SF number of SF output attributes + * is more than 16." + */ + _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH); +} + +static void +ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + /* WaDisableEarlyCull:ivb */ + wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); + + /* WaDisablePSDDualDispatchEnable:ivb */ + if (IS_IVB_GT1(i915)) + wa_masked_en(wal, + GEN7_HALF_SLICE_CHICKEN1, + GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); + + /* WaDisable_RenderCache_OperationalFlush:ivb */ + wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); + + /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ + wa_masked_dis(wal, + GEN7_COMMON_SLICE_CHICKEN1, + GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); + + /* WaApplyL3ControlAndL3ChickenMode:ivb */ + wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL); + wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); + + /* WaForceL3Serialization:ivb */ + wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); + + /* + * WaVSThreadDispatchOverride:ivb,vlv + * + * This actually overrides the dispatch + * mode for all thread types. + */ + wa_write_masked_or(wal, GEN7_FF_THREAD_MODE, + GEN7_FF_SCHED_MASK, + GEN7_FF_TS_SCHED_HW | + GEN7_FF_VS_SCHED_HW | + GEN7_FF_DS_SCHED_HW); + + if (0) { /* causes HiZ corruption on ivb:gt1 */ + /* enable HiZ Raw Stall Optimization */ + wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); + } + + /* WaDisable4x2SubspanOptimization:ivb */ + wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + wa_add(wal, GEN7_GT_MODE, 0, + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), + GEN6_WIZ_HASHING_16x4); +} + +static void +vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + /* WaDisableEarlyCull:vlv */ + wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); + + /* WaPsdDispatchEnable:vlv */ + /* WaDisablePSDDualDispatchEnable:vlv */ + wa_masked_en(wal, + GEN7_HALF_SLICE_CHICKEN1, + GEN7_MAX_PS_THREAD_DEP | + GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); + + /* WaDisable_RenderCache_OperationalFlush:vlv */ + wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); + + /* WaForceL3Serialization:vlv */ + wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); + + /* + * WaVSThreadDispatchOverride:ivb,vlv + * + * This actually overrides the dispatch + * mode for all thread types. + */ + wa_write_masked_or(wal, + GEN7_FF_THREAD_MODE, + GEN7_FF_SCHED_MASK, + GEN7_FF_TS_SCHED_HW | + GEN7_FF_VS_SCHED_HW | + GEN7_FF_DS_SCHED_HW); + + /* + * BSpec says this must be set, even though + * WaDisable4x2SubspanOptimization isn't listed for VLV. + */ + wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + wa_add(wal, GEN7_GT_MODE, 0, + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), + GEN6_WIZ_HASHING_16x4); + + /* + * WaIncreaseL3CreditsForVLVB0:vlv + * This is the hardware default actually. + */ + wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); +} + +static void +hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + /* L3 caching of data atomics doesn't work -- disable it. */ + wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); + + wa_add(wal, + HSW_ROW_CHICKEN3, 0, + _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), + 0 /* XXX does this reg exist? */); + + /* WaVSRefCountFullforceMissDisable:hsw */ + wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); + + wa_masked_dis(wal, + CACHE_MODE_0_GEN7, + /* WaDisable_RenderCache_OperationalFlush:hsw */ + RC_OP_FLUSH_ENABLE | + /* enable HiZ Raw Stall Optimization */ + HIZ_RAW_STALL_OPT_DISABLE); + + /* WaDisable4x2SubspanOptimization:hsw */ + wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + wa_add(wal, GEN7_GT_MODE, 0, + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), + GEN6_WIZ_HASHING_16x4); + + /* WaSampleCChickenBitEnable:hsw */ + wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE); +} + +static void gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { /* WaDisableKillLogic:bxt,skl,kbl */ @@ -963,6 +1190,20 @@ gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) bxt_gt_workarounds_init(i915, wal); else if (IS_SKYLAKE(i915)) skl_gt_workarounds_init(i915, wal); + else if (IS_HASWELL(i915)) + hsw_gt_workarounds_init(i915, wal); + else if (IS_VALLEYVIEW(i915)) + vlv_gt_workarounds_init(i915, wal); + else if (IS_IVYBRIDGE(i915)) + ivb_gt_workarounds_init(i915, wal); + else if (IS_GEN(i915, 6)) + snb_gt_workarounds_init(i915, wal); + else if (IS_GEN(i915, 5)) + ilk_gt_workarounds_init(i915, wal); + else if (IS_G4X(i915)) + g4x_gt_workarounds_init(i915, wal); + else if (IS_GEN(i915, 4)) + gen4_gt_workarounds_init(i915, wal); else if (INTEL_GEN(i915) <= 8) return; else diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 2b2efff6e19d..4aa4cc917d8b 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -310,22 +310,20 @@ static bool wait_until_running(struct hang *h, struct i915_request *rq) 1000)); } -static void engine_heartbeat_disable(struct intel_engine_cs *engine, - unsigned long *saved) +static void engine_heartbeat_disable(struct intel_engine_cs *engine) { - *saved = engine->props.heartbeat_interval_ms; engine->props.heartbeat_interval_ms = 0; intel_engine_pm_get(engine); intel_engine_park_heartbeat(engine); } -static void engine_heartbeat_enable(struct intel_engine_cs *engine, - unsigned long saved) +static void engine_heartbeat_enable(struct intel_engine_cs *engine) { intel_engine_pm_put(engine); - engine->props.heartbeat_interval_ms = saved; + engine->props.heartbeat_interval_ms = + engine->defaults.heartbeat_interval_ms; } static int igt_hang_sanitycheck(void *arg) @@ -473,7 +471,6 @@ static int igt_reset_nop_engine(void *arg) for_each_engine(engine, gt, id) { unsigned int reset_count, reset_engine_count, count; struct intel_context *ce; - unsigned long heartbeat; IGT_TIMEOUT(end_time); int err; @@ -485,7 +482,7 @@ static int igt_reset_nop_engine(void *arg) reset_engine_count = i915_reset_engine_count(global, engine); count = 0; - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); set_bit(I915_RESET_ENGINE + id, >->reset.flags); do { int i; @@ -529,7 +526,7 @@ static int igt_reset_nop_engine(void *arg) } } while (time_before(jiffies, end_time)); clear_bit(I915_RESET_ENGINE + id, >->reset.flags); - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); pr_info("%s(%s): %d resets\n", __func__, engine->name, count); @@ -564,7 +561,6 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) for_each_engine(engine, gt, id) { unsigned int reset_count, reset_engine_count; - unsigned long heartbeat; IGT_TIMEOUT(end_time); if (active && !intel_engine_can_store_dword(engine)) @@ -580,7 +576,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) reset_count = i915_reset_count(global); reset_engine_count = i915_reset_engine_count(global, engine); - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); set_bit(I915_RESET_ENGINE + id, >->reset.flags); do { if (active) { @@ -632,7 +628,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) } } while (time_before(jiffies, end_time)); clear_bit(I915_RESET_ENGINE + id, >->reset.flags); - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); if (err) break; @@ -789,7 +785,6 @@ static int __igt_reset_engines(struct intel_gt *gt, struct active_engine threads[I915_NUM_ENGINES] = {}; unsigned long device = i915_reset_count(global); unsigned long count = 0, reported; - unsigned long heartbeat; IGT_TIMEOUT(end_time); if (flags & TEST_ACTIVE && @@ -832,7 +827,7 @@ static int __igt_reset_engines(struct intel_gt *gt, yield(); /* start all threads before we begin */ - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); set_bit(I915_RESET_ENGINE + id, >->reset.flags); do { struct i915_request *rq = NULL; @@ -906,7 +901,7 @@ static int __igt_reset_engines(struct intel_gt *gt, } } while (time_before(jiffies, end_time)); clear_bit(I915_RESET_ENGINE + id, >->reset.flags); - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); pr_info("i915_reset_engine(%s:%s): %lu resets\n", engine->name, test_name, count); diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 824f99c4cc7c..924bc01ef526 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -51,22 +51,20 @@ static struct i915_vma *create_scratch(struct intel_gt *gt) return vma; } -static void engine_heartbeat_disable(struct intel_engine_cs *engine, - unsigned long *saved) +static void engine_heartbeat_disable(struct intel_engine_cs *engine) { - *saved = engine->props.heartbeat_interval_ms; engine->props.heartbeat_interval_ms = 0; intel_engine_pm_get(engine); intel_engine_park_heartbeat(engine); } -static void engine_heartbeat_enable(struct intel_engine_cs *engine, - unsigned long saved) +static void engine_heartbeat_enable(struct intel_engine_cs *engine) { intel_engine_pm_put(engine); - engine->props.heartbeat_interval_ms = saved; + engine->props.heartbeat_interval_ms = + engine->defaults.heartbeat_interval_ms; } static bool is_active(struct i915_request *rq) @@ -224,7 +222,6 @@ static int live_unlite_restore(struct intel_gt *gt, int prio) struct intel_context *ce[2] = {}; struct i915_request *rq[2]; struct igt_live_test t; - unsigned long saved; int n; if (prio && !intel_engine_has_preemption(engine)) @@ -237,7 +234,7 @@ static int live_unlite_restore(struct intel_gt *gt, int prio) err = -EIO; break; } - engine_heartbeat_disable(engine, &saved); + engine_heartbeat_disable(engine); for (n = 0; n < ARRAY_SIZE(ce); n++) { struct intel_context *tmp; @@ -345,7 +342,7 @@ err_ce: intel_context_put(ce[n]); } - engine_heartbeat_enable(engine, saved); + engine_heartbeat_enable(engine); if (igt_live_test_end(&t)) err = -EIO; if (err) @@ -466,7 +463,6 @@ static int live_hold_reset(void *arg) for_each_engine(engine, gt, id) { struct intel_context *ce; - unsigned long heartbeat; struct i915_request *rq; ce = intel_context_create(engine); @@ -475,7 +471,7 @@ static int live_hold_reset(void *arg) break; } - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); if (IS_ERR(rq)) { @@ -535,7 +531,7 @@ static int live_hold_reset(void *arg) i915_request_put(rq); out: - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); intel_context_put(ce); if (err) break; @@ -580,10 +576,9 @@ static int live_error_interrupt(void *arg) for_each_engine(engine, gt, id) { const struct error_phase *p; - unsigned long heartbeat; int err = 0; - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); for (p = phases; p->error[0] != GOOD; p++) { struct i915_request *client[ARRAY_SIZE(phases->error)]; @@ -682,7 +677,7 @@ out: } } - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); if (err) { intel_gt_set_wedged(gt); return err; @@ -828,7 +823,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer, } } - err = release_queue(outer, vma, n, INT_MAX); + err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER); if (err) goto out; @@ -895,16 +890,14 @@ static int live_timeslice_preempt(void *arg) enum intel_engine_id id; for_each_engine(engine, gt, id) { - unsigned long saved; - if (!intel_engine_has_preemption(engine)) continue; memset(vaddr, 0, PAGE_SIZE); - engine_heartbeat_disable(engine, &saved); + engine_heartbeat_disable(engine); err = slice_semaphore_queue(engine, vma, count); - engine_heartbeat_enable(engine, saved); + engine_heartbeat_enable(engine); if (err) goto err_pin; @@ -1009,7 +1002,6 @@ static int live_timeslice_rewind(void *arg) enum { X = 1, Z, Y }; struct i915_request *rq[3] = {}; struct intel_context *ce; - unsigned long heartbeat; unsigned long timeslice; int i, err = 0; u32 *slot; @@ -1028,7 +1020,7 @@ static int live_timeslice_rewind(void *arg) * Expect execution/evaluation order XZY */ - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); timeslice = xchg(&engine->props.timeslice_duration_ms, 1); slot = memset32(engine->status_page.addr + 1000, 0, 4); @@ -1122,7 +1114,7 @@ err: wmb(); engine->props.timeslice_duration_ms = timeslice; - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); for (i = 0; i < 3; i++) i915_request_put(rq[i]); if (igt_flush_test(gt->i915)) @@ -1202,12 +1194,11 @@ static int live_timeslice_queue(void *arg) .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), }; struct i915_request *rq, *nop; - unsigned long saved; if (!intel_engine_has_preemption(engine)) continue; - engine_heartbeat_disable(engine, &saved); + engine_heartbeat_disable(engine); memset(vaddr, 0, PAGE_SIZE); /* ELSP[0]: semaphore wait */ @@ -1284,7 +1275,7 @@ static int live_timeslice_queue(void *arg) err_rq: i915_request_put(rq); err_heartbeat: - engine_heartbeat_enable(engine, saved); + engine_heartbeat_enable(engine); if (err) break; } @@ -1298,6 +1289,121 @@ err_obj: return err; } +static int live_timeslice_nopreempt(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + + /* + * We should not timeslice into a request that is marked with + * I915_REQUEST_NOPREEMPT. + */ + if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + for_each_engine(engine, gt, id) { + struct intel_context *ce; + struct i915_request *rq; + unsigned long timeslice; + + if (!intel_engine_has_preemption(engine)) + continue; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + break; + } + + engine_heartbeat_disable(engine); + timeslice = xchg(&engine->props.timeslice_duration_ms, 1); + + /* Create an unpreemptible spinner */ + + rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK); + intel_context_put(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_heartbeat; + } + + i915_request_get(rq); + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + i915_request_put(rq); + err = -ETIME; + goto out_spin; + } + + set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags); + i915_request_put(rq); + + /* Followed by a maximum priority barrier (heartbeat) */ + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(rq); + goto out_spin; + } + + rq = intel_context_create_request(ce); + intel_context_put(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_spin; + } + + rq->sched.attr.priority = I915_PRIORITY_BARRIER; + i915_request_get(rq); + i915_request_add(rq); + + /* + * Wait until the barrier is in ELSP, and we know timeslicing + * will have been activated. + */ + if (wait_for_submit(engine, rq, HZ / 2)) { + i915_request_put(rq); + err = -ETIME; + goto out_spin; + } + + /* + * Since the ELSP[0] request is unpreemptible, it should not + * allow the maximum priority barrier through. Wait long + * enough to see if it is timesliced in by mistake. + */ + if (i915_request_wait(rq, 0, timeslice_threshold(engine)) >= 0) { + pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n", + engine->name); + err = -EINVAL; + } + i915_request_put(rq); + +out_spin: + igt_spinner_end(&spin); +out_heartbeat: + xchg(&engine->props.timeslice_duration_ms, timeslice); + engine_heartbeat_enable(engine); + if (err) + break; + + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + } + + igt_spinner_fini(&spin); + return err; +} + static int live_busywait_preempt(void *arg) { struct intel_gt *gt = arg; @@ -4153,7 +4259,6 @@ static int reset_virtual_engine(struct intel_gt *gt, { struct intel_engine_cs *engine; struct intel_context *ve; - unsigned long *heartbeat; struct igt_spinner spin; struct i915_request *rq; unsigned int n; @@ -4165,15 +4270,9 @@ static int reset_virtual_engine(struct intel_gt *gt, * descendents are not executed while the capture is in progress. */ - heartbeat = kmalloc_array(nsibling, sizeof(*heartbeat), GFP_KERNEL); - if (!heartbeat) + if (igt_spinner_init(&spin, gt)) return -ENOMEM; - if (igt_spinner_init(&spin, gt)) { - err = -ENOMEM; - goto out_free; - } - ve = intel_execlists_create_virtual(siblings, nsibling); if (IS_ERR(ve)) { err = PTR_ERR(ve); @@ -4181,7 +4280,7 @@ static int reset_virtual_engine(struct intel_gt *gt, } for (n = 0; n < nsibling; n++) - engine_heartbeat_disable(siblings[n], &heartbeat[n]); + engine_heartbeat_disable(siblings[n]); rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK); if (IS_ERR(rq)) { @@ -4252,13 +4351,11 @@ out_rq: i915_request_put(rq); out_heartbeat: for (n = 0; n < nsibling; n++) - engine_heartbeat_enable(siblings[n], heartbeat[n]); + engine_heartbeat_enable(siblings[n]); intel_context_put(ve); out_spin: igt_spinner_fini(&spin); -out_free: - kfree(heartbeat); return err; } @@ -4314,6 +4411,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_timeslice_preempt), SUBTEST(live_timeslice_rewind), SUBTEST(live_timeslice_queue), + SUBTEST(live_timeslice_nopreempt), SUBTEST(live_busywait_preempt), SUBTEST(live_preempt), SUBTEST(live_late_preempt), @@ -4932,9 +5030,7 @@ static int live_lrc_gpr(void *arg) return PTR_ERR(scratch); for_each_engine(engine, gt, id) { - unsigned long heartbeat; - - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); err = __live_lrc_gpr(engine, scratch, false); if (err) @@ -4945,7 +5041,7 @@ static int live_lrc_gpr(void *arg) goto err; err: - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); if (igt_flush_test(gt->i915)) err = -EIO; if (err) @@ -5092,10 +5188,9 @@ static int live_lrc_timestamp(void *arg) */ for_each_engine(data.engine, gt, id) { - unsigned long heartbeat; int i, err = 0; - engine_heartbeat_disable(data.engine, &heartbeat); + engine_heartbeat_disable(data.engine); for (i = 0; i < ARRAY_SIZE(data.ce); i++) { struct intel_context *tmp; @@ -5128,7 +5223,7 @@ static int live_lrc_timestamp(void *arg) } err: - engine_heartbeat_enable(data.engine, heartbeat); + engine_heartbeat_enable(data.engine); for (i = 0; i < ARRAY_SIZE(data.ce); i++) { if (!data.ce[i]) break; diff --git a/drivers/gpu/drm/i915/gt/selftest_mocs.c b/drivers/gpu/drm/i915/gt/selftest_mocs.c index 8831ffee2061..63f87d8608c3 100644 --- a/drivers/gpu/drm/i915/gt/selftest_mocs.c +++ b/drivers/gpu/drm/i915/gt/selftest_mocs.c @@ -18,6 +18,20 @@ struct live_mocs { void *vaddr; }; +static struct intel_context *mocs_context_create(struct intel_engine_cs *engine) +{ + struct intel_context *ce; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return ce; + + /* We build large requests to read the registers from the ring */ + ce->ring = __intel_context_ring_size(SZ_16K); + + return ce; +} + static int request_add_sync(struct i915_request *rq, int err) { i915_request_get(rq); @@ -301,7 +315,7 @@ static int live_mocs_clean(void *arg) for_each_engine(engine, gt, id) { struct intel_context *ce; - ce = intel_context_create(engine); + ce = mocs_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); break; @@ -395,7 +409,7 @@ static int live_mocs_reset(void *arg) for_each_engine(engine, gt, id) { struct intel_context *ce; - ce = intel_context_create(engine); + ce = mocs_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); break; diff --git a/drivers/gpu/drm/i915/gt/selftest_ring.c b/drivers/gpu/drm/i915/gt/selftest_ring.c new file mode 100644 index 000000000000..2a8c534dc125 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_ring.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2020 Intel Corporation + */ + +static struct intel_ring *mock_ring(unsigned long sz) +{ + struct intel_ring *ring; + + ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL); + if (!ring) + return NULL; + + kref_init(&ring->ref); + ring->size = sz; + ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(sz); + ring->effective_size = sz; + ring->vaddr = (void *)(ring + 1); + atomic_set(&ring->pin_count, 1); + + intel_ring_update_space(ring); + + return ring; +} + +static void mock_ring_free(struct intel_ring *ring) +{ + kfree(ring); +} + +static int check_ring_direction(struct intel_ring *ring, + u32 next, u32 prev, + int expected) +{ + int result; + + result = intel_ring_direction(ring, next, prev); + if (result < 0) + result = -1; + else if (result > 0) + result = 1; + + if (result != expected) { + pr_err("intel_ring_direction(%u, %u):%d != %d\n", + next, prev, result, expected); + return -EINVAL; + } + + return 0; +} + +static int check_ring_step(struct intel_ring *ring, u32 x, u32 step) +{ + u32 prev = x, next = intel_ring_wrap(ring, x + step); + int err = 0; + + err |= check_ring_direction(ring, next, next, 0); + err |= check_ring_direction(ring, prev, prev, 0); + err |= check_ring_direction(ring, next, prev, 1); + err |= check_ring_direction(ring, prev, next, -1); + + return err; +} + +static int check_ring_offset(struct intel_ring *ring, u32 x, u32 step) +{ + int err = 0; + + err |= check_ring_step(ring, x, step); + err |= check_ring_step(ring, intel_ring_wrap(ring, x + 1), step); + err |= check_ring_step(ring, intel_ring_wrap(ring, x - 1), step); + + return err; +} + +static int igt_ring_direction(void *dummy) +{ + struct intel_ring *ring; + unsigned int half = 2048; + int step, err = 0; + + ring = mock_ring(2 * half); + if (!ring) + return -ENOMEM; + + GEM_BUG_ON(ring->size != 2 * half); + + /* Precision of wrap detection is limited to ring->size / 2 */ + for (step = 1; step < half; step <<= 1) { + err |= check_ring_offset(ring, 0, step); + err |= check_ring_offset(ring, half, step); + } + err |= check_ring_step(ring, 0, half - 64); + + /* And check unwrapped handling for good measure */ + err |= check_ring_offset(ring, 0, 2 * half + 64); + err |= check_ring_offset(ring, 3 * half, 1); + + mock_ring_free(ring); + return err; +} + +int intel_ring_mock_selftests(void) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_ring_direction), + }; + + return i915_subtests(tests, NULL); +} diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 6275d69aa9cc..5049c3dd08a6 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -20,24 +20,20 @@ /* Try to isolate the impact of cstates from determing frequency response */ #define CPU_LATENCY 0 /* -1 to disable pm_qos, 0 to disable cstates */ -static unsigned long engine_heartbeat_disable(struct intel_engine_cs *engine) +static void engine_heartbeat_disable(struct intel_engine_cs *engine) { - unsigned long old; - - old = fetch_and_zero(&engine->props.heartbeat_interval_ms); + engine->props.heartbeat_interval_ms = 0; intel_engine_pm_get(engine); intel_engine_park_heartbeat(engine); - - return old; } -static void engine_heartbeat_enable(struct intel_engine_cs *engine, - unsigned long saved) +static void engine_heartbeat_enable(struct intel_engine_cs *engine) { intel_engine_pm_put(engine); - engine->props.heartbeat_interval_ms = saved; + engine->props.heartbeat_interval_ms = + engine->defaults.heartbeat_interval_ms; } static void dummy_rps_work(struct work_struct *wrk) @@ -246,7 +242,6 @@ int live_rps_clock_interval(void *arg) intel_gt_check_clock_frequency(gt); for_each_engine(engine, gt, id) { - unsigned long saved_heartbeat; struct i915_request *rq; u32 cycles; u64 dt; @@ -254,13 +249,13 @@ int live_rps_clock_interval(void *arg) if (!intel_engine_can_store_dword(engine)) continue; - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); rq = igt_spinner_create_request(&spin, engine->kernel_context, MI_NOOP); if (IS_ERR(rq)) { - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); err = PTR_ERR(rq); break; } @@ -271,7 +266,7 @@ int live_rps_clock_interval(void *arg) pr_err("%s: RPS spinner did not start\n", engine->name); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); intel_gt_set_wedged(engine->gt); err = -EIO; break; @@ -327,7 +322,7 @@ int live_rps_clock_interval(void *arg) intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); if (err == 0) { u64 time = intel_gt_pm_interval_to_ns(gt, cycles); @@ -405,7 +400,6 @@ int live_rps_control(void *arg) intel_gt_pm_get(gt); for_each_engine(engine, gt, id) { - unsigned long saved_heartbeat; struct i915_request *rq; ktime_t min_dt, max_dt; int f, limit; @@ -414,7 +408,7 @@ int live_rps_control(void *arg) if (!intel_engine_can_store_dword(engine)) continue; - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); rq = igt_spinner_create_request(&spin, engine->kernel_context, @@ -430,7 +424,7 @@ int live_rps_control(void *arg) pr_err("%s: RPS spinner did not start\n", engine->name); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); intel_gt_set_wedged(engine->gt); err = -EIO; break; @@ -440,7 +434,7 @@ int live_rps_control(void *arg) pr_err("%s: could not set minimum frequency [%x], only %x!\n", engine->name, rps->min_freq, read_cagf(rps)); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); show_pstate_limits(rps); err = -EINVAL; break; @@ -457,7 +451,7 @@ int live_rps_control(void *arg) pr_err("%s: could not restore minimum frequency [%x], only %x!\n", engine->name, rps->min_freq, read_cagf(rps)); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); show_pstate_limits(rps); err = -EINVAL; break; @@ -472,7 +466,7 @@ int live_rps_control(void *arg) min_dt = ktime_sub(ktime_get(), min_dt); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); pr_info("%s: range:[%x:%uMHz, %x:%uMHz] limit:[%x:%uMHz], %x:%x response %lluns:%lluns\n", engine->name, @@ -635,7 +629,6 @@ int live_rps_frequency_cs(void *arg) rps->work.func = dummy_rps_work; for_each_engine(engine, gt, id) { - unsigned long saved_heartbeat; struct i915_request *rq; struct i915_vma *vma; u32 *cancel, *cntr; @@ -644,14 +637,14 @@ int live_rps_frequency_cs(void *arg) int freq; } min, max; - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); vma = create_spin_counter(engine, engine->kernel_context->vm, false, &cancel, &cntr); if (IS_ERR(vma)) { err = PTR_ERR(vma); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); break; } @@ -732,7 +725,7 @@ err_vma: i915_vma_unpin(vma); i915_vma_put(vma); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); if (igt_flush_test(gt->i915)) err = -EIO; if (err) @@ -778,7 +771,6 @@ int live_rps_frequency_srm(void *arg) rps->work.func = dummy_rps_work; for_each_engine(engine, gt, id) { - unsigned long saved_heartbeat; struct i915_request *rq; struct i915_vma *vma; u32 *cancel, *cntr; @@ -787,14 +779,14 @@ int live_rps_frequency_srm(void *arg) int freq; } min, max; - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); vma = create_spin_counter(engine, engine->kernel_context->vm, true, &cancel, &cntr); if (IS_ERR(vma)) { err = PTR_ERR(vma); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); break; } @@ -874,7 +866,7 @@ err_vma: i915_vma_unpin(vma); i915_vma_put(vma); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); if (igt_flush_test(gt->i915)) err = -EIO; if (err) @@ -1066,16 +1058,14 @@ int live_rps_interrupt(void *arg) for_each_engine(engine, gt, id) { /* Keep the engine busy with a spinner; expect an UP! */ if (pm_events & GEN6_PM_RP_UP_THRESHOLD) { - unsigned long saved_heartbeat; - intel_gt_pm_wait_for_idle(engine->gt); GEM_BUG_ON(intel_rps_is_active(rps)); - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); err = __rps_up_interrupt(rps, engine, &spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); if (err) goto out; @@ -1084,15 +1074,13 @@ int live_rps_interrupt(void *arg) /* Keep the engine awake but idle and check for DOWN */ if (pm_events & GEN6_PM_RP_DOWN_THRESHOLD) { - unsigned long saved_heartbeat; - - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); intel_rc6_disable(>->rc6); err = __rps_down_interrupt(rps, engine); intel_rc6_enable(>->rc6); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); if (err) goto out; } @@ -1168,7 +1156,6 @@ int live_rps_power(void *arg) rps->work.func = dummy_rps_work; for_each_engine(engine, gt, id) { - unsigned long saved_heartbeat; struct i915_request *rq; struct { u64 power; @@ -1178,13 +1165,13 @@ int live_rps_power(void *arg) if (!intel_engine_can_store_dword(engine)) continue; - saved_heartbeat = engine_heartbeat_disable(engine); + engine_heartbeat_disable(engine); rq = igt_spinner_create_request(&spin, engine->kernel_context, MI_NOOP); if (IS_ERR(rq)) { - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); err = PTR_ERR(rq); break; } @@ -1195,7 +1182,7 @@ int live_rps_power(void *arg) pr_err("%s: RPS spinner did not start\n", engine->name); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); intel_gt_set_wedged(engine->gt); err = -EIO; break; @@ -1208,7 +1195,7 @@ int live_rps_power(void *arg) min.power = measure_power_at(rps, &min.freq); igt_spinner_end(&spin); - engine_heartbeat_enable(engine, saved_heartbeat); + engine_heartbeat_enable(engine); pr_info("%s: min:%llumW @ %uMHz, max:%llumW @ %uMHz\n", engine->name, diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index c2578a0f2f14..ef1c35073dc0 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -751,22 +751,20 @@ out_free: return err; } -static void engine_heartbeat_disable(struct intel_engine_cs *engine, - unsigned long *saved) +static void engine_heartbeat_disable(struct intel_engine_cs *engine) { - *saved = engine->props.heartbeat_interval_ms; engine->props.heartbeat_interval_ms = 0; intel_engine_pm_get(engine); intel_engine_park_heartbeat(engine); } -static void engine_heartbeat_enable(struct intel_engine_cs *engine, - unsigned long saved) +static void engine_heartbeat_enable(struct intel_engine_cs *engine) { intel_engine_pm_put(engine); - engine->props.heartbeat_interval_ms = saved; + engine->props.heartbeat_interval_ms = + engine->defaults.heartbeat_interval_ms; } static int live_hwsp_rollover_kernel(void *arg) @@ -785,10 +783,9 @@ static int live_hwsp_rollover_kernel(void *arg) struct intel_context *ce = engine->kernel_context; struct intel_timeline *tl = ce->timeline; struct i915_request *rq[3] = {}; - unsigned long heartbeat; int i; - engine_heartbeat_disable(engine, &heartbeat); + engine_heartbeat_disable(engine); if (intel_gt_wait_for_idle(gt, HZ / 2)) { err = -EIO; goto out; @@ -839,7 +836,7 @@ static int live_hwsp_rollover_kernel(void *arg) out: for (i = 0; i < ARRAY_SIZE(rq); i++) i915_request_put(rq[i]); - engine_heartbeat_enable(engine, heartbeat); + engine_heartbeat_enable(engine); if (err) break; } diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index 5ed323254ee1..32785463ec9e 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -623,6 +623,8 @@ err_request: err = -EINVAL; goto out_unpin; } + } else { + rsvd = 0; } expect = results[0]; diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index eee530453aa6..ad8a9df49f29 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -31,7 +31,7 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/mm.h> -#include <linux/mmu_context.h> +#include <linux/kthread.h> #include <linux/sched/mm.h> #include <linux/types.h> #include <linux/list.h> diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 1d5ff88078bd..7d361623ff67 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -124,7 +124,7 @@ int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) */ low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; - num_types = sizeof(vgpu_types) / sizeof(vgpu_types[0]); + num_types = ARRAY_SIZE(vgpu_types); gvt->types = kcalloc(num_types, sizeof(struct intel_vgpu_type), GFP_KERNEL); diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index 189b573d02be..372354d33f55 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -572,6 +572,9 @@ struct drm_i915_reg_descriptor { #define REG32(_reg, ...) \ { .addr = (_reg), __VA_ARGS__ } +#define REG32_IDX(_reg, idx) \ + { .addr = _reg(idx) } + /* * Convenience macro for adding 64-bit registers. * @@ -669,6 +672,7 @@ static const struct drm_i915_reg_descriptor gen9_blt_regs[] = { REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), REG32(BCS_SWCTRL), REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), + REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE), REG64_IDX(BCS_GPR, 0), REG64_IDX(BCS_GPR, 1), REG64_IDX(BCS_GPR, 2), diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 8e45ca3d2ede..55b97c3a3dde 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -47,20 +47,16 @@ static int compat_i915_getparam(struct file *file, unsigned int cmd, unsigned long arg) { struct drm_i915_getparam32 req32; - drm_i915_getparam_t __user *request; + struct drm_i915_getparam req; if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) return -EFAULT; - request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(request, sizeof(*request)) || - __put_user(req32.param, &request->param) || - __put_user((void __user *)(unsigned long)req32.value, - &request->value)) - return -EFAULT; + req.param = req32.param; + req.value = compat_ptr(req32.value); - return drm_ioctl(file, DRM_IOCTL_I915_GETPARAM, - (unsigned long)request); + return drm_ioctl_kernel(file, i915_getparam_ioctl, &req, + DRM_RENDER_ALLOW); } static drm_ioctl_compat_t *i915_compat_ioctls[] = { diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 4dc601dffc08..284cf078135a 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -3125,6 +3125,7 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv) val = I915_READ(GEN11_DE_HPD_IMR); val &= ~hotplug_irqs; + val |= ~enabled_irqs & hotplug_irqs; I915_WRITE(GEN11_DE_HPD_IMR, val); POSTING_READ(GEN11_DE_HPD_IMR); diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c index b6376b25ef63..43039dc8c607 100644 --- a/drivers/gpu/drm/i915/i915_mm.c +++ b/drivers/gpu/drm/i915/i915_mm.c @@ -25,7 +25,6 @@ #include <linux/mm.h> #include <linux/io-mapping.h> -#include <asm/pgtable.h> #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index add00ec1f787..02559da61e6e 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -65,7 +65,7 @@ i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400, "Override/Ignore selection of SDVO panel mode in the VBT " "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); -i915_param_named_unsafe(reset, int, 0600, +i915_param_named_unsafe(reset, uint, 0600, "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])"); i915_param_named_unsafe(vbt_firmware, charp, 0400, @@ -173,7 +173,7 @@ i915_param_named(enable_gvt, bool, 0400, #endif #if IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) -i915_param_named_unsafe(fake_lmem_start, ulong, 0600, +i915_param_named_unsafe(fake_lmem_start, ulong, 0400, "Fake LMEM start offset (default: 0)"); #endif diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 45323732f099..4f21bfffbf0e 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -64,7 +64,7 @@ struct drm_printer; param(int, mmio_debug, -IS_ENABLED(CONFIG_DRM_I915_DEBUG_MMIO), 0600) \ param(int, edp_vswing, 0, 0400) \ param(unsigned int, reset, 3, 0600) \ - param(unsigned int, inject_probe_failure, 0, 0600) \ + param(unsigned int, inject_probe_failure, 0, 0) \ param(int, fastboot, -1, 0600) \ param(int, enable_dpcd_backlight, -1, 0600) \ param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \ diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 75c60c2afb7e..25329b7600c9 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -3676,7 +3676,7 @@ static int read_properties_unlocked(struct i915_perf *perf, * buffered data written by the GPU besides periodic OA metrics. * * Note we copy the properties from userspace outside of the i915 perf - * mutex to avoid an awkward lockdep with mmap_sem. + * mutex to avoid an awkward lockdep with mmap_lock. * * Most of the implementation details are handled by * i915_perf_open_ioctl_locked() after taking the &perf->lock @@ -3896,9 +3896,6 @@ static struct i915_oa_reg *alloc_oa_regs(struct i915_perf *perf, if (!n_regs) return NULL; - if (!access_ok(regs, n_regs * sizeof(u32) * 2)) - return ERR_PTR(-EFAULT); - /* No is_valid function means we're not allowing any register to be programmed. */ GEM_BUG_ON(!is_valid); if (!is_valid) diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index e991a707bdb7..962ded9ce73f 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -269,12 +269,48 @@ static bool exclusive_mmio_access(const struct drm_i915_private *i915) return IS_GEN(i915, 7); } +static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns) +{ + struct intel_engine_pmu *pmu = &engine->pmu; + bool busy; + u32 val; + + val = ENGINE_READ_FW(engine, RING_CTL); + if (val == 0) /* powerwell off => engine idle */ + return; + + if (val & RING_WAIT) + add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); + if (val & RING_WAIT_SEMAPHORE) + add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); + + /* No need to sample when busy stats are supported. */ + if (intel_engine_supports_stats(engine)) + return; + + /* + * While waiting on a semaphore or event, MI_MODE reports the + * ring as idle. However, previously using the seqno, and with + * execlists sampling, we account for the ring waiting as the + * engine being busy. Therefore, we record the sample as being + * busy if either waiting or !idle. + */ + busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT); + if (!busy) { + val = ENGINE_READ_FW(engine, RING_MI_MODE); + busy = !(val & MODE_IDLE); + } + if (busy) + add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); +} + static void engines_sample(struct intel_gt *gt, unsigned int period_ns) { struct drm_i915_private *i915 = gt->i915; struct intel_engine_cs *engine; enum intel_engine_id id; + unsigned long flags; if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0) return; @@ -283,53 +319,17 @@ engines_sample(struct intel_gt *gt, unsigned int period_ns) return; for_each_engine(engine, gt, id) { - struct intel_engine_pmu *pmu = &engine->pmu; - spinlock_t *mmio_lock; - unsigned long flags; - bool busy; - u32 val; - if (!intel_engine_pm_get_if_awake(engine)) continue; - mmio_lock = NULL; - if (exclusive_mmio_access(i915)) - mmio_lock = &engine->uncore->lock; - - if (unlikely(mmio_lock)) - spin_lock_irqsave(mmio_lock, flags); - - val = ENGINE_READ_FW(engine, RING_CTL); - if (val == 0) /* powerwell off => engine idle */ - goto skip; - - if (val & RING_WAIT) - add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); - if (val & RING_WAIT_SEMAPHORE) - add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); - - /* No need to sample when busy stats are supported. */ - if (intel_engine_supports_stats(engine)) - goto skip; - - /* - * While waiting on a semaphore or event, MI_MODE reports the - * ring as idle. However, previously using the seqno, and with - * execlists sampling, we account for the ring waiting as the - * engine being busy. Therefore, we record the sample as being - * busy if either waiting or !idle. - */ - busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT); - if (!busy) { - val = ENGINE_READ_FW(engine, RING_MI_MODE); - busy = !(val & MODE_IDLE); + if (exclusive_mmio_access(i915)) { + spin_lock_irqsave(&engine->uncore->lock, flags); + engine_sample(engine, period_ns); + spin_unlock_irqrestore(&engine->uncore->lock, flags); + } else { + engine_sample(engine, period_ns); } - if (busy) - add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); -skip: - if (unlikely(mmio_lock)) - spin_unlock_irqrestore(mmio_lock, flags); intel_engine_pm_put_async(engine); } } diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h index 5003a71113cb..8aa7866ec6b6 100644 --- a/drivers/gpu/drm/i915/i915_priolist_types.h +++ b/drivers/gpu/drm/i915/i915_priolist_types.h @@ -42,7 +42,7 @@ enum { * active request. */ #define I915_PRIORITY_UNPREEMPTABLE INT_MAX -#define I915_PRIORITY_BARRIER INT_MAX +#define I915_PRIORITY_BARRIER (I915_PRIORITY_UNPREEMPTABLE - 1) struct i915_priolist { struct list_head requests[I915_PRIORITY_COUNT]; diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index ef25ce6e395e..e75c528ebbe0 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -25,10 +25,6 @@ static int copy_query_item(void *query_hdr, size_t query_sz, query_sz)) return -EFAULT; - if (!access_ok(u64_to_user_ptr(query_item->data_ptr), - total_length)) - return -EFAULT; - return 0; } @@ -72,20 +68,20 @@ static int query_topology_info(struct drm_i915_private *dev_priv, topo.eu_offset = slice_length + subslice_length; topo.eu_stride = sseu->eu_stride; - if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr), + if (copy_to_user(u64_to_user_ptr(query_item->data_ptr), &topo, sizeof(topo))) return -EFAULT; - if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)), + if (copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)), &sseu->slice_mask, slice_length)) return -EFAULT; - if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + + if (copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo) + slice_length), sseu->subslice_mask, subslice_length)) return -EFAULT; - if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + + if (copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo) + slice_length + subslice_length), sseu->eu_mask, eu_length)) @@ -131,14 +127,14 @@ query_engine_info(struct drm_i915_private *i915, info.engine.engine_instance = engine->uabi_instance; info.capabilities = engine->uabi_capabilities; - if (__copy_to_user(info_ptr, &info, sizeof(info))) + if (copy_to_user(info_ptr, &info, sizeof(info))) return -EFAULT; query.num_engines++; info_ptr++; } - if (__copy_to_user(query_ptr, &query, sizeof(query))) + if (copy_to_user(query_ptr, &query, sizeof(query))) return -EFAULT; return len; @@ -158,10 +154,6 @@ static int can_copy_perf_config_registers_or_number(u32 user_n_regs, if (user_n_regs < kernel_n_regs) return -EINVAL; - if (!access_ok(u64_to_user_ptr(user_regs_ptr), - 2 * sizeof(u32) * kernel_n_regs)) - return -EFAULT; - return 0; } @@ -170,6 +162,7 @@ static int copy_perf_config_registers_or_number(const struct i915_oa_reg *kernel u64 user_regs_ptr, u32 *user_n_regs) { + u32 __user *p = u64_to_user_ptr(user_regs_ptr); u32 r; if (*user_n_regs == 0) { @@ -179,25 +172,19 @@ static int copy_perf_config_registers_or_number(const struct i915_oa_reg *kernel *user_n_regs = kernel_n_regs; - for (r = 0; r < kernel_n_regs; r++) { - u32 __user *user_reg_ptr = - u64_to_user_ptr(user_regs_ptr + sizeof(u32) * r * 2); - u32 __user *user_val_ptr = - u64_to_user_ptr(user_regs_ptr + sizeof(u32) * r * 2 + - sizeof(u32)); - int ret; - - ret = __put_user(i915_mmio_reg_offset(kernel_regs[r].addr), - user_reg_ptr); - if (ret) - return -EFAULT; + if (!user_write_access_begin(p, 2 * sizeof(u32) * kernel_n_regs)) + return -EFAULT; - ret = __put_user(kernel_regs[r].value, user_val_ptr); - if (ret) - return -EFAULT; + for (r = 0; r < kernel_n_regs; r++, p += 2) { + unsafe_put_user(i915_mmio_reg_offset(kernel_regs[r].addr), + p, Efault); + unsafe_put_user(kernel_regs[r].value, p + 1, Efault); } - + user_write_access_end(); return 0; +Efault: + user_write_access_end(); + return -EFAULT; } static int query_perf_config_data(struct drm_i915_private *i915, @@ -233,10 +220,7 @@ static int query_perf_config_data(struct drm_i915_private *i915, return -EINVAL; } - if (!access_ok(user_query_config_ptr, total_size)) - return -EFAULT; - - if (__get_user(flags, &user_query_config_ptr->flags)) + if (get_user(flags, &user_query_config_ptr->flags)) return -EFAULT; if (flags != 0) @@ -249,7 +233,7 @@ static int query_perf_config_data(struct drm_i915_private *i915, BUILD_BUG_ON(sizeof(user_query_config_ptr->uuid) >= sizeof(uuid)); memset(&uuid, 0, sizeof(uuid)); - if (__copy_from_user(uuid, user_query_config_ptr->uuid, + if (copy_from_user(uuid, user_query_config_ptr->uuid, sizeof(user_query_config_ptr->uuid))) return -EFAULT; @@ -263,7 +247,7 @@ static int query_perf_config_data(struct drm_i915_private *i915, } rcu_read_unlock(); } else { - if (__get_user(config_id, &user_query_config_ptr->config)) + if (get_user(config_id, &user_query_config_ptr->config)) return -EFAULT; oa_config = i915_perf_get_oa_config(perf, config_id); @@ -271,8 +255,7 @@ static int query_perf_config_data(struct drm_i915_private *i915, if (!oa_config) return -ENOENT; - if (__copy_from_user(&user_config, user_config_ptr, - sizeof(user_config))) { + if (copy_from_user(&user_config, user_config_ptr, sizeof(user_config))) { ret = -EFAULT; goto out; } @@ -318,8 +301,7 @@ static int query_perf_config_data(struct drm_i915_private *i915, memcpy(user_config.uuid, oa_config->uuid, sizeof(user_config.uuid)); - if (__copy_to_user(user_config_ptr, &user_config, - sizeof(user_config))) { + if (copy_to_user(user_config_ptr, &user_config, sizeof(user_config))) { ret = -EFAULT; goto out; } diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 6c076a24eb82..06cd1d28a176 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -186,7 +186,7 @@ typedef struct { #define INVALID_MMIO_REG _MMIO(0) -static inline u32 i915_mmio_reg_offset(i915_reg_t reg) +static __always_inline u32 i915_mmio_reg_offset(i915_reg_t reg) { return reg.reg; } @@ -7896,7 +7896,7 @@ enum { /* GEN7 chicken */ #define GEN7_COMMON_SLICE_CHICKEN1 _MMIO(0x7010) - #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC ((1 << 10) | (1 << 26)) + #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC (1 << 10) #define GEN9_RHWO_OPTIMIZATION_DISABLE (1 << 14) #define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 526c1e9acbd5..def62100e666 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -121,8 +121,39 @@ static void i915_fence_release(struct dma_fence *fence) i915_sw_fence_fini(&rq->submit); i915_sw_fence_fini(&rq->semaphore); - /* Keep one request on each engine for reserved use under mempressure */ - if (!cmpxchg(&rq->engine->request_pool, NULL, rq)) + /* + * Keep one request on each engine for reserved use under mempressure + * + * We do not hold a reference to the engine here and so have to be + * very careful in what rq->engine we poke. The virtual engine is + * referenced via the rq->context and we released that ref during + * i915_request_retire(), ergo we must not dereference a virtual + * engine here. Not that we would want to, as the only consumer of + * the reserved engine->request_pool is the power management parking, + * which must-not-fail, and that is only run on the physical engines. + * + * Since the request must have been executed to be have completed, + * we know that it will have been processed by the HW and will + * not be unsubmitted again, so rq->engine and rq->execution_mask + * at this point is stable. rq->execution_mask will be a single + * bit if the last and _only_ engine it could execution on was a + * physical engine, if it's multiple bits then it started on and + * could still be on a virtual engine. Thus if the mask is not a + * power-of-two we assume that rq->engine may still be a virtual + * engine and so a dangling invalid pointer that we cannot dereference + * + * For example, consider the flow of a bonded request through a virtual + * engine. The request is created with a wide engine mask (all engines + * that we might execute on). On processing the bond, the request mask + * is reduced to one or more engines. If the request is subsequently + * bound to a single engine, it will then be constrained to only + * execute on that engine and never returned to the virtual engine + * after timeslicing away, see __unwind_incomplete_requests(). Thus we + * know that if the rq->execution_mask is a single bit, rq->engine + * can be a physical engine with the exact corresponding mask. + */ + if (is_power_of_2(rq->execution_mask) && + !cmpxchg(&rq->engine->request_pool, NULL, rq)) return; kmem_cache_free(global.slab_requests, rq); @@ -326,6 +357,53 @@ void i915_request_retire_upto(struct i915_request *rq) } while (i915_request_retire(tmp) && tmp != rq); } +static struct i915_request * const * +__engine_active(struct intel_engine_cs *engine) +{ + return READ_ONCE(engine->execlists.active); +} + +static bool __request_in_flight(const struct i915_request *signal) +{ + struct i915_request * const *port, *rq; + bool inflight = false; + + if (!i915_request_is_ready(signal)) + return false; + + /* + * Even if we have unwound the request, it may still be on + * the GPU (preempt-to-busy). If that request is inside an + * unpreemptible critical section, it will not be removed. Some + * GPU functions may even be stuck waiting for the paired request + * (__await_execution) to be submitted and cannot be preempted + * until the bond is executing. + * + * As we know that there are always preemption points between + * requests, we know that only the currently executing request + * may be still active even though we have cleared the flag. + * However, we can't rely on our tracking of ELSP[0] to known + * which request is currently active and so maybe stuck, as + * the tracking maybe an event behind. Instead assume that + * if the context is still inflight, then it is still active + * even if the active flag has been cleared. + */ + if (!intel_context_inflight(signal->context)) + return false; + + rcu_read_lock(); + for (port = __engine_active(signal->engine); (rq = *port); port++) { + if (rq->context == signal->context) { + inflight = i915_seqno_passed(rq->fence.seqno, + signal->fence.seqno); + break; + } + } + rcu_read_unlock(); + + return inflight; +} + static int __await_execution(struct i915_request *rq, struct i915_request *signal, @@ -356,7 +434,7 @@ __await_execution(struct i915_request *rq, } spin_lock_irq(&signal->lock); - if (i915_request_is_active(signal)) { + if (i915_request_is_active(signal) || __request_in_flight(signal)) { if (hook) { hook(rq, &signal->fence); i915_request_put(signal); @@ -1022,37 +1100,91 @@ await_fence: I915_FENCE_GFP); } +static bool intel_timeline_sync_has_start(struct intel_timeline *tl, + struct dma_fence *fence) +{ + return __intel_timeline_sync_is_later(tl, + fence->context, + fence->seqno - 1); +} + +static int intel_timeline_sync_set_start(struct intel_timeline *tl, + const struct dma_fence *fence) +{ + return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1); +} + static int -i915_request_await_request(struct i915_request *to, struct i915_request *from) +__i915_request_await_execution(struct i915_request *to, + struct i915_request *from, + void (*hook)(struct i915_request *rq, + struct dma_fence *signal)) { - int ret; + int err; - GEM_BUG_ON(to == from); - GEM_BUG_ON(to->timeline == from->timeline); + GEM_BUG_ON(intel_context_is_barrier(from->context)); - if (i915_request_completed(from)) { - i915_sw_fence_set_error_once(&to->submit, from->fence.error); + /* Submit both requests at the same time */ + err = __await_execution(to, from, hook, I915_FENCE_GFP); + if (err) + return err; + + /* Squash repeated depenendices to the same timelines */ + if (intel_timeline_sync_has_start(i915_request_timeline(to), + &from->fence)) return 0; + + /* + * Wait until the start of this request. + * + * The execution cb fires when we submit the request to HW. But in + * many cases this may be long before the request itself is ready to + * run (consider that we submit 2 requests for the same context, where + * the request of interest is behind an indefinite spinner). So we hook + * up to both to reduce our queues and keep the execution lag minimised + * in the worst case, though we hope that the await_start is elided. + */ + err = i915_request_await_start(to, from); + if (err < 0) + return err; + + /* + * Ensure both start together [after all semaphores in signal] + * + * Now that we are queued to the HW at roughly the same time (thanks + * to the execute cb) and are ready to run at roughly the same time + * (thanks to the await start), our signaler may still be indefinitely + * delayed by waiting on a semaphore from a remote engine. If our + * signaler depends on a semaphore, so indirectly do we, and we do not + * want to start our payload until our signaler also starts theirs. + * So we wait. + * + * However, there is also a second condition for which we need to wait + * for the precise start of the signaler. Consider that the signaler + * was submitted in a chain of requests following another context + * (with just an ordinary intra-engine fence dependency between the + * two). In this case the signaler is queued to HW, but not for + * immediate execution, and so we must wait until it reaches the + * active slot. + */ + if (intel_engine_has_semaphores(to->engine) && + !i915_request_has_initial_breadcrumb(to)) { + err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); + if (err < 0) + return err; } + /* Couple the dependency tree for PI on this exposed to->fence */ if (to->engine->schedule) { - ret = i915_sched_node_add_dependency(&to->sched, + err = i915_sched_node_add_dependency(&to->sched, &from->sched, - I915_DEPENDENCY_EXTERNAL); - if (ret < 0) - return ret; + I915_DEPENDENCY_WEAK); + if (err < 0) + return err; } - if (to->engine == from->engine) - ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, - &from->submit, - I915_FENCE_GFP); - else - ret = emit_semaphore_wait(to, from, I915_FENCE_GFP); - if (ret < 0) - return ret; - - return 0; + return intel_timeline_sync_set_start(i915_request_timeline(to), + &from->fence); } static void mark_external(struct i915_request *rq) @@ -1105,23 +1237,20 @@ i915_request_await_external(struct i915_request *rq, struct dma_fence *fence) } int -i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) +i915_request_await_execution(struct i915_request *rq, + struct dma_fence *fence, + void (*hook)(struct i915_request *rq, + struct dma_fence *signal)) { struct dma_fence **child = &fence; unsigned int nchild = 1; int ret; - /* - * Note that if the fence-array was created in signal-on-any mode, - * we should *not* decompose it into its individual fences. However, - * we don't currently store which mode the fence-array is operating - * in. Fortunately, the only user of signal-on-any is private to - * amdgpu and we should not see any incoming fence-array from - * sync-file being in signal-on-any mode. - */ if (dma_fence_is_array(fence)) { struct dma_fence_array *array = to_dma_fence_array(fence); + /* XXX Error for signal-on-any fence arrays */ + child = array->fences; nchild = array->num_fences; GEM_BUG_ON(!nchild); @@ -1134,138 +1263,95 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) continue; } - /* - * Requests on the same timeline are explicitly ordered, along - * with their dependencies, by i915_request_add() which ensures - * that requests are submitted in-order through each ring. - */ if (fence->context == rq->fence.context) continue; - /* Squash repeated waits to the same timelines */ - if (fence->context && - intel_timeline_sync_is_later(i915_request_timeline(rq), - fence)) - continue; + /* + * We don't squash repeated fence dependencies here as we + * want to run our callback in all cases. + */ if (dma_fence_is_i915(fence)) - ret = i915_request_await_request(rq, to_request(fence)); + ret = __i915_request_await_execution(rq, + to_request(fence), + hook); else ret = i915_request_await_external(rq, fence); if (ret < 0) return ret; - - /* Record the latest fence used against each timeline */ - if (fence->context) - intel_timeline_sync_set(i915_request_timeline(rq), - fence); } while (--nchild); return 0; } -static bool intel_timeline_sync_has_start(struct intel_timeline *tl, - struct dma_fence *fence) -{ - return __intel_timeline_sync_is_later(tl, - fence->context, - fence->seqno - 1); -} - -static int intel_timeline_sync_set_start(struct intel_timeline *tl, - const struct dma_fence *fence) +static int +await_request_submit(struct i915_request *to, struct i915_request *from) { - return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1); + /* + * If we are waiting on a virtual engine, then it may be + * constrained to execute on a single engine *prior* to submission. + * When it is submitted, it will be first submitted to the virtual + * engine and then passed to the physical engine. We cannot allow + * the waiter to be submitted immediately to the physical engine + * as it may then bypass the virtual request. + */ + if (to->engine == READ_ONCE(from->engine)) + return i915_sw_fence_await_sw_fence_gfp(&to->submit, + &from->submit, + I915_FENCE_GFP); + else + return __i915_request_await_execution(to, from, NULL); } static int -__i915_request_await_execution(struct i915_request *to, - struct i915_request *from, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal)) +i915_request_await_request(struct i915_request *to, struct i915_request *from) { - int err; - - GEM_BUG_ON(intel_context_is_barrier(from->context)); + int ret; - /* Submit both requests at the same time */ - err = __await_execution(to, from, hook, I915_FENCE_GFP); - if (err) - return err; + GEM_BUG_ON(to == from); + GEM_BUG_ON(to->timeline == from->timeline); - /* Squash repeated depenendices to the same timelines */ - if (intel_timeline_sync_has_start(i915_request_timeline(to), - &from->fence)) + if (i915_request_completed(from)) { + i915_sw_fence_set_error_once(&to->submit, from->fence.error); return 0; - - /* - * Wait until the start of this request. - * - * The execution cb fires when we submit the request to HW. But in - * many cases this may be long before the request itself is ready to - * run (consider that we submit 2 requests for the same context, where - * the request of interest is behind an indefinite spinner). So we hook - * up to both to reduce our queues and keep the execution lag minimised - * in the worst case, though we hope that the await_start is elided. - */ - err = i915_request_await_start(to, from); - if (err < 0) - return err; - - /* - * Ensure both start together [after all semaphores in signal] - * - * Now that we are queued to the HW at roughly the same time (thanks - * to the execute cb) and are ready to run at roughly the same time - * (thanks to the await start), our signaler may still be indefinitely - * delayed by waiting on a semaphore from a remote engine. If our - * signaler depends on a semaphore, so indirectly do we, and we do not - * want to start our payload until our signaler also starts theirs. - * So we wait. - * - * However, there is also a second condition for which we need to wait - * for the precise start of the signaler. Consider that the signaler - * was submitted in a chain of requests following another context - * (with just an ordinary intra-engine fence dependency between the - * two). In this case the signaler is queued to HW, but not for - * immediate execution, and so we must wait until it reaches the - * active slot. - */ - if (intel_engine_has_semaphores(to->engine) && - !i915_request_has_initial_breadcrumb(to)) { - err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); - if (err < 0) - return err; } - /* Couple the dependency tree for PI on this exposed to->fence */ if (to->engine->schedule) { - err = i915_sched_node_add_dependency(&to->sched, + ret = i915_sched_node_add_dependency(&to->sched, &from->sched, - I915_DEPENDENCY_WEAK); - if (err < 0) - return err; + I915_DEPENDENCY_EXTERNAL); + if (ret < 0) + return ret; } - return intel_timeline_sync_set_start(i915_request_timeline(to), - &from->fence); + if (is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask))) + ret = await_request_submit(to, from); + else + ret = emit_semaphore_wait(to, from, I915_FENCE_GFP); + if (ret < 0) + return ret; + + return 0; } int -i915_request_await_execution(struct i915_request *rq, - struct dma_fence *fence, - void (*hook)(struct i915_request *rq, - struct dma_fence *signal)) +i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) { struct dma_fence **child = &fence; unsigned int nchild = 1; int ret; + /* + * Note that if the fence-array was created in signal-on-any mode, + * we should *not* decompose it into its individual fences. However, + * we don't currently store which mode the fence-array is operating + * in. Fortunately, the only user of signal-on-any is private to + * amdgpu and we should not see any incoming fence-array from + * sync-file being in signal-on-any mode. + */ if (dma_fence_is_array(fence)) { struct dma_fence_array *array = to_dma_fence_array(fence); - /* XXX Error for signal-on-any fence arrays */ - child = array->fences; nchild = array->num_fences; GEM_BUG_ON(!nchild); @@ -1278,22 +1364,31 @@ i915_request_await_execution(struct i915_request *rq, continue; } + /* + * Requests on the same timeline are explicitly ordered, along + * with their dependencies, by i915_request_add() which ensures + * that requests are submitted in-order through each ring. + */ if (fence->context == rq->fence.context) continue; - /* - * We don't squash repeated fence dependencies here as we - * want to run our callback in all cases. - */ + /* Squash repeated waits to the same timelines */ + if (fence->context && + intel_timeline_sync_is_later(i915_request_timeline(rq), + fence)) + continue; if (dma_fence_is_i915(fence)) - ret = __i915_request_await_execution(rq, - to_request(fence), - hook); + ret = i915_request_await_request(rq, to_request(fence)); else ret = i915_request_await_external(rq, fence); if (ret < 0) return ret; + + /* Record the latest fence used against each timeline */ + if (fence->context) + intel_timeline_sync_set(i915_request_timeline(rq), + fence); } while (--nchild); return 0; diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index f4ea318781f0..cbb880b10c65 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -209,14 +209,6 @@ static void kick_submission(struct intel_engine_cs *engine, if (!inflight) goto unlock; - ENGINE_TRACE(engine, - "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n", - prio, - rq->fence.context, rq->fence.seqno, - inflight->fence.context, inflight->fence.seqno, - inflight->sched.attr.priority); - engine->execlists.queue_priority_hint = prio; - /* * If we are already the currently executing context, don't * bother evaluating if we should preempt ourselves. @@ -224,6 +216,14 @@ static void kick_submission(struct intel_engine_cs *engine, if (inflight->context == rq->context) goto unlock; + ENGINE_TRACE(engine, + "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n", + prio, + rq->fence.context, rq->fence.seqno, + inflight->fence.context, inflight->fence.seqno, + inflight->sched.attr.priority); + + engine->execlists.queue_priority_hint = prio; if (need_preempt(prio, rq_prio(inflight))) tasklet_hi_schedule(&engine->execlists.tasklet); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 696491d71a1d..07f663cd2d1c 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -6830,16 +6830,6 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) I915_WRITE(ILK_DISPLAY_CHICKEN2, I915_READ(ILK_DISPLAY_CHICKEN2) | ILK_ELPIN_409_SELECT); - I915_WRITE(_3D_CHICKEN2, - _3D_CHICKEN2_WM_READ_PIPELINED << 16 | - _3D_CHICKEN2_WM_READ_PIPELINED); - - /* WaDisableRenderCachePipelinedFlush:ilk */ - I915_WRITE(CACHE_MODE_0, - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); - - /* WaDisable_RenderCache_OperationalFlush:ilk */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); g4x_disable_trickle_feed(dev_priv); @@ -6902,27 +6892,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) I915_READ(ILK_DISPLAY_CHICKEN2) | ILK_ELPIN_409_SELECT); - /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ - I915_WRITE(_3D_CHICKEN, - _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); - - /* WaDisable_RenderCache_OperationalFlush:snb */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - /* - * BSpec recoomends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN6_GT_MODE, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - - I915_WRITE(CACHE_MODE_0, - _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); - I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | GEN6_BLBUNIT_CLOCK_GATE_DISABLE | @@ -6945,18 +6914,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - /* WaStripsFansDisableFastClipPerformanceFix:snb */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); - - /* - * Bspec says: - * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and - * 3DSTATE_SF number of SF output attributes is more than 16." - */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); - /* * According to the spec the following bits should be * set in order to enable memory self-refresh and fbc: @@ -6986,24 +6943,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) gen6_check_mch_setup(dev_priv); } -static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) -{ - u32 reg = I915_READ(GEN7_FF_THREAD_MODE); - - /* - * WaVSThreadDispatchOverride:ivb,vlv - * - * This actually overrides the dispatch - * mode for all thread types. - */ - reg &= ~GEN7_FF_SCHED_MASK; - reg |= GEN7_FF_TS_SCHED_HW; - reg |= GEN7_FF_VS_SCHED_HW; - reg |= GEN7_FF_DS_SCHED_HW; - - I915_WRITE(GEN7_FF_THREAD_MODE, reg); -} - static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) { /* @@ -7230,45 +7169,10 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) static void hsw_init_clock_gating(struct drm_i915_private *dev_priv) { - /* L3 caching of data atomics doesn't work -- disable it. */ - I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); - I915_WRITE(HSW_ROW_CHICKEN3, - _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE)); - /* This is required by WaCatErrorRejectionIssue:hsw */ I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, - I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | - GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - - /* WaVSRefCountFullforceMissDisable:hsw */ - I915_WRITE(GEN7_FF_THREAD_MODE, - I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); - - /* WaDisable_RenderCache_OperationalFlush:hsw */ - I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - /* enable HiZ Raw Stall Optimization */ - I915_WRITE(CACHE_MODE_0_GEN7, - _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); - - /* WaDisable4x2SubspanOptimization:hsw */ - I915_WRITE(CACHE_MODE_1, - _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); - - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN7_GT_MODE, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - - /* WaSampleCChickenBitEnable:hsw */ - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE)); + I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | + GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* WaSwitchSolVfFArbitrationPriority:hsw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -7282,32 +7186,11 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); - /* WaDisableEarlyCull:ivb */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); - /* WaDisableBackToBackFlipFix:ivb */ I915_WRITE(IVB_CHICKEN3, CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); - /* WaDisablePSDDualDispatchEnable:ivb */ - if (IS_IVB_GT1(dev_priv)) - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, - _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - - /* WaDisable_RenderCache_OperationalFlush:ivb */ - I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:ivb */ - I915_WRITE(GEN7_L3CNTLREG1, - GEN7_WA_FOR_GEN7_L3_CONTROL); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, - GEN7_WA_L3_CHICKEN_MODE); if (IS_IVB_GT1(dev_priv)) I915_WRITE(GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); @@ -7319,10 +7202,6 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); } - /* WaForceL3Serialization:ivb */ - I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & - ~L3SQ_URB_READ_CAM_MATCH_DISABLE); - /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:ivb workaround. @@ -7337,29 +7216,6 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) g4x_disable_trickle_feed(dev_priv); - gen7_setup_fixed_func_scheduler(dev_priv); - - if (0) { /* causes HiZ corruption on ivb:gt1 */ - /* enable HiZ Raw Stall Optimization */ - I915_WRITE(CACHE_MODE_0_GEN7, - _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); - } - - /* WaDisable4x2SubspanOptimization:ivb */ - I915_WRITE(CACHE_MODE_1, - _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); - - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN7_GT_MODE, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); snpcr &= ~GEN6_MBC_SNPCR_MASK; snpcr |= GEN6_MBC_SNPCR_MED; @@ -7373,28 +7229,11 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) static void vlv_init_clock_gating(struct drm_i915_private *dev_priv) { - /* WaDisableEarlyCull:vlv */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); - /* WaDisableBackToBackFlipFix:vlv */ I915_WRITE(IVB_CHICKEN3, CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); - /* WaPsdDispatchEnable:vlv */ - /* WaDisablePSDDualDispatchEnable:vlv */ - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, - _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | - GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - - /* WaDisable_RenderCache_OperationalFlush:vlv */ - I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - /* WaForceL3Serialization:vlv */ - I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & - ~L3SQ_URB_READ_CAM_MATCH_DISABLE); - /* WaDisableDopClockGating:vlv */ I915_WRITE(GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); @@ -7404,8 +7243,6 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - gen7_setup_fixed_func_scheduler(dev_priv); - /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:vlv workaround. @@ -7420,30 +7257,6 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv) I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); /* - * BSpec says this must be set, even though - * WaDisable4x2SubspanOptimization isn't listed for VLV. - */ - I915_WRITE(CACHE_MODE_1, - _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); - - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN7_GT_MODE, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - - /* - * WaIncreaseL3CreditsForVLVB0:vlv - * This is the hardware default actually. - */ - I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); - - /* * WaDisableVLVClockGating_VBIIssue:vlv * Disable clock gating on th GCFG unit to prevent a delay * in the reporting of vblank events. @@ -7495,13 +7308,6 @@ static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; I915_WRITE(DSPCLK_GATE_D, dspclk_gate); - /* WaDisableRenderCachePipelinedFlush */ - I915_WRITE(CACHE_MODE_0, - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); - - /* WaDisable_RenderCache_OperationalFlush:g4x */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - g4x_disable_trickle_feed(dev_priv); } @@ -7517,11 +7323,6 @@ static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_write(uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); - - /* WaDisable_RenderCache_OperationalFlush:gen4 */ - intel_uncore_write(uncore, - CACHE_MODE_0, - _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } static void i965g_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7534,9 +7335,6 @@ static void i965g_init_clock_gating(struct drm_i915_private *dev_priv) I915_WRITE(RENCLK_GATE_D2, 0); I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); - - /* WaDisable_RenderCache_OperationalFlush:gen4 */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 6a2be7d0dd95..6090ce35226b 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests) selftest(scatterlist, scatterlist_mock_selftests) selftest(syncmap, i915_syncmap_mock_selftests) selftest(uncore, intel_uncore_mock_selftests) +selftest(ring, intel_ring_mock_selftests) selftest(engine, intel_engine_cs_mock_selftests) selftest(timelines, intel_timeline_mock_selftests) selftest(requests, i915_request_mock_selftests) diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig index fa5ffc4fe823..c420f5a3d33b 100644 --- a/drivers/gpu/drm/mediatek/Kconfig +++ b/drivers/gpu/drm/mediatek/Kconfig @@ -11,6 +11,7 @@ config DRM_MEDIATEK select DRM_MIPI_DSI select DRM_PANEL select MEMORY + select MTK_MMSYS select MTK_SMI select VIDEOMODE_HELPERS help diff --git a/drivers/gpu/drm/mediatek/mtk_disp_color.c b/drivers/gpu/drm/mediatek/mtk_disp_color.c index 6fb0d6983a4a..3ae9c810845b 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_color.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_color.c @@ -119,7 +119,10 @@ static int mtk_disp_color_probe(struct platform_device *pdev) ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id, &mtk_disp_color_funcs); if (ret) { - dev_err(dev, "Failed to initialize component: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to initialize component: %d\n", + ret); + return ret; } diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c index 891d80c73e04..28651bc579bc 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c @@ -386,7 +386,10 @@ static int mtk_disp_ovl_probe(struct platform_device *pdev) ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id, &mtk_disp_ovl_funcs); if (ret) { - dev_err(dev, "Failed to initialize component: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to initialize component: %d\n", + ret); + return ret; } diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c index 0cb848d64206..e04319fedf46 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c @@ -294,7 +294,10 @@ static int mtk_disp_rdma_probe(struct platform_device *pdev) ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id, &mtk_disp_rdma_funcs); if (ret) { - dev_err(dev, "Failed to initialize component: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to initialize component: %d\n", + ret); + return ret; } diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c index 945c3ac92998..d4f0fb7ad312 100644 --- a/drivers/gpu/drm/mediatek/mtk_dpi.c +++ b/drivers/gpu/drm/mediatek/mtk_dpi.c @@ -739,21 +739,27 @@ static int mtk_dpi_probe(struct platform_device *pdev) dpi->engine_clk = devm_clk_get(dev, "engine"); if (IS_ERR(dpi->engine_clk)) { ret = PTR_ERR(dpi->engine_clk); - dev_err(dev, "Failed to get engine clock: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get engine clock: %d\n", ret); + return ret; } dpi->pixel_clk = devm_clk_get(dev, "pixel"); if (IS_ERR(dpi->pixel_clk)) { ret = PTR_ERR(dpi->pixel_clk); - dev_err(dev, "Failed to get pixel clock: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get pixel clock: %d\n", ret); + return ret; } dpi->tvd_clk = devm_clk_get(dev, "pll"); if (IS_ERR(dpi->tvd_clk)) { ret = PTR_ERR(dpi->tvd_clk); - dev_err(dev, "Failed to get tvdpll clock: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get tvdpll clock: %d\n", ret); + return ret; } diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c index fe85e487e477..fe46c4bac64d 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c @@ -6,6 +6,7 @@ #include <linux/clk.h> #include <linux/pm_runtime.h> #include <linux/soc/mediatek/mtk-cmdq.h> +#include <linux/soc/mediatek/mtk-mmsys.h> #include <asm/barrier.h> #include <soc/mediatek/smi.h> @@ -28,7 +29,7 @@ * @enabled: records whether crtc_enable succeeded * @planes: array of 4 drm_plane structures, one for each overlay plane * @pending_planes: whether any plane has pending changes to be applied - * @config_regs: memory mapped mmsys configuration register space + * @mmsys_dev: pointer to the mmsys device for configuration registers * @mutex: handle to one of the ten disp_mutex streams * @ddp_comp_nr: number of components in ddp_comp * @ddp_comp: array of pointers the mtk_ddp_comp structures used by this crtc @@ -50,7 +51,7 @@ struct mtk_drm_crtc { u32 cmdq_event; #endif - void __iomem *config_regs; + struct device *mmsys_dev; struct mtk_disp_mutex *mutex; unsigned int ddp_comp_nr; struct mtk_ddp_comp **ddp_comp; @@ -300,9 +301,9 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc) DRM_DEBUG_DRIVER("mediatek_ddp_ddp_path_setup\n"); for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) { - mtk_ddp_add_comp_to_path(mtk_crtc->config_regs, - mtk_crtc->ddp_comp[i]->id, - mtk_crtc->ddp_comp[i + 1]->id); + mtk_mmsys_ddp_connect(mtk_crtc->mmsys_dev, + mtk_crtc->ddp_comp[i]->id, + mtk_crtc->ddp_comp[i + 1]->id); mtk_disp_mutex_add_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id); } @@ -360,9 +361,9 @@ static void mtk_crtc_ddp_hw_fini(struct mtk_drm_crtc *mtk_crtc) mtk_crtc->ddp_comp[i]->id); mtk_disp_mutex_disable(mtk_crtc->mutex); for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) { - mtk_ddp_remove_comp_from_path(mtk_crtc->config_regs, - mtk_crtc->ddp_comp[i]->id, - mtk_crtc->ddp_comp[i + 1]->id); + mtk_mmsys_ddp_disconnect(mtk_crtc->mmsys_dev, + mtk_crtc->ddp_comp[i]->id, + mtk_crtc->ddp_comp[i + 1]->id); mtk_disp_mutex_remove_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id); } @@ -766,7 +767,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev, if (!mtk_crtc) return -ENOMEM; - mtk_crtc->config_regs = priv->config_regs; + mtk_crtc->mmsys_dev = priv->mmsys_dev; mtk_crtc->ddp_comp_nr = path_len; mtk_crtc->ddp_comp = devm_kmalloc_array(dev, mtk_crtc->ddp_comp_nr, sizeof(*mtk_crtc->ddp_comp), diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c index 13035c906035..014c1bbe1df2 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c @@ -13,26 +13,6 @@ #include "mtk_drm_ddp.h" #include "mtk_drm_ddp_comp.h" -#define DISP_REG_CONFIG_DISP_OVL0_MOUT_EN 0x040 -#define DISP_REG_CONFIG_DISP_OVL1_MOUT_EN 0x044 -#define DISP_REG_CONFIG_DISP_OD_MOUT_EN 0x048 -#define DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN 0x04c -#define DISP_REG_CONFIG_DISP_UFOE_MOUT_EN 0x050 -#define DISP_REG_CONFIG_DISP_COLOR0_SEL_IN 0x084 -#define DISP_REG_CONFIG_DISP_COLOR1_SEL_IN 0x088 -#define DISP_REG_CONFIG_DSIE_SEL_IN 0x0a4 -#define DISP_REG_CONFIG_DSIO_SEL_IN 0x0a8 -#define DISP_REG_CONFIG_DPI_SEL_IN 0x0ac -#define DISP_REG_CONFIG_DISP_RDMA2_SOUT 0x0b8 -#define DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN 0x0c4 -#define DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN 0x0c8 -#define DISP_REG_CONFIG_MMSYS_CG_CON0 0x100 - -#define DISP_REG_CONFIG_DISP_OVL_MOUT_EN 0x030 -#define DISP_REG_CONFIG_OUT_SEL 0x04c -#define DISP_REG_CONFIG_DSI_SEL 0x050 -#define DISP_REG_CONFIG_DPI_SEL 0x064 - #define MT2701_DISP_MUTEX0_MOD0 0x2c #define MT2701_DISP_MUTEX0_SOF0 0x30 @@ -94,48 +74,6 @@ #define MUTEX_SOF_DSI2 5 #define MUTEX_SOF_DSI3 6 -#define OVL0_MOUT_EN_COLOR0 0x1 -#define OD_MOUT_EN_RDMA0 0x1 -#define OD1_MOUT_EN_RDMA1 BIT(16) -#define UFOE_MOUT_EN_DSI0 0x1 -#define COLOR0_SEL_IN_OVL0 0x1 -#define OVL1_MOUT_EN_COLOR1 0x1 -#define GAMMA_MOUT_EN_RDMA1 0x1 -#define RDMA0_SOUT_DPI0 0x2 -#define RDMA0_SOUT_DPI1 0x3 -#define RDMA0_SOUT_DSI1 0x1 -#define RDMA0_SOUT_DSI2 0x4 -#define RDMA0_SOUT_DSI3 0x5 -#define RDMA1_SOUT_DPI0 0x2 -#define RDMA1_SOUT_DPI1 0x3 -#define RDMA1_SOUT_DSI1 0x1 -#define RDMA1_SOUT_DSI2 0x4 -#define RDMA1_SOUT_DSI3 0x5 -#define RDMA2_SOUT_DPI0 0x2 -#define RDMA2_SOUT_DPI1 0x3 -#define RDMA2_SOUT_DSI1 0x1 -#define RDMA2_SOUT_DSI2 0x4 -#define RDMA2_SOUT_DSI3 0x5 -#define DPI0_SEL_IN_RDMA1 0x1 -#define DPI0_SEL_IN_RDMA2 0x3 -#define DPI1_SEL_IN_RDMA1 (0x1 << 8) -#define DPI1_SEL_IN_RDMA2 (0x3 << 8) -#define DSI0_SEL_IN_RDMA1 0x1 -#define DSI0_SEL_IN_RDMA2 0x4 -#define DSI1_SEL_IN_RDMA1 0x1 -#define DSI1_SEL_IN_RDMA2 0x4 -#define DSI2_SEL_IN_RDMA1 (0x1 << 16) -#define DSI2_SEL_IN_RDMA2 (0x4 << 16) -#define DSI3_SEL_IN_RDMA1 (0x1 << 16) -#define DSI3_SEL_IN_RDMA2 (0x4 << 16) -#define COLOR1_SEL_IN_OVL1 0x1 - -#define OVL_MOUT_EN_RDMA 0x1 -#define BLS_TO_DSI_RDMA1_TO_DPI1 0x8 -#define BLS_TO_DPI_RDMA1_TO_DSI 0x2 -#define DSI_SEL_IN_BLS 0x0 -#define DPI_SEL_IN_BLS 0x0 -#define DSI_SEL_IN_RDMA 0x1 struct mtk_disp_mutex { int id; @@ -246,200 +184,6 @@ static const struct mtk_ddp_data mt8173_ddp_driver_data = { .mutex_sof_reg = MT2701_DISP_MUTEX0_SOF0, }; -static unsigned int mtk_ddp_mout_en(enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next, - unsigned int *addr) -{ - unsigned int value; - - if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) { - *addr = DISP_REG_CONFIG_DISP_OVL0_MOUT_EN; - value = OVL0_MOUT_EN_COLOR0; - } else if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_RDMA0) { - *addr = DISP_REG_CONFIG_DISP_OVL_MOUT_EN; - value = OVL_MOUT_EN_RDMA; - } else if (cur == DDP_COMPONENT_OD0 && next == DDP_COMPONENT_RDMA0) { - *addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN; - value = OD_MOUT_EN_RDMA0; - } else if (cur == DDP_COMPONENT_UFOE && next == DDP_COMPONENT_DSI0) { - *addr = DISP_REG_CONFIG_DISP_UFOE_MOUT_EN; - value = UFOE_MOUT_EN_DSI0; - } else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) { - *addr = DISP_REG_CONFIG_DISP_OVL1_MOUT_EN; - value = OVL1_MOUT_EN_COLOR1; - } else if (cur == DDP_COMPONENT_GAMMA && next == DDP_COMPONENT_RDMA1) { - *addr = DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN; - value = GAMMA_MOUT_EN_RDMA1; - } else if (cur == DDP_COMPONENT_OD1 && next == DDP_COMPONENT_RDMA1) { - *addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN; - value = OD1_MOUT_EN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI0) { - *addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN; - value = RDMA0_SOUT_DPI0; - } else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN; - value = RDMA0_SOUT_DPI1; - } else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN; - value = RDMA0_SOUT_DSI1; - } else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI2) { - *addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN; - value = RDMA0_SOUT_DSI2; - } else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI3) { - *addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN; - value = RDMA0_SOUT_DSI3; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN; - value = RDMA1_SOUT_DSI1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) { - *addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN; - value = RDMA1_SOUT_DSI2; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) { - *addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN; - value = RDMA1_SOUT_DSI3; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) { - *addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN; - value = RDMA1_SOUT_DPI0; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN; - value = RDMA1_SOUT_DPI1; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) { - *addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT; - value = RDMA2_SOUT_DPI0; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT; - value = RDMA2_SOUT_DPI1; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) { - *addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT; - value = RDMA2_SOUT_DSI1; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) { - *addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT; - value = RDMA2_SOUT_DSI2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) { - *addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT; - value = RDMA2_SOUT_DSI3; - } else { - value = 0; - } - - return value; -} - -static unsigned int mtk_ddp_sel_in(enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next, - unsigned int *addr) -{ - unsigned int value; - - if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) { - *addr = DISP_REG_CONFIG_DISP_COLOR0_SEL_IN; - value = COLOR0_SEL_IN_OVL0; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) { - *addr = DISP_REG_CONFIG_DPI_SEL_IN; - value = DPI0_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) { - *addr = DISP_REG_CONFIG_DPI_SEL_IN; - value = DPI1_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI0) { - *addr = DISP_REG_CONFIG_DSIE_SEL_IN; - value = DSI0_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) { - *addr = DISP_REG_CONFIG_DSIO_SEL_IN; - value = DSI1_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) { - *addr = DISP_REG_CONFIG_DSIE_SEL_IN; - value = DSI2_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) { - *addr = DISP_REG_CONFIG_DSIO_SEL_IN; - value = DSI3_SEL_IN_RDMA1; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) { - *addr = DISP_REG_CONFIG_DPI_SEL_IN; - value = DPI0_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) { - *addr = DISP_REG_CONFIG_DPI_SEL_IN; - value = DPI1_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI0) { - *addr = DISP_REG_CONFIG_DSIE_SEL_IN; - value = DSI0_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) { - *addr = DISP_REG_CONFIG_DSIO_SEL_IN; - value = DSI1_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) { - *addr = DISP_REG_CONFIG_DSIE_SEL_IN; - value = DSI2_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) { - *addr = DISP_REG_CONFIG_DSIE_SEL_IN; - value = DSI3_SEL_IN_RDMA2; - } else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) { - *addr = DISP_REG_CONFIG_DISP_COLOR1_SEL_IN; - value = COLOR1_SEL_IN_OVL1; - } else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) { - *addr = DISP_REG_CONFIG_DSI_SEL; - value = DSI_SEL_IN_BLS; - } else { - value = 0; - } - - return value; -} - -static void mtk_ddp_sout_sel(void __iomem *config_regs, - enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next) -{ - if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) { - writel_relaxed(BLS_TO_DSI_RDMA1_TO_DPI1, - config_regs + DISP_REG_CONFIG_OUT_SEL); - } else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DPI0) { - writel_relaxed(BLS_TO_DPI_RDMA1_TO_DSI, - config_regs + DISP_REG_CONFIG_OUT_SEL); - writel_relaxed(DSI_SEL_IN_RDMA, - config_regs + DISP_REG_CONFIG_DSI_SEL); - writel_relaxed(DPI_SEL_IN_BLS, - config_regs + DISP_REG_CONFIG_DPI_SEL); - } -} - -void mtk_ddp_add_comp_to_path(void __iomem *config_regs, - enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next) -{ - unsigned int addr, value, reg; - - value = mtk_ddp_mout_en(cur, next, &addr); - if (value) { - reg = readl_relaxed(config_regs + addr) | value; - writel_relaxed(reg, config_regs + addr); - } - - mtk_ddp_sout_sel(config_regs, cur, next); - - value = mtk_ddp_sel_in(cur, next, &addr); - if (value) { - reg = readl_relaxed(config_regs + addr) | value; - writel_relaxed(reg, config_regs + addr); - } -} - -void mtk_ddp_remove_comp_from_path(void __iomem *config_regs, - enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next) -{ - unsigned int addr, value, reg; - - value = mtk_ddp_mout_en(cur, next, &addr); - if (value) { - reg = readl_relaxed(config_regs + addr) & ~value; - writel_relaxed(reg, config_regs + addr); - } - - value = mtk_ddp_sel_in(cur, next, &addr); - if (value) { - reg = readl_relaxed(config_regs + addr) & ~value; - writel_relaxed(reg, config_regs + addr); - } -} - struct mtk_disp_mutex *mtk_disp_mutex_get(struct device *dev, unsigned int id) { struct mtk_ddp *ddp = dev_get_drvdata(dev); @@ -628,7 +372,8 @@ static int mtk_ddp_probe(struct platform_device *pdev) if (!ddp->data->no_clk) { ddp->clk = devm_clk_get(dev, NULL); if (IS_ERR(ddp->clk)) { - dev_err(dev, "Failed to get clock\n"); + if (PTR_ERR(ddp->clk) != -EPROBE_DEFER) + dev_err(dev, "Failed to get clock\n"); return PTR_ERR(ddp->clk); } } diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp.h b/drivers/gpu/drm/mediatek/mtk_drm_ddp.h index 827be424a148..6b691a57be4a 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_ddp.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp.h @@ -12,13 +12,6 @@ struct regmap; struct device; struct mtk_disp_mutex; -void mtk_ddp_add_comp_to_path(void __iomem *config_regs, - enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next); -void mtk_ddp_remove_comp_from_path(void __iomem *config_regs, - enum mtk_ddp_comp_id cur, - enum mtk_ddp_comp_id next); - struct mtk_disp_mutex *mtk_disp_mutex_get(struct device *dev, unsigned int id); int mtk_disp_mutex_prepare(struct mtk_disp_mutex *mutex); void mtk_disp_mutex_add_comp(struct mtk_disp_mutex *mutex, diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index ce570283b55f..6bd369434d9d 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -10,6 +10,7 @@ #include <linux/of_address.h> #include <linux/of_platform.h> #include <linux/pm_runtime.h> +#include <linux/soc/mediatek/mtk-mmsys.h> #include <linux/dma-mapping.h> #include <drm/drm_atomic.h> @@ -418,11 +419,22 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = { { } }; +static const struct of_device_id mtk_drm_of_ids[] = { + { .compatible = "mediatek,mt2701-mmsys", + .data = &mt2701_mmsys_driver_data}, + { .compatible = "mediatek,mt2712-mmsys", + .data = &mt2712_mmsys_driver_data}, + { .compatible = "mediatek,mt8173-mmsys", + .data = &mt8173_mmsys_driver_data}, + { } +}; + static int mtk_drm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *phandle = dev->parent->of_node; + const struct of_device_id *of_id; struct mtk_drm_private *private; - struct resource *mem; struct device_node *node; struct component_match *match = NULL; int ret; @@ -433,18 +445,20 @@ static int mtk_drm_probe(struct platform_device *pdev) return -ENOMEM; private->data = of_device_get_match_data(dev); - - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - private->config_regs = devm_ioremap_resource(dev, mem); - if (IS_ERR(private->config_regs)) { - ret = PTR_ERR(private->config_regs); - dev_err(dev, "Failed to ioremap mmsys-config resource: %d\n", - ret); - return ret; + private->mmsys_dev = dev->parent; + if (!private->mmsys_dev) { + dev_err(dev, "Failed to get MMSYS device\n"); + return -ENODEV; } + of_id = of_match_node(mtk_drm_of_ids, phandle); + if (!of_id) + return -ENODEV; + + private->data = of_id->data; + /* Iterate over sibling DISP function blocks */ - for_each_child_of_node(dev->of_node->parent, node) { + for_each_child_of_node(phandle->parent, node) { const struct of_device_id *of_id; enum mtk_ddp_comp_type comp_type; int comp_id; @@ -578,22 +592,11 @@ static int mtk_drm_sys_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(mtk_drm_pm_ops, mtk_drm_sys_suspend, mtk_drm_sys_resume); -static const struct of_device_id mtk_drm_of_ids[] = { - { .compatible = "mediatek,mt2701-mmsys", - .data = &mt2701_mmsys_driver_data}, - { .compatible = "mediatek,mt2712-mmsys", - .data = &mt2712_mmsys_driver_data}, - { .compatible = "mediatek,mt8173-mmsys", - .data = &mt8173_mmsys_driver_data}, - { } -}; - static struct platform_driver mtk_drm_platform_driver = { .probe = mtk_drm_probe, .remove = mtk_drm_remove, .driver = { .name = "mediatek-drm", - .of_match_table = mtk_drm_of_ids, .pm = &mtk_drm_pm_ops, }, }; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h index 17bc99b9f5d4..b5be63e53176 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h @@ -39,7 +39,7 @@ struct mtk_drm_private { struct device_node *mutex_node; struct device *mutex_dev; - void __iomem *config_regs; + struct device *mmsys_dev; struct device_node *comp_node[DDP_COMPONENT_ID_MAX]; struct mtk_ddp_comp *ddp_comp[DDP_COMPONENT_ID_MAX]; const struct mtk_mmsys_driver_data *data; diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c index a9a25087112f..270bf22c98fe 100644 --- a/drivers/gpu/drm/mediatek/mtk_dsi.c +++ b/drivers/gpu/drm/mediatek/mtk_dsi.c @@ -1186,14 +1186,18 @@ static int mtk_dsi_probe(struct platform_device *pdev) dsi->engine_clk = devm_clk_get(dev, "engine"); if (IS_ERR(dsi->engine_clk)) { ret = PTR_ERR(dsi->engine_clk); - dev_err(dev, "Failed to get engine clock: %d\n", ret); + + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get engine clock: %d\n", ret); goto err_unregister_host; } dsi->digital_clk = devm_clk_get(dev, "digital"); if (IS_ERR(dsi->digital_clk)) { ret = PTR_ERR(dsi->digital_clk); - dev_err(dev, "Failed to get digital clock: %d\n", ret); + + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get digital clock: %d\n", ret); goto err_unregister_host; } diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c index 7bc086ec74f7..5feb760617cb 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c @@ -1470,7 +1470,9 @@ static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi, ret = mtk_hdmi_get_all_clk(hdmi, np); if (ret) { - dev_err(dev, "Failed to get clocks: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get clocks: %d\n", ret); + return ret; } diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile index 1579cf0d828f..42f8aae28b31 100644 --- a/drivers/gpu/drm/msm/Makefile +++ b/drivers/gpu/drm/msm/Makefile @@ -65,6 +65,7 @@ msm-y := \ disp/dpu1/dpu_hw_lm.o \ disp/dpu1/dpu_hw_pingpong.o \ disp/dpu1/dpu_hw_sspp.o \ + disp/dpu1/dpu_hw_dspp.o \ disp/dpu1/dpu_hw_top.o \ disp/dpu1/dpu_hw_util.o \ disp/dpu1/dpu_hw_vbif.o \ diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index 1f83bc18d500..60f6472a3e58 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -401,6 +401,21 @@ static struct msm_gpu_state *a2xx_gpu_state_get(struct msm_gpu *gpu) return state; } +static struct msm_gem_address_space * +a2xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev) +{ + struct msm_mmu *mmu = msm_gpummu_new(&pdev->dev, gpu); + struct msm_gem_address_space *aspace; + + aspace = msm_gem_address_space_create(mmu, "gpu", SZ_16M, + SZ_16M + 0xfff * SZ_64K); + + if (IS_ERR(aspace) && !IS_ERR(mmu)) + mmu->funcs->destroy(mmu); + + return aspace; +} + /* Register offset defines for A2XX - copy of A3XX */ static const unsigned int a2xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE), @@ -429,6 +444,7 @@ static const struct adreno_gpu_funcs funcs = { #endif .gpu_state_get = a2xx_gpu_state_get, .gpu_state_put = adreno_gpu_state_put, + .create_address_space = a2xx_create_address_space, }, }; diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c index b67f88872726..0a5ea9f56cb8 100644 --- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c @@ -441,6 +441,7 @@ static const struct adreno_gpu_funcs funcs = { #endif .gpu_state_get = a3xx_gpu_state_get, .gpu_state_put = adreno_gpu_state_put, + .create_address_space = adreno_iommu_create_address_space, }, }; diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c index 253d8d85daad..b9b26b2bf9c5 100644 --- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c @@ -66,19 +66,22 @@ static void a4xx_enable_hwcg(struct msm_gpu *gpu) } } - for (i = 0; i < 4; i++) { - gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU(i), - 0x00000922); - } + /* No CCU for A405 */ + if (!adreno_is_a405(adreno_gpu)) { + for (i = 0; i < 4; i++) { + gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU(i), + 0x00000922); + } - for (i = 0; i < 4; i++) { - gpu_write(gpu, REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU(i), - 0x00000000); - } + for (i = 0; i < 4; i++) { + gpu_write(gpu, REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU(i), + 0x00000000); + } - for (i = 0; i < 4; i++) { - gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1(i), - 0x00000001); + for (i = 0; i < 4; i++) { + gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1(i), + 0x00000001); + } } gpu_write(gpu, REG_A4XX_RBBM_CLOCK_MODE_GPC, 0x02222222); @@ -137,7 +140,9 @@ static int a4xx_hw_init(struct msm_gpu *gpu) uint32_t *ptr, len; int i, ret; - if (adreno_is_a420(adreno_gpu)) { + if (adreno_is_a405(adreno_gpu)) { + gpu_write(gpu, REG_A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); + } else if (adreno_is_a420(adreno_gpu)) { gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT, 0x0001001F); gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT_CONF, 0x000000A4); gpu_write(gpu, REG_A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001); @@ -440,6 +445,52 @@ static const unsigned int a4xx_registers[] = { ~0 /* sentinel */ }; +static const unsigned int a405_registers[] = { + /* RBBM */ + 0x0000, 0x0002, 0x0004, 0x0021, 0x0023, 0x0024, 0x0026, 0x0026, + 0x0028, 0x002B, 0x002E, 0x0034, 0x0037, 0x0044, 0x0047, 0x0066, + 0x0068, 0x0095, 0x009C, 0x0170, 0x0174, 0x01AF, + /* CP */ + 0x0200, 0x0233, 0x0240, 0x0250, 0x04C0, 0x04DD, 0x0500, 0x050B, + 0x0578, 0x058F, + /* VSC */ + 0x0C00, 0x0C03, 0x0C08, 0x0C41, 0x0C50, 0x0C51, + /* GRAS */ + 0x0C80, 0x0C81, 0x0C88, 0x0C8F, + /* RB */ + 0x0CC0, 0x0CC0, 0x0CC4, 0x0CD2, + /* PC */ + 0x0D00, 0x0D0C, 0x0D10, 0x0D17, 0x0D20, 0x0D23, + /* VFD */ + 0x0E40, 0x0E4A, + /* VPC */ + 0x0E60, 0x0E61, 0x0E63, 0x0E68, + /* UCHE */ + 0x0E80, 0x0E84, 0x0E88, 0x0E95, + /* GRAS CTX 0 */ + 0x2000, 0x2004, 0x2008, 0x2067, 0x2070, 0x2078, 0x207B, 0x216E, + /* PC CTX 0 */ + 0x21C0, 0x21C6, 0x21D0, 0x21D0, 0x21D9, 0x21D9, 0x21E5, 0x21E7, + /* VFD CTX 0 */ + 0x2200, 0x2204, 0x2208, 0x22A9, + /* GRAS CTX 1 */ + 0x2400, 0x2404, 0x2408, 0x2467, 0x2470, 0x2478, 0x247B, 0x256E, + /* PC CTX 1 */ + 0x25C0, 0x25C6, 0x25D0, 0x25D0, 0x25D9, 0x25D9, 0x25E5, 0x25E7, + /* VFD CTX 1 */ + 0x2600, 0x2604, 0x2608, 0x26A9, + /* VBIF version 0x20050000*/ + 0x3000, 0x3007, 0x302C, 0x302C, 0x3030, 0x3030, 0x3034, 0x3036, + 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, 0x3049, 0x3049, + 0x3058, 0x3058, 0x305B, 0x3061, 0x3064, 0x3068, 0x306C, 0x306D, + 0x3080, 0x3088, 0x308B, 0x308C, 0x3090, 0x3094, 0x3098, 0x3098, + 0x309C, 0x309C, 0x30C0, 0x30C0, 0x30C8, 0x30C8, 0x30D0, 0x30D0, + 0x30D8, 0x30D8, 0x30E0, 0x30E0, 0x3100, 0x3100, 0x3108, 0x3108, + 0x3110, 0x3110, 0x3118, 0x3118, 0x3120, 0x3120, 0x3124, 0x3125, + 0x3129, 0x3129, 0x340C, 0x340C, 0x3410, 0x3410, + ~0 /* sentinel */ +}; + static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu) { struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL); @@ -532,6 +583,7 @@ static const struct adreno_gpu_funcs funcs = { #endif .gpu_state_get = a4xx_gpu_state_get, .gpu_state_put = adreno_gpu_state_put, + .create_address_space = adreno_iommu_create_address_space, }, .get_timestamp = a4xx_get_timestamp, }; @@ -563,13 +615,14 @@ struct msm_gpu *a4xx_gpu_init(struct drm_device *dev) gpu->perfcntrs = NULL; gpu->num_perfcntrs = 0; - adreno_gpu->registers = a4xx_registers; - adreno_gpu->reg_offsets = a4xx_register_offsets; - ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); if (ret) goto fail; + adreno_gpu->registers = adreno_is_a405(adreno_gpu) ? a405_registers : + a4xx_registers; + adreno_gpu->reg_offsets = a4xx_register_offsets; + /* if needed, allocate gmem: */ if (adreno_is_a4xx(adreno_gpu)) { ret = adreno_gpu_ocmem_init(dev->dev, adreno_gpu, diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index 724024a2243a..d95970a73fb4 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -1404,6 +1404,10 @@ static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu) { u64 busy_cycles, busy_time; + /* Only read the gpu busy if the hardware is already active */ + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) == 0) + return 0; + busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO, REG_A5XX_RBBM_PERFCTR_RBBM_0_HI); @@ -1412,6 +1416,8 @@ static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu) gpu->devfreq.busy_cycles = busy_cycles; + pm_runtime_put(&gpu->pdev->dev); + if (WARN_ON(busy_time > ~0LU)) return ~0LU; @@ -1439,6 +1445,7 @@ static const struct adreno_gpu_funcs funcs = { .gpu_busy = a5xx_gpu_busy, .gpu_state_get = a5xx_gpu_state_get, .gpu_state_put = a5xx_gpu_state_put, + .create_address_space = adreno_iommu_create_address_space, }, .get_timestamp = a5xx_get_timestamp, }; diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h index ed78fee2a262..47840b73cdda 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h @@ -1047,6 +1047,8 @@ enum a6xx_tex_type { #define REG_A6XX_CP_MISC_CNTL 0x00000840 +#define REG_A6XX_CP_APRIV_CNTL 0x00000844 + #define REG_A6XX_CP_ROQ_THRESHOLDS_1 0x000008c1 #define REG_A6XX_CP_ROQ_THRESHOLDS_2 0x000008c2 @@ -1764,6 +1766,8 @@ static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val) #define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL 0x00000010 +#define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00000011 + #define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL 0x0000001f #define REG_A6XX_RBBM_INT_CLEAR_CMD 0x00000037 @@ -2418,6 +2422,16 @@ static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val) #define REG_A6XX_TPL1_NC_MODE_CNTL 0x0000b604 +#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0 0x0000b608 + +#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1 0x0000b609 + +#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2 0x0000b60a + +#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3 0x0000b60b + +#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4 0x0000b60c + #define REG_A6XX_TPL1_PERFCTR_TP_SEL_0 0x0000b610 #define REG_A6XX_TPL1_PERFCTR_TP_SEL_1 0x0000b611 diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index c4e71abbdd53..096be97ce9f9 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -2,14 +2,16 @@ /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */ #include <linux/clk.h> -#include <linux/dma-mapping.h> #include <linux/interconnect.h> #include <linux/pm_domain.h> #include <linux/pm_opp.h> #include <soc/qcom/cmd-db.h> +#include <drm/drm_gem.h> #include "a6xx_gpu.h" #include "a6xx_gmu.xml.h" +#include "msm_gem.h" +#include "msm_mmu.h" static void a6xx_gmu_fault(struct a6xx_gmu *gmu) { @@ -127,8 +129,6 @@ static void __a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index) if (ret) dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret); - gmu->freq = gmu->gpu_freqs[index]; - /* * Eventually we will want to scale the path vote with the frequency but * for now leave it at max so that the performance is nominal. @@ -151,8 +151,21 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, unsigned long freq) break; gmu->current_perf_index = perf_index; + gmu->freq = gmu->gpu_freqs[perf_index]; + + /* + * This can get called from devfreq while the hardware is idle. Don't + * bring up the power if it isn't already active + */ + if (pm_runtime_get_if_in_use(gmu->dev) == 0) + return; - __a6xx_gmu_set_freq(gmu, perf_index); + if (gmu->legacy) + __a6xx_gmu_set_freq(gmu, perf_index); + else + a6xx_hfi_set_freq(gmu, perf_index); + + pm_runtime_put(gmu->dev); } unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu) @@ -196,6 +209,12 @@ static int a6xx_gmu_start(struct a6xx_gmu *gmu) u32 val; gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + + /* Set the log wptr index + * note: downstream saves the value in poweroff and restores it here + */ + gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0); + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0); ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val, @@ -232,8 +251,13 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) switch (state) { case GMU_OOB_GPU_SET: - request = GMU_OOB_GPU_SET_REQUEST; - ack = GMU_OOB_GPU_SET_ACK; + if (gmu->legacy) { + request = GMU_OOB_GPU_SET_REQUEST; + ack = GMU_OOB_GPU_SET_ACK; + } else { + request = GMU_OOB_GPU_SET_REQUEST_NEW; + ack = GMU_OOB_GPU_SET_ACK_NEW; + } name = "GPU_SET"; break; case GMU_OOB_BOOT_SLUMBER: @@ -272,6 +296,13 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) /* Clear a pending OOB state in the GMU */ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) { + if (!gmu->legacy) { + WARN_ON(state != GMU_OOB_GPU_SET); + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, + 1 << GMU_OOB_GPU_SET_CLEAR_NEW); + return; + } + switch (state) { case GMU_OOB_GPU_SET: gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, @@ -294,6 +325,9 @@ static int a6xx_sptprac_enable(struct a6xx_gmu *gmu) int ret; u32 val; + if (!gmu->legacy) + return 0; + gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000); ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, @@ -313,6 +347,9 @@ static void a6xx_sptprac_disable(struct a6xx_gmu *gmu) u32 val; int ret; + if (!gmu->legacy) + return; + /* Make sure retention is on */ gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11)); @@ -356,6 +393,11 @@ static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) if (gmu->idle_level < GMU_IDLE_STATE_SPTP) a6xx_sptprac_disable(gmu); + if (!gmu->legacy) { + ret = a6xx_hfi_send_prep_slumber(gmu); + goto out; + } + /* Tell the GMU to get ready to slumber */ gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1); @@ -371,6 +413,7 @@ static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) } } +out: /* Put fence into allow mode */ gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); return ret; @@ -392,7 +435,7 @@ static int a6xx_rpmh_start(struct a6xx_gmu *gmu) return ret; } - ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val, + ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val, !val, 100, 10000); if (ret) { @@ -418,7 +461,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); - ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, + ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, val, val & (1 << 16), 100, 10000); if (ret) DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n"); @@ -441,32 +484,48 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) struct platform_device *pdev = to_platform_device(gmu->dev); void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc"); void __iomem *seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq"); + uint32_t pdc_address_offset; if (!pdcptr || !seqptr) goto err; + if (adreno_is_a618(adreno_gpu) || adreno_is_a640(adreno_gpu)) + pdc_address_offset = 0x30090; + else if (adreno_is_a650(adreno_gpu)) + pdc_address_offset = 0x300a0; + else + pdc_address_offset = 0x30080; + /* Disable SDE clock gating */ - gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24)); + gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24)); /* Setup RSC PDC handshake for sleep and wakeup */ - gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000); - gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0); - gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0); - gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520); - gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510); - gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514); + gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000); + gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520); + gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510); + gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514); /* Load RSC sequencer uCode for sleep and wakeup */ - gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0); - gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7); - gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1); - gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2); - gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8); + if (adreno_is_a650(adreno_gpu)) { + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xeaaae5a0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xe1a1ebab); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e0a581); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xecac82e2); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020edad); + } else { + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2); + gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8); + } /* Load PDC sequencer uCode for power up and power down sequence */ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1); @@ -487,10 +546,7 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0); pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108); - if (adreno_is_a618(adreno_gpu)) - pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30090); - else - pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080); + pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset); pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0); pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7); @@ -502,17 +558,12 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108); pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000); - if (adreno_is_a618(adreno_gpu)) + if (adreno_is_a618(adreno_gpu) || adreno_is_a650(adreno_gpu)) pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2); else pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3); - - pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108); - if (adreno_is_a618(adreno_gpu)) - pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30090); - else - pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080); + pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset); pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3); /* Setup GPU PDC */ @@ -542,6 +593,8 @@ static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) { /* Disable GMU WB/RB buffer */ gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1); + gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1); + gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1); gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400); @@ -571,14 +624,95 @@ static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE); } +struct block_header { + u32 addr; + u32 size; + u32 type; + u32 value; + u32 data[]; +}; + +/* this should be a general kernel helper */ +static int in_range(u32 addr, u32 start, u32 size) +{ + return addr >= start && addr < start + size; +} + +static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk) +{ + if (!in_range(blk->addr, bo->iova, bo->size)) + return false; + + memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size); + return true; +} + +static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu) +{ + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU]; + const struct block_header *blk; + u32 reg_offset; + + u32 itcm_base = 0x00000000; + u32 dtcm_base = 0x00040000; + + if (adreno_is_a650(adreno_gpu)) + dtcm_base = 0x10004000; + + if (gmu->legacy) { + /* Sanity check the size of the firmware that was loaded */ + if (fw_image->size > 0x8000) { + DRM_DEV_ERROR(gmu->dev, + "GMU firmware is bigger than the available region\n"); + return -EINVAL; + } + + gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START, + (u32*) fw_image->data, fw_image->size); + return 0; + } + + + for (blk = (const struct block_header *) fw_image->data; + (const u8*) blk < fw_image->data + fw_image->size; + blk = (const struct block_header *) &blk->data[blk->size >> 2]) { + if (blk->size == 0) + continue; + + if (in_range(blk->addr, itcm_base, SZ_16K)) { + reg_offset = (blk->addr - itcm_base) >> 2; + gmu_write_bulk(gmu, + REG_A6XX_GMU_CM3_ITCM_START + reg_offset, + blk->data, blk->size); + } else if (in_range(blk->addr, dtcm_base, SZ_16K)) { + reg_offset = (blk->addr - dtcm_base) >> 2; + gmu_write_bulk(gmu, + REG_A6XX_GMU_CM3_DTCM_START + reg_offset, + blk->data, blk->size); + } else if (!fw_block_mem(&gmu->icache, blk) && + !fw_block_mem(&gmu->dcache, blk) && + !fw_block_mem(&gmu->dummy, blk)) { + DRM_DEV_ERROR(gmu->dev, + "failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n", + blk->addr, blk->size, blk->data[0]); + } + } + + return 0; +} + static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) { static bool rpmh_init; struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; - int i, ret; + int ret; u32 chipid; - u32 *image; + + if (adreno_is_a650(adreno_gpu)) + gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1); if (state == GMU_WARM_BOOT) { ret = a6xx_rpmh_start(gmu); @@ -589,13 +723,6 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) "GMU firmware is not loaded\n")) return -ENOENT; - /* Sanity check the size of the firmware that was loaded */ - if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) { - DRM_DEV_ERROR(gmu->dev, - "GMU firmware is bigger than the available region\n"); - return -EINVAL; - } - /* Turn on register retention */ gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); @@ -609,18 +736,16 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) return ret; } - image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data; - - for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++) - gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i, - image[i]); + ret = a6xx_gmu_fw_load(gmu); + if (ret) + return ret; } gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0); gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02); /* Write the iova of the HFI table */ - gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova); + gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova); gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1); gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0, @@ -633,6 +758,9 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid); + gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG, + gmu->log.iova | (gmu->log.size / SZ_4K - 1)); + /* Set up the lowest idle level on the GMU */ a6xx_gmu_power_config(gmu); @@ -640,9 +768,11 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) if (ret) return ret; - ret = a6xx_gmu_gfx_rail_on(gmu); - if (ret) - return ret; + if (gmu->legacy) { + ret = a6xx_gmu_gfx_rail_on(gmu); + if (ret) + return ret; + } /* Enable SPTP_PC if the CPU is responsible for it */ if (gmu->idle_level < GMU_IDLE_STATE_SPTP) { @@ -683,13 +813,13 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) u32 val; /* Make sure there are no outstanding RPMh votes */ - gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val, + gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val, (val & 1), 100, 10000); - gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val, + gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val, (val & 1), 100, 10000); - gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val, + gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val, (val & 1), 100, 10000); - gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val, + gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val, (val & 1), 100, 1000); } @@ -744,6 +874,13 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ? GMU_WARM_BOOT : GMU_COLD_BOOT; + /* + * Warm boot path does not work on newer GPUs + * Presumably this is because icache/dcache regions must be restored + */ + if (!gmu->legacy) + status = GMU_COLD_BOOT; + ret = a6xx_gmu_fw_start(gmu, status); if (ret) goto out; @@ -761,7 +898,10 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) enable_irq(gmu->hfi_irq); /* Set the GPU to the current freq */ - __a6xx_gmu_set_freq(gmu, gmu->current_perf_index); + if (gmu->legacy) + __a6xx_gmu_set_freq(gmu, gmu->current_perf_index); + else + a6xx_hfi_set_freq(gmu, gmu->current_perf_index); /* * "enable" the GX power domain which won't actually do anything but it @@ -919,34 +1059,75 @@ int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu) return 0; } -static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo) +static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu) { - if (IS_ERR_OR_NULL(bo)) - return; - - dma_free_wc(gmu->dev, bo->size, bo->virt, bo->iova); - kfree(bo); + msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace, false); + msm_gem_kernel_put(gmu->debug.obj, gmu->aspace, false); + msm_gem_kernel_put(gmu->icache.obj, gmu->aspace, false); + msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace, false); + msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace, false); + msm_gem_kernel_put(gmu->log.obj, gmu->aspace, false); + + gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu); + msm_gem_address_space_put(gmu->aspace); } -static struct a6xx_gmu_bo *a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, - size_t size) +static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo, + size_t size, u64 iova) { - struct a6xx_gmu_bo *bo; + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct drm_device *dev = a6xx_gpu->base.base.dev; + uint32_t flags = MSM_BO_WC; + u64 range_start, range_end; + int ret; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - if (!bo) - return ERR_PTR(-ENOMEM); + size = PAGE_ALIGN(size); + if (!iova) { + /* no fixed address - use GMU's uncached range */ + range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */ + range_end = 0x80000000; + } else { + /* range for fixed address */ + range_start = iova; + range_end = iova + size; + /* use IOMMU_PRIV for icache/dcache */ + flags |= MSM_BO_MAP_PRIV; + } - bo->size = PAGE_ALIGN(size); + bo->obj = msm_gem_new(dev, size, flags); + if (IS_ERR(bo->obj)) + return PTR_ERR(bo->obj); - bo->virt = dma_alloc_wc(gmu->dev, bo->size, &bo->iova, GFP_KERNEL); + ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova, + range_start >> PAGE_SHIFT, range_end >> PAGE_SHIFT); + if (ret) { + drm_gem_object_put(bo->obj); + return ret; + } + + bo->virt = msm_gem_get_vaddr(bo->obj); + bo->size = size; + + return 0; +} + +static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu) +{ + struct iommu_domain *domain; + struct msm_mmu *mmu; - if (!bo->virt) { - kfree(bo); - return ERR_PTR(-ENOMEM); + domain = iommu_domain_alloc(&platform_bus_type); + if (!domain) + return -ENODEV; + + mmu = msm_iommu_new(gmu->dev, domain); + gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x7fffffff); + if (IS_ERR(gmu->aspace)) { + iommu_domain_free(domain); + return PTR_ERR(gmu->aspace); } - return bo; + return 0; } /* Return the 'arc-level' for the given frequency */ @@ -1011,8 +1192,8 @@ static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes, if (j == pri_count) { DRM_DEV_ERROR(dev, - "Level %u not found in in the RPMh list\n", - level); + "Level %u not found in the RPMh list\n", + level); DRM_DEV_ERROR(dev, "Available levels:\n"); for (j = 0; j < pri_count; j++) DRM_DEV_ERROR(dev, " %u\n", pri[j]); @@ -1190,6 +1371,7 @@ static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) { struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + struct platform_device *pdev = to_platform_device(gmu->dev); if (!gmu->initialized) return; @@ -1202,9 +1384,12 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) } iounmap(gmu->mmio); + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) + iounmap(gmu->rscc); gmu->mmio = NULL; + gmu->rscc = NULL; - a6xx_gmu_memory_free(gmu, gmu->hfi); + a6xx_gmu_memory_free(gmu); free_irq(gmu->gmu_irq, gmu); free_irq(gmu->hfi_irq, gmu); @@ -1217,6 +1402,7 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) { + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; struct platform_device *pdev = of_find_device_by_node(node); int ret; @@ -1226,15 +1412,7 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) gmu->dev = &pdev->dev; - /* Pass force_dma false to require the DT to set the dma region */ - ret = of_dma_configure(gmu->dev, node, false); - if (ret) - return ret; - - /* Set the mask after the of_dma_configure() */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(31)); - if (ret) - return ret; + of_dma_configure(gmu->dev, node, true); /* Fow now, don't do anything fancy until we get our feet under us */ gmu->idle_level = GMU_IDLE_STATE_ACTIVE; @@ -1246,20 +1424,64 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) if (ret) goto err_put_device; + ret = a6xx_gmu_memory_probe(gmu); + if (ret) + goto err_put_device; + + /* Allocate memory for the GMU dummy page */ + ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, SZ_4K, 0x60000000); + if (ret) + goto err_memory; + + if (adreno_is_a650(adreno_gpu)) { + ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, + SZ_16M - SZ_16K, 0x04000); + if (ret) + goto err_memory; + } else if (adreno_is_a640(adreno_gpu)) { + ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, + SZ_256K - SZ_16K, 0x04000); + if (ret) + goto err_memory; + + ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache, + SZ_256K - SZ_16K, 0x44000); + if (ret) + goto err_memory; + } else { + /* HFI v1, has sptprac */ + gmu->legacy = true; + + /* Allocate memory for the GMU debug region */ + ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0); + if (ret) + goto err_memory; + } + /* Allocate memory for for the HFI queues */ - gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K); - if (IS_ERR(gmu->hfi)) + ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0); + if (ret) goto err_memory; - /* Allocate memory for the GMU debug region */ - gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K); - if (IS_ERR(gmu->debug)) + /* Allocate memory for the GMU log region */ + ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_4K, 0); + if (ret) goto err_memory; /* Map the GMU registers */ gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); - if (IS_ERR(gmu->mmio)) + if (IS_ERR(gmu->mmio)) { + ret = PTR_ERR(gmu->mmio); goto err_memory; + } + + if (adreno_is_a650(adreno_gpu)) { + gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc"); + if (IS_ERR(gmu->rscc)) + goto err_mmio; + } else { + gmu->rscc = gmu->mmio + 0x23000; + } /* Get the HFI and GMU interrupts */ gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq); @@ -1286,13 +1508,15 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) err_mmio: iounmap(gmu->mmio); + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) + iounmap(gmu->rscc); free_irq(gmu->gmu_irq, gmu); free_irq(gmu->hfi_irq, gmu); -err_memory: - a6xx_gmu_memory_free(gmu, gmu->hfi); ret = -ENODEV; +err_memory: + a6xx_gmu_memory_free(gmu); err_put_device: /* Drop reference taken in of_find_device_by_node */ put_device(gmu->dev); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 4af65a36d5ca..47df4745db50 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -10,9 +10,10 @@ #include "a6xx_hfi.h" struct a6xx_gmu_bo { + struct drm_gem_object *obj; void *virt; size_t size; - dma_addr_t iova; + u64 iova; }; /* @@ -43,7 +44,10 @@ struct a6xx_gmu_bo { struct a6xx_gmu { struct device *dev; + struct msm_gem_address_space *aspace; + void * __iomem mmio; + void * __iomem rscc; int hfi_irq; int gmu_irq; @@ -52,8 +56,12 @@ struct a6xx_gmu { int idle_level; - struct a6xx_gmu_bo *hfi; - struct a6xx_gmu_bo *debug; + struct a6xx_gmu_bo hfi; + struct a6xx_gmu_bo debug; + struct a6xx_gmu_bo icache; + struct a6xx_gmu_bo dcache; + struct a6xx_gmu_bo dummy; + struct a6xx_gmu_bo log; int nr_clocks; struct clk_bulk_data *clocks; @@ -76,6 +84,7 @@ struct a6xx_gmu { bool initialized; bool hung; + bool legacy; /* a618 or a630 */ }; static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset) @@ -88,6 +97,13 @@ static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value) return msm_writel(value, gmu->mmio + (offset << 2)); } +static inline void +gmu_write_bulk(struct a6xx_gmu *gmu, u32 offset, const u32 *data, u32 size) +{ + memcpy_toio(gmu->mmio + (offset << 2), data, size); + wmb(); +} + static inline void gmu_rmw(struct a6xx_gmu *gmu, u32 reg, u32 mask, u32 or) { u32 val = gmu_read(gmu, reg); @@ -111,6 +127,15 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi) readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \ interval, timeout) +static inline void gmu_write_rscc(struct a6xx_gmu *gmu, u32 offset, u32 value) +{ + return msm_writel(value, gmu->rscc + (offset << 2)); +} + +#define gmu_poll_timeout_rscc(gmu, addr, val, cond, interval, timeout) \ + readl_poll_timeout((gmu)->rscc + ((addr) << 2), val, cond, \ + interval, timeout) + /* * These are the available OOB (out of band requests) to the GMU where "out of * band" means that the CPU talks to the GMU directly and not through HFI. @@ -156,10 +181,16 @@ enum a6xx_gmu_oob_state { #define GMU_OOB_GPU_SET_ACK 24 #define GMU_OOB_GPU_SET_CLEAR 24 +#define GMU_OOB_GPU_SET_REQUEST_NEW 30 +#define GMU_OOB_GPU_SET_ACK_NEW 31 +#define GMU_OOB_GPU_SET_CLEAR_NEW 31 + void a6xx_hfi_init(struct a6xx_gmu *gmu); int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state); void a6xx_hfi_stop(struct a6xx_gmu *gmu); +int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu); +int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, int index); bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu); bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h index 1cc1c135236b..176ae94d9fe6 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h @@ -101,6 +101,10 @@ static inline uint32_t A6XX_HFI_IRQ_OOB_MASK(uint32_t val) #define REG_A6XX_GMU_DCVS_RETURN 0x000023ff +#define REG_A6XX_GMU_ICACHE_CONFIG 0x00004c00 + +#define REG_A6XX_GMU_DCACHE_CONFIG 0x00004c01 + #define REG_A6XX_GMU_SYS_BUS_CONFIG 0x00004c0f #define REG_A6XX_GMU_CM3_SYSRESET 0x00005000 @@ -199,6 +203,12 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val) #define REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE 0x000050ec +#define REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF 0x000050f0 + +#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG 0x00005100 + +#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP 0x00005101 + #define REG_A6XX_GMU_BOOT_KMD_LM_HANDSHAKE 0x000051f0 #define REG_A6XX_GMU_LLM_GLM_SLEEP_CTRL 0x00005157 @@ -330,8 +340,6 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val) #define REG_A6XX_GMU_AO_SPARE_CNTL 0x00009316 -#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0 0x00008c04 - #define REG_A6XX_GMU_RSCC_CONTROL_REQ 0x00009307 #define REG_A6XX_GMU_RSCC_CONTROL_ACK 0x00009308 @@ -344,39 +352,41 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val) #define REG_A6XX_GPU_CC_GX_DOMAIN_MISC 0x00009d42 -#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR 0x00008c08 +#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0 0x00000004 + +#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR 0x00000008 -#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO 0x00008c09 +#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO 0x00000009 -#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI 0x00008c0a +#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI 0x0000000a -#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0 0x00008c0b +#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0 0x0000000b -#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR 0x00008c0d +#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR 0x0000000d -#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA 0x00008c0e +#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA 0x0000000e -#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0 0x00008c82 +#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0 0x00000082 -#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0 0x00008c83 +#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0 0x00000083 -#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0 0x00008c89 +#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0 0x00000089 -#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0 0x00008c8c +#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0 0x0000008c -#define REG_A6XX_RSCC_OVERRIDE_START_ADDR 0x00008d00 +#define REG_A6XX_RSCC_OVERRIDE_START_ADDR 0x00000100 -#define REG_A6XX_RSCC_SEQ_BUSY_DRV0 0x00008d01 +#define REG_A6XX_RSCC_SEQ_BUSY_DRV0 0x00000101 -#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0 0x00008d80 +#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0 0x00000180 -#define REG_A6XX_RSCC_TCS0_DRV0_STATUS 0x00008f46 +#define REG_A6XX_RSCC_TCS0_DRV0_STATUS 0x00000346 -#define REG_A6XX_RSCC_TCS1_DRV0_STATUS 0x000090ae +#define REG_A6XX_RSCC_TCS1_DRV0_STATUS 0x000003ee -#define REG_A6XX_RSCC_TCS2_DRV0_STATUS 0x00009216 +#define REG_A6XX_RSCC_TCS2_DRV0_STATUS 0x00000496 -#define REG_A6XX_RSCC_TCS3_DRV0_STATUS 0x0000937e +#define REG_A6XX_RSCC_TCS3_DRV0_STATUS 0x0000053e #endif /* A6XX_GMU_XML */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 68af24150de5..a1589e040c57 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -414,7 +414,17 @@ static int a6xx_hw_init(struct msm_gpu *gpu) a6xx_set_hwcg(gpu, true); /* VBIF/GBIF start*/ - gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3); + if (adreno_is_a640(adreno_gpu) || adreno_is_a650(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620); + gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620); + gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620); + gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620); + gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620); + gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3); + } else { + gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3); + } + if (adreno_is_a630(adreno_gpu)) gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); @@ -429,25 +439,35 @@ static int a6xx_hw_init(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000); gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); - /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ - gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO, - REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000); + if (!adreno_is_a650(adreno_gpu)) { + /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ + gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO, + REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000); - gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO, - REG_A6XX_UCHE_GMEM_RANGE_MAX_HI, - 0x00100000 + adreno_gpu->gmem - 1); + gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO, + REG_A6XX_UCHE_GMEM_RANGE_MAX_HI, + 0x00100000 + adreno_gpu->gmem - 1); + } gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804); gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4); - gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); + if (adreno_is_a640(adreno_gpu) || adreno_is_a650(adreno_gpu)) + gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140); + else + gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); /* Setting the mem pool size */ gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128); /* Setting the primFifo thresholds default values */ - gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11)); + if (adreno_is_a650(adreno_gpu)) + gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300000); + else if (adreno_is_a640(adreno_gpu)) + gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200000); + else + gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11)); /* Set the AHB default slave response to "ERROR" */ gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1); @@ -471,6 +491,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1); + /* Set weights for bicubic filtering */ + if (adreno_is_a650(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0); + gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1, + 0x3fe05ff4); + gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2, + 0x3fa0ebee); + gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3, + 0x3f5193ed); + gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4, + 0x3f0243f0); + } + /* Protect registers from the CP */ gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003); @@ -508,6 +541,11 @@ static int a6xx_hw_init(struct msm_gpu *gpu) A6XX_PROTECT_RDONLY(0x980, 0x4)); gpu_write(gpu, REG_A6XX_CP_PROTECT(25), A6XX_PROTECT_RW(0xa630, 0x0)); + if (adreno_is_a650(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL, + (1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1)); + } + /* Enable interrupts */ gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK); @@ -566,8 +604,10 @@ out: */ a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); - /* Take the GMU out of its special boot mode */ - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); + if (a6xx_gpu->gmu.legacy) { + /* Take the GMU out of its special boot mode */ + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); + } return ret; } @@ -810,6 +850,11 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu) struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); u64 busy_cycles, busy_time; + + /* Only read the gpu busy if the hardware is already active */ + if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0) + return 0; + busy_cycles = gmu_read64(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H); @@ -819,6 +864,8 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu) gpu->devfreq.busy_cycles = busy_cycles; + pm_runtime_put(a6xx_gpu->gmu.dev); + if (WARN_ON(busy_time > ~0LU)) return ~0LU; @@ -846,6 +893,7 @@ static const struct adreno_gpu_funcs funcs = { #if defined(CONFIG_DRM_MSM_GPU_STATE) .gpu_state_get = a6xx_gpu_state_get, .gpu_state_put = a6xx_gpu_state_put, + .create_address_space = adreno_iommu_create_address_space, #endif }, .get_timestamp = a6xx_get_timestamp, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c index e450e0b97211..9921e632f1ca 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c @@ -17,10 +17,14 @@ static const char * const a6xx_hfi_msg_id[] = { HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE), HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE), HFI_MSG_ID(HFI_H2F_MSG_TEST), + HFI_MSG_ID(HFI_H2F_MSG_START), + HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START), + HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE), + HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER), }; -static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data, - u32 dwords) +static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu, + struct a6xx_hfi_queue *queue, u32 *data, u32 dwords) { struct a6xx_hfi_queue_header *header = queue->header; u32 i, hdr, index = header->read_index; @@ -48,6 +52,9 @@ static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data, index = (index + 1) % header->size; } + if (!gmu->legacy) + index = ALIGN(index, 4) % header->size; + header->read_index = index; return HFI_HEADER_SIZE(hdr); } @@ -73,6 +80,12 @@ static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, index = (index + 1) % header->size; } + /* Cookify any non used data at the end of the write buffer */ + if (!gmu->legacy) { + for (; index % 4; index = (index + 1) % header->size) + queue->data[index] = 0xfafafafa; + } + header->write_index = index; spin_unlock(&queue->lock); @@ -106,7 +119,7 @@ static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum, struct a6xx_hfi_msg_response resp; /* Get the next packet */ - ret = a6xx_hfi_queue_read(queue, (u32 *) &resp, + ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp, sizeof(resp) >> 2); /* If the queue is empty our response never made it */ @@ -176,8 +189,8 @@ static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state) { struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 }; - msg.dbg_buffer_addr = (u32) gmu->debug->iova; - msg.dbg_buffer_size = (u32) gmu->debug->size; + msg.dbg_buffer_addr = (u32) gmu->debug.iova; + msg.dbg_buffer_size = (u32) gmu->debug.size; msg.boot_state = boot_state; return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg), @@ -195,6 +208,28 @@ static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version) version, sizeof(*version)); } +static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_perf_table_v1 msg = { 0 }; + int i; + + msg.num_gpu_levels = gmu->nr_gpu_freqs; + msg.num_gmu_levels = gmu->nr_gmu_freqs; + + for (i = 0; i < gmu->nr_gpu_freqs; i++) { + msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; + msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; + } + + for (i = 0; i < gmu->nr_gmu_freqs; i++) { + msg.cx_votes[i].vote = gmu->cx_arc_votes[i]; + msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000; + } + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg), + NULL, 0); +} + static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu) { struct a6xx_hfi_msg_perf_table msg = { 0 }; @@ -205,6 +240,7 @@ static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu) for (i = 0; i < gmu->nr_gpu_freqs; i++) { msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; + msg.gx_votes[i].acd = 0xffffffff; msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; } @@ -306,7 +342,45 @@ static int a6xx_hfi_send_test(struct a6xx_gmu *gmu) NULL, 0); } -int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) +static int a6xx_hfi_send_start(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_start msg = { 0 }; + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg), + NULL, 0); +} + +static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_core_fw_start msg = { 0 }; + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg, + sizeof(msg), NULL, 0); +} + +int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, int index) +{ + struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 }; + + msg.ack_type = 1; /* blocking */ + msg.freq = index; + msg.bw = 0; /* TODO: bus scaling */ + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg, + sizeof(msg), NULL, 0); +} + +int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_prep_slumber_cmd msg = { 0 }; + + /* TODO: should freq and bw fields be non-zero ? */ + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg, + sizeof(msg), NULL, 0); +} + +static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state) { int ret; @@ -324,7 +398,7 @@ int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) * the GMU firmware */ - ret = a6xx_hfi_send_perf_table(gmu); + ret = a6xx_hfi_send_perf_table_v1(gmu); if (ret) return ret; @@ -341,6 +415,37 @@ int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) return 0; } +int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) +{ + int ret; + + if (gmu->legacy) + return a6xx_hfi_start_v1(gmu, boot_state); + + + ret = a6xx_hfi_send_perf_table(gmu); + if (ret) + return ret; + + ret = a6xx_hfi_send_bw_table(gmu); + if (ret) + return ret; + + ret = a6xx_hfi_send_core_fw_start(gmu); + if (ret) + return ret; + + /* + * Downstream driver sends this in its "a6xx_hw_init" equivalent, + * but seems to be no harm in sending it here + */ + ret = a6xx_hfi_send_start(gmu); + if (ret) + return ret; + + return 0; +} + void a6xx_hfi_stop(struct a6xx_gmu *gmu) { int i; @@ -385,7 +490,7 @@ static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue, void a6xx_hfi_init(struct a6xx_gmu *gmu) { - struct a6xx_gmu_bo *hfi = gmu->hfi; + struct a6xx_gmu_bo *hfi = &gmu->hfi; struct a6xx_hfi_queue_table_header *table = hfi->virt; struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table); u64 offset; @@ -415,5 +520,5 @@ void a6xx_hfi_init(struct a6xx_gmu *gmu) /* GMU response queue */ offset += SZ_4K; a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset, - hfi->iova + offset, 4); + hfi->iova + offset, gmu->legacy ? 4 : 1); } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h index 60d1319fa44f..2bd670ca42d6 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h @@ -51,7 +51,8 @@ struct a6xx_hfi_queue { /* HFI message types */ #define HFI_MSG_CMD 0 -#define HFI_MSG_ACK 2 +#define HFI_MSG_ACK 1 +#define HFI_MSG_ACK_V1 2 #define HFI_F2H_MSG_ACK 126 @@ -94,7 +95,13 @@ struct perf_level { u32 freq; }; -struct a6xx_hfi_msg_perf_table { +struct perf_gx_level { + u32 vote; + u32 acd; + u32 freq; +}; + +struct a6xx_hfi_msg_perf_table_v1 { u32 header; u32 num_gpu_levels; u32 num_gmu_levels; @@ -103,6 +110,15 @@ struct a6xx_hfi_msg_perf_table { struct perf_level cx_votes[4]; }; +struct a6xx_hfi_msg_perf_table { + u32 header; + u32 num_gpu_levels; + u32 num_gmu_levels; + + struct perf_gx_level gx_votes[16]; + struct perf_level cx_votes[4]; +}; + #define HFI_H2F_MSG_BW_TABLE 3 struct a6xx_hfi_msg_bw_table { @@ -124,4 +140,34 @@ struct a6xx_hfi_msg_test { u32 header; }; +#define HFI_H2F_MSG_START 10 + +struct a6xx_hfi_msg_start { + u32 header; +}; + +#define HFI_H2F_MSG_CORE_FW_START 14 + +struct a6xx_hfi_msg_core_fw_start { + u32 header; + u32 handle; +}; + +#define HFI_H2F_MSG_GX_BW_PERF_VOTE 30 + +struct a6xx_hfi_gx_bw_perf_vote_cmd { + u32 header; + u32 ack_type; + u32 freq; + u32 bw; +}; + +#define HFI_H2F_MSG_PREPARE_SLUMBER 33 + +struct a6xx_hfi_prep_slumber_cmd { + u32 header; + u32 bw; + u32 freq; +}; + #endif diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index cb3a6e597d76..7732f03d9e3a 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -93,6 +93,17 @@ static const struct adreno_info gpulist[] = { .inactive_period = DRM_MSM_INACTIVE_PERIOD, .init = a3xx_gpu_init, }, { + .rev = ADRENO_REV(4, 0, 5, ANY_ID), + .revn = 405, + .name = "A405", + .fw = { + [ADRENO_FW_PM4] = "a420_pm4.fw", + [ADRENO_FW_PFP] = "a420_pfp.fw", + }, + .gmem = SZ_256K, + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a4xx_gpu_init, + }, { .rev = ADRENO_REV(4, 2, 0, ANY_ID), .revn = 420, .name = "A420", @@ -189,6 +200,30 @@ static const struct adreno_info gpulist[] = { .inactive_period = DRM_MSM_INACTIVE_PERIOD, .init = a6xx_gpu_init, .zapfw = "a630_zap.mdt", + }, { + .rev = ADRENO_REV(6, 4, 0, ANY_ID), + .revn = 640, + .name = "A640", + .fw = { + [ADRENO_FW_SQE] = "a630_sqe.fw", + [ADRENO_FW_GMU] = "a640_gmu.bin", + }, + .gmem = SZ_1M, + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a6xx_gpu_init, + .zapfw = "a640_zap.mdt", + }, { + .rev = ADRENO_REV(6, 5, 0, ANY_ID), + .revn = 650, + .name = "A650", + .fw = { + [ADRENO_FW_SQE] = "a650_sqe.fw", + [ADRENO_FW_GMU] = "a650_gmu.bin", + }, + .gmem = SZ_1M + SZ_128K, + .inactive_period = DRM_MSM_INACTIVE_PERIOD, + .init = a6xx_gpu_init, + .zapfw = "a650_zap.mdt", }, }; diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 1d5c43c22269..89673c7ed473 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -185,6 +185,23 @@ int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid) return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid); } +struct msm_gem_address_space * +adreno_iommu_create_address_space(struct msm_gpu *gpu, + struct platform_device *pdev) +{ + struct iommu_domain *iommu = iommu_domain_alloc(&platform_bus_type); + struct msm_mmu *mmu = msm_iommu_new(&pdev->dev, iommu); + struct msm_gem_address_space *aspace; + + aspace = msm_gem_address_space_create(mmu, "gpu", SZ_16M, + 0xfffffff); + + if (IS_ERR(aspace) && !IS_ERR(mmu)) + mmu->funcs->destroy(mmu); + + return aspace; +} + int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); @@ -197,7 +214,7 @@ int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value) *value = adreno_gpu->gmem; return 0; case MSM_PARAM_GMEM_BASE: - *value = 0x100000; + *value = !adreno_is_a650(adreno_gpu) ? 0x100000 : 0; return 0; case MSM_PARAM_CHIP_ID: *value = adreno_gpu->rev.patchid | @@ -459,7 +476,7 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, break; /* fall-thru */ case MSM_SUBMIT_CMD_BUF: - OUT_PKT3(ring, adreno_is_a430(adreno_gpu) ? + OUT_PKT3(ring, adreno_is_a4xx(adreno_gpu) ? CP_INDIRECT_BUFFER_PFE : CP_INDIRECT_BUFFER_PFD, 2); OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); OUT_RING(ring, submit->cmd[i].size); @@ -988,12 +1005,6 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, adreno_gpu_config.ioname = "kgsl_3d0_reg_memory"; - adreno_gpu_config.va_start = SZ_16M; - adreno_gpu_config.va_end = 0xffffffff; - /* maximum range of a2xx mmu */ - if (adreno_is_a2xx(adreno_gpu)) - adreno_gpu_config.va_end = SZ_16M + 0xfff * SZ_64K; - adreno_gpu_config.nr_rings = nr_rings; adreno_get_pwrlevels(&pdev->dev, gpu); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 9ff4e550e7bd..2f5d2c3acc3a 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -202,6 +202,11 @@ static inline bool adreno_is_a4xx(struct adreno_gpu *gpu) return (gpu->revn >= 400) && (gpu->revn < 500); } +static inline int adreno_is_a405(struct adreno_gpu *gpu) +{ + return gpu->revn == 405; +} + static inline int adreno_is_a420(struct adreno_gpu *gpu) { return gpu->revn == 420; @@ -237,6 +242,16 @@ static inline int adreno_is_a630(struct adreno_gpu *gpu) return gpu->revn == 630; } +static inline int adreno_is_a640(struct adreno_gpu *gpu) +{ + return gpu->revn == 640; +} + +static inline int adreno_is_a650(struct adreno_gpu *gpu) +{ + return gpu->revn == 650; +} + int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value); const struct firmware *adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname); @@ -273,6 +288,14 @@ int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state); int adreno_gpu_state_put(struct msm_gpu_state *state); /* + * Common helper function to initialize the default address space for arm-smmu + * attached targets + */ +struct msm_gem_address_space * +adreno_iommu_create_address_space(struct msm_gpu *gpu, + struct platform_device *pdev); + +/* * For a5xx and a6xx targets load the zap shader that is used to pull the GPU * out of secure mode */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c index 11f2bebe3869..7c230f719ad3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c @@ -36,22 +36,6 @@ static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc) return to_dpu_kms(priv->kms); } -static bool _dpu_core_video_mode_intf_connected(struct drm_crtc *crtc) -{ - struct drm_crtc *tmp_crtc; - - drm_for_each_crtc(tmp_crtc, crtc->dev) { - if ((dpu_crtc_get_intf_mode(tmp_crtc) == INTF_MODE_VIDEO) && - tmp_crtc->enabled) { - DPU_DEBUG("video interface connected crtc:%d\n", - tmp_crtc->base.id); - return true; - } - } - - return false; -} - static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms, struct drm_crtc *crtc, struct drm_crtc_state *state, @@ -94,7 +78,6 @@ int dpu_core_perf_crtc_check(struct drm_crtc *crtc, u32 bw, threshold; u64 bw_sum_of_intfs = 0; enum dpu_crtc_client_type curr_client_type; - bool is_video_mode; struct dpu_crtc_state *dpu_cstate; struct drm_crtc *tmp_crtc; struct dpu_kms *kms; @@ -144,11 +127,7 @@ int dpu_core_perf_crtc_check(struct drm_crtc *crtc, bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000); DPU_DEBUG("calculated bandwidth=%uk\n", bw); - is_video_mode = dpu_crtc_get_intf_mode(crtc) == INTF_MODE_VIDEO; - threshold = (is_video_mode || - _dpu_core_video_mode_intf_connected(crtc)) ? - kms->catalog->perf.max_bw_low : - kms->catalog->perf.max_bw_high; + threshold = kms->catalog->perf.max_bw_high; DPU_DEBUG("final threshold bw limit = %d\n", threshold); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index 17448505a9b5..e15b42a780e0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -9,6 +9,7 @@ #include <linux/sort.h> #include <linux/debugfs.h> #include <linux/ktime.h> +#include <linux/bits.h> #include <drm/drm_crtc.h> #include <drm/drm_flip_work.h> @@ -20,6 +21,7 @@ #include "dpu_kms.h" #include "dpu_hw_lm.h" #include "dpu_hw_ctl.h" +#include "dpu_hw_dspp.h" #include "dpu_crtc.h" #include "dpu_plane.h" #include "dpu_encoder.h" @@ -40,6 +42,9 @@ /* timeout in ms waiting for frame done */ #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS 60 +#define CONVERT_S3_15(val) \ + (((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0)) + static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc) { struct msm_drm_private *priv = crtc->dev->dev_private; @@ -88,11 +93,9 @@ static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer, static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc) { - struct dpu_crtc *dpu_crtc; struct dpu_crtc_state *crtc_state; int lm_idx, lm_horiz_position; - dpu_crtc = to_dpu_crtc(crtc); crtc_state = to_dpu_crtc_state(crtc->state); lm_horiz_position = 0; @@ -422,6 +425,74 @@ static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc, drm_mode_debug_printmodeline(adj_mode); } +static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state, + struct dpu_hw_pcc_cfg *cfg) +{ + struct drm_color_ctm *ctm; + + memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg)); + + ctm = (struct drm_color_ctm *)state->ctm->data; + + if (!ctm) + return; + + cfg->r.r = CONVERT_S3_15(ctm->matrix[0]); + cfg->g.r = CONVERT_S3_15(ctm->matrix[1]); + cfg->b.r = CONVERT_S3_15(ctm->matrix[2]); + + cfg->r.g = CONVERT_S3_15(ctm->matrix[3]); + cfg->g.g = CONVERT_S3_15(ctm->matrix[4]); + cfg->b.g = CONVERT_S3_15(ctm->matrix[5]); + + cfg->r.b = CONVERT_S3_15(ctm->matrix[6]); + cfg->g.b = CONVERT_S3_15(ctm->matrix[7]); + cfg->b.b = CONVERT_S3_15(ctm->matrix[8]); +} + +static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc) +{ + struct drm_crtc_state *state = crtc->state; + struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state); + struct dpu_crtc_mixer *mixer = cstate->mixers; + struct dpu_hw_pcc_cfg cfg; + struct dpu_hw_ctl *ctl; + struct dpu_hw_mixer *lm; + struct dpu_hw_dspp *dspp; + int i; + + + if (!state->color_mgmt_changed) + return; + + for (i = 0; i < cstate->num_mixers; i++) { + ctl = mixer[i].lm_ctl; + lm = mixer[i].hw_lm; + dspp = mixer[i].hw_dspp; + + if (!dspp || !dspp->ops.setup_pcc) + continue; + + if (!state->ctm) { + dspp->ops.setup_pcc(dspp, NULL); + } else { + _dpu_crtc_get_pcc_coeff(state, &cfg); + dspp->ops.setup_pcc(dspp, &cfg); + } + + mixer[i].flush_mask |= ctl->ops.get_bitmask_dspp(ctl, + mixer[i].hw_dspp->idx); + + /* stage config flush mask */ + ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask); + + DPU_DEBUG("lm %d, ctl %d, flush mask 0x%x\n", + mixer[i].hw_lm->idx - DSPP_0, + ctl->idx - CTL_0, + mixer[i].flush_mask); + } +} + static void dpu_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_crtc_state *old_state) { @@ -430,7 +501,6 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_encoder *encoder; struct drm_device *dev; unsigned long flags; - struct dpu_crtc_smmu_state_data *smmu_state; if (!crtc) { DPU_ERROR("invalid crtc\n"); @@ -448,7 +518,6 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc, dpu_crtc = to_dpu_crtc(crtc); cstate = to_dpu_crtc_state(crtc->state); dev = crtc->dev; - smmu_state = &dpu_crtc->smmu_state; _dpu_crtc_setup_lm_bounds(crtc, crtc->state); @@ -475,6 +544,8 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc, _dpu_crtc_blend_setup(crtc); + _dpu_crtc_setup_cp_blocks(crtc); + /* * PP_DONE irq is only used by command mode for now. * It is better to request pending before FLUSH and START trigger @@ -491,7 +562,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_device *dev; struct drm_plane *plane; struct msm_drm_private *priv; - struct msm_drm_thread *event_thread; unsigned long flags; struct dpu_crtc_state *cstate; @@ -513,8 +583,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc, return; } - event_thread = &priv->event_thread[crtc->index]; - if (dpu_crtc->event) { DPU_DEBUG("already received dpu_crtc->event\n"); } else { @@ -567,7 +635,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc, static void dpu_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *state) { - struct dpu_crtc *dpu_crtc; struct dpu_crtc_state *cstate; if (!crtc || !state) { @@ -575,7 +642,6 @@ static void dpu_crtc_destroy_state(struct drm_crtc *crtc, return; } - dpu_crtc = to_dpu_crtc(crtc); cstate = to_dpu_crtc_state(state); DPU_DEBUG("crtc%d\n", crtc->base.id); @@ -662,11 +728,9 @@ static void dpu_crtc_reset(struct drm_crtc *crtc) /** * dpu_crtc_duplicate_state - state duplicate hook * @crtc: Pointer to drm crtc structure - * @Returns: Pointer to new drm_crtc_state structure */ static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc) { - struct dpu_crtc *dpu_crtc; struct dpu_crtc_state *cstate, *old_cstate; if (!crtc || !crtc->state) { @@ -674,7 +738,6 @@ static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc) return NULL; } - dpu_crtc = to_dpu_crtc(crtc); old_cstate = to_dpu_crtc_state(crtc->state); cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL); if (!cstate) { @@ -693,9 +756,7 @@ static void dpu_crtc_disable(struct drm_crtc *crtc, { struct dpu_crtc *dpu_crtc; struct dpu_crtc_state *cstate; - struct drm_display_mode *mode; struct drm_encoder *encoder; - struct msm_drm_private *priv; unsigned long flags; bool release_bandwidth = false; @@ -705,8 +766,6 @@ static void dpu_crtc_disable(struct drm_crtc *crtc, } dpu_crtc = to_dpu_crtc(crtc); cstate = to_dpu_crtc_state(crtc->state); - mode = &cstate->base.adjusted_mode; - priv = crtc->dev->dev_private; DRM_DEBUG_KMS("crtc%d\n", crtc->base.id); @@ -768,14 +827,12 @@ static void dpu_crtc_enable(struct drm_crtc *crtc, { struct dpu_crtc *dpu_crtc; struct drm_encoder *encoder; - struct msm_drm_private *priv; bool request_bandwidth; if (!crtc) { DPU_ERROR("invalid crtc\n"); return; } - priv = crtc->dev->dev_private; pm_runtime_get_sync(crtc->dev->dev); @@ -1319,6 +1376,8 @@ struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane, drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs); + drm_crtc_enable_color_mgmt(crtc, 0, true, 0); + /* save user friendly CRTC name for later */ snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h index 5174e86124cc..cec3474340e8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h @@ -73,12 +73,14 @@ struct dpu_crtc_smmu_state_data { * struct dpu_crtc_mixer: stores the map for each virtual pipeline in the CRTC * @hw_lm: LM HW Driver context * @lm_ctl: CTL Path HW driver context + * @lm_dspp: DSPP HW driver context * @mixer_op_mode: mixer blending operation mode * @flush_mask: mixer flush mask for ctl, mixer and pipe */ struct dpu_crtc_mixer { struct dpu_hw_mixer *hw_lm; struct dpu_hw_ctl *lm_ctl; + struct dpu_hw_dspp *hw_dspp; u32 mixer_op_mode; u32 flush_mask; }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index a1b79ee2bd9d..63976dcd2ac8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -20,6 +20,7 @@ #include "dpu_hw_catalog.h" #include "dpu_hw_intf.h" #include "dpu_hw_ctl.h" +#include "dpu_hw_dspp.h" #include "dpu_formats.h" #include "dpu_encoder_phys.h" #include "dpu_crtc.h" @@ -536,6 +537,7 @@ static struct msm_display_topology dpu_encoder_get_topology( * 1 LM, 1 INTF * 2 LM, 1 INTF (stream merge to support high resolution interfaces) * + * Adding color blocks only to primary interface */ if (intf_count == 2) topology.num_lm = 2; @@ -544,6 +546,9 @@ static struct msm_display_topology dpu_encoder_get_topology( else topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1; + if (dpu_enc->disp_info.intf_type == DRM_MODE_ENCODER_DSI) + topology.num_dspp = topology.num_lm; + topology.num_enc = 0; topology.num_intf = intf_count; @@ -959,7 +964,8 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc, struct dpu_hw_blk *hw_pp[MAX_CHANNELS_PER_ENC]; struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_ENC]; struct dpu_hw_blk *hw_lm[MAX_CHANNELS_PER_ENC]; - int num_lm, num_ctl, num_pp; + struct dpu_hw_blk *hw_dspp[MAX_CHANNELS_PER_ENC] = { NULL }; + int num_lm, num_ctl, num_pp, num_dspp; int i, j; if (!drm_enc) { @@ -1008,6 +1014,9 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc, drm_enc->base.id, DPU_HW_BLK_CTL, hw_ctl, ARRAY_SIZE(hw_ctl)); num_lm = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state, drm_enc->base.id, DPU_HW_BLK_LM, hw_lm, ARRAY_SIZE(hw_lm)); + num_dspp = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state, + drm_enc->base.id, DPU_HW_BLK_DSPP, hw_dspp, + ARRAY_SIZE(hw_dspp)); for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) dpu_enc->hw_pp[i] = i < num_pp ? to_dpu_hw_pingpong(hw_pp[i]) @@ -1020,6 +1029,7 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc, cstate->mixers[i].hw_lm = to_dpu_hw_mixer(hw_lm[i]); cstate->mixers[i].lm_ctl = to_dpu_hw_ctl(hw_ctl[ctl_idx]); + cstate->mixers[i].hw_dspp = to_dpu_hw_dspp(hw_dspp[i]); } cstate->num_mixers = num_lm; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index c567917541e8..29d4fde3172b 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -41,6 +41,8 @@ #define PINGPONG_SDM845_SPLIT_MASK \ (PINGPONG_SDM845_MASK | BIT(DPU_PINGPONG_TE2)) +#define DSPP_SC7180_MASK BIT(DPU_DSPP_PCC) + #define DEFAULT_PIXEL_RAM_SIZE (50 * 1024) #define DEFAULT_DPU_LINE_WIDTH 2048 #define DEFAULT_DPU_OUTPUT_LINE_WIDTH 2560 @@ -291,29 +293,30 @@ static const struct dpu_lm_sub_blks sdm845_lm_sblk = { }, }; -#define LM_BLK(_name, _id, _base, _fmask, _sblk, _pp, _lmpair) \ +#define LM_BLK(_name, _id, _base, _fmask, _sblk, _pp, _lmpair, _dspp) \ { \ .name = _name, .id = _id, \ .base = _base, .len = 0x320, \ .features = _fmask, \ .sblk = _sblk, \ .pingpong = _pp, \ - .lm_pair_mask = (1 << _lmpair) \ + .lm_pair_mask = (1 << _lmpair), \ + .dspp = _dspp \ } static const struct dpu_lm_cfg sdm845_lm[] = { LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_0, LM_1), + &sdm845_lm_sblk, PINGPONG_0, LM_1, 0), LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_1, LM_0), + &sdm845_lm_sblk, PINGPONG_1, LM_0, 0), LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_2, LM_5), + &sdm845_lm_sblk, PINGPONG_2, LM_5, 0), LM_BLK("lm_3", LM_3, 0x0, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_MAX, 0), + &sdm845_lm_sblk, PINGPONG_MAX, 0, 0), LM_BLK("lm_4", LM_4, 0x0, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_MAX, 0), + &sdm845_lm_sblk, PINGPONG_MAX, 0, 0), LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK, - &sdm845_lm_sblk, PINGPONG_3, LM_2), + &sdm845_lm_sblk, PINGPONG_3, LM_2, 0), }; /* SC7180 */ @@ -328,11 +331,30 @@ static const struct dpu_lm_sub_blks sc7180_lm_sblk = { static const struct dpu_lm_cfg sc7180_lm[] = { LM_BLK("lm_0", LM_0, 0x44000, MIXER_SC7180_MASK, - &sc7180_lm_sblk, PINGPONG_0, LM_1), + &sc7180_lm_sblk, PINGPONG_0, LM_1, DSPP_0), LM_BLK("lm_1", LM_1, 0x45000, MIXER_SC7180_MASK, - &sc7180_lm_sblk, PINGPONG_1, LM_0), + &sc7180_lm_sblk, PINGPONG_1, LM_0, 0), +}; + +/************************************************************* + * DSPP sub blocks config + *************************************************************/ +static const struct dpu_dspp_sub_blks sc7180_dspp_sblk = { + .pcc = {.id = DPU_DSPP_PCC, .base = 0x1700, + .len = 0x90, .version = 0x10000}, }; +#define DSPP_BLK(_name, _id, _base) \ + {\ + .name = _name, .id = _id, \ + .base = _base, .len = 0x1800, \ + .features = DSPP_SC7180_MASK, \ + .sblk = &sc7180_dspp_sblk \ + } + +static const struct dpu_dspp_cfg sc7180_dspp[] = { + DSPP_BLK("dspp_0", DSPP_0, 0x54000), +}; /************************************************************* * PINGPONG sub blocks config *************************************************************/ @@ -515,8 +537,8 @@ static const struct dpu_perf_cfg sdm845_perf_data = { }; static const struct dpu_perf_cfg sc7180_perf_data = { - .max_bw_low = 3900000, - .max_bw_high = 5500000, + .max_bw_low = 6800000, + .max_bw_high = 6800000, .min_core_ib = 2400000, .min_llcc_ib = 800000, .min_dram_ib = 800000, @@ -587,6 +609,8 @@ static void sc7180_cfg_init(struct dpu_mdss_cfg *dpu_cfg) .sspp = sc7180_sspp, .mixer_count = ARRAY_SIZE(sc7180_lm), .mixer = sc7180_lm, + .dspp_count = ARRAY_SIZE(sc7180_dspp), + .dspp = sc7180_dspp, .pingpong_count = ARRAY_SIZE(sc7180_pp), .pingpong = sc7180_pp, .intf_count = ARRAY_SIZE(sc7180_intf), diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h index 09df7d87dd43..f7de43838c69 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h @@ -146,6 +146,17 @@ enum { }; /** + * DSPP sub-blocks + * @DPU_DSPP_PCC Panel color correction block + * @DPU_DSPP_GC Gamma correction block + */ +enum { + DPU_DSPP_PCC = 0x1, + DPU_DSPP_GC, + DPU_DSPP_MAX +}; + +/** * PINGPONG sub-blocks * @DPU_PINGPONG_TE Tear check block * @DPU_PINGPONG_TE2 Additional tear check block for split pipes @@ -377,6 +388,16 @@ struct dpu_lm_sub_blks { struct dpu_pp_blk gc; }; +/** + * struct dpu_dspp_sub_blks: Information of DSPP block + * @gc : gamma correction block + * @pcc: pixel color correction block + */ +struct dpu_dspp_sub_blks { + struct dpu_pp_blk gc; + struct dpu_pp_blk pcc; +}; + struct dpu_pingpong_sub_blks { struct dpu_pp_blk te; struct dpu_pp_blk te2; @@ -471,10 +492,24 @@ struct dpu_lm_cfg { DPU_HW_BLK_INFO; const struct dpu_lm_sub_blks *sblk; u32 pingpong; + u32 dspp; unsigned long lm_pair_mask; }; /** + * struct dpu_dspp_cfg - information of DSPP blocks + * @id enum identifying this block + * @base register offset of this block + * @features bit mask identifying sub-blocks/features + * supported by this block + * @sblk sub-blocks information + */ +struct dpu_dspp_cfg { + DPU_HW_BLK_INFO; + const struct dpu_dspp_sub_blks *sblk; +}; + +/** * struct dpu_pingpong_cfg - information of PING-PONG blocks * @id enum identifying this block * @base register offset of this block @@ -688,6 +723,9 @@ struct dpu_mdss_cfg { u32 ad_count; + u32 dspp_count; + const struct dpu_dspp_cfg *dspp; + /* Add additional block data structures here */ struct dpu_perf_cfg perf; @@ -716,6 +754,7 @@ struct dpu_mdss_hw_cfg_handler { #define BLK_PINGPONG(s) ((s)->pingpong) #define BLK_INTF(s) ((s)->intf) #define BLK_AD(s) ((s)->ad) +#define BLK_DSPP(s) ((s)->dspp) /** * dpu_hw_catalog_init - dpu hardware catalog init API retrieves diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c index 831e5f7a9b7f..613ae8f0cfcd 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c @@ -272,6 +272,31 @@ static int dpu_hw_ctl_active_get_bitmask_intf(struct dpu_hw_ctl *ctx, return 0; } +static uint32_t dpu_hw_ctl_get_bitmask_dspp(struct dpu_hw_ctl *ctx, + enum dpu_dspp dspp) +{ + uint32_t flushbits = 0; + + switch (dspp) { + case DSPP_0: + flushbits = BIT(13); + break; + case DSPP_1: + flushbits = BIT(14); + break; + case DSPP_2: + flushbits = BIT(15); + break; + case DSPP_3: + flushbits = BIT(21); + break; + default: + return 0; + } + + return flushbits; +} + static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us) { struct dpu_hw_blk_reg_map *c = &ctx->hw; @@ -548,6 +573,7 @@ static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops, ops->setup_blendstage = dpu_hw_ctl_setup_blendstage; ops->get_bitmask_sspp = dpu_hw_ctl_get_bitmask_sspp; ops->get_bitmask_mixer = dpu_hw_ctl_get_bitmask_mixer; + ops->get_bitmask_dspp = dpu_hw_ctl_get_bitmask_dspp; }; static struct dpu_hw_blk_ops dpu_hw_ops; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h index 09e1263c72e2..ec579b470a80 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h @@ -139,6 +139,9 @@ struct dpu_hw_ctl_ops { uint32_t (*get_bitmask_mixer)(struct dpu_hw_ctl *ctx, enum dpu_lm blk); + uint32_t (*get_bitmask_dspp)(struct dpu_hw_ctl *ctx, + enum dpu_dspp blk); + /** * Query the value of the intf flush mask * No effect on hardware diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c new file mode 100644 index 000000000000..a7a24539921f --- /dev/null +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. + */ + +#include "dpu_hwio.h" +#include "dpu_hw_catalog.h" +#include "dpu_hw_lm.h" +#include "dpu_hw_dspp.h" +#include "dpu_kms.h" + + +/* DSPP_PCC */ +#define PCC_EN BIT(0) +#define PCC_DIS 0 +#define PCC_RED_R_OFF 0x10 +#define PCC_RED_G_OFF 0x1C +#define PCC_RED_B_OFF 0x28 +#define PCC_GREEN_R_OFF 0x14 +#define PCC_GREEN_G_OFF 0x20 +#define PCC_GREEN_B_OFF 0x2C +#define PCC_BLUE_R_OFF 0x18 +#define PCC_BLUE_G_OFF 0x24 +#define PCC_BLUE_B_OFF 0x30 + +static void dpu_setup_dspp_pcc(struct dpu_hw_dspp *ctx, + struct dpu_hw_pcc_cfg *cfg) +{ + + u32 base = ctx->cap->sblk->pcc.base; + + if (!ctx || !base) { + DRM_ERROR("invalid ctx %pK pcc base 0x%x\n", ctx, base); + return; + } + + if (!cfg) { + DRM_DEBUG_DRIVER("disable pcc feature\n"); + DPU_REG_WRITE(&ctx->hw, base, PCC_DIS); + return; + } + + DPU_REG_WRITE(&ctx->hw, base + PCC_RED_R_OFF, cfg->r.r); + DPU_REG_WRITE(&ctx->hw, base + PCC_RED_G_OFF, cfg->r.g); + DPU_REG_WRITE(&ctx->hw, base + PCC_RED_B_OFF, cfg->r.b); + + DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_R_OFF, cfg->g.r); + DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_G_OFF, cfg->g.g); + DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_B_OFF, cfg->g.b); + + DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_R_OFF, cfg->b.r); + DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_G_OFF, cfg->b.g); + DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_B_OFF, cfg->b.b); + + DPU_REG_WRITE(&ctx->hw, base, PCC_EN); +} + +static void _setup_dspp_ops(struct dpu_hw_dspp *c, + unsigned long features) +{ + if (test_bit(DPU_DSPP_PCC, &features) && + IS_SC7180_TARGET(c->hw.hwversion)) + c->ops.setup_pcc = dpu_setup_dspp_pcc; +} + +static const struct dpu_dspp_cfg *_dspp_offset(enum dpu_dspp dspp, + const struct dpu_mdss_cfg *m, + void __iomem *addr, + struct dpu_hw_blk_reg_map *b) +{ + int i; + + if (!m || !addr || !b) + return ERR_PTR(-EINVAL); + + for (i = 0; i < m->dspp_count; i++) { + if (dspp == m->dspp[i].id) { + b->base_off = addr; + b->blk_off = m->dspp[i].base; + b->length = m->dspp[i].len; + b->hwversion = m->hwversion; + b->log_mask = DPU_DBG_MASK_DSPP; + return &m->dspp[i]; + } + } + + return ERR_PTR(-EINVAL); +} + +static struct dpu_hw_blk_ops dpu_hw_ops; + +struct dpu_hw_dspp *dpu_hw_dspp_init(enum dpu_dspp idx, + void __iomem *addr, + const struct dpu_mdss_cfg *m) +{ + struct dpu_hw_dspp *c; + const struct dpu_dspp_cfg *cfg; + + if (!addr || !m) + return ERR_PTR(-EINVAL); + + c = kzalloc(sizeof(*c), GFP_KERNEL); + if (!c) + return ERR_PTR(-ENOMEM); + + cfg = _dspp_offset(idx, m, addr, &c->hw); + if (IS_ERR_OR_NULL(cfg)) { + kfree(c); + return ERR_PTR(-EINVAL); + } + + /* Assign ops */ + c->idx = idx; + c->cap = cfg; + _setup_dspp_ops(c, c->cap->features); + + dpu_hw_blk_init(&c->base, DPU_HW_BLK_DSPP, idx, &dpu_hw_ops); + + return c; +} + +void dpu_hw_dspp_destroy(struct dpu_hw_dspp *dspp) +{ + if (dspp) + dpu_hw_blk_destroy(&dspp->base); + + kfree(dspp); +} + + diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h new file mode 100644 index 000000000000..7fa189cfcb06 --- /dev/null +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. + */ + +#ifndef _DPU_HW_DSPP_H +#define _DPU_HW_DSPP_H + +#include "dpu_hw_blk.h" + +struct dpu_hw_dspp; + +/** + * struct dpu_hw_pcc_coeff - PCC coefficient structure for each color + * component. + * @r: red coefficient. + * @g: green coefficient. + * @b: blue coefficient. + */ + +struct dpu_hw_pcc_coeff { + __u32 r; + __u32 g; + __u32 b; +}; + +/** + * struct dpu_hw_pcc - pcc feature structure + * @r: red coefficients. + * @g: green coefficients. + * @b: blue coefficients. + */ +struct dpu_hw_pcc_cfg { + struct dpu_hw_pcc_coeff r; + struct dpu_hw_pcc_coeff g; + struct dpu_hw_pcc_coeff b; +}; + +/** + * struct dpu_hw_dspp_ops - interface to the dspp hardware driver functions + * Caller must call the init function to get the dspp context for each dspp + * Assumption is these functions will be called after clocks are enabled + */ +struct dpu_hw_dspp_ops { + /** + * setup_pcc - setup dspp pcc + * @ctx: Pointer to dspp context + * @cfg: Pointer to configuration + */ + void (*setup_pcc)(struct dpu_hw_dspp *ctx, struct dpu_hw_pcc_cfg *cfg); + +}; + +/** + * struct dpu_hw_dspp - dspp description + * @base: Hardware block base structure + * @hw: Block hardware details + * @idx: DSPP index + * @cap: Pointer to layer_cfg + * @ops: Pointer to operations possible for this DSPP + */ +struct dpu_hw_dspp { + struct dpu_hw_blk base; + struct dpu_hw_blk_reg_map hw; + + /* dspp */ + int idx; + const struct dpu_dspp_cfg *cap; + + /* Ops */ + struct dpu_hw_dspp_ops ops; +}; + +/** + * dpu_hw_dspp - convert base object dpu_hw_base to container + * @hw: Pointer to base hardware block + * return: Pointer to hardware block container + */ +static inline struct dpu_hw_dspp *to_dpu_hw_dspp(struct dpu_hw_blk *hw) +{ + return container_of(hw, struct dpu_hw_dspp, base); +} + +/** + * dpu_hw_dspp_init - initializes the dspp hw driver object. + * should be called once before accessing every dspp. + * @idx: DSPP index for which driver object is required + * @addr: Mapped register io address of MDP + * @Return: pointer to structure or ERR_PTR + */ +struct dpu_hw_dspp *dpu_hw_dspp_init(enum dpu_dspp idx, + void __iomem *addr, const struct dpu_mdss_cfg *m); + +/** + * dpu_hw_dspp_destroy(): Destroys DSPP driver context + * @dspp: Pointer to DSPP driver context + */ +void dpu_hw_dspp_destroy(struct dpu_hw_dspp *dspp); + +#endif /*_DPU_HW_DSPP_H */ + diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h index 686882132bf6..402dc5832361 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h @@ -95,6 +95,7 @@ enum dpu_hw_blk_type { DPU_HW_BLK_PINGPONG, DPU_HW_BLK_INTF, DPU_HW_BLK_WB, + DPU_HW_BLK_DSPP, DPU_HW_BLK_MAX, }; @@ -425,5 +426,6 @@ struct dpu_mdss_color { #define DPU_DBG_MASK_TOP (1 << 7) #define DPU_DBG_MASK_VBIF (1 << 8) #define DPU_DBG_MASK_ROT (1 << 9) +#define DPU_DBG_MASK_DSPP (1 << 10) #endif /* _DPU_HW_MDSS_H */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index ce19f1d39367..b8615d4fe8a3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -772,29 +772,21 @@ static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms) { struct iommu_domain *domain; struct msm_gem_address_space *aspace; - int ret; + struct msm_mmu *mmu; domain = iommu_domain_alloc(&platform_bus_type); if (!domain) return 0; - domain->geometry.aperture_start = 0x1000; - domain->geometry.aperture_end = 0xffffffff; + mmu = msm_iommu_new(dpu_kms->dev->dev, domain); + aspace = msm_gem_address_space_create(mmu, "dpu1", + 0x1000, 0xfffffff); - aspace = msm_gem_address_space_create(dpu_kms->dev->dev, - domain, "dpu1"); if (IS_ERR(aspace)) { - iommu_domain_free(domain); + mmu->funcs->destroy(mmu); return PTR_ERR(aspace); } - ret = aspace->mmu->funcs->attach(aspace->mmu); - if (ret) { - DPU_ERROR("failed to attach iommu %d\n", ret); - msm_gem_address_space_put(aspace); - return ret; - } - dpu_kms->base.aspace = aspace; return 0; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h index 9aba2910d83a..a3b122bfb676 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h @@ -158,6 +158,7 @@ struct dpu_global_state { uint32_t mixer_to_enc_id[LM_MAX - LM_0]; uint32_t ctl_to_enc_id[CTL_MAX - CTL_0]; uint32_t intf_to_enc_id[INTF_MAX - INTF_0]; + uint32_t dspp_to_enc_id[DSPP_MAX - DSPP_0]; }; struct dpu_global_state diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c index 9b62451b01ee..9b2b5044e8e0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c @@ -9,6 +9,7 @@ #include "dpu_hw_ctl.h" #include "dpu_hw_pingpong.h" #include "dpu_hw_intf.h" +#include "dpu_hw_dspp.h" #include "dpu_encoder.h" #include "dpu_trace.h" @@ -174,6 +175,23 @@ int dpu_rm_init(struct dpu_rm *rm, rm->ctl_blks[ctl->id - CTL_0] = &hw->base; } + for (i = 0; i < cat->dspp_count; i++) { + struct dpu_hw_dspp *hw; + const struct dpu_dspp_cfg *dspp = &cat->dspp[i]; + + if (dspp->id < DSPP_0 || dspp->id >= DSPP_MAX) { + DPU_ERROR("skip dspp %d with invalid id\n", dspp->id); + continue; + } + hw = dpu_hw_dspp_init(dspp->id, mmio, cat); + if (IS_ERR_OR_NULL(hw)) { + rc = PTR_ERR(hw); + DPU_ERROR("failed dspp object creation: err %d\n", rc); + goto fail; + } + rm->dspp_blks[dspp->id - DSPP_0] = &hw->base; + } + return 0; fail: @@ -222,12 +240,17 @@ static bool _dpu_rm_check_lm_peer(struct dpu_rm *rm, int primary_idx, * if lm, and all other hardwired blocks connected to the lm (pp) is * available and appropriate * @pp_idx: output parameter, index of pingpong block attached to the layer - * mixer in rm->pongpong_blks[]. + * mixer in rm->pingpong_blks[]. + * @dspp_idx: output parameter, index of dspp block attached to the layer + * mixer in rm->dspp_blks[]. + * @reqs: input parameter, rm requirements for HW blocks needed in the + * datapath. * @Return: true if lm matches all requirements, false otherwise */ static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm, struct dpu_global_state *global_state, - uint32_t enc_id, int lm_idx, int *pp_idx) + uint32_t enc_id, int lm_idx, int *pp_idx, int *dspp_idx, + struct dpu_rm_requirements *reqs) { const struct dpu_lm_cfg *lm_cfg; int idx; @@ -251,6 +274,23 @@ static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm, return false; } *pp_idx = idx; + + if (!reqs->topology.num_dspp) + return true; + + idx = lm_cfg->dspp - DSPP_0; + if (idx < 0 || idx >= ARRAY_SIZE(rm->dspp_blks)) { + DPU_ERROR("failed to get dspp on lm %d\n", lm_cfg->dspp); + return false; + } + + if (reserved_by_other(global_state->dspp_to_enc_id, idx, enc_id)) { + DPU_DEBUG("lm %d dspp %d already reserved\n", lm_cfg->id, + lm_cfg->dspp); + return false; + } + *dspp_idx = idx; + return true; } @@ -262,6 +302,7 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, { int lm_idx[MAX_BLOCKS]; int pp_idx[MAX_BLOCKS]; + int dspp_idx[MAX_BLOCKS] = {0}; int i, j, lm_count = 0; if (!reqs->topology.num_lm) { @@ -279,7 +320,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, lm_idx[lm_count] = i; if (!_dpu_rm_check_lm_and_get_connected_blks(rm, global_state, - enc_id, i, &pp_idx[lm_count])) { + enc_id, i, &pp_idx[lm_count], + &dspp_idx[lm_count], reqs)) { continue; } @@ -299,7 +341,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, if (!_dpu_rm_check_lm_and_get_connected_blks(rm, global_state, enc_id, j, - &pp_idx[lm_count])) { + &pp_idx[lm_count], &dspp_idx[lm_count], + reqs)) { continue; } @@ -316,6 +359,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm, for (i = 0; i < lm_count; i++) { global_state->mixer_to_enc_id[lm_idx[i]] = enc_id; global_state->pingpong_to_enc_id[pp_idx[i]] = enc_id; + global_state->dspp_to_enc_id[dspp_idx[i]] = + reqs->topology.num_dspp ? enc_id : 0; trace_dpu_rm_reserve_lms(lm_idx[i] + LM_0, enc_id, pp_idx[i] + PINGPONG_0); @@ -560,6 +605,11 @@ int dpu_rm_get_assigned_resources(struct dpu_rm *rm, hw_to_enc_id = global_state->intf_to_enc_id; max_blks = ARRAY_SIZE(rm->intf_blks); break; + case DPU_HW_BLK_DSPP: + hw_blks = rm->dspp_blks; + hw_to_enc_id = global_state->dspp_to_enc_id; + max_blks = ARRAY_SIZE(rm->dspp_blks); + break; default: DPU_ERROR("blk type %d not managed by rm\n", type); return 0; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h index 6d2b04f306f0..08726bb1063a 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h @@ -19,6 +19,7 @@ struct dpu_global_state; * @mixer_blks: array of layer mixer hardware resources * @ctl_blks: array of ctl hardware resources * @intf_blks: array of intf hardware resources + * @dspp_blks: array of dspp hardware resources * @lm_max_width: cached layer mixer maximum width * @rm_lock: resource manager mutex */ @@ -27,6 +28,7 @@ struct dpu_rm { struct dpu_hw_blk *mixer_blks[LM_MAX - LM_0]; struct dpu_hw_blk *ctl_blks[CTL_MAX - CTL_0]; struct dpu_hw_blk *intf_blks[INTF_MAX - INTF_0]; + struct dpu_hw_blk *dspp_blks[DSPP_MAX - DSPP_0]; uint32_t lm_max_width; }; diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c index dda05436f716..08897184b1d9 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c @@ -510,18 +510,20 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev) mdelay(16); if (config->iommu) { - aspace = msm_gem_address_space_create(&pdev->dev, - config->iommu, "mdp4"); + struct msm_mmu *mmu = msm_iommu_new(&pdev->dev, + config->iommu); + + aspace = msm_gem_address_space_create(mmu, + "mdp4", 0x1000, 0xffffffff); + if (IS_ERR(aspace)) { + if (!IS_ERR(mmu)) + mmu->funcs->destroy(mmu); ret = PTR_ERR(aspace); goto fail; } kms->aspace = aspace; - - ret = aspace->mmu->funcs->attach(aspace->mmu); - if (ret) - goto fail; } else { DRM_DEV_INFO(dev->dev, "no iommu, fallback to phys " "contig buffers for scanout\n"); @@ -569,10 +571,6 @@ static struct mdp4_platform_config *mdp4_get_config(struct platform_device *dev) /* TODO: Chips that aren't apq8064 have a 200 Mhz max_clk */ config.max_clk = 266667000; config.iommu = iommu_domain_alloc(&platform_bus_type); - if (config.iommu) { - config.iommu->geometry.aperture_start = 0x1000; - config.iommu->geometry.aperture_end = 0xffffffff; - } return &config; } diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c index e3c4c250238b..25a13a2a57a9 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c @@ -342,6 +342,81 @@ static const struct mdp5_cfg_hw msm8x16_config = { .max_clk = 320000000, }; +static const struct mdp5_cfg_hw msm8x36_config = { + .name = "msm8x36", + .mdp = { + .count = 1, + .base = { 0x0 }, + .caps = MDP_CAP_SMP | + 0, + }, + .smp = { + .mmb_count = 8, + .mmb_size = 10240, + .clients = { + [SSPP_VIG0] = 1, [SSPP_DMA0] = 4, + [SSPP_RGB0] = 7, [SSPP_RGB1] = 8, + }, + }, + .ctl = { + .count = 3, + .base = { 0x01000, 0x01200, 0x01400 }, + .flush_hw_mask = 0x4003ffff, + }, + .pipe_vig = { + .count = 1, + .base = { 0x04000 }, + .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP | + MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC | + MDP_PIPE_CAP_DECIMATION, + }, + .pipe_rgb = { + .count = 2, + .base = { 0x14000, 0x16000 }, + .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP | + MDP_PIPE_CAP_DECIMATION, + }, + .pipe_dma = { + .count = 1, + .base = { 0x24000 }, + .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP, + }, + .lm = { + .count = 2, + .base = { 0x44000, 0x47000 }, + .instances = { + { .id = 0, .pp = 0, .dspp = 0, + .caps = MDP_LM_CAP_DISPLAY, }, + { .id = 1, .pp = -1, .dspp = -1, + .caps = MDP_LM_CAP_WB, }, + }, + .nb_stages = 8, + .max_width = 2560, + .max_height = 0xFFFF, + }, + .pp = { + .count = 1, + .base = { 0x70000 }, + }, + .ad = { + .count = 1, + .base = { 0x78000 }, + }, + .dspp = { + .count = 1, + .base = { 0x54000 }, + }, + .intf = { + .base = { 0x00000, 0x6a800, 0x6b000 }, + .connect = { + [0] = INTF_DISABLED, + [1] = INTF_DSI, + [2] = INTF_DSI, + }, + }, + .max_clk = 366670000, +}; + static const struct mdp5_cfg_hw msm8x94_config = { .name = "msm8x94", .mdp = { @@ -840,6 +915,7 @@ static const struct mdp5_cfg_handler cfg_handlers_v1[] = { { .revision = 2, .config = { .hw = &msm8x74v2_config } }, { .revision = 3, .config = { .hw = &apq8084_config } }, { .revision = 6, .config = { .hw = &msm8x16_config } }, + { .revision = 8, .config = { .hw = &msm8x36_config } }, { .revision = 9, .config = { .hw = &msm8x94_config } }, { .revision = 7, .config = { .hw = &msm8x96_config } }, { .revision = 11, .config = { .hw = &msm8x76_config } }, @@ -941,10 +1017,6 @@ static struct mdp5_cfg_platform *mdp5_get_config(struct platform_device *dev) static struct mdp5_cfg_platform config = {}; config.iommu = iommu_domain_alloc(&platform_bus_type); - if (config.iommu) { - config.iommu->geometry.aperture_start = 0x1000; - config.iommu->geometry.aperture_end = 0xffffffff; - } return &config; } diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c index 998bef1190a3..b5fed67c4651 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c @@ -959,7 +959,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc, if (!ctl) return -EINVAL; - /* don't support LM cursors when we we have source split enabled */ + /* don't support LM cursors when we have source split enabled */ if (mdp5_cstate->pipeline.r_mixer) return -EINVAL; @@ -1030,7 +1030,7 @@ static int mdp5_crtc_cursor_move(struct drm_crtc *crtc, int x, int y) return -EINVAL; } - /* don't support LM cursors when we we have source split enabled */ + /* don't support LM cursors when we have source split enabled */ if (mdp5_cstate->pipeline.r_mixer) return -EINVAL; diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index c902c6503675..19ec48695ffb 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -624,25 +624,25 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev) mdelay(16); if (config->platform.iommu) { + struct msm_mmu *mmu; + iommu_dev = &pdev->dev; if (!dev_iommu_fwspec_get(iommu_dev)) iommu_dev = iommu_dev->parent; - aspace = msm_gem_address_space_create(iommu_dev, - config->platform.iommu, "mdp5"); + mmu = msm_iommu_new(iommu_dev, config->platform.iommu); + + aspace = msm_gem_address_space_create(mmu, "mdp5", + 0x1000, 0xffffffff); + if (IS_ERR(aspace)) { + if (!IS_ERR(mmu)) + mmu->funcs->destroy(mmu); ret = PTR_ERR(aspace); goto fail; } kms->aspace = aspace; - - ret = aspace->mmu->funcs->attach(aspace->mmu); - if (ret) { - DRM_DEV_ERROR(&pdev->dev, "failed to attach iommu: %d\n", - ret); - goto fail; - } } else { DRM_DEV_INFO(&pdev->dev, "no iommu, fallback to phys contig buffers for scanout\n"); @@ -935,7 +935,8 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev) return 0; fail: - mdp5_destroy(pdev); + if (mdp5_kms) + mdp5_destroy(pdev); return ret; } diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 29295dee2a2e..f6ce40bf3699 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -37,9 +37,10 @@ * - 1.4.0 - softpin, MSM_RELOC_BO_DUMP, and GEM_INFO support to set/get * GEM object's debug name * - 1.5.0 - Add SUBMITQUERY_QUERY ioctl + * - 1.6.0 - Syncobj support */ #define MSM_VERSION_MAJOR 1 -#define MSM_VERSION_MINOR 5 +#define MSM_VERSION_MINOR 6 #define MSM_VERSION_PATCHLEVEL 0 static const struct drm_mode_config_funcs mode_config_funcs = { @@ -1002,7 +1003,8 @@ static struct drm_driver msm_driver = { .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_ATOMIC | - DRIVER_MODESET, + DRIVER_MODESET | + DRIVER_SYNCOBJ, .open = msm_open, .postclose = msm_postclose, .lastclose = drm_fb_helper_lastclose, diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 194d900a460e..e2d6a6056418 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -105,6 +105,7 @@ struct msm_display_topology { u32 num_lm; u32 num_enc; u32 num_intf; + u32 num_dspp; }; /** @@ -236,7 +237,8 @@ int msm_crtc_enable_vblank(struct drm_crtc *crtc); void msm_crtc_disable_vblank(struct drm_crtc *crtc); int msm_gem_init_vma(struct msm_gem_address_space *aspace, - struct msm_gem_vma *vma, int npages); + struct msm_gem_vma *vma, int npages, + u64 range_start, u64 range_end); void msm_gem_purge_vma(struct msm_gem_address_space *aspace, struct msm_gem_vma *vma); void msm_gem_unmap_vma(struct msm_gem_address_space *aspace, @@ -250,12 +252,8 @@ void msm_gem_close_vma(struct msm_gem_address_space *aspace, void msm_gem_address_space_put(struct msm_gem_address_space *aspace); struct msm_gem_address_space * -msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain, - const char *name); - -struct msm_gem_address_space * -msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu, - const char *name, uint64_t va_start, uint64_t va_end); +msm_gem_address_space_create(struct msm_mmu *mmu, const char *name, + u64 va_start, u64 size); int msm_register_mmu(struct drm_device *dev, struct msm_mmu *mmu); void msm_unregister_mmu(struct drm_device *dev, struct msm_mmu *mmu); @@ -276,6 +274,9 @@ vm_fault_t msm_gem_fault(struct vm_fault *vmf); uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj); int msm_gem_get_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova); +int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj, + struct msm_gem_address_space *aspace, uint64_t *iova, + u64 range_start, u64 range_end); int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova); uint64_t msm_gem_iova(struct drm_gem_object *obj, diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 5a6a79fbc9d6..6277fde13df9 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -389,7 +389,8 @@ put_iova(struct drm_gem_object *obj) } static int msm_gem_get_iova_locked(struct drm_gem_object *obj, - struct msm_gem_address_space *aspace, uint64_t *iova) + struct msm_gem_address_space *aspace, uint64_t *iova, + u64 range_start, u64 range_end) { struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; @@ -404,7 +405,8 @@ static int msm_gem_get_iova_locked(struct drm_gem_object *obj, if (IS_ERR(vma)) return PTR_ERR(vma); - ret = msm_gem_init_vma(aspace, vma, obj->size >> PAGE_SHIFT); + ret = msm_gem_init_vma(aspace, vma, obj->size >> PAGE_SHIFT, + range_start, range_end); if (ret) { del_vma(vma); return ret; @@ -426,6 +428,9 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, if (!(msm_obj->flags & MSM_BO_GPU_READONLY)) prot |= IOMMU_WRITE; + if (msm_obj->flags & MSM_BO_MAP_PRIV) + prot |= IOMMU_PRIV; + WARN_ON(!mutex_is_locked(&msm_obj->lock)); if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) @@ -443,9 +448,13 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, msm_obj->sgt, obj->size >> PAGE_SHIFT); } -/* get iova and pin it. Should have a matching put */ -int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, - struct msm_gem_address_space *aspace, uint64_t *iova) +/* + * get iova and pin it. Should have a matching put + * limits iova to specified range (in pages) + */ +int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj, + struct msm_gem_address_space *aspace, uint64_t *iova, + u64 range_start, u64 range_end) { struct msm_gem_object *msm_obj = to_msm_bo(obj); u64 local; @@ -453,7 +462,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, mutex_lock(&msm_obj->lock); - ret = msm_gem_get_iova_locked(obj, aspace, &local); + ret = msm_gem_get_iova_locked(obj, aspace, &local, + range_start, range_end); if (!ret) ret = msm_gem_pin_iova(obj, aspace); @@ -465,6 +475,13 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, return ret; } +/* get iova and pin it. Should have a matching put */ +int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, + struct msm_gem_address_space *aspace, uint64_t *iova) +{ + return msm_gem_get_and_pin_iova_range(obj, aspace, iova, 0, U64_MAX); +} + /* * Get an iova but don't pin it. Doesn't need a put because iovas are currently * valid for the life of the object @@ -476,7 +493,7 @@ int msm_gem_get_iova(struct drm_gem_object *obj, int ret; mutex_lock(&msm_obj->lock); - ret = msm_gem_get_iova_locked(obj, aspace, iova); + ret = msm_gem_get_iova_locked(obj, aspace, iova, 0, U64_MAX); mutex_unlock(&msm_obj->lock); return ret; diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 30584eaf8cc8..972490b14ba5 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -13,6 +13,7 @@ /* Additional internal-use only BO flags: */ #define MSM_BO_STOLEN 0x10000000 /* try to use stolen/splash memory */ +#define MSM_BO_MAP_PRIV 0x20000000 /* use IOMMU_PRIV when mapping */ struct msm_gem_address_space { const char *name; diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index 385d4965a8d0..6630aa817505 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -8,7 +8,9 @@ #include <linux/sync_file.h> #include <linux/uaccess.h> +#include <drm/drm_drv.h> #include <drm/drm_file.h> +#include <drm/drm_syncobj.h> #include "msm_drv.h" #include "msm_gpu.h" @@ -391,6 +393,186 @@ static void submit_cleanup(struct msm_gem_submit *submit) } } + +struct msm_submit_post_dep { + struct drm_syncobj *syncobj; + uint64_t point; + struct dma_fence_chain *chain; +}; + +static struct drm_syncobj **msm_wait_deps(struct drm_device *dev, + struct drm_file *file, + uint64_t in_syncobjs_addr, + uint32_t nr_in_syncobjs, + size_t syncobj_stride, + struct msm_ringbuffer *ring) +{ + struct drm_syncobj **syncobjs = NULL; + struct drm_msm_gem_submit_syncobj syncobj_desc = {0}; + int ret = 0; + uint32_t i, j; + + syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs), + GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + if (!syncobjs) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < nr_in_syncobjs; ++i) { + uint64_t address = in_syncobjs_addr + i * syncobj_stride; + struct dma_fence *fence; + + if (copy_from_user(&syncobj_desc, + u64_to_user_ptr(address), + min(syncobj_stride, sizeof(syncobj_desc)))) { + ret = -EFAULT; + break; + } + + if (syncobj_desc.point && + !drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) { + ret = -EOPNOTSUPP; + break; + } + + if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) { + ret = -EINVAL; + break; + } + + ret = drm_syncobj_find_fence(file, syncobj_desc.handle, + syncobj_desc.point, 0, &fence); + if (ret) + break; + + if (!dma_fence_match_context(fence, ring->fctx->context)) + ret = dma_fence_wait(fence, true); + + dma_fence_put(fence); + if (ret) + break; + + if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) { + syncobjs[i] = + drm_syncobj_find(file, syncobj_desc.handle); + if (!syncobjs[i]) { + ret = -EINVAL; + break; + } + } + } + + if (ret) { + for (j = 0; j <= i; ++j) { + if (syncobjs[j]) + drm_syncobj_put(syncobjs[j]); + } + kfree(syncobjs); + return ERR_PTR(ret); + } + return syncobjs; +} + +static void msm_reset_syncobjs(struct drm_syncobj **syncobjs, + uint32_t nr_syncobjs) +{ + uint32_t i; + + for (i = 0; syncobjs && i < nr_syncobjs; ++i) { + if (syncobjs[i]) + drm_syncobj_replace_fence(syncobjs[i], NULL); + } +} + +static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev, + struct drm_file *file, + uint64_t syncobjs_addr, + uint32_t nr_syncobjs, + size_t syncobj_stride) +{ + struct msm_submit_post_dep *post_deps; + struct drm_msm_gem_submit_syncobj syncobj_desc = {0}; + int ret = 0; + uint32_t i, j; + + post_deps = kmalloc_array(nr_syncobjs, sizeof(*post_deps), + GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + if (!post_deps) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < nr_syncobjs; ++i) { + uint64_t address = syncobjs_addr + i * syncobj_stride; + + if (copy_from_user(&syncobj_desc, + u64_to_user_ptr(address), + min(syncobj_stride, sizeof(syncobj_desc)))) { + ret = -EFAULT; + break; + } + + post_deps[i].point = syncobj_desc.point; + post_deps[i].chain = NULL; + + if (syncobj_desc.flags) { + ret = -EINVAL; + break; + } + + if (syncobj_desc.point) { + if (!drm_core_check_feature(dev, + DRIVER_SYNCOBJ_TIMELINE)) { + ret = -EOPNOTSUPP; + break; + } + + post_deps[i].chain = + kmalloc(sizeof(*post_deps[i].chain), + GFP_KERNEL); + if (!post_deps[i].chain) { + ret = -ENOMEM; + break; + } + } + + post_deps[i].syncobj = + drm_syncobj_find(file, syncobj_desc.handle); + if (!post_deps[i].syncobj) { + ret = -EINVAL; + break; + } + } + + if (ret) { + for (j = 0; j <= i; ++j) { + kfree(post_deps[j].chain); + if (post_deps[j].syncobj) + drm_syncobj_put(post_deps[j].syncobj); + } + + kfree(post_deps); + return ERR_PTR(ret); + } + + return post_deps; +} + +static void msm_process_post_deps(struct msm_submit_post_dep *post_deps, + uint32_t count, struct dma_fence *fence) +{ + uint32_t i; + + for (i = 0; post_deps && i < count; ++i) { + if (post_deps[i].chain) { + drm_syncobj_add_point(post_deps[i].syncobj, + post_deps[i].chain, + fence, post_deps[i].point); + post_deps[i].chain = NULL; + } else { + drm_syncobj_replace_fence(post_deps[i].syncobj, + fence); + } + } +} + int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file) { @@ -403,6 +585,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct sync_file *sync_file = NULL; struct msm_gpu_submitqueue *queue; struct msm_ringbuffer *ring; + struct msm_submit_post_dep *post_deps = NULL; + struct drm_syncobj **syncobjs_to_reset = NULL; int out_fence_fd = -1; struct pid *pid = get_pid(task_pid(current)); bool has_ww_ticket = false; @@ -411,6 +595,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, if (!gpu) return -ENXIO; + if (args->pad) + return -EINVAL; + /* for now, we just have 3d pipe.. eventually this would need to * be more clever to dispatch to appropriate gpu module: */ @@ -458,9 +645,29 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, return ret; } + if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) { + syncobjs_to_reset = msm_wait_deps(dev, file, + args->in_syncobjs, + args->nr_in_syncobjs, + args->syncobj_stride, ring); + if (IS_ERR(syncobjs_to_reset)) + return PTR_ERR(syncobjs_to_reset); + } + + if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) { + post_deps = msm_parse_post_deps(dev, file, + args->out_syncobjs, + args->nr_out_syncobjs, + args->syncobj_stride); + if (IS_ERR(post_deps)) { + ret = PTR_ERR(post_deps); + goto out_post_unlock; + } + } + ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) - return ret; + goto out_post_unlock; if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) { out_fence_fd = get_unused_fd_flags(O_CLOEXEC); @@ -587,6 +794,11 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, args->fence_fd = out_fence_fd; } + msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs); + msm_process_post_deps(post_deps, args->nr_out_syncobjs, + submit->fence); + + out: submit_cleanup(submit); if (has_ww_ticket) @@ -597,5 +809,23 @@ out_unlock: if (ret && (out_fence_fd >= 0)) put_unused_fd(out_fence_fd); mutex_unlock(&dev->struct_mutex); + +out_post_unlock: + if (!IS_ERR_OR_NULL(post_deps)) { + for (i = 0; i < args->nr_out_syncobjs; ++i) { + kfree(post_deps[i].chain); + drm_syncobj_put(post_deps[i].syncobj); + } + kfree(post_deps); + } + + if (!IS_ERR_OR_NULL(syncobjs_to_reset)) { + for (i = 0; i < args->nr_in_syncobjs; ++i) { + if (syncobjs_to_reset[i]) + drm_syncobj_put(syncobjs_to_reset[i]); + } + kfree(syncobjs_to_reset); + } + return ret; } diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index 1af5354bcd46..5f6a11211b64 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -103,7 +103,8 @@ void msm_gem_close_vma(struct msm_gem_address_space *aspace, /* Initialize a new vma and allocate an iova for it */ int msm_gem_init_vma(struct msm_gem_address_space *aspace, - struct msm_gem_vma *vma, int npages) + struct msm_gem_vma *vma, int npages, + u64 range_start, u64 range_end) { int ret; @@ -111,7 +112,8 @@ int msm_gem_init_vma(struct msm_gem_address_space *aspace, return -EBUSY; spin_lock(&aspace->lock); - ret = drm_mm_insert_node(&aspace->mm, &vma->node, npages); + ret = drm_mm_insert_node_in_range(&aspace->mm, &vma->node, npages, 0, + 0, range_start, range_end, 0); spin_unlock(&aspace->lock); if (ret) @@ -125,37 +127,14 @@ int msm_gem_init_vma(struct msm_gem_address_space *aspace, return 0; } - struct msm_gem_address_space * -msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain, - const char *name) +msm_gem_address_space_create(struct msm_mmu *mmu, const char *name, + u64 va_start, u64 size) { struct msm_gem_address_space *aspace; - u64 size = domain->geometry.aperture_end - - domain->geometry.aperture_start; - - aspace = kzalloc(sizeof(*aspace), GFP_KERNEL); - if (!aspace) - return ERR_PTR(-ENOMEM); - - spin_lock_init(&aspace->lock); - aspace->name = name; - aspace->mmu = msm_iommu_new(dev, domain); - - drm_mm_init(&aspace->mm, (domain->geometry.aperture_start >> PAGE_SHIFT), - size >> PAGE_SHIFT); - kref_init(&aspace->kref); - - return aspace; -} - -struct msm_gem_address_space * -msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu, - const char *name, uint64_t va_start, uint64_t va_end) -{ - struct msm_gem_address_space *aspace; - u64 size = va_end - va_start; + if (IS_ERR(mmu)) + return ERR_CAST(mmu); aspace = kzalloc(sizeof(*aspace), GFP_KERNEL); if (!aspace) @@ -163,10 +142,9 @@ msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu, spin_lock_init(&aspace->lock); aspace->name = name; - aspace->mmu = msm_gpummu_new(dev, gpu); + aspace->mmu = mmu; - drm_mm_init(&aspace->mm, (va_start >> PAGE_SHIFT), - size >> PAGE_SHIFT); + drm_mm_init(&aspace->mm, va_start >> PAGE_SHIFT, size >> PAGE_SHIFT); kref_init(&aspace->kref); diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 615c5cda5389..a22d30622306 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -821,51 +821,6 @@ static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu) return 0; } -static struct msm_gem_address_space * -msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev, - uint64_t va_start, uint64_t va_end) -{ - struct msm_gem_address_space *aspace; - int ret; - - /* - * Setup IOMMU.. eventually we will (I think) do this once per context - * and have separate page tables per context. For now, to keep things - * simple and to get something working, just use a single address space: - */ - if (!adreno_is_a2xx(to_adreno_gpu(gpu))) { - struct iommu_domain *iommu = iommu_domain_alloc(&platform_bus_type); - if (!iommu) - return NULL; - - iommu->geometry.aperture_start = va_start; - iommu->geometry.aperture_end = va_end; - - DRM_DEV_INFO(gpu->dev->dev, "%s: using IOMMU\n", gpu->name); - - aspace = msm_gem_address_space_create(&pdev->dev, iommu, "gpu"); - if (IS_ERR(aspace)) - iommu_domain_free(iommu); - } else { - aspace = msm_gem_address_space_create_a2xx(&pdev->dev, gpu, "gpu", - va_start, va_end); - } - - if (IS_ERR(aspace)) { - DRM_DEV_ERROR(gpu->dev->dev, "failed to init mmu: %ld\n", - PTR_ERR(aspace)); - return ERR_CAST(aspace); - } - - ret = aspace->mmu->funcs->attach(aspace->mmu); - if (ret) { - msm_gem_address_space_put(aspace); - return ERR_PTR(ret); - } - - return aspace; -} - int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs, const char *name, struct msm_gpu_config *config) @@ -938,8 +893,8 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, msm_devfreq_init(gpu); - gpu->aspace = msm_gpu_create_address_space(gpu, pdev, - config->va_start, config->va_end); + + gpu->aspace = gpu->funcs->create_address_space(gpu, pdev); if (gpu->aspace == NULL) DRM_DEV_INFO(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name); diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index 6ccae4ba905c..429cb40f7931 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -21,8 +21,6 @@ struct msm_gpu_state; struct msm_gpu_config { const char *ioname; - uint64_t va_start; - uint64_t va_end; unsigned int nr_rings; }; @@ -64,6 +62,8 @@ struct msm_gpu_funcs { int (*gpu_state_put)(struct msm_gpu_state *state); unsigned long (*gpu_get_freq)(struct msm_gpu *gpu); void (*gpu_set_freq)(struct msm_gpu *gpu, unsigned long freq); + struct msm_gem_address_space *(*create_address_space) + (struct msm_gpu *gpu, struct platform_device *pdev); }; struct msm_gpu { diff --git a/drivers/gpu/drm/msm/msm_gpummu.c b/drivers/gpu/drm/msm/msm_gpummu.c index 34980d8eb7ad..310a31b05faa 100644 --- a/drivers/gpu/drm/msm/msm_gpummu.c +++ b/drivers/gpu/drm/msm/msm_gpummu.c @@ -21,17 +21,12 @@ struct msm_gpummu { #define GPUMMU_PAGE_SIZE SZ_4K #define TABLE_SIZE (sizeof(uint32_t) * GPUMMU_VA_RANGE / GPUMMU_PAGE_SIZE) -static int msm_gpummu_attach(struct msm_mmu *mmu) -{ - return 0; -} - static void msm_gpummu_detach(struct msm_mmu *mmu) { } static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova, - struct sg_table *sgt, unsigned len, int prot) + struct sg_table *sgt, size_t len, int prot) { struct msm_gpummu *gpummu = to_msm_gpummu(mmu); unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE; @@ -59,7 +54,7 @@ static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova, return 0; } -static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, unsigned len) +static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len) { struct msm_gpummu *gpummu = to_msm_gpummu(mmu); unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE; @@ -85,7 +80,6 @@ static void msm_gpummu_destroy(struct msm_mmu *mmu) } static const struct msm_mmu_funcs funcs = { - .attach = msm_gpummu_attach, .detach = msm_gpummu_detach, .map = msm_gpummu_map, .unmap = msm_gpummu_unmap, diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c index ad58cfe5998e..3a381a9674c9 100644 --- a/drivers/gpu/drm/msm/msm_iommu.c +++ b/drivers/gpu/drm/msm/msm_iommu.c @@ -23,13 +23,6 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev, return 0; } -static int msm_iommu_attach(struct msm_mmu *mmu) -{ - struct msm_iommu *iommu = to_msm_iommu(mmu); - - return iommu_attach_device(iommu->domain, mmu->dev); -} - static void msm_iommu_detach(struct msm_mmu *mmu) { struct msm_iommu *iommu = to_msm_iommu(mmu); @@ -38,7 +31,7 @@ static void msm_iommu_detach(struct msm_mmu *mmu) } static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova, - struct sg_table *sgt, unsigned len, int prot) + struct sg_table *sgt, size_t len, int prot) { struct msm_iommu *iommu = to_msm_iommu(mmu); size_t ret; @@ -49,7 +42,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova, return (ret == len) ? 0 : -EINVAL; } -static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, unsigned len) +static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len) { struct msm_iommu *iommu = to_msm_iommu(mmu); @@ -66,7 +59,6 @@ static void msm_iommu_destroy(struct msm_mmu *mmu) } static const struct msm_mmu_funcs funcs = { - .attach = msm_iommu_attach, .detach = msm_iommu_detach, .map = msm_iommu_map, .unmap = msm_iommu_unmap, @@ -76,6 +68,10 @@ static const struct msm_mmu_funcs funcs = { struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain) { struct msm_iommu *iommu; + int ret; + + if (!domain) + return ERR_PTR(-ENODEV); iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); if (!iommu) @@ -85,5 +81,11 @@ struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain) msm_mmu_init(&iommu->base, dev, &funcs); iommu_set_fault_handler(domain, msm_fault_handler, iommu); + ret = iommu_attach_device(iommu->domain, dev); + if (ret) { + kfree(iommu); + return ERR_PTR(ret); + } + return &iommu->base; } diff --git a/drivers/gpu/drm/msm/msm_mmu.h b/drivers/gpu/drm/msm/msm_mmu.h index 67a623f14319..3a534ee59bf6 100644 --- a/drivers/gpu/drm/msm/msm_mmu.h +++ b/drivers/gpu/drm/msm/msm_mmu.h @@ -10,11 +10,10 @@ #include <linux/iommu.h> struct msm_mmu_funcs { - int (*attach)(struct msm_mmu *mmu); void (*detach)(struct msm_mmu *mmu); int (*map)(struct msm_mmu *mmu, uint64_t iova, struct sg_table *sgt, - unsigned len, int prot); - int (*unmap)(struct msm_mmu *mmu, uint64_t iova, unsigned len); + size_t len, int prot); + int (*unmap)(struct msm_mmu *mmu, uint64_t iova, size_t len); void (*destroy)(struct msm_mmu *mmu); }; diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c index 732f65df5c4f..fea30e7aa9e8 100644 --- a/drivers/gpu/drm/msm/msm_rd.c +++ b/drivers/gpu/drm/msm/msm_rd.c @@ -29,8 +29,6 @@ * or shader programs (if not emitted inline in cmdstream). */ -#ifdef CONFIG_DEBUG_FS - #include <linux/circ_buf.h> #include <linux/debugfs.h> #include <linux/kfifo.h> @@ -47,6 +45,8 @@ bool rd_full = false; MODULE_PARM_DESC(rd_full, "If true, $debugfs/.../rd will snapshot all buffer contents"); module_param_named(rd_full, rd_full, bool, 0600); +#ifdef CONFIG_DEBUG_FS + enum rd_sect_type { RD_NONE, RD_TEST, /* ascii text */ diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 7622490d8602..d472942102f5 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -277,7 +277,7 @@ nv50_outp_release(struct nouveau_encoder *nv_encoder) } static int -nv50_outp_acquire(struct nouveau_encoder *nv_encoder) +nv50_outp_acquire(struct nouveau_encoder *nv_encoder, bool hda) { struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev); struct nv50_disp *disp = nv50_disp(drm->dev); @@ -289,6 +289,7 @@ nv50_outp_acquire(struct nouveau_encoder *nv_encoder) .base.method = NV50_DISP_MTHD_V1_ACQUIRE, .base.hasht = nv_encoder->dcb->hasht, .base.hashm = nv_encoder->dcb->hashm, + .info.hda = hda, }; int ret; @@ -393,7 +394,7 @@ nv50_dac_enable(struct drm_encoder *encoder) struct nv50_head_atom *asyh = nv50_head_atom(nv_crtc->base.state); struct nv50_core *core = nv50_disp(encoder->dev)->core; - nv50_outp_acquire(nv_encoder); + nv50_outp_acquire(nv_encoder, false); core->func->dac->ctrl(core, nv_encoder->or, 1 << nv_crtc->index, asyh); asyh->or.depth = 0; @@ -510,7 +511,7 @@ nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id, if (!nv_connector || !nv_crtc || nv_encoder->or != port || nv_crtc->index != dev_id) continue; - *enabled = drm_detect_monitor_audio(nv_connector->edid); + *enabled = nv_encoder->audio; if (*enabled) { ret = drm_eld_size(nv_connector->base.eld); memcpy(buf, nv_connector->base.eld, @@ -600,6 +601,7 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) (0x0100 << nv_crtc->index), }; + nv_encoder->audio = false; nvif_mthd(&disp->disp->object, 0, &args, sizeof(args)); nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or, @@ -636,6 +638,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) nvif_mthd(&disp->disp->object, 0, &args, sizeof(args.base) + drm_eld_size(args.data)); + nv_encoder->audio = true; nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or, nv_crtc->index); @@ -966,7 +969,7 @@ nv50_msto_enable(struct drm_encoder *encoder) DRM_DEBUG_KMS("Failed to allocate VCPI\n"); if (!mstm->links++) - nv50_outp_acquire(mstm->outp); + nv50_outp_acquire(mstm->outp, false /*XXX: MST audio.*/); if (mstm->outp->link & 1) proto = 0x8; @@ -1560,12 +1563,18 @@ nv50_sor_enable(struct drm_encoder *encoder) struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_connector *nv_connector; struct nvbios *bios = &drm->vbios; + bool hda = false; u8 proto = 0xf; u8 depth = 0x0; nv_connector = nouveau_encoder_connector_get(nv_encoder); nv_encoder->crtc = encoder->crtc; - nv50_outp_acquire(nv_encoder); + + if ((disp->disp->object.oclass == GT214_DISP || + disp->disp->object.oclass >= GF110_DISP) && + drm_detect_monitor_audio(nv_connector->edid)) + hda = true; + nv50_outp_acquire(nv_encoder, hda); switch (nv_encoder->dcb->type) { case DCB_OUTPUT_TMDS: @@ -1775,7 +1784,7 @@ nv50_pior_enable(struct drm_encoder *encoder) u8 owner = 1 << nv_crtc->index; u8 proto; - nv50_outp_acquire(nv_encoder); + nv50_outp_acquire(nv_encoder, false); switch (asyh->or.bpc) { case 10: asyh->or.depth = 0x6; break; diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c index e25ead56052c..99b9b681736d 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c +++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c @@ -192,6 +192,8 @@ nv50_wndw_atomic_check_release(struct nv50_wndw *wndw, wndw->func->release(wndw, asyw, asyh); asyw->ntfy.handle = 0; asyw->sema.handle = 0; + asyw->xlut.handle = 0; + memset(asyw->image.handle, 0x00, sizeof(asyw->image.handle)); } static int @@ -519,7 +521,8 @@ nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state) return PTR_ERR(ctxdma); } - asyw->image.handle[0] = ctxdma->object.handle; + if (asyw->visible) + asyw->image.handle[0] = ctxdma->object.handle; } asyw->state.fence = dma_resv_get_excl_rcu(nvbo->bo.base.resv); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index 38bf4f38e869..53800fb46582 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -46,7 +46,8 @@ struct nv50_disp_acquire_v0 { __u8 version; __u8 or; __u8 link; - __u8 pad03[5]; + __u8 hda; + __u8 pad04[4]; }; struct nv50_disp_dac_load_v0 { diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index de51733b0476..a72c412ac8b1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -52,6 +52,7 @@ struct nouveau_encoder { * actually programmed on the hw, not the proposed crtc */ struct drm_crtc *crtc; u32 ctrl; + bool audio; struct drm_display_mode mode; int last_dpms; diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index 22f054f7ee3e..ba9f9359c30e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -175,10 +175,10 @@ nouveau_svmm_bind(struct drm_device *dev, void *data, */ mm = get_task_mm(current); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (!cli->svm.svmm) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return -EINVAL; } @@ -205,7 +205,7 @@ nouveau_svmm_bind(struct drm_device *dev, void *data, */ args->result = 0; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); return 0; @@ -355,7 +355,7 @@ nouveau_svmm_init(struct drm_device *dev, void *data, if (ret) goto out_free; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); svmm->notifier.ops = &nouveau_mn_ops; ret = __mmu_notifier_register(&svmm->notifier, current->mm); if (ret) @@ -364,12 +364,12 @@ nouveau_svmm_init(struct drm_device *dev, void *data, cli->svm.svmm = svmm; cli->svm.cli = cli; - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); mutex_unlock(&cli->mutex); return 0; out_mm_unlock: - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); out_free: mutex_unlock(&cli->mutex); kfree(svmm); @@ -571,9 +571,9 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, return -EBUSY; range.notifier_seq = mmu_interval_read_begin(range.notifier); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); ret = hmm_range_fault(&range); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (ret) { /* * FIXME: the input PFN_REQ flags are destroyed on @@ -705,18 +705,18 @@ nouveau_svm_fault(struct nvif_notify *notify) /* Intersect fault window with the CPU VMA, cancelling * the fault if the address is invalid. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma_intersection(mm, start, limit); if (!vma) { SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); continue; } start = max_t(u64, start, vma->vm_start); limit = min_t(u64, limit, vma->vm_end); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); if (buffer->fault[fi]->addr != start) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index 571687ba85b8..cf075311cdd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -39,6 +39,7 @@ nvkm-y += nvkm/engine/disp/sorgf119.o nvkm-y += nvkm/engine/disp/sorgk104.o nvkm-y += nvkm/engine/disp/sorgm107.o nvkm-y += nvkm/engine/disp/sorgm200.o +nvkm-y += nvkm/engine/disp/sorgp100.o nvkm-y += nvkm/engine/disp/sorgv100.o nvkm-y += nvkm/engine/disp/sortu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c index fd6216684f6d..8471de3f3b61 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c @@ -36,7 +36,7 @@ gp100_disp = { .super = gf119_disp_super, .root = &gp100_disp_root_oclass, .head = { .cnt = gf119_head_cnt, .new = gf119_head_new }, - .sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new }, + .sor = { .cnt = gf119_sor_cnt, .new = gp100_sor_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c index 3468ddec1270..a3779c5046ea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c @@ -63,7 +63,7 @@ gp102_disp = { .super = gf119_disp_super, .root = &gp102_disp_root_oclass, .head = { .cnt = gf119_head_cnt, .new = gf119_head_new }, - .sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new }, + .sor = { .cnt = gf119_sor_cnt, .new = gp100_sor_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c index 9b16a08eb4d9..bf6d41fb0c9f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c @@ -27,10 +27,10 @@ void gm200_hdmi_scdc(struct nvkm_ior *ior, int head, u8 scdc) { struct nvkm_device *device = ior->disp->engine.subdev.device; - const u32 hoff = head * 0x800; + const u32 soff = nv50_ior_base(ior); const u32 ctrl = scdc & 0x3; - nvkm_mask(device, 0x61c5bc + hoff, 0x00000003, ctrl); + nvkm_mask(device, 0x61c5bc + soff, 0x00000003, ctrl); ior->tmds.high_speed = !!(scdc & 0x2); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index c1d7a36e4d3c..1a200a9ba4e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -201,6 +201,7 @@ int gf119_sor_new(struct nvkm_disp *, int); int gk104_sor_new(struct nvkm_disp *, int); int gm107_sor_new(struct nvkm_disp *, int); int gm200_sor_new(struct nvkm_disp *, int); +int gp100_sor_new(struct nvkm_disp *, int); int gv100_sor_cnt(struct nvkm_disp *, unsigned long *); int gv100_sor_new(struct nvkm_disp *, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c index c62030c96fba..dcf08249374a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c @@ -111,8 +111,44 @@ nvkm_outp_acquire_ior(struct nvkm_outp *outp, u8 user, struct nvkm_ior *ior) return 0; } +static inline int +nvkm_outp_acquire_hda(struct nvkm_outp *outp, enum nvkm_ior_type type, + u8 user, bool hda) +{ + struct nvkm_ior *ior; + + /* First preference is to reuse the OR that is currently armed + * on HW, if any, in order to prevent unnecessary switching. + */ + list_for_each_entry(ior, &outp->disp->ior, head) { + if (!ior->identity && !!ior->func->hda.hpd == hda && + !ior->asy.outp && ior->arm.outp == outp) + return nvkm_outp_acquire_ior(outp, user, ior); + } + + /* Failing that, a completely unused OR is the next best thing. */ + list_for_each_entry(ior, &outp->disp->ior, head) { + if (!ior->identity && !!ior->func->hda.hpd == hda && + !ior->asy.outp && ior->type == type && !ior->arm.outp && + (ior->func->route.set || ior->id == __ffs(outp->info.or))) + return nvkm_outp_acquire_ior(outp, user, ior); + } + + /* Last resort is to assign an OR that's already active on HW, + * but will be released during the next modeset. + */ + list_for_each_entry(ior, &outp->disp->ior, head) { + if (!ior->identity && !!ior->func->hda.hpd == hda && + !ior->asy.outp && ior->type == type && + (ior->func->route.set || ior->id == __ffs(outp->info.or))) + return nvkm_outp_acquire_ior(outp, user, ior); + } + + return -ENOSPC; +} + int -nvkm_outp_acquire(struct nvkm_outp *outp, u8 user) +nvkm_outp_acquire(struct nvkm_outp *outp, u8 user, bool hda) { struct nvkm_ior *ior = outp->ior; enum nvkm_ior_proto proto; @@ -137,32 +173,25 @@ nvkm_outp_acquire(struct nvkm_outp *outp, u8 user) return nvkm_outp_acquire_ior(outp, user, ior); } - /* First preference is to reuse the OR that is currently armed - * on HW, if any, in order to prevent unnecessary switching. + /* If we don't need HDA, first try to acquire an OR that doesn't + * support it to leave free the ones that do. */ - list_for_each_entry(ior, &outp->disp->ior, head) { - if (!ior->identity && !ior->asy.outp && ior->arm.outp == outp) - return nvkm_outp_acquire_ior(outp, user, ior); - } + if (!hda) { + if (!nvkm_outp_acquire_hda(outp, type, user, false)) + return 0; - /* Failing that, a completely unused OR is the next best thing. */ - list_for_each_entry(ior, &outp->disp->ior, head) { - if (!ior->identity && - !ior->asy.outp && ior->type == type && !ior->arm.outp && - (ior->func->route.set || ior->id == __ffs(outp->info.or))) - return nvkm_outp_acquire_ior(outp, user, ior); + /* Use a HDA-supporting SOR anyway. */ + return nvkm_outp_acquire_hda(outp, type, user, true); } - /* Last resort is to assign an OR that's already active on HW, - * but will be released during the next modeset. - */ - list_for_each_entry(ior, &outp->disp->ior, head) { - if (!ior->identity && !ior->asy.outp && ior->type == type && - (ior->func->route.set || ior->id == __ffs(outp->info.or))) - return nvkm_outp_acquire_ior(outp, user, ior); - } + /* We want HDA, try to acquire an OR that supports it. */ + if (!nvkm_outp_acquire_hda(outp, type, user, true)) + return 0; - return -ENOSPC; + /* There weren't any free ORs that support HDA, grab one that + * doesn't and at least allow display to work still. + */ + return nvkm_outp_acquire_hda(outp, type, user, false); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index 721b068b87ef..ee028d30cfe7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -32,7 +32,7 @@ int nvkm_outp_new(struct nvkm_disp *, int index, struct dcb_output *, void nvkm_outp_del(struct nvkm_outp **); void nvkm_outp_init(struct nvkm_outp *); void nvkm_outp_fini(struct nvkm_outp *); -int nvkm_outp_acquire(struct nvkm_outp *, u8 user); +int nvkm_outp_acquire(struct nvkm_outp *, u8 user, bool hda); void nvkm_outp_release(struct nvkm_outp *, u8 user); void nvkm_outp_route(struct nvkm_disp *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index a7672ef17d3b..fb5de44e4b8d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -99,7 +99,7 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) } *args = data; int ret = -ENOSYS; if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER); + ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.hda); if (ret == 0) { args->v0.or = outp->ior->id; args->v0.link = outp->ior->asy.link; @@ -119,7 +119,7 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { if (args->v0.data & 0xfff00000) return -EINVAL; - ret = nvkm_outp_acquire(outp, NVKM_OUTP_PRIV); + ret = nvkm_outp_acquire(outp, NVKM_OUTP_PRIV, false); if (ret) return ret; ret = outp->ior->func->sense(outp->ior, args->v0.data); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c index cf2075db742a..4dd7f382968e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c @@ -89,7 +89,7 @@ gm200_sor_route_get(struct nvkm_outp *outp, int *link) } static const struct nvkm_ior_func -gm200_sor = { +gm200_sor_hda = { .route = { .get = gm200_sor_route_get, .set = gm200_sor_route_set, @@ -119,8 +119,42 @@ gm200_sor = { }, }; +static const struct nvkm_ior_func +gm200_sor = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gf119_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gk104_hdmi_ctrl, + .scdc = gm200_hdmi_scdc, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = gf119_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .vcpi = gf119_sor_dp_vcpi, + .audio = gf119_sor_dp_audio, + .audio_sym = gf119_sor_dp_audio_sym, + .watermark = gf119_sor_dp_watermark, + }, +}; + int gm200_sor_new(struct nvkm_disp *disp, int id) { + struct nvkm_device *device = disp->engine.subdev.device; + u32 hda; + + if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000)) + hda = nvkm_rd32(device, 0x101034); + + if (hda & BIT(id)) + return nvkm_ior_new_(&gm200_sor_hda, disp, SOR, id); return nvkm_ior_new_(&gm200_sor, disp, SOR, id); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c new file mode 100644 index 000000000000..c54f88317a07 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "ior.h" + +static const struct nvkm_ior_func +gp100_sor_hda = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gf119_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gk104_hdmi_ctrl, + .scdc = gm200_hdmi_scdc, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = gf119_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .vcpi = gf119_sor_dp_vcpi, + .audio = gf119_sor_dp_audio, + .audio_sym = gf119_sor_dp_audio_sym, + .watermark = gf119_sor_dp_watermark, + }, + .hda = { + .hpd = gf119_hda_hpd, + .eld = gf119_hda_eld, + .device_entry = gf119_hda_device_entry, + }, +}; + +static const struct nvkm_ior_func +gp100_sor = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gf119_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gk104_hdmi_ctrl, + .scdc = gm200_hdmi_scdc, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = gf119_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .vcpi = gf119_sor_dp_vcpi, + .audio = gf119_sor_dp_audio, + .audio_sym = gf119_sor_dp_audio_sym, + .watermark = gf119_sor_dp_watermark, + }, +}; + +int +gp100_sor_new(struct nvkm_disp *disp, int id) +{ + struct nvkm_device *device = disp->engine.subdev.device; + u32 hda; + + if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000)) + hda = nvkm_rd32(device, 0x10ebb0) >> 8; + + if (hda & BIT(id)) + return nvkm_ior_new_(&gp100_sor_hda, disp, SOR, id); + return nvkm_ior_new_(&gp100_sor, disp, SOR, id); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c index d11a0dff10c6..4441187e8ec9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c @@ -78,7 +78,7 @@ gv100_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state) } static const struct nvkm_ior_func -gv100_sor = { +gv100_sor_hda = { .route = { .get = gm200_sor_route_get, .set = gm200_sor_route_set, @@ -107,9 +107,42 @@ gv100_sor = { }, }; +static const struct nvkm_ior_func +gv100_sor = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gv100_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gv100_hdmi_ctrl, + .scdc = gm200_hdmi_scdc, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = gf119_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .audio = gv100_sor_dp_audio, + .audio_sym = gv100_sor_dp_audio_sym, + .watermark = gv100_sor_dp_watermark, + }, +}; + int gv100_sor_new(struct nvkm_disp *disp, int id) { + struct nvkm_device *device = disp->engine.subdev.device; + u32 hda; + + if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000)) + hda = nvkm_rd32(device, 0x118fb0) >> 8; + + if (hda & BIT(id)) + return nvkm_ior_new_(&gv100_sor_hda, disp, SOR, id); return nvkm_ior_new_(&gv100_sor, disp, SOR, id); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c index fa6d74251237..59865a934c4b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c @@ -62,7 +62,7 @@ tu102_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux) } static const struct nvkm_ior_func -tu102_sor = { +tu102_sor_hda = { .route = { .get = gm200_sor_route_get, .set = gm200_sor_route_set, @@ -92,8 +92,38 @@ tu102_sor = { }, }; +static const struct nvkm_ior_func +tu102_sor = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gv100_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gv100_hdmi_ctrl, + .scdc = gm200_hdmi_scdc, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = tu102_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .vcpi = tu102_sor_dp_vcpi, + .audio = gv100_sor_dp_audio, + .audio_sym = gv100_sor_dp_audio_sym, + .watermark = gv100_sor_dp_watermark, + }, +}; + int tu102_sor_new(struct nvkm_disp *disp, int id) { + struct nvkm_device *device = disp->engine.subdev.device; + u32 hda = nvkm_rd32(device, 0x08a15c); + if (hda & BIT(id)) + return nvkm_ior_new_(&tu102_sor_hda, disp, SOR, id); return nvkm_ior_new_(&tu102_sor, disp, SOR, id); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c index ec330d791d15..e56880f3e3bd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c @@ -352,7 +352,7 @@ gk20a_gr_load(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif) static const struct gf100_gr_fwif gk20a_gr_fwif[] = { - { -1, gk20a_gr_load, &gk20a_gr }, + { 0, gk20a_gr_load, &gk20a_gr }, {} }; diff --git a/drivers/gpu/drm/omapdrm/dss/Kconfig b/drivers/gpu/drm/omapdrm/dss/Kconfig index 72ae79c0c9b4..2658c521b702 100644 --- a/drivers/gpu/drm/omapdrm/dss/Kconfig +++ b/drivers/gpu/drm/omapdrm/dss/Kconfig @@ -71,7 +71,7 @@ config OMAP4_DSS_HDMI_CEC depends on OMAP4_DSS_HDMI select CEC_CORE default y - ---help--- + help When selected the HDMI transmitter will support the CEC feature. config OMAP5_DSS_HDMI diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 0d0ab8e0ff3b..cc31d187042e 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -196,12 +196,12 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p) p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm, &p->validated); if (need_mmap_lock) - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring); if (need_mmap_lock) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return r; } diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index 068c3e5da173..3c8f570a20ee 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -342,17 +342,17 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data, } if (args->flags & RADEON_GEM_USERPTR_VALIDATE) { - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); r = radeon_bo_reserve(bo, true); if (r) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto release_object; } radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_GTT); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); radeon_bo_unreserve(bo); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (r) goto release_object; } diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi.h b/drivers/gpu/drm/sun4i/sun4i_hdmi.h index 7ad3f06c127e..00ca35f07ba5 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi.h +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi.h @@ -148,7 +148,7 @@ #define SUN4I_HDMI_DDC_CMD_IMPLICIT_WRITE 3 #define SUN4I_HDMI_DDC_CLK_REG 0x528 -#define SUN4I_HDMI_DDC_CLK_M(m) (((m) & 0x7) << 3) +#define SUN4I_HDMI_DDC_CLK_M(m) (((m) & 0xf) << 3) #define SUN4I_HDMI_DDC_CLK_N(n) ((n) & 0x7) #define SUN4I_HDMI_DDC_LINE_CTRL_REG 0x540 diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c index 2ff780114106..12430b9d4e93 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c @@ -33,7 +33,7 @@ static unsigned long sun4i_ddc_calc_divider(unsigned long rate, unsigned long best_rate = 0; u8 best_m = 0, best_n = 0, _m, _n; - for (_m = 0; _m < 8; _m++) { + for (_m = 0; _m < 16; _m++) { for (_n = 0; _n < 8; _n++) { unsigned long tmp_rate; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 52d2b71f1588..f09b096ba4fd 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -257,54 +257,6 @@ static int ttm_copy_io_page(void *dst, void *src, unsigned long page) return 0; } -#ifdef CONFIG_X86 -#define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot) -#define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr) -#else -#define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0, __prot) -#define __ttm_kunmap_atomic(__addr) vunmap(__addr) -#endif - - -/** - * ttm_kmap_atomic_prot - Efficient kernel map of a single page with - * specified page protection. - * - * @page: The page to map. - * @prot: The page protection. - * - * This function maps a TTM page using the kmap_atomic api if available, - * otherwise falls back to vmap. The user must make sure that the - * specified page does not have an aliased mapping with a different caching - * policy unless the architecture explicitly allows it. Also mapping and - * unmapping using this api must be correctly nested. Unmapping should - * occur in the reverse order of mapping. - */ -void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot) -{ - if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) - return kmap_atomic(page); - else - return __ttm_kmap_atomic_prot(page, prot); -} -EXPORT_SYMBOL(ttm_kmap_atomic_prot); - -/** - * ttm_kunmap_atomic_prot - Unmap a page that was mapped using - * ttm_kmap_atomic_prot. - * - * @addr: The virtual address from the map. - * @prot: The page protection. - */ -void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot) -{ - if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) - kunmap_atomic(addr); - else - __ttm_kunmap_atomic(addr); -} -EXPORT_SYMBOL(ttm_kunmap_atomic_prot); - static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, unsigned long page, pgprot_t prot) @@ -316,13 +268,13 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, return -ENOMEM; src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); - dst = ttm_kmap_atomic_prot(d, prot); + dst = kmap_atomic_prot(d, prot); if (!dst) return -ENOMEM; memcpy_fromio(dst, src, PAGE_SIZE); - ttm_kunmap_atomic_prot(dst, prot); + kunmap_atomic(dst); return 0; } @@ -338,13 +290,13 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, return -ENOMEM; dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); - src = ttm_kmap_atomic_prot(s, prot); + src = kmap_atomic_prot(s, prot); if (!src) return -ENOMEM; memcpy_toio(dst, src, PAGE_SIZE); - ttm_kunmap_atomic_prot(src, prot); + kunmap_atomic(src); return 0; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 0ad30b112982..a43aa7275f12 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -58,7 +58,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, goto out_clear; /* - * If possible, avoid waiting for GPU with mmap_sem + * If possible, avoid waiting for GPU with mmap_lock * held. We only do this if the fault allows retry and this * is the first attempt. */ @@ -68,7 +68,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, goto out_unlock; ttm_bo_get(bo); - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); (void) dma_fence_wait(bo->moving, true); dma_resv_unlock(bo->base.resv); ttm_bo_put(bo); @@ -131,20 +131,20 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, { /* * Work around locking order reversal in fault / nopfn - * between mmap_sem and bo_reserve: Perform a trylock operation + * between mmap_lock and bo_reserve: Perform a trylock operation * for reserve, and if it fails, retry the fault after waiting * for the buffer to become unreserved. */ if (unlikely(!dma_resv_trylock(bo->base.resv))) { /* * If the fault allows retry and this is the first - * fault attempt, we try to release the mmap_sem + * fault attempt, we try to release the mmap_lock * before waiting */ if (fault_flag_allow_retry_first(vmf->flags)) { if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { ttm_bo_get(bo); - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); if (!dma_resv_lock_interruptible(bo->base.resv, NULL)) dma_resv_unlock(bo->base.resv); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c index bb46ca0c458f..1629427d5734 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c @@ -27,6 +27,7 @@ **************************************************************************/ #include "vmwgfx_drv.h" +#include <linux/highmem.h> /* * Template that implements find_first_diff() for a generic @@ -374,12 +375,12 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d, copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset); if (unmap_src) { - ttm_kunmap_atomic_prot(d->src_addr, d->src_prot); + kunmap_atomic(d->src_addr); d->src_addr = NULL; } if (unmap_dst) { - ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot); + kunmap_atomic(d->dst_addr); d->dst_addr = NULL; } @@ -388,8 +389,8 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d, return -EINVAL; d->dst_addr = - ttm_kmap_atomic_prot(d->dst_pages[dst_page], - d->dst_prot); + kmap_atomic_prot(d->dst_pages[dst_page], + d->dst_prot); if (!d->dst_addr) return -ENOMEM; @@ -401,8 +402,8 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d, return -EINVAL; d->src_addr = - ttm_kmap_atomic_prot(d->src_pages[src_page], - d->src_prot); + kmap_atomic_prot(d->src_pages[src_page], + d->src_prot); if (!d->src_addr) return -ENOMEM; @@ -499,9 +500,9 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst, } out: if (d.src_addr) - ttm_kunmap_atomic_prot(d.src_addr, d.src_prot); + kunmap_atomic(d.src_addr); if (d.dst_addr) - ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot); + kunmap_atomic(d.dst_addr); return ret; } |