summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm
diff options
context:
space:
mode:
authorDaniel Vetter <daniel.vetter@ffwll.ch>2017-11-21 11:33:10 +0100
committerDaniel Vetter <daniel.vetter@ffwll.ch>2017-11-21 14:17:56 +0100
commit70c5f93669249886b151812076509f30569aff80 (patch)
tree05dce785f3a70e022b91016c87e4092f143a2fef /drivers/gpu/drm
parentc83ecfa5851f4d35be88f32dabb3a53f51cf5c32 (diff)
parentf150891fd9878ef0d9197c4e8451ce67c3bdd014 (diff)
Merge airlied/drm-next into drm-misc-next
Bake in the conflict between the drm_print.h extraction and the addition of DRM_DEBUG_LEASES since we lost it a few too many times. Also fix a new use of drm_plane_helper_check_state in msm to follow Ville's conversion in commit a01cb8ba3f6282934cff65e89ab36b18b14cbe27 Author: Ville Syrjälä <ville.syrjala@linux.intel.com> Date: Wed Nov 1 22:16:19 2017 +0200 drm: Move drm_plane_helper_check_state() into drm_atomic_helper.c Acked-by: Ville Syrjälä <ville.syrjala@linux.intel.com> Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Diffstat (limited to 'drivers/gpu/drm')
-rw-r--r--drivers/gpu/drm/Makefile2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/Makefile2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h41
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c161
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c56
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c225
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c106
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c178
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c63
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.h13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c83
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c59
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c104
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c109
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h (renamed from drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.h)24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c227
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c61
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c109
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h191
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c94
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h17
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ci_dpm.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cz_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_virtual.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c195
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/iceland_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/kv_dpm.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c44
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c44
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v10_0.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dpm.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/tonga_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c515
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vega10_ih.c11
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vid.h10
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Kconfig2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c8
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cik_int.h3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_chardev.c12
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c14
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c108
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c459
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h24
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c20
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c20
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_events.c615
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_events.h18
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c83
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c20
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c34
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c7
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c29
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h118
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_process.c169
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c73
-rw-r--r--drivers/gpu/drm/amd/include/amd_shared.h109
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h15
-rw-r--r--drivers/gpu/drm/amd/include/atombios.h1
-rw-r--r--drivers/gpu/drm/amd/include/cgs_common.h8
-rw-r--r--drivers/gpu/drm/amd/lib/Kconfig3
-rw-r--r--drivers/gpu/drm/amd/lib/chash.c4
-rw-r--r--drivers/gpu/drm/amd/powerplay/amd_powerplay.c283
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c103
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c1
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c2489
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h4
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c6
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c2
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c138
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h2
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c55
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c60
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h1
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c8
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c35
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c4
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.h3
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h9
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/fiji_pwrvirus.h10299
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/hwmgr.h6
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h11792
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/pp_instance.h3
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h7
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smumgr.h5
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h5
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/Makefile6
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/ci_smc.c2753
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c2736
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c2486
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h53
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c2415
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h9
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c2568
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c2491
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c2344
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c2262
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c48
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h4
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c3261
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.h62
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c3117
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h20
-rw-r--r--drivers/gpu/drm/amd/scheduler/gpu_scheduler.c47
-rw-r--r--drivers/gpu/drm/amd/scheduler/gpu_scheduler.h19
-rw-r--r--drivers/gpu/drm/ast/ast_mode.c2
-rw-r--r--drivers/gpu/drm/bochs/bochs_kms.c2
-rw-r--r--drivers/gpu/drm/cirrus/cirrus_mode.c2
-rw-r--r--drivers/gpu/drm/drm_atomic.c15
-rw-r--r--drivers/gpu/drm/drm_atomic_helper.c2
-rw-r--r--drivers/gpu/drm/drm_auth.c32
-rw-r--r--drivers/gpu/drm/drm_color_mgmt.c4
-rw-r--r--drivers/gpu/drm/drm_connector.c2
-rw-r--r--drivers/gpu/drm/drm_crtc.c8
-rw-r--r--drivers/gpu/drm/drm_crtc_internal.h1
-rw-r--r--drivers/gpu/drm/drm_dp_dual_mode_helper.c16
-rw-r--r--drivers/gpu/drm/drm_drv.c3
-rw-r--r--drivers/gpu/drm/drm_encoder.c7
-rw-r--r--drivers/gpu/drm/drm_framebuffer.c10
-rw-r--r--drivers/gpu/drm/drm_internal.h7
-rw-r--r--drivers/gpu/drm/drm_ioctl.c8
-rw-r--r--drivers/gpu/drm/drm_lease.c767
-rw-r--r--drivers/gpu/drm/drm_mode_config.c23
-rw-r--r--drivers/gpu/drm/drm_mode_object.c32
-rw-r--r--drivers/gpu/drm/drm_modeset_lock.c2
-rw-r--r--drivers/gpu/drm/drm_plane.c68
-rw-r--r--drivers/gpu/drm/drm_probe_helper.c2
-rw-r--r--drivers/gpu/drm/drm_property.c6
-rw-r--r--drivers/gpu/drm/drm_syncobj.c82
-rw-r--r--drivers/gpu/drm/drm_vblank.c457
-rw-r--r--drivers/gpu/drm/etnaviv/Kconfig2
-rw-r--r--drivers/gpu/drm/etnaviv/Makefile3
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_buffer.c36
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c15
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h6
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_drv.c43
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_drv.h6
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_gem.c19
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c69
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_gpu.c217
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_gpu.h13
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_iommu.c197
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_iommu.h7
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c120
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_mmu.c106
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_mmu.h36
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_perfmon.c495
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_perfmon.h49
-rw-r--r--drivers/gpu/drm/exynos/Kconfig1
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_crtc.c15
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_drv.c8
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_drv.h3
-rw-r--r--drivers/gpu/drm/exynos/exynos_hdmi.c310
-rw-r--r--drivers/gpu/drm/exynos/exynos_mixer.c460
-rw-r--r--drivers/gpu/drm/exynos/regs-hdmi.h8
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c2
-rw-r--r--drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c3
-rw-r--r--drivers/gpu/drm/i915/Kconfig1
-rw-r--r--drivers/gpu/drm/i915/Makefile8
-rw-r--r--drivers/gpu/drm/i915/gvt/cfg_space.c30
-rw-r--r--drivers/gpu/drm/i915/gvt/cmd_parser.c40
-rw-r--r--drivers/gpu/drm/i915/gvt/execlist.c130
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.c5
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.h14
-rw-r--r--drivers/gpu/drm/i915/gvt/handlers.c70
-rw-r--r--drivers/gpu/drm/i915/gvt/kvmgt.c44
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio.c47
-rw-r--r--drivers/gpu/drm/i915/gvt/reg.h3
-rw-r--r--drivers/gpu/drm/i915/gvt/render.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/sched_policy.c22
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.c115
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.h2
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c638
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c90
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h314
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c583
-rw-r--r--drivers/gpu/drm/i915/i915_gem_clflush.c1
-rw-r--r--drivers/gpu/drm/i915/i915_gem_context.c123
-rw-r--r--drivers/gpu/drm/i915/i915_gem_dmabuf.c18
-rw-r--r--drivers/gpu/drm/i915/i915_gem_evict.c77
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c29
-rw-r--r--drivers/gpu/drm/i915/i915_gem_fence_reg.c40
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c614
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.h66
-rw-r--r--drivers/gpu/drm/i915/i915_gem_internal.c18
-rw-r--r--drivers/gpu/drm/i915/i915_gem_object.h42
-rw-r--r--drivers/gpu/drm/i915/i915_gem_render_state.c2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_request.c52
-rw-r--r--drivers/gpu/drm/i915/i915_gem_request.h48
-rw-r--r--drivers/gpu/drm/i915/i915_gem_shrinker.c131
-rw-r--r--drivers/gpu/drm/i915/i915_gem_stolen.c21
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_userptr.c144
-rw-r--r--drivers/gpu/drm/i915/i915_gemfs.c74
-rw-r--r--drivers/gpu/drm/i915/i915_gemfs.h34
-rw-r--r--drivers/gpu/drm/i915/i915_gpu_error.c43
-rw-r--r--drivers/gpu/drm/i915/i915_guc_submission.c476
-rw-r--r--drivers/gpu/drm/i915/i915_guc_submission.h80
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c1093
-rw-r--r--drivers/gpu/drm/i915/i915_oa_cflgt2.c109
-rw-r--r--drivers/gpu/drm/i915/i915_oa_cflgt2.h34
-rw-r--r--drivers/gpu/drm/i915/i915_params.c207
-rw-r--r--drivers/gpu/drm/i915/i915_params.h85
-rw-r--r--drivers/gpu/drm/i915/i915_pci.c94
-rw-r--r--drivers/gpu/drm/i915/i915_perf.c18
-rw-r--r--drivers/gpu/drm/i915/i915_pvinfo.h1
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h70
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c2
-rw-r--r--drivers/gpu/drm/i915/i915_sw_fence.c45
-rw-r--r--drivers/gpu/drm/i915/i915_sysfs.c76
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h116
-rw-r--r--drivers/gpu/drm/i915/i915_utils.h18
-rw-r--r--drivers/gpu/drm/i915/i915_vgpu.h6
-rw-r--r--drivers/gpu/drm/i915/i915_vma.c144
-rw-r--r--drivers/gpu/drm/i915/i915_vma.h49
-rw-r--r--drivers/gpu/drm/i915/intel_audio.c7
-rw-r--r--drivers/gpu/drm/i915/intel_bios.c233
-rw-r--r--drivers/gpu/drm/i915/intel_breadcrumbs.c20
-rw-r--r--drivers/gpu/drm/i915/intel_cdclk.c46
-rw-r--r--drivers/gpu/drm/i915/intel_color.c16
-rw-r--r--drivers/gpu/drm/i915/intel_crt.c114
-rw-r--r--drivers/gpu/drm/i915/intel_csr.c41
-rw-r--r--drivers/gpu/drm/i915/intel_ddi.c906
-rw-r--r--drivers/gpu/drm/i915/intel_device_info.c41
-rw-r--r--drivers/gpu/drm/i915/intel_display.c459
-rw-r--r--drivers/gpu/drm/i915/intel_dp.c137
-rw-r--r--drivers/gpu/drm/i915/intel_dp_aux_backlight.c2
-rw-r--r--drivers/gpu/drm/i915/intel_dp_mst.c60
-rw-r--r--drivers/gpu/drm/i915/intel_dpio_phy.c20
-rw-r--r--drivers/gpu/drm/i915/intel_dpll_mgr.c2
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h48
-rw-r--r--drivers/gpu/drm/i915/intel_dsi.c54
-rw-r--r--drivers/gpu/drm/i915/intel_dvo.c33
-rw-r--r--drivers/gpu/drm/i915/intel_engine_cs.c493
-rw-r--r--drivers/gpu/drm/i915/intel_fbc.c24
-rw-r--r--drivers/gpu/drm/i915/intel_fbdev.c4
-rw-r--r--drivers/gpu/drm/i915/intel_fifo_underrun.c14
-rw-r--r--drivers/gpu/drm/i915/intel_guc.c369
-rw-r--r--drivers/gpu/drm/i915/intel_guc.h120
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fw.c (renamed from drivers/gpu/drm/i915/intel_guc_loader.c)252
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fw.h33
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fwif.h62
-rw-r--r--drivers/gpu/drm/i915/intel_guc_log.c32
-rw-r--r--drivers/gpu/drm/i915/intel_guc_log.h59
-rw-r--r--drivers/gpu/drm/i915/intel_gvt.c12
-rw-r--r--drivers/gpu/drm/i915/intel_hangcheck.c2
-rw-r--r--drivers/gpu/drm/i915/intel_hdmi.c26
-rw-r--r--drivers/gpu/drm/i915/intel_huc.c172
-rw-r--r--drivers/gpu/drm/i915/intel_huc.h41
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.c705
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.h37
-rw-r--r--drivers/gpu/drm/i915/intel_lspcon.c22
-rw-r--r--drivers/gpu/drm/i915/intel_lvds.c27
-rw-r--r--drivers/gpu/drm/i915/intel_modes.c17
-rw-r--r--drivers/gpu/drm/i915/intel_opregion.c2
-rw-r--r--drivers/gpu/drm/i915/intel_overlay.c2
-rw-r--r--drivers/gpu/drm/i915/intel_panel.c8
-rw-r--r--drivers/gpu/drm/i915/intel_pipe_crc.c15
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c1078
-rw-r--r--drivers/gpu/drm/i915/intel_psr.c399
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c194
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.h174
-rw-r--r--drivers/gpu/drm/i915/intel_runtime_pm.c48
-rw-r--r--drivers/gpu/drm/i915/intel_sideband.c6
-rw-r--r--drivers/gpu/drm/i915/intel_sprite.c12
-rw-r--r--drivers/gpu/drm/i915/intel_tv.c2
-rw-r--r--drivers/gpu/drm/i915/intel_uc.c363
-rw-r--r--drivers/gpu/drm/i915/intel_uc.h247
-rw-r--r--drivers/gpu/drm/i915/intel_uc_fw.c318
-rw-r--r--drivers/gpu/drm/i915/intel_uc_fw.h121
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.c268
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.h18
-rw-r--r--drivers/gpu/drm/i915/intel_vbt_defs.h8
-rw-r--r--drivers/gpu/drm/i915/selftests/huge_gem_object.c14
-rw-r--r--drivers/gpu/drm/i915/selftests/huge_pages.c1734
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_context.c2
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_evict.c162
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_gtt.c15
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_object.c8
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_request.c12
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_timeline.c2
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_live_selftests.h2
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_mock_selftests.h1
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_random.c5
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_random.h5
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_sw_fence.c42
-rw-r--r--drivers/gpu/drm/i915/selftests/intel_hangcheck.c50
-rw-r--r--drivers/gpu/drm/i915/selftests/lib_sw_fence.c78
-rw-r--r--drivers/gpu/drm/i915/selftests/lib_sw_fence.h42
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_context.c6
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_engine.c8
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gem_device.c20
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gtt.c11
-rw-r--r--drivers/gpu/drm/i915/selftests/scatterlist.c15
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_mode.c2
-rw-r--r--drivers/gpu/drm/msm/Makefile4
-rw-r--r--drivers/gpu/drm/msm/adreno/a3xx_gpu.c10
-rw-r--r--drivers/gpu/drm/msm/adreno/a4xx_gpu.c10
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_gpu.c261
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_gpu.h109
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_power.c9
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_preempt.c305
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_device.c78
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_gpu.c297
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_gpu.h66
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_cfg.c8
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_host.c32
-rw-r--r--drivers/gpu/drm/msm/dsi/phy/dsi_phy.c2
-rw-r--r--drivers/gpu/drm/msm/edp/edp_ctrl.c22
-rw-r--r--drivers/gpu/drm/msm/hdmi/hdmi.c10
-rw-r--r--drivers/gpu/drm/msm/hdmi/hdmi_phy.c2
-rw-r--r--drivers/gpu/drm/msm/hdmi/hdmi_phy_8960.c2
-rw-r--r--drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c4
-rw-r--r--drivers/gpu/drm/msm/hdmi/hdmi_phy_8x74.c3
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c7
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c2
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c2
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c162
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c2
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c10
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c8
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.c71
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.h7
-rw-r--r--drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c233
-rw-r--r--drivers/gpu/drm/msm/msm_atomic.c41
-rw-r--r--drivers/gpu/drm/msm/msm_drv.c83
-rw-r--r--drivers/gpu/drm/msm/msm_drv.h37
-rw-r--r--drivers/gpu/drm/msm/msm_fence.c2
-rw-r--r--drivers/gpu/drm/msm/msm_fence.h2
-rw-r--r--drivers/gpu/drm/msm/msm_gem.c37
-rw-r--r--drivers/gpu/drm/msm/msm_gem.h5
-rw-r--r--drivers/gpu/drm/msm/msm_gem_submit.c61
-rw-r--r--drivers/gpu/drm/msm/msm_gpu.c238
-rw-r--r--drivers/gpu/drm/msm/msm_gpu.h51
-rw-r--r--drivers/gpu/drm/msm/msm_rd.c154
-rw-r--r--drivers/gpu/drm/msm/msm_ringbuffer.c36
-rw-r--r--drivers/gpu/drm/msm/msm_ringbuffer.h33
-rw-r--r--drivers/gpu/drm/msm/msm_submitqueue.c152
-rw-r--r--drivers/gpu/drm/nouveau/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/Kconfig7
-rw-r--r--drivers/gpu/drm/nouveau/dispnv04/disp.c2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl506e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl506f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl826e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl826f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl906f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cla06f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/class.h17
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/device.h5
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0008.h42
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if000a.h22
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if000b.h11
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if000c.h64
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if000d.h12
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if500b.h25
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if500d.h21
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if900b.h23
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if900d.h21
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/ifb00d.h27
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/ifc00d.h21
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/ioctl.h10
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/mem.h18
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/mmu.h56
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/object.h7
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/os.h14
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/vmm.h42
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/client.h3
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/device.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/engine.h1
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h14
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/memory.h64
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/mm.h32
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/object.h41
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/oclass.h31
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/os.h19
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h1
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h1
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h5
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h3
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h9
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h36
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h1
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h7
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h140
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h1
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_abi16.c42
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_abi16.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bios.c10
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bo.c391
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bo.h17
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_chan.c51
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_chan.h3
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_connector.c4
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_display.h5
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dma.c15
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dma.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_drm.c161
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_drv.h33
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fbcon.c14
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fence.c58
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fence.h7
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_gem.c123
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_gem.h3
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_mem.c198
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_mem.h30
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_sgdma.c54
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_ttm.c280
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vmm.c135
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vmm.h31
-rw-r--r--drivers/gpu/drm/nouveau/nv50_display.c17
-rw-r--r--drivers/gpu/drm/nouveau/nv50_fbcon.c9
-rw-r--r--drivers/gpu/drm/nouveau/nv84_fence.c43
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_fbcon.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvif/Kbuild3
-rw-r--r--drivers/gpu/drm/nouveau/nvif/mem.c88
-rw-r--r--drivers/gpu/drm/nouveau/nvif/mmu.c117
-rw-r--r--drivers/gpu/drm/nouveau/nvif/object.c87
-rw-r--r--drivers/gpu/drm/nouveau/nvif/vmm.c167
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/client.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/engine.c10
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c47
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/ioctl.c15
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/memory.c100
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/mm.c5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/object.c14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/oproxy.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/ramht.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/base.c98
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c24
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/user.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/falcon.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c8
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c43
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c19
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c45
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c51
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c123
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c83
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/base.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/v1.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c60
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c10
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c151
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h16
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c65
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c42
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c174
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h16
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c12
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c35
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c24
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c30
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c144
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h11
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c94
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c98
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c266
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c61
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c71
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c329
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c25
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild29
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c694
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c228
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c97
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c (renamed from drivers/gpu/drm/amd/powerplay/smumgr/ci_smc.h)57
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c (renamed from drivers/gpu/drm/radeon/radeon_kfd.h)44
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.0
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c45
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c242
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h23
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c94
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c69
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c88
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c122
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c104
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c190
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c231
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c192
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h26
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c178
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c352
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c1513
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h310
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c403
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c102
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c71
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c185
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c347
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c49
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c140
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c113
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c230
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c385
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c43
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c (renamed from drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.h)48
-rw-r--r--drivers/gpu/drm/omapdrm/displays/connector-hdmi.c8
-rw-r--r--drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c18
-rw-r--r--drivers/gpu/drm/omapdrm/dss/Kconfig8
-rw-r--r--drivers/gpu/drm/omapdrm/dss/Makefile1
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi.h7
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi4.c62
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi4_cec.c381
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi4_cec.h55
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi4_core.c7
-rw-r--r--drivers/gpu/drm/omapdrm/dss/hdmi4_core.h4
-rw-r--r--drivers/gpu/drm/omapdrm/dss/omapdss.h1
-rw-r--r--drivers/gpu/drm/qxl/qxl_cmd.c22
-rw-r--r--drivers/gpu/drm/qxl/qxl_display.c49
-rw-r--r--drivers/gpu/drm/qxl/qxl_drv.h28
-rw-r--r--drivers/gpu/drm/qxl/qxl_dumb.c1
-rw-r--r--drivers/gpu/drm/qxl/qxl_fb.c13
-rw-r--r--drivers/gpu/drm/qxl/qxl_release.c5
-rw-r--r--drivers/gpu/drm/qxl/qxl_ttm.c4
-rw-r--r--drivers/gpu/drm/radeon/Makefile3
-rw-r--r--drivers/gpu/drm/radeon/atombios_dp.c46
-rw-r--r--drivers/gpu/drm/radeon/ci_dpm.c22
-rw-r--r--drivers/gpu/drm/radeon/ci_dpm.h1
-rw-r--r--drivers/gpu/drm/radeon/ci_smc.c21
-rw-r--r--drivers/gpu/drm/radeon/cik.c14
-rw-r--r--drivers/gpu/drm/radeon/cikd.h2
-rw-r--r--drivers/gpu/drm/radeon/r100.c2
-rw-r--r--drivers/gpu/drm/radeon/r600_cs.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon.h3
-rw-r--r--drivers/gpu/drm/radeon/radeon_connectors.c16
-rw-r--r--drivers/gpu/drm/radeon/radeon_drv.c10
-rw-r--r--drivers/gpu/drm/radeon/radeon_kfd.c901
-rw-r--r--drivers/gpu/drm/radeon/radeon_kms.c7
-rw-r--r--drivers/gpu/drm/radeon/radeon_mode.h4
-rw-r--r--drivers/gpu/drm/rockchip/Kconfig2
-rw-r--r--drivers/gpu/drm/rockchip/analogix_dp-rockchip.c14
-rw-r--r--drivers/gpu/drm/tegra/Kconfig1
-rw-r--r--drivers/gpu/drm/tegra/dc.c84
-rw-r--r--drivers/gpu/drm/tegra/dc.h120
-rw-r--r--drivers/gpu/drm/tegra/drm.c30
-rw-r--r--drivers/gpu/drm/tegra/drm.h106
-rw-r--r--drivers/gpu/drm/tegra/gr2d.c2
-rw-r--r--drivers/gpu/drm/tegra/gr3d.c2
-rw-r--r--drivers/gpu/drm/tegra/hdmi.c17
-rw-r--r--drivers/gpu/drm/tegra/output.c6
-rw-r--r--drivers/gpu/drm/tegra/sor.c5
-rw-r--r--drivers/gpu/drm/tegra/vic.c22
-rw-r--r--drivers/gpu/drm/tilcdc/tilcdc_crtc.c51
-rw-r--r--drivers/gpu/drm/tilcdc/tilcdc_panel.c2
-rw-r--r--drivers/gpu/drm/tilcdc/tilcdc_slave_compat.c1
-rw-r--r--drivers/gpu/drm/tilcdc/tilcdc_tfp410.c4
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c13
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_util.c1
-rw-r--r--drivers/gpu/drm/ttm/ttm_memory.c12
-rw-r--r--drivers/gpu/drm/ttm/ttm_page_alloc.c380
-rw-r--r--drivers/gpu/drm/ttm/ttm_page_alloc_dma.c236
-rw-r--r--drivers/gpu/drm/udl/udl_connector.c2
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c4
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_kms.c2
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c4
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c4
709 files changed, 47108 insertions, 49663 deletions
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index a3fdc5a68dff..81ff79336623 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -17,7 +17,7 @@ drm-y := drm_auth.o drm_bufs.o drm_cache.o \
drm_encoder.o drm_mode_object.o drm_property.o \
drm_plane.o drm_color_mgmt.o drm_print.o \
drm_dumb_buffers.o drm_mode_config.o drm_vblank.o \
- drm_syncobj.o
+ drm_syncobj.o drm_lease.o
drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o
drm-$(CONFIG_DRM_VM) += drm_vm.o
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 25a95c95df14..ef9a3b6d7b62 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -25,7 +25,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
- amdgpu_queue_mgr.o amdgpu_vf_error.o
+ amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sched.o
# add asic specific block
amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index ebfc267467ee..cbcb6a153aba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
extern int amdgpu_param_buf_per_se;
extern int amdgpu_job_hang_limit;
extern int amdgpu_lbpw;
+extern int amdgpu_compute_multipipe;
#ifdef CONFIG_DRM_AMDGPU_SI
extern int amdgpu_si_support;
@@ -731,10 +732,14 @@ struct amdgpu_ctx {
struct amdgpu_device *adev;
struct amdgpu_queue_mgr queue_mgr;
unsigned reset_counter;
+ uint32_t vram_lost_counter;
spinlock_t ring_lock;
struct dma_fence **fences;
struct amdgpu_ctx_ring rings[AMDGPU_MAX_RINGS];
- bool preamble_presented;
+ bool preamble_presented;
+ enum amd_sched_priority init_priority;
+ enum amd_sched_priority override_priority;
+ struct mutex lock;
};
struct amdgpu_ctx_mgr {
@@ -751,13 +756,18 @@ int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
struct dma_fence *fence, uint64_t *seq);
struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
struct amdgpu_ring *ring, uint64_t seq);
+void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
+ enum amd_sched_priority priority);
int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp);
+int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id);
+
void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
+
/*
* file private structure
*/
@@ -769,7 +779,6 @@ struct amdgpu_fpriv {
struct mutex bo_list_lock;
struct idr bo_list_handles;
struct amdgpu_ctx_mgr ctx_mgr;
- u32 vram_lost_counter;
};
/*
@@ -870,7 +879,7 @@ struct amdgpu_mec {
struct amdgpu_kiq {
u64 eop_gpu_addr;
struct amdgpu_bo *eop_obj;
- struct mutex ring_mutex;
+ spinlock_t ring_lock;
struct amdgpu_ring ring;
struct amdgpu_irq_src irq;
};
@@ -1034,6 +1043,10 @@ struct amdgpu_gfx {
bool in_suspend;
/* NGG */
struct amdgpu_ngg ngg;
+
+ /* pipe reservation */
+ struct mutex pipe_reserve_mutex;
+ DECLARE_BITMAP (pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
};
int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -1112,6 +1125,7 @@ struct amdgpu_job {
uint32_t gds_base, gds_size;
uint32_t gws_base, gws_size;
uint32_t oa_base, oa_size;
+ uint32_t vram_lost_counter;
/* user fence handling */
uint64_t uf_addr;
@@ -1137,7 +1151,7 @@ static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
/*
* Writeback
*/
-#define AMDGPU_MAX_WB 1024 /* Reserve at most 1024 WB slots for amdgpu-owned rings. */
+#define AMDGPU_MAX_WB 512 /* Reserve at most 512 WB slots for amdgpu-owned rings. */
struct amdgpu_wb {
struct amdgpu_bo *wb_obj;
@@ -1310,6 +1324,8 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp);
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp);
@@ -1376,6 +1392,18 @@ struct amdgpu_atcs {
};
/*
+ * Firmware VRAM reservation
+ */
+struct amdgpu_fw_vram_usage {
+ u64 start_offset;
+ u64 size;
+ struct amdgpu_bo *reserved_bo;
+ void *va;
+};
+
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
+
+/*
* CGS
*/
struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
@@ -1524,7 +1552,6 @@ struct amdgpu_device {
/* powerplay */
struct amd_powerplay powerplay;
- bool pp_enabled;
bool pp_force_state_enabled;
/* dpm */
@@ -1580,6 +1607,8 @@ struct amdgpu_device {
struct delayed_work late_init_work;
struct amdgpu_virt virt;
+ /* firmware VRAM reservation */
+ struct amdgpu_fw_vram_usage fw_vram_usage;
/* link all shadow bo */
struct list_head shadow_list;
@@ -1831,8 +1860,6 @@ static inline bool amdgpu_has_atpx(void) { return false; }
extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
extern const int amdgpu_max_kms_ioctl;
-bool amdgpu_kms_vram_lost(struct amdgpu_device *adev,
- struct amdgpu_fpriv *fpriv);
int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags);
void amdgpu_driver_unload_kms(struct drm_device *dev);
void amdgpu_driver_lastclose_kms(struct drm_device *dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index a52795d9b458..c04f44a90392 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -35,41 +35,50 @@
#include "acp_gfx_if.h"
-#define ACP_TILE_ON_MASK 0x03
-#define ACP_TILE_OFF_MASK 0x02
-#define ACP_TILE_ON_RETAIN_REG_MASK 0x1f
-#define ACP_TILE_OFF_RETAIN_REG_MASK 0x20
-
-#define ACP_TILE_P1_MASK 0x3e
-#define ACP_TILE_P2_MASK 0x3d
-#define ACP_TILE_DSP0_MASK 0x3b
-#define ACP_TILE_DSP1_MASK 0x37
-
-#define ACP_TILE_DSP2_MASK 0x2f
-
-#define ACP_DMA_REGS_END 0x146c0
-#define ACP_I2S_PLAY_REGS_START 0x14840
-#define ACP_I2S_PLAY_REGS_END 0x148b4
-#define ACP_I2S_CAP_REGS_START 0x148b8
-#define ACP_I2S_CAP_REGS_END 0x1496c
-
-#define ACP_I2S_COMP1_CAP_REG_OFFSET 0xac
-#define ACP_I2S_COMP2_CAP_REG_OFFSET 0xa8
-#define ACP_I2S_COMP1_PLAY_REG_OFFSET 0x6c
-#define ACP_I2S_COMP2_PLAY_REG_OFFSET 0x68
-
-#define mmACP_PGFSM_RETAIN_REG 0x51c9
-#define mmACP_PGFSM_CONFIG_REG 0x51ca
-#define mmACP_PGFSM_READ_REG_0 0x51cc
-
-#define mmACP_MEM_SHUT_DOWN_REQ_LO 0x51f8
-#define mmACP_MEM_SHUT_DOWN_REQ_HI 0x51f9
-#define mmACP_MEM_SHUT_DOWN_STS_LO 0x51fa
-#define mmACP_MEM_SHUT_DOWN_STS_HI 0x51fb
-
-#define ACP_TIMEOUT_LOOP 0x000000FF
-#define ACP_DEVS 3
-#define ACP_SRC_ID 162
+#define ACP_TILE_ON_MASK 0x03
+#define ACP_TILE_OFF_MASK 0x02
+#define ACP_TILE_ON_RETAIN_REG_MASK 0x1f
+#define ACP_TILE_OFF_RETAIN_REG_MASK 0x20
+
+#define ACP_TILE_P1_MASK 0x3e
+#define ACP_TILE_P2_MASK 0x3d
+#define ACP_TILE_DSP0_MASK 0x3b
+#define ACP_TILE_DSP1_MASK 0x37
+
+#define ACP_TILE_DSP2_MASK 0x2f
+
+#define ACP_DMA_REGS_END 0x146c0
+#define ACP_I2S_PLAY_REGS_START 0x14840
+#define ACP_I2S_PLAY_REGS_END 0x148b4
+#define ACP_I2S_CAP_REGS_START 0x148b8
+#define ACP_I2S_CAP_REGS_END 0x1496c
+
+#define ACP_I2S_COMP1_CAP_REG_OFFSET 0xac
+#define ACP_I2S_COMP2_CAP_REG_OFFSET 0xa8
+#define ACP_I2S_COMP1_PLAY_REG_OFFSET 0x6c
+#define ACP_I2S_COMP2_PLAY_REG_OFFSET 0x68
+
+#define mmACP_PGFSM_RETAIN_REG 0x51c9
+#define mmACP_PGFSM_CONFIG_REG 0x51ca
+#define mmACP_PGFSM_READ_REG_0 0x51cc
+
+#define mmACP_MEM_SHUT_DOWN_REQ_LO 0x51f8
+#define mmACP_MEM_SHUT_DOWN_REQ_HI 0x51f9
+#define mmACP_MEM_SHUT_DOWN_STS_LO 0x51fa
+#define mmACP_MEM_SHUT_DOWN_STS_HI 0x51fb
+
+#define mmACP_CONTROL 0x5131
+#define mmACP_STATUS 0x5133
+#define mmACP_SOFT_RESET 0x5134
+#define ACP_CONTROL__ClkEn_MASK 0x1
+#define ACP_SOFT_RESET__SoftResetAud_MASK 0x100
+#define ACP_SOFT_RESET__SoftResetAudDone_MASK 0x1000000
+#define ACP_CLOCK_EN_TIME_OUT_VALUE 0x000000FF
+#define ACP_SOFT_RESET_DONE_TIME_OUT_VALUE 0x000000FF
+
+#define ACP_TIMEOUT_LOOP 0x000000FF
+#define ACP_DEVS 3
+#define ACP_SRC_ID 162
enum {
ACP_TILE_P1 = 0,
@@ -260,6 +269,8 @@ static int acp_hw_init(void *handle)
{
int r, i;
uint64_t acp_base;
+ u32 val = 0;
+ u32 count = 0;
struct device *dev;
struct i2s_platform_data *i2s_pdata;
@@ -371,6 +382,8 @@ static int acp_hw_init(void *handle)
adev->acp.acp_cell[0].name = "acp_audio_dma";
adev->acp.acp_cell[0].num_resources = 4;
adev->acp.acp_cell[0].resources = &adev->acp.acp_res[0];
+ adev->acp.acp_cell[0].platform_data = &adev->asic_type;
+ adev->acp.acp_cell[0].pdata_size = sizeof(adev->asic_type);
adev->acp.acp_cell[1].name = "designware-i2s";
adev->acp.acp_cell[1].num_resources = 1;
@@ -400,6 +413,46 @@ static int acp_hw_init(void *handle)
}
}
+ /* Assert Soft reset of ACP */
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+
+ val |= ACP_SOFT_RESET__SoftResetAud_MASK;
+ cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
+ count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE;
+ while (true) {
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+ if (ACP_SOFT_RESET__SoftResetAudDone_MASK ==
+ (val & ACP_SOFT_RESET__SoftResetAudDone_MASK))
+ break;
+ if (--count == 0) {
+ dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+ return -ETIMEDOUT;
+ }
+ udelay(100);
+ }
+ /* Enable clock to ACP and wait until the clock is enabled */
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL);
+ val = val | ACP_CONTROL__ClkEn_MASK;
+ cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val);
+
+ count = ACP_CLOCK_EN_TIME_OUT_VALUE;
+
+ while (true) {
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS);
+ if (val & (u32) 0x1)
+ break;
+ if (--count == 0) {
+ dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+ return -ETIMEDOUT;
+ }
+ udelay(100);
+ }
+ /* Deassert the SOFT RESET flags */
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+ val &= ~ACP_SOFT_RESET__SoftResetAud_MASK;
+ cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
return 0;
}
@@ -412,6 +465,8 @@ static int acp_hw_init(void *handle)
static int acp_hw_fini(void *handle)
{
int i, ret;
+ u32 val = 0;
+ u32 count = 0;
struct device *dev;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -419,6 +474,42 @@ static int acp_hw_fini(void *handle)
if (!adev->acp.acp_cell)
return 0;
+ /* Assert Soft reset of ACP */
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+
+ val |= ACP_SOFT_RESET__SoftResetAud_MASK;
+ cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
+ count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE;
+ while (true) {
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+ if (ACP_SOFT_RESET__SoftResetAudDone_MASK ==
+ (val & ACP_SOFT_RESET__SoftResetAudDone_MASK))
+ break;
+ if (--count == 0) {
+ dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+ return -ETIMEDOUT;
+ }
+ udelay(100);
+ }
+ /* Disable ACP clock */
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL);
+ val &= ~ACP_CONTROL__ClkEn_MASK;
+ cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val);
+
+ count = ACP_CLOCK_EN_TIME_OUT_VALUE;
+
+ while (true) {
+ val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS);
+ if (val & (u32) 0x1)
+ break;
+ if (--count == 0) {
+ dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+ return -ETIMEDOUT;
+ }
+ udelay(100);
+ }
+
if (adev->acp.acp_genpd) {
for (i = 0; i < ACP_DEVS ; i++) {
dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index dc7e25cce741..47d1c132ac40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -338,6 +338,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
struct cik_mqd *m;
uint32_t *mqd_hqd;
uint32_t reg, wptr_val, data;
+ bool valid_wptr = false;
m = get_mqd(mqd);
@@ -356,7 +357,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
- if (read_user_wptr(mm, wptr, wptr_val))
+ /* read_user_ptr may take the mm->mmap_sem.
+ * release srbm_mutex to avoid circular dependency between
+ * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+ */
+ release_queue(kgd);
+ valid_wptr = read_user_wptr(mm, wptr, wptr_val);
+ acquire_queue(kgd, pipe_id, queue_id);
+ if (valid_wptr)
WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index c678c69936a0..056929b8ccd0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -292,6 +292,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
struct vi_mqd *m;
uint32_t *mqd_hqd;
uint32_t reg, wptr_val, data;
+ bool valid_wptr = false;
m = get_mqd(mqd);
@@ -339,7 +340,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
- if (read_user_wptr(mm, wptr, wptr_val))
+ /* read_user_ptr may take the mm->mmap_sem.
+ * release srbm_mutex to avoid circular dependency between
+ * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+ */
+ release_queue(kgd);
+ valid_wptr = read_user_wptr(mm, wptr, wptr_val);
+ acquire_queue(kgd, pipe_id, queue_id);
+ if (valid_wptr)
WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index ce443586a0c7..f450b69323fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1766,34 +1766,32 @@ bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev)
return true;
}
-/* Atom needs data in little endian format
- * so swap as appropriate when copying data to
- * or from atom. Note that atom operates on
- * dw units.
+/* Atom needs data in little endian format so swap as appropriate when copying
+ * data to or from atom. Note that atom operates on dw units.
+ *
+ * Use to_le=true when sending data to atom and provide at least
+ * ALIGN(num_bytes,4) bytes in the dst buffer.
+ *
+ * Use to_le=false when receiving data from atom and provide ALIGN(num_bytes,4)
+ * byes in the src buffer.
*/
void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le)
{
#ifdef __BIG_ENDIAN
- u8 src_tmp[20], dst_tmp[20]; /* used for byteswapping */
- u32 *dst32, *src32;
+ u32 src_tmp[5], dst_tmp[5];
int i;
+ u8 align_num_bytes = ALIGN(num_bytes, 4);
- memcpy(src_tmp, src, num_bytes);
- src32 = (u32 *)src_tmp;
- dst32 = (u32 *)dst_tmp;
if (to_le) {
- for (i = 0; i < ((num_bytes + 3) / 4); i++)
- dst32[i] = cpu_to_le32(src32[i]);
- memcpy(dst, dst_tmp, num_bytes);
+ memcpy(src_tmp, src, num_bytes);
+ for (i = 0; i < align_num_bytes / 4; i++)
+ dst_tmp[i] = cpu_to_le32(src_tmp[i]);
+ memcpy(dst, dst_tmp, align_num_bytes);
} else {
- u8 dws = num_bytes & ~3;
- for (i = 0; i < ((num_bytes + 3) / 4); i++)
- dst32[i] = le32_to_cpu(src32[i]);
- memcpy(dst, dst_tmp, dws);
- if (num_bytes % 4) {
- for (i = 0; i < (num_bytes % 4); i++)
- dst[dws+i] = dst_tmp[dws+i];
- }
+ memcpy(src_tmp, src, align_num_bytes);
+ for (i = 0; i < align_num_bytes / 4; i++)
+ dst_tmp[i] = le32_to_cpu(src_tmp[i]);
+ memcpy(dst, dst_tmp, num_bytes);
}
#else
memcpy(dst, src, num_bytes);
@@ -1807,6 +1805,8 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
uint16_t data_offset;
int usage_bytes = 0;
struct _ATOM_VRAM_USAGE_BY_FIRMWARE *firmware_usage;
+ u64 start_addr;
+ u64 size;
if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
firmware_usage = (struct _ATOM_VRAM_USAGE_BY_FIRMWARE *)(ctx->bios + data_offset);
@@ -1815,7 +1815,21 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
le32_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware),
le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb));
- usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+ start_addr = firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware;
+ size = firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb;
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->fw_vram_usage.start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->fw_vram_usage.size = size << 10;
+ /* Use the default scratch size */
+ usage_bytes = 0;
+ } else {
+ usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+ }
}
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index f9ffe8ef0cd6..ff8efd0f8fd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -71,19 +71,33 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
+ struct vram_usagebyfirmware_v2_1 * firmware_usage;
+ uint32_t start_addr, size;
uint16_t data_offset;
int usage_bytes = 0;
if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- struct vram_usagebyfirmware_v2_1 *firmware_usage =
- (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
-
+ firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
le32_to_cpu(firmware_usage->start_address_in_kb),
le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
le16_to_cpu(firmware_usage->used_by_driver_in_kb));
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) * 1024;
+ start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->fw_vram_usage.start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->fw_vram_usage.size = size << 10;
+ /* Use the default scratch size */
+ usage_bytes = 0;
+ } else {
+ usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ }
}
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 383204e911a4..a7afe553e0a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -42,6 +42,28 @@ struct amdgpu_cgs_device {
struct amdgpu_device *adev = \
((struct amdgpu_cgs_device *)cgs_device)->adev
+static void *amdgpu_cgs_register_pp_handle(struct cgs_device *cgs_device,
+ int (*call_back_func)(struct amd_pp_init *, void **))
+{
+ CGS_FUNC_ADEV;
+ struct amd_pp_init pp_init;
+ struct amd_powerplay *amd_pp;
+
+ if (call_back_func == NULL)
+ return NULL;
+
+ amd_pp = &(adev->powerplay);
+ pp_init.chip_family = adev->family;
+ pp_init.chip_id = adev->asic_type;
+ pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
+ pp_init.feature_mask = amdgpu_pp_feature_mask;
+ pp_init.device = cgs_device;
+ if (call_back_func(&pp_init, &(amd_pp->pp_handle)))
+ return NULL;
+
+ return adev->powerplay.pp_handle;
+}
+
static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
enum cgs_gpu_mem_type type,
uint64_t size, uint64_t align,
@@ -1179,6 +1201,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
.is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
.enter_safe_mode = amdgpu_cgs_enter_safe_mode,
.lock_grbm_idx = amdgpu_cgs_lock_grbm_idx,
+ .register_pp_handle = amdgpu_cgs_register_pp_handle,
};
static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 126be9e0f05f..8ca3783f2deb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -231,7 +231,7 @@ amdgpu_connector_update_scratch_regs(struct drm_connector *connector,
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev,
+ encoder = drm_encoder_find(connector->dev, NULL,
connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -256,7 +256,7 @@ amdgpu_connector_find_encoder(struct drm_connector *connector,
for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev,
+ encoder = drm_encoder_find(connector->dev, NULL,
connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -371,7 +371,7 @@ amdgpu_connector_best_single_encoder(struct drm_connector *connector)
/* pick the encoder ids */
if (enc_id)
- return drm_encoder_find(connector->dev, enc_id);
+ return drm_encoder_find(connector->dev, NULL, enc_id);
return NULL;
}
@@ -1076,7 +1076,7 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+ encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -1133,7 +1133,7 @@ amdgpu_connector_dvi_encoder(struct drm_connector *connector)
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+ encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -1152,7 +1152,7 @@ amdgpu_connector_dvi_encoder(struct drm_connector *connector)
/* then check use digitial */
/* pick the first one */
if (enc_id)
- return drm_encoder_find(connector->dev, enc_id);
+ return drm_encoder_find(connector->dev, NULL, enc_id);
return NULL;
}
@@ -1293,7 +1293,7 @@ u16 amdgpu_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *conn
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev,
+ encoder = drm_encoder_find(connector->dev, NULL,
connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -1322,7 +1322,7 @@ static bool amdgpu_connector_encoder_is_hbr2(struct drm_connector *connector)
for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev,
+ encoder = drm_encoder_find(connector->dev, NULL,
connector->encoder_ids[i]);
if (!encoder)
continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index c6a214f1e991..f7fceb63413c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -25,6 +25,7 @@
* Jerome Glisse <glisse@freedesktop.org>
*/
#include <linux/pagemap.h>
+#include <linux/sync_file.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include <drm/drm_syncobj.h>
@@ -89,12 +90,14 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
goto free_chunk;
}
+ mutex_lock(&p->ctx->lock);
+
/* get chunks */
chunk_array_user = u64_to_user_ptr(cs->in.chunks);
if (copy_from_user(chunk_array, chunk_array_user,
sizeof(uint64_t)*cs->in.num_chunks)) {
ret = -EFAULT;
- goto put_ctx;
+ goto free_chunk;
}
p->nchunks = cs->in.num_chunks;
@@ -102,7 +105,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
GFP_KERNEL);
if (!p->chunks) {
ret = -ENOMEM;
- goto put_ctx;
+ goto free_chunk;
}
for (i = 0; i < p->nchunks; i++) {
@@ -169,6 +172,11 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
if (ret)
goto free_all_kdata;
+ if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
+ ret = -ECANCELED;
+ goto free_all_kdata;
+ }
+
if (p->uf_entry.robj)
p->job->uf_addr = uf_offset;
kfree(chunk_array);
@@ -182,8 +190,6 @@ free_partial_kdata:
kfree(p->chunks);
p->chunks = NULL;
p->nchunks = 0;
-put_ctx:
- amdgpu_ctx_put(p->ctx);
free_chunk:
kfree(chunk_array);
@@ -704,7 +710,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
list_for_each_entry(e, &p->validated, tv.head) {
struct reservation_object *resv = e->robj->tbo.resv;
- r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
+ r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp,
+ amdgpu_bo_explicit_sync(e->robj));
if (r)
return r;
@@ -735,8 +742,10 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
dma_fence_put(parser->fence);
- if (parser->ctx)
+ if (parser->ctx) {
+ mutex_unlock(&parser->ctx->lock);
amdgpu_ctx_put(parser->ctx);
+ }
if (parser->bo_list)
amdgpu_bo_list_put(parser->bo_list);
@@ -843,14 +852,58 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
struct amdgpu_vm *vm = &fpriv->vm;
struct amdgpu_ring *ring = p->job->ring;
- int i, r;
+ int r;
/* Only for UVD/VCE VM emulation */
- if (ring->funcs->parse_cs) {
- for (i = 0; i < p->job->num_ibs; i++) {
- r = amdgpu_ring_parse_cs(ring, p, i);
+ if (p->job->ring->funcs->parse_cs) {
+ unsigned i, j;
+
+ for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
+ struct drm_amdgpu_cs_chunk_ib *chunk_ib;
+ struct amdgpu_bo_va_mapping *m;
+ struct amdgpu_bo *aobj = NULL;
+ struct amdgpu_cs_chunk *chunk;
+ struct amdgpu_ib *ib;
+ uint64_t offset;
+ uint8_t *kptr;
+
+ chunk = &p->chunks[i];
+ ib = &p->job->ibs[j];
+ chunk_ib = chunk->kdata;
+
+ if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
+ continue;
+
+ r = amdgpu_cs_find_mapping(p, chunk_ib->va_start,
+ &aobj, &m);
+ if (r) {
+ DRM_ERROR("IB va_start is invalid\n");
+ return r;
+ }
+
+ if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
+ (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
+ DRM_ERROR("IB va_start+ib_bytes is invalid\n");
+ return -EINVAL;
+ }
+
+ /* the IB should be reserved at this point */
+ r = amdgpu_bo_kmap(aobj, (void **)&kptr);
+ if (r) {
+ return r;
+ }
+
+ offset = m->start * AMDGPU_GPU_PAGE_SIZE;
+ kptr += chunk_ib->va_start - offset;
+
+ memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
+ amdgpu_bo_kunmap(aobj);
+
+ r = amdgpu_ring_parse_cs(ring, p, j);
if (r)
return r;
+
+ j++;
}
}
@@ -917,54 +970,18 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
parser->job->ring = ring;
- if (ring->funcs->parse_cs) {
- struct amdgpu_bo_va_mapping *m;
- struct amdgpu_bo *aobj = NULL;
- uint64_t offset;
- uint8_t *kptr;
-
- r = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
- &aobj, &m);
- if (r) {
- DRM_ERROR("IB va_start is invalid\n");
- return r;
- }
-
- if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
- (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
- DRM_ERROR("IB va_start+ib_bytes is invalid\n");
- return -EINVAL;
- }
-
- /* the IB should be reserved at this point */
- r = amdgpu_bo_kmap(aobj, (void **)&kptr);
- if (r) {
- return r;
- }
-
- offset = m->start * AMDGPU_GPU_PAGE_SIZE;
- kptr += chunk_ib->va_start - offset;
-
- r = amdgpu_ib_get(adev, vm, chunk_ib->ib_bytes, ib);
- if (r) {
- DRM_ERROR("Failed to get ib !\n");
- return r;
- }
-
- memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
- amdgpu_bo_kunmap(aobj);
- } else {
- r = amdgpu_ib_get(adev, vm, 0, ib);
- if (r) {
- DRM_ERROR("Failed to get ib !\n");
- return r;
- }
-
+ r = amdgpu_ib_get(adev, vm,
+ ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0,
+ ib);
+ if (r) {
+ DRM_ERROR("Failed to get ib !\n");
+ return r;
}
ib->gpu_addr = chunk_ib->va_start;
ib->length_dw = chunk_ib->ib_bytes / 4;
ib->flags = chunk_ib->flags;
+
j++;
}
@@ -974,7 +991,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
return -EINVAL;
- return 0;
+ return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->job->ring->idx);
}
static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
@@ -1175,6 +1192,8 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
job->uf_sequence = seq;
amdgpu_job_free_resources(job);
+ amdgpu_ring_priority_get(job->ring,
+ amd_sched_get_job_priority(&job->base));
trace_amdgpu_cs_ioctl(job);
amd_sched_entity_push_job(&job->base);
@@ -1188,7 +1207,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
struct amdgpu_device *adev = dev->dev_private;
- struct amdgpu_fpriv *fpriv = filp->driver_priv;
union drm_amdgpu_cs *cs = data;
struct amdgpu_cs_parser parser = {};
bool reserved_buffers = false;
@@ -1196,8 +1214,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
if (!adev->accel_working)
return -EBUSY;
- if (amdgpu_kms_vram_lost(adev, fpriv))
- return -ENODEV;
parser.adev = adev;
parser.filp = filp;
@@ -1208,6 +1224,10 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
goto out;
}
+ r = amdgpu_cs_ib_fill(adev, &parser);
+ if (r)
+ goto out;
+
r = amdgpu_cs_parser_bos(&parser, data);
if (r) {
if (r == -ENOMEM)
@@ -1218,9 +1238,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
}
reserved_buffers = true;
- r = amdgpu_cs_ib_fill(adev, &parser);
- if (r)
- goto out;
r = amdgpu_cs_dependencies(adev, &parser);
if (r) {
@@ -1256,16 +1273,12 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
{
union drm_amdgpu_wait_cs *wait = data;
struct amdgpu_device *adev = dev->dev_private;
- struct amdgpu_fpriv *fpriv = filp->driver_priv;
unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
struct amdgpu_ring *ring = NULL;
struct amdgpu_ctx *ctx;
struct dma_fence *fence;
long r;
- if (amdgpu_kms_vram_lost(adev, fpriv))
- return -ENODEV;
-
ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
if (ctx == NULL)
return -EINVAL;
@@ -1283,6 +1296,8 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
r = PTR_ERR(fence);
else if (fence) {
r = dma_fence_wait_timeout(fence, true, timeout);
+ if (r > 0 && fence->error)
+ r = fence->error;
dma_fence_put(fence);
} else
r = 1;
@@ -1330,6 +1345,62 @@ static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
return fence;
}
+int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct amdgpu_device *adev = dev->dev_private;
+ union drm_amdgpu_fence_to_handle *info = data;
+ struct dma_fence *fence;
+ struct drm_syncobj *syncobj;
+ struct sync_file *sync_file;
+ int fd, r;
+
+ fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
+ if (IS_ERR(fence))
+ return PTR_ERR(fence);
+
+ switch (info->in.what) {
+ case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
+ r = drm_syncobj_create(&syncobj, 0, fence);
+ dma_fence_put(fence);
+ if (r)
+ return r;
+ r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
+ drm_syncobj_put(syncobj);
+ return r;
+
+ case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
+ r = drm_syncobj_create(&syncobj, 0, fence);
+ dma_fence_put(fence);
+ if (r)
+ return r;
+ r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle);
+ drm_syncobj_put(syncobj);
+ return r;
+
+ case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
+ fd = get_unused_fd_flags(O_CLOEXEC);
+ if (fd < 0) {
+ dma_fence_put(fence);
+ return fd;
+ }
+
+ sync_file = sync_file_create(fence);
+ dma_fence_put(fence);
+ if (!sync_file) {
+ put_unused_fd(fd);
+ return -ENOMEM;
+ }
+
+ fd_install(fd, sync_file->file);
+ info->out.handle = fd;
+ return 0;
+
+ default:
+ return -EINVAL;
+ }
+}
+
/**
* amdgpu_cs_wait_all_fence - wait on all fences to signal
*
@@ -1364,6 +1435,9 @@ static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
if (r == 0)
break;
+
+ if (fence->error)
+ return fence->error;
}
memset(wait, 0, sizeof(*wait));
@@ -1424,7 +1498,7 @@ out:
wait->out.status = (r > 0);
wait->out.first_signaled = first;
/* set return value 0 to indicate success */
- r = 0;
+ r = array[first]->error;
err_free_fence_array:
for (i = 0; i < fence_count; i++)
@@ -1445,15 +1519,12 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp)
{
struct amdgpu_device *adev = dev->dev_private;
- struct amdgpu_fpriv *fpriv = filp->driver_priv;
union drm_amdgpu_wait_fences *wait = data;
uint32_t fence_count = wait->in.fence_count;
struct drm_amdgpu_fence *fences_user;
struct drm_amdgpu_fence *fences;
int r;
- if (amdgpu_kms_vram_lost(adev, fpriv))
- return -ENODEV;
/* Get the fences from userspace */
fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
GFP_KERNEL);
@@ -1511,14 +1582,14 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
return -EINVAL;
- r = amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
- if (unlikely(r))
- return r;
-
- if ((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
- return 0;
+ if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
+ (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+ amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
+ r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false,
+ false);
+ if (r)
+ return r;
+ }
- (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
- amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
- return ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false, false);
+ return amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 75c933b1a432..c184468e2b2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -23,13 +23,41 @@
*/
#include <drm/drmP.h>
+#include <drm/drm_auth.h>
#include "amdgpu.h"
+#include "amdgpu_sched.h"
-static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
+static int amdgpu_ctx_priority_permit(struct drm_file *filp,
+ enum amd_sched_priority priority)
+{
+ /* NORMAL and below are accessible by everyone */
+ if (priority <= AMD_SCHED_PRIORITY_NORMAL)
+ return 0;
+
+ if (capable(CAP_SYS_NICE))
+ return 0;
+
+ if (drm_is_current_master(filp))
+ return 0;
+
+ return -EACCES;
+}
+
+static int amdgpu_ctx_init(struct amdgpu_device *adev,
+ enum amd_sched_priority priority,
+ struct drm_file *filp,
+ struct amdgpu_ctx *ctx)
{
unsigned i, j;
int r;
+ if (priority < 0 || priority >= AMD_SCHED_PRIORITY_MAX)
+ return -EINVAL;
+
+ r = amdgpu_ctx_priority_permit(filp, priority);
+ if (r)
+ return r;
+
memset(ctx, 0, sizeof(*ctx));
ctx->adev = adev;
kref_init(&ctx->refcount);
@@ -39,19 +67,24 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
if (!ctx->fences)
return -ENOMEM;
+ mutex_init(&ctx->lock);
+
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
ctx->rings[i].sequence = 1;
ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i];
}
ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+ ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
+ ctx->init_priority = priority;
+ ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
/* create context entity for each ring */
for (i = 0; i < adev->num_rings; i++) {
struct amdgpu_ring *ring = adev->rings[i];
struct amd_sched_rq *rq;
- rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
+ rq = &ring->sched.sched_rq[priority];
if (ring == &adev->gfx.kiq.ring)
continue;
@@ -96,10 +129,14 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
&ctx->rings[i].entity);
amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr);
+
+ mutex_destroy(&ctx->lock);
}
static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
struct amdgpu_fpriv *fpriv,
+ struct drm_file *filp,
+ enum amd_sched_priority priority,
uint32_t *id)
{
struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
@@ -117,8 +154,9 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
kfree(ctx);
return r;
}
+
*id = (uint32_t)r;
- r = amdgpu_ctx_init(adev, ctx);
+ r = amdgpu_ctx_init(adev, priority, filp, ctx);
if (r) {
idr_remove(&mgr->ctx_handles, *id);
*id = 0;
@@ -193,6 +231,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
{
int r;
uint32_t id;
+ enum amd_sched_priority priority;
union drm_amdgpu_ctx *args = data;
struct amdgpu_device *adev = dev->dev_private;
@@ -200,10 +239,16 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
r = 0;
id = args->in.ctx_id;
+ priority = amdgpu_to_sched_priority(args->in.priority);
+
+ /* For backwards compatibility reasons, we need to accept
+ * ioctls with garbage in the priority field */
+ if (priority == AMD_SCHED_PRIORITY_INVALID)
+ priority = AMD_SCHED_PRIORITY_NORMAL;
switch (args->in.op) {
case AMDGPU_CTX_OP_ALLOC_CTX:
- r = amdgpu_ctx_alloc(adev, fpriv, &id);
+ r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
args->out.alloc.ctx_id = id;
break;
case AMDGPU_CTX_OP_FREE_CTX:
@@ -256,12 +301,8 @@ int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
idx = seq & (amdgpu_sched_jobs - 1);
other = cring->fences[idx];
- if (other) {
- signed long r;
- r = dma_fence_wait_timeout(other, true, MAX_SCHEDULE_TIMEOUT);
- if (r < 0)
- return r;
- }
+ if (other)
+ BUG_ON(!dma_fence_is_signaled(other));
dma_fence_get(fence);
@@ -305,6 +346,51 @@ struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
return fence;
}
+void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
+ enum amd_sched_priority priority)
+{
+ int i;
+ struct amdgpu_device *adev = ctx->adev;
+ struct amd_sched_rq *rq;
+ struct amd_sched_entity *entity;
+ struct amdgpu_ring *ring;
+ enum amd_sched_priority ctx_prio;
+
+ ctx->override_priority = priority;
+
+ ctx_prio = (ctx->override_priority == AMD_SCHED_PRIORITY_UNSET) ?
+ ctx->init_priority : ctx->override_priority;
+
+ for (i = 0; i < adev->num_rings; i++) {
+ ring = adev->rings[i];
+ entity = &ctx->rings[i].entity;
+ rq = &ring->sched.sched_rq[ctx_prio];
+
+ if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+ continue;
+
+ amd_sched_entity_set_rq(entity, rq);
+ }
+}
+
+int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id)
+{
+ struct amdgpu_ctx_ring *cring = &ctx->rings[ring_id];
+ unsigned idx = cring->sequence & (amdgpu_sched_jobs - 1);
+ struct dma_fence *other = cring->fences[idx];
+
+ if (other) {
+ signed long r;
+ r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
+ if (r < 0) {
+ DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+ return r;
+ }
+ }
+
+ return 0;
+}
+
void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
{
mutex_init(&mgr->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3e84ddf9e3b5..efcacb827de7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -56,6 +56,7 @@
#include "amdgpu_vf_error.h"
#include "amdgpu_amdkfd.h"
+#include "amdgpu_pm.h"
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
@@ -108,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
{
uint32_t ret;
- if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
- BUG_ON(in_interrupt());
+ if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
return amdgpu_virt_kiq_rreg(adev, reg);
- }
if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
@@ -136,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
adev->last_mm_index = v;
}
- if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
- BUG_ON(in_interrupt());
+ if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
return amdgpu_virt_kiq_wreg(adev, reg, v);
- }
if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
@@ -549,7 +546,7 @@ int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
if (offset < adev->wb.num_wb) {
__set_bit(offset, adev->wb.used);
- *wb = offset * 8; /* convert to dw offset */
+ *wb = offset << 3; /* convert to dw offset */
return 0;
} else {
return -EINVAL;
@@ -567,7 +564,7 @@ int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
{
if (wb < adev->wb.num_wb)
- __clear_bit(wb, adev->wb.used);
+ __clear_bit(wb >> 3, adev->wb.used);
}
/**
@@ -657,42 +654,96 @@ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
}
/*
- * GPU helpers function.
+ * Firmware Reservation functions
*/
/**
- * amdgpu_need_post - check if the hw need post or not
+ * amdgpu_fw_reserve_vram_fini - free fw reserved vram
*
* @adev: amdgpu_device pointer
*
- * Check if the asic has been initialized (all asics) at driver startup
- * or post is needed if hw reset is performed.
- * Returns true if need or false if not.
+ * free fw reserved vram if it has been reserved.
*/
-bool amdgpu_need_post(struct amdgpu_device *adev)
+void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
{
- uint32_t reg;
+ amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
+ NULL, &adev->fw_vram_usage.va);
+}
- if (adev->has_hw_reset) {
- adev->has_hw_reset = false;
- return true;
- }
+/**
+ * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from fw.
+ */
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
+{
+ int r = 0;
+ u64 gpu_addr;
+ u64 vram_size = adev->mc.visible_vram_size;
- /* bios scratch used on CIK+ */
- if (adev->asic_type >= CHIP_BONAIRE)
- return amdgpu_atombios_scratch_need_asic_init(adev);
+ adev->fw_vram_usage.va = NULL;
+ adev->fw_vram_usage.reserved_bo = NULL;
- /* check MEM_SIZE for older asics */
- reg = amdgpu_asic_get_config_memsize(adev);
+ if (adev->fw_vram_usage.size > 0 &&
+ adev->fw_vram_usage.size <= vram_size) {
- if ((reg != 0) && (reg != 0xffffffff))
- return false;
+ r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
+ PAGE_SIZE, true, 0,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+ AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
+ &adev->fw_vram_usage.reserved_bo);
+ if (r)
+ goto error_create;
- return true;
+ r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
+ if (r)
+ goto error_reserve;
+ r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ adev->fw_vram_usage.start_offset,
+ (adev->fw_vram_usage.start_offset +
+ adev->fw_vram_usage.size), &gpu_addr);
+ if (r)
+ goto error_pin;
+ r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
+ &adev->fw_vram_usage.va);
+ if (r)
+ goto error_kmap;
+ amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+ }
+ return r;
+
+error_kmap:
+ amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
+error_pin:
+ amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+error_reserve:
+ amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
+error_create:
+ adev->fw_vram_usage.va = NULL;
+ adev->fw_vram_usage.reserved_bo = NULL;
+ return r;
}
-static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
+
+/*
+ * GPU helpers function.
+ */
+/**
+ * amdgpu_need_post - check if the hw need post or not
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Check if the asic has been initialized (all asics) at driver startup
+ * or post is needed if hw reset is performed.
+ * Returns true if need or false if not.
+ */
+bool amdgpu_need_post(struct amdgpu_device *adev)
{
+ uint32_t reg;
+
if (amdgpu_sriov_vf(adev))
return false;
@@ -715,7 +766,23 @@ static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
return true;
}
}
- return amdgpu_need_post(adev);
+
+ if (adev->has_hw_reset) {
+ adev->has_hw_reset = false;
+ return true;
+ }
+
+ /* bios scratch used on CIK+ */
+ if (adev->asic_type >= CHIP_BONAIRE)
+ return amdgpu_atombios_scratch_need_asic_init(adev);
+
+ /* check MEM_SIZE for older asics */
+ reg = amdgpu_asic_get_config_memsize(adev);
+
+ if ((reg != 0) && (reg != 0xffffffff))
+ return false;
+
+ return true;
}
/**
@@ -1879,6 +1946,7 @@ static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
static enum amd_ip_block_type ip_order[] = {
AMD_IP_BLOCK_TYPE_SMC,
+ AMD_IP_BLOCK_TYPE_PSP,
AMD_IP_BLOCK_TYPE_DCE,
AMD_IP_BLOCK_TYPE_GFX,
AMD_IP_BLOCK_TYPE_SDMA,
@@ -1964,12 +2032,17 @@ static int amdgpu_resume(struct amdgpu_device *adev)
static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
{
- if (adev->is_atom_fw) {
- if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
- adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
- } else {
- if (amdgpu_atombios_has_gpu_virtualization_table(adev))
- adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+ if (amdgpu_sriov_vf(adev)) {
+ if (adev->is_atom_fw) {
+ if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+ } else {
+ if (amdgpu_atombios_has_gpu_virtualization_table(adev))
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+ }
+
+ if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
}
}
@@ -2010,6 +2083,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
adev->vm_manager.vm_pte_num_rings = 0;
adev->gart.gart_funcs = NULL;
adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
+ bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
adev->smc_rreg = &amdgpu_invalid_rreg;
adev->smc_wreg = &amdgpu_invalid_wreg;
@@ -2038,8 +2112,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
mutex_init(&adev->pm.mutex);
mutex_init(&adev->gfx.gpu_clock_mutex);
mutex_init(&adev->srbm_mutex);
+ mutex_init(&adev->gfx.pipe_reserve_mutex);
mutex_init(&adev->grbm_idx_mutex);
mutex_init(&adev->mn_lock);
+ mutex_init(&adev->virt.vf_errors.lock);
hash_init(adev->mn_hash);
amdgpu_check_arguments(adev);
@@ -2125,7 +2201,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_atombios_init(adev);
if (r) {
dev_err(adev->dev, "amdgpu_atombios_init failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
goto failed;
}
@@ -2133,10 +2209,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
amdgpu_device_detect_sriov_bios(adev);
/* Post card if necessary */
- if (amdgpu_vpost_needed(adev)) {
+ if (amdgpu_need_post(adev)) {
if (!adev->bios) {
dev_err(adev->dev, "no vBIOS found\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
r = -EINVAL;
goto failed;
}
@@ -2144,7 +2219,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
if (r) {
dev_err(adev->dev, "gpu post error!\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
goto failed;
}
} else {
@@ -2156,7 +2230,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_atomfirmware_get_clock_info(adev);
if (r) {
dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
goto failed;
}
} else {
@@ -2164,7 +2238,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_atombios_get_clock_info(adev);
if (r) {
dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
goto failed;
}
/* init i2c buses */
@@ -2175,7 +2249,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_fence_driver_init(adev);
if (r) {
dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
goto failed;
}
@@ -2185,7 +2259,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_init(adev);
if (r) {
dev_err(adev->dev, "amdgpu_init failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
amdgpu_fini(adev);
goto failed;
}
@@ -2205,7 +2279,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_ib_pool_init(adev);
if (r) {
dev_err(adev->dev, "IB initialization failed (%d).\n", r);
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
goto failed;
}
@@ -2213,8 +2287,15 @@ int amdgpu_device_init(struct amdgpu_device *adev,
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
+ if (amdgpu_sriov_vf(adev))
+ amdgpu_virt_init_data_exchange(adev);
+
amdgpu_fbdev_init(adev);
+ r = amdgpu_pm_sysfs_init(adev);
+ if (r)
+ DRM_ERROR("registering pm debugfs failed (%d).\n", r);
+
r = amdgpu_gem_debugfs_init(adev);
if (r)
DRM_ERROR("registering gem debugfs failed (%d).\n", r);
@@ -2254,7 +2335,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
r = amdgpu_late_init(adev);
if (r) {
dev_err(adev->dev, "amdgpu_late_init failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
goto failed;
}
@@ -2286,6 +2367,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
/* evict vram memory */
amdgpu_bo_evict_vram(adev);
amdgpu_ib_pool_fini(adev);
+ amdgpu_fw_reserve_vram_fini(adev);
amdgpu_fence_driver_fini(adev);
amdgpu_fbdev_fini(adev);
r = amdgpu_fini(adev);
@@ -2311,6 +2393,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
iounmap(adev->rmmio);
adev->rmmio = NULL;
amdgpu_doorbell_fini(adev);
+ amdgpu_pm_sysfs_fini(adev);
amdgpu_debugfs_regs_cleanup(adev);
}
@@ -2537,6 +2620,9 @@ static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
int i;
bool asic_hang = false;
+ if (amdgpu_sriov_vf(adev))
+ return true;
+
for (i = 0; i < adev->num_ip_blocks; i++) {
if (!adev->ip_blocks[i].status.valid)
continue;
@@ -2936,7 +3022,6 @@ out:
}
} else {
dev_err(adev->dev, "asic resume failed (%d).\n", r);
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
if (adev->rings[i] && adev->rings[i]->sched.thread) {
kthread_unpark(adev->rings[i]->sched.thread);
@@ -2950,7 +3035,6 @@ out:
if (r) {
/* bad news, how to tell it to userspace ? */
dev_info(adev->dev, "GPU reset failed\n");
- amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
}
else {
dev_info(adev->dev, "GPU reset successed!\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index f79f9ea58b17..7279fb5c3abc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -356,6 +356,10 @@ enum amdgpu_pcie_gen {
((adev)->powerplay.pp_funcs->switch_power_profile(\
(adev)->powerplay.pp_handle, type))
+#define amdgpu_dpm_set_clockgating_by_smu(adev, msg_id) \
+ ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
+ (adev)->powerplay.pp_handle, msg_id))
+
struct amdgpu_dpm {
struct amdgpu_ps *ps;
/* number of valid power states */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 4f98960e47f9..dd2f060d62a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -70,9 +70,12 @@
* - 3.18.0 - Export gpu always on cu bitmap
* - 3.19.0 - Add support for UVD MJPEG decode
* - 3.20.0 - Add support for local BOs
+ * - 3.21.0 - Add DRM_AMDGPU_FENCE_TO_HANDLE ioctl
+ * - 3.22.0 - Add DRM_AMDGPU_SCHED ioctl
+ * - 3.23.0 - Add query for VRAM lost counter
*/
#define KMS_DRIVER_MAJOR 3
-#define KMS_DRIVER_MINOR 20
+#define KMS_DRIVER_MINOR 23
#define KMS_DRIVER_PATCHLEVEL 0
int amdgpu_vram_limit = 0;
@@ -122,6 +125,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
int amdgpu_param_buf_per_se = 0;
int amdgpu_job_hang_limit = 0;
int amdgpu_lbpw = -1;
+int amdgpu_compute_multipipe = -1;
MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -265,6 +269,9 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(lbpw, amdgpu_lbpw, int, 0444);
+MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
+module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
+
#ifdef CONFIG_DRM_AMDGPU_SI
#if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 333bad749067..fb9f88ef6059 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -169,6 +169,32 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
}
/**
+ * amdgpu_fence_emit_polling - emit a fence on the requeste ring
+ *
+ * @ring: ring the fence is associated with
+ * @s: resulting sequence number
+ *
+ * Emits a fence command on the requested ring (all asics).
+ * Used For polling fence.
+ * Returns 0 on success, -ENOMEM on failure.
+ */
+int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s)
+{
+ uint32_t seq;
+
+ if (!s)
+ return -EINVAL;
+
+ seq = ++ring->fence_drv.sync_seq;
+ amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
+ seq, AMDGPU_FENCE_FLAG_INT);
+
+ *s = seq;
+
+ return 0;
+}
+
+/**
* amdgpu_fence_schedule_fallback - schedule fallback check
*
* @ring: pointer to struct amdgpu_ring
@@ -282,6 +308,30 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
}
/**
+ * amdgpu_fence_wait_polling - busy wait for givn sequence number
+ *
+ * @ring: ring index the fence is associated with
+ * @wait_seq: sequence number to wait
+ * @timeout: the timeout for waiting in usecs
+ *
+ * Wait for all fences on the requested ring to signal (all asics).
+ * Returns left time if no timeout, 0 or minus if timeout.
+ */
+signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
+ uint32_t wait_seq,
+ signed long timeout)
+{
+ uint32_t seq;
+
+ do {
+ seq = amdgpu_fence_read(ring);
+ udelay(5);
+ timeout -= 5;
+ } while ((int32_t)(wait_seq - seq) > 0 && timeout > 0);
+
+ return timeout > 0 ? timeout : 0;
+}
+/**
* amdgpu_fence_count_emitted - get the count of emitted fences
*
* @ring: ring the fence is associated with
@@ -641,6 +691,19 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
atomic_read(&ring->fence_drv.last_seq));
seq_printf(m, "Last emitted 0x%08x\n",
ring->fence_drv.sync_seq);
+
+ if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
+ continue;
+
+ /* set in CP_VMID_PREEMPT and preemption occurred */
+ seq_printf(m, "Last preempted 0x%08x\n",
+ le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
+ /* set in CP_VMID_RESET and reset occurred */
+ seq_printf(m, "Last reset 0x%08x\n",
+ le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
+ /* Both preemption and reset occurred */
+ seq_printf(m, "Last both 0x%08x\n",
+ le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
}
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index f4370081f6e6..fe818501c520 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -332,12 +332,13 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
adev->gart.pages[p] = pagelist[i];
#endif
- if (adev->gart.ptr) {
- r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags,
- adev->gart.ptr);
- if (r)
- return r;
- }
+ if (!adev->gart.ptr)
+ return 0;
+
+ r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags,
+ adev->gart.ptr);
+ if (r)
+ return r;
mb();
amdgpu_gart_flush_gpu_tlb(adev, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index b0d45c8e6bb3..fb72edc4c026 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -212,7 +212,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
AMDGPU_GEM_CREATE_CPU_GTT_USWC |
AMDGPU_GEM_CREATE_VRAM_CLEARED |
- AMDGPU_GEM_CREATE_VM_ALWAYS_VALID))
+ AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
+ AMDGPU_GEM_CREATE_EXPLICIT_SYNC))
+
return -EINVAL;
/* reject invalid gem domains */
@@ -577,11 +579,6 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
args->operation);
return -EINVAL;
}
- if ((args->operation == AMDGPU_VA_OP_MAP) ||
- (args->operation == AMDGPU_VA_OP_REPLACE)) {
- if (amdgpu_kms_vram_lost(adev, fpriv))
- return -ENODEV;
- }
INIT_LIST_HEAD(&list);
INIT_LIST_HEAD(&duplicates);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 4fcd98e65998..ef043361009f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -109,9 +109,26 @@ void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_s
}
}
+static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev)
+{
+ if (amdgpu_compute_multipipe != -1) {
+ DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
+ amdgpu_compute_multipipe);
+ return amdgpu_compute_multipipe == 1;
+ }
+
+ /* FIXME: spreading the queues across pipes causes perf regressions
+ * on POLARIS11 compute workloads */
+ if (adev->asic_type == CHIP_POLARIS11)
+ return false;
+
+ return adev->gfx.mec.num_mec > 1;
+}
+
void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
{
int i, queue, pipe, mec;
+ bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev);
/* policy for amdgpu compute queue ownership */
for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
@@ -125,8 +142,7 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
if (mec >= adev->gfx.mec.num_mec)
break;
- /* FIXME: spreading the queues across pipes causes perf regressions */
- if (0) {
+ if (multipipe_policy) {
/* policy: amdgpu owns the first two queues of the first MEC */
if (mec == 0 && queue < 2)
set_bit(i, adev->gfx.mec.queue_bitmap);
@@ -185,7 +201,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
int r = 0;
- mutex_init(&kiq->ring_mutex);
+ spin_lock_init(&kiq->ring_lock);
r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 0d15eb7d31d7..33535d347734 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -169,7 +169,8 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
int r;
spin_lock(&mgr->lock);
- if (atomic64_read(&mgr->available) < mem->num_pages) {
+ if ((&tbo->mem == mem || tbo->mem.mem_type != TTM_PL_TT) &&
+ atomic64_read(&mgr->available) < mem->num_pages) {
spin_unlock(&mgr->lock);
return 0;
}
@@ -244,8 +245,9 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
{
struct amdgpu_gtt_mgr *mgr = man->priv;
+ s64 result = man->size - atomic64_read(&mgr->available);
- return (u64)(man->size - atomic64_read(&mgr->available)) * PAGE_SIZE;
+ return (result > 0 ? result : 0) * PAGE_SIZE;
}
/**
@@ -265,7 +267,7 @@ static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
drm_mm_print(&mgr->mm, printer);
spin_unlock(&mgr->lock);
- drm_printf(printer, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
+ drm_printf(printer, "man size:%llu pages, gtt available:%lld pages, usage:%lluMB\n",
man->size, (u64)atomic64_read(&mgr->available),
amdgpu_gtt_mgr_usage(man) >> 20);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 4510627ae83e..0cfc68db575b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -65,6 +65,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
amdgpu_sync_create(&(*job)->sync);
amdgpu_sync_create(&(*job)->dep_sync);
amdgpu_sync_create(&(*job)->sched_sync);
+ (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
return 0;
}
@@ -103,6 +104,7 @@ static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
{
struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
+ amdgpu_ring_priority_put(job->ring, amd_sched_get_job_priority(s_job));
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->dep_sync);
@@ -139,6 +141,8 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
job->fence_ctx = entity->fence_context;
*f = dma_fence_get(&job->base.s_fence->finished);
amdgpu_job_free_resources(job);
+ amdgpu_ring_priority_get(job->ring,
+ amd_sched_get_job_priority(&job->base));
amd_sched_entity_push_job(&job->base);
return 0;
@@ -177,8 +181,8 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
{
struct dma_fence *fence = NULL;
+ struct amdgpu_device *adev;
struct amdgpu_job *job;
- struct amdgpu_fpriv *fpriv = NULL;
int r;
if (!sched_job) {
@@ -186,23 +190,25 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
return NULL;
}
job = to_amdgpu_job(sched_job);
+ adev = job->adev;
BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
trace_amdgpu_sched_run_job(job);
- if (job->vm)
- fpriv = container_of(job->vm, struct amdgpu_fpriv, vm);
/* skip ib schedule when vram is lost */
- if (fpriv && amdgpu_kms_vram_lost(job->adev, fpriv))
+ if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+ dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
DRM_ERROR("Skip scheduling IBs!\n");
- else {
- r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job, &fence);
+ } else {
+ r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
+ &fence);
if (r)
DRM_ERROR("Error scheduling IBs (%d)\n", r);
}
/* if gpu reset, hw fence will be replaced here */
dma_fence_put(job->fence);
job->fence = dma_fence_get(fence);
+
amdgpu_job_free_resources(job);
return fence;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 4fd06f8d9768..6f0b26dae3b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -28,6 +28,7 @@
#include <drm/drmP.h>
#include "amdgpu.h"
#include <drm/amdgpu_drm.h>
+#include "amdgpu_sched.h"
#include "amdgpu_uvd.h"
#include "amdgpu_vce.h"
@@ -269,7 +270,6 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
struct amdgpu_device *adev = dev->dev_private;
- struct amdgpu_fpriv *fpriv = filp->driver_priv;
struct drm_amdgpu_info *info = data;
struct amdgpu_mode_info *minfo = &adev->mode_info;
void __user *out = (void __user *)(uintptr_t)info->return_pointer;
@@ -282,8 +282,6 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
if (!info->return_size || !info->return_pointer)
return -EINVAL;
- if (amdgpu_kms_vram_lost(adev, fpriv))
- return -ENODEV;
switch (info->query) {
case AMDGPU_INFO_ACCEL_WORKING:
@@ -765,6 +763,9 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
}
return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
}
+ case AMDGPU_INFO_VRAM_LOST_COUNTER:
+ ui32 = atomic_read(&adev->vram_lost_counter);
+ return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
default:
DRM_DEBUG_KMS("Invalid request %d\n", info->query);
return -EINVAL;
@@ -791,12 +792,6 @@ void amdgpu_driver_lastclose_kms(struct drm_device *dev)
vga_switcheroo_process_delayed_switch();
}
-bool amdgpu_kms_vram_lost(struct amdgpu_device *adev,
- struct amdgpu_fpriv *fpriv)
-{
- return fpriv->vram_lost_counter != atomic_read(&adev->vram_lost_counter);
-}
-
/**
* amdgpu_driver_open_kms - drm callback for open
*
@@ -853,7 +848,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
- fpriv->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
file_priv->driver_priv = fpriv;
out_suspend:
@@ -1023,7 +1017,9 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(AMDGPU_VM, amdgpu_vm_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_SCHED, amdgpu_sched_ioctl, DRM_MASTER),
DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_FENCE_TO_HANDLE, amdgpu_cs_fence_to_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
/* KMS */
DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6982baeccd14..ea25164e7f4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -40,9 +40,7 @@
static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
- struct amdgpu_bo *bo;
-
- bo = container_of(tbo, struct amdgpu_bo, tbo);
+ struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
amdgpu_bo_kunmap(bo);
@@ -371,6 +369,9 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
&bo->placement, page_align, !kernel, NULL,
acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
+ if (unlikely(r != 0))
+ return r;
+
bytes_moved = atomic64_read(&adev->num_bytes_moved) -
initial_bytes_moved;
if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
@@ -380,9 +381,6 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
else
amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
- if (unlikely(r != 0))
- return r;
-
if (kernel)
bo->tbo.priority = 1;
@@ -884,7 +882,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
return;
- abo = container_of(bo, struct amdgpu_bo, tbo);
+ abo = ttm_to_amdgpu_bo(bo);
amdgpu_vm_bo_invalidate(adev, abo, evict);
amdgpu_bo_kunmap(abo);
@@ -911,7 +909,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
return 0;
- abo = container_of(bo, struct amdgpu_bo, tbo);
+ abo = ttm_to_amdgpu_bo(bo);
/* Remember that this BO was accessed by the CPU */
abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 39b6bf6fb051..428aae048f4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -94,6 +94,11 @@ struct amdgpu_bo {
};
};
+static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo)
+{
+ return container_of(tbo, struct amdgpu_bo, tbo);
+}
+
/**
* amdgpu_mem_type_to_domain - return domain corresponding to mem_type
* @mem_type: ttm memory type
@@ -188,6 +193,14 @@ static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
}
}
+/**
+ * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
+ */
+static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
+{
+ return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;
+}
+
int amdgpu_bo_create(struct amdgpu_device *adev,
unsigned long size, int byte_align,
bool kernel, u32 domain, u64 flags,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index f6ce52956e6d..a59e04f3eeba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -64,10 +64,6 @@ static const struct cg_flag_name clocks[] = {
void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
{
- if (adev->pp_enabled)
- /* TODO */
- return;
-
if (adev->pm.dpm_enabled) {
mutex_lock(&adev->pm.mutex);
if (power_supply_is_system_supplied() > 0)
@@ -118,7 +114,7 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
goto fail;
}
- if (adev->pp_enabled) {
+ if (adev->powerplay.pp_funcs->dispatch_tasks) {
amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
} else {
mutex_lock(&adev->pm.mutex);
@@ -303,7 +299,8 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
if (strlen(buf) == 1)
adev->pp_force_state_enabled = false;
- else if (adev->pp_enabled) {
+ else if (adev->powerplay.pp_funcs->dispatch_tasks &&
+ adev->powerplay.pp_funcs->get_pp_num_states) {
struct pp_states_info data;
ret = kstrtoul(buf, 0, &idx);
@@ -531,7 +528,7 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
if (adev->powerplay.pp_funcs->set_sclk_od)
amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
- if (adev->pp_enabled) {
+ if (adev->powerplay.pp_funcs->dispatch_tasks) {
amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
} else {
adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
@@ -575,7 +572,7 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
if (adev->powerplay.pp_funcs->set_mclk_od)
amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
- if (adev->pp_enabled) {
+ if (adev->powerplay.pp_funcs->dispatch_tasks) {
amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
} else {
adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
@@ -959,9 +956,6 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
return 0;
- if (adev->pp_enabled)
- return effective_mode;
-
/* Skip fan attributes if fan is not present */
if (adev->pm.no_fan &&
(attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
@@ -1317,6 +1311,9 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
if (adev->pm.sysfs_initialized)
return 0;
+ if (adev->pm.dpm_enabled == 0)
+ return 0;
+
if (adev->powerplay.pp_funcs->get_temperature == NULL)
return 0;
@@ -1341,27 +1338,26 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
return ret;
}
- if (adev->pp_enabled) {
- ret = device_create_file(adev->dev, &dev_attr_pp_num_states);
- if (ret) {
- DRM_ERROR("failed to create device file pp_num_states\n");
- return ret;
- }
- ret = device_create_file(adev->dev, &dev_attr_pp_cur_state);
- if (ret) {
- DRM_ERROR("failed to create device file pp_cur_state\n");
- return ret;
- }
- ret = device_create_file(adev->dev, &dev_attr_pp_force_state);
- if (ret) {
- DRM_ERROR("failed to create device file pp_force_state\n");
- return ret;
- }
- ret = device_create_file(adev->dev, &dev_attr_pp_table);
- if (ret) {
- DRM_ERROR("failed to create device file pp_table\n");
- return ret;
- }
+
+ ret = device_create_file(adev->dev, &dev_attr_pp_num_states);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_num_states\n");
+ return ret;
+ }
+ ret = device_create_file(adev->dev, &dev_attr_pp_cur_state);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_cur_state\n");
+ return ret;
+ }
+ ret = device_create_file(adev->dev, &dev_attr_pp_force_state);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_force_state\n");
+ return ret;
+ }
+ ret = device_create_file(adev->dev, &dev_attr_pp_table);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_table\n");
+ return ret;
}
ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
@@ -1417,16 +1413,19 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
{
+ if (adev->pm.dpm_enabled == 0)
+ return;
+
if (adev->pm.int_hwmon_dev)
hwmon_device_unregister(adev->pm.int_hwmon_dev);
device_remove_file(adev->dev, &dev_attr_power_dpm_state);
device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level);
- if (adev->pp_enabled) {
- device_remove_file(adev->dev, &dev_attr_pp_num_states);
- device_remove_file(adev->dev, &dev_attr_pp_cur_state);
- device_remove_file(adev->dev, &dev_attr_pp_force_state);
- device_remove_file(adev->dev, &dev_attr_pp_table);
- }
+
+ device_remove_file(adev->dev, &dev_attr_pp_num_states);
+ device_remove_file(adev->dev, &dev_attr_pp_cur_state);
+ device_remove_file(adev->dev, &dev_attr_pp_force_state);
+ device_remove_file(adev->dev, &dev_attr_pp_table);
+
device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie);
@@ -1457,7 +1456,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
amdgpu_fence_wait_empty(ring);
}
- if (adev->pp_enabled) {
+ if (adev->powerplay.pp_funcs->dispatch_tasks) {
amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL, NULL);
} else {
mutex_lock(&adev->pm.mutex);
@@ -1592,15 +1591,15 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
if ((adev->flags & AMD_IS_PX) &&
(ddev->switch_power_state != DRM_SWITCH_POWER_ON)) {
seq_printf(m, "PX asic powered off\n");
- } else if (adev->pp_enabled) {
- return amdgpu_debugfs_pm_info_pp(m, adev);
- } else {
+ } else if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) {
mutex_lock(&adev->pm.mutex);
if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level)
adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m);
else
seq_printf(m, "Debugfs support not implemented for this asic\n");
mutex_unlock(&adev->pm.mutex);
+ } else {
+ return amdgpu_debugfs_pm_info_pp(m, adev);
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 2d2f0960b025..5f5aa5fddc16 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -34,24 +34,6 @@
#include "cik_dpm.h"
#include "vi_dpm.h"
-static int amdgpu_create_pp_handle(struct amdgpu_device *adev)
-{
- struct amd_pp_init pp_init;
- struct amd_powerplay *amd_pp;
- int ret;
-
- amd_pp = &(adev->powerplay);
- pp_init.chip_family = adev->family;
- pp_init.chip_id = adev->asic_type;
- pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
- pp_init.feature_mask = amdgpu_pp_feature_mask;
- pp_init.device = amdgpu_cgs_create_device(adev);
- ret = amd_powerplay_create(&pp_init, &(amd_pp->pp_handle));
- if (ret)
- return -EINVAL;
- return 0;
-}
-
static int amdgpu_pp_early_init(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -59,7 +41,6 @@ static int amdgpu_pp_early_init(void *handle)
int ret = 0;
amd_pp = &(adev->powerplay);
- adev->pp_enabled = false;
amd_pp->pp_handle = (void *)adev;
switch (adev->asic_type) {
@@ -73,9 +54,7 @@ static int amdgpu_pp_early_init(void *handle)
case CHIP_STONEY:
case CHIP_VEGA10:
case CHIP_RAVEN:
- adev->pp_enabled = true;
- if (amdgpu_create_pp_handle(adev))
- return -EINVAL;
+ amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
amd_pp->ip_funcs = &pp_ip_funcs;
amd_pp->pp_funcs = &pp_dpm_funcs;
break;
@@ -97,9 +76,7 @@ static int amdgpu_pp_early_init(void *handle)
amd_pp->ip_funcs = &ci_dpm_ip_funcs;
amd_pp->pp_funcs = &ci_dpm_funcs;
} else {
- adev->pp_enabled = true;
- if (amdgpu_create_pp_handle(adev))
- return -EINVAL;
+ amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
amd_pp->ip_funcs = &pp_ip_funcs;
amd_pp->pp_funcs = &pp_dpm_funcs;
}
@@ -118,12 +95,9 @@ static int amdgpu_pp_early_init(void *handle)
if (adev->powerplay.ip_funcs->early_init)
ret = adev->powerplay.ip_funcs->early_init(
- adev->powerplay.pp_handle);
+ amd_pp->cgs_device ? amd_pp->cgs_device :
+ amd_pp->pp_handle);
- if (ret == PP_DPM_DISABLED) {
- adev->pm.dpm_enabled = false;
- return 0;
- }
return ret;
}
@@ -137,11 +111,6 @@ static int amdgpu_pp_late_init(void *handle)
ret = adev->powerplay.ip_funcs->late_init(
adev->powerplay.pp_handle);
- if (adev->pp_enabled && adev->pm.dpm_enabled) {
- amdgpu_pm_sysfs_init(adev);
- amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
- }
-
return ret;
}
@@ -176,21 +145,13 @@ static int amdgpu_pp_hw_init(void *handle)
int ret = 0;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
amdgpu_ucode_init_bo(adev);
if (adev->powerplay.ip_funcs->hw_init)
ret = adev->powerplay.ip_funcs->hw_init(
adev->powerplay.pp_handle);
- if (ret == PP_DPM_DISABLED) {
- adev->pm.dpm_enabled = false;
- return 0;
- }
-
- if ((amdgpu_dpm != 0) && !amdgpu_sriov_vf(adev))
- adev->pm.dpm_enabled = true;
-
return ret;
}
@@ -199,14 +160,11 @@ static int amdgpu_pp_hw_fini(void *handle)
int ret = 0;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (adev->pp_enabled && adev->pm.dpm_enabled)
- amdgpu_pm_sysfs_fini(adev);
-
if (adev->powerplay.ip_funcs->hw_fini)
ret = adev->powerplay.ip_funcs->hw_fini(
adev->powerplay.pp_handle);
- if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
amdgpu_ucode_fini_bo(adev);
return ret;
@@ -220,9 +178,8 @@ static void amdgpu_pp_late_fini(void *handle)
adev->powerplay.ip_funcs->late_fini(
adev->powerplay.pp_handle);
-
- if (adev->pp_enabled)
- amd_powerplay_destroy(adev->powerplay.pp_handle);
+ if (adev->powerplay.cgs_device)
+ amdgpu_cgs_destroy_device(adev->powerplay.cgs_device);
}
static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index befc09b68543..190e28cb827e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -121,7 +121,7 @@ static enum amdgpu_ring_type amdgpu_hw_ip_to_ring_type(int hw_ip)
static int amdgpu_lru_map(struct amdgpu_device *adev,
struct amdgpu_queue_mapper *mapper,
- int user_ring,
+ int user_ring, bool lru_pipe_order,
struct amdgpu_ring **out_ring)
{
int r, i, j;
@@ -139,7 +139,7 @@ static int amdgpu_lru_map(struct amdgpu_device *adev,
}
r = amdgpu_ring_lru_get(adev, ring_type, ring_blacklist,
- j, out_ring);
+ j, lru_pipe_order, out_ring);
if (r)
return r;
@@ -284,8 +284,10 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
r = amdgpu_identity_map(adev, mapper, ring, out_ring);
break;
case AMDGPU_HW_IP_DMA:
+ r = amdgpu_lru_map(adev, mapper, ring, false, out_ring);
+ break;
case AMDGPU_HW_IP_COMPUTE:
- r = amdgpu_lru_map(adev, mapper, ring, out_ring);
+ r = amdgpu_lru_map(adev, mapper, ring, true, out_ring);
break;
default:
*out_ring = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 5ce65280b396..a98fbbb4739f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -136,7 +136,8 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
if (ring->funcs->end_use)
ring->funcs->end_use(ring);
- amdgpu_ring_lru_touch(ring->adev, ring);
+ if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
+ amdgpu_ring_lru_touch(ring->adev, ring);
}
/**
@@ -155,6 +156,75 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
}
/**
+ * amdgpu_ring_priority_put - restore a ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Release a request for executing at @priority
+ */
+void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
+ enum amd_sched_priority priority)
+{
+ int i;
+
+ if (!ring->funcs->set_priority)
+ return;
+
+ if (atomic_dec_return(&ring->num_jobs[priority]) > 0)
+ return;
+
+ /* no need to restore if the job is already at the lowest priority */
+ if (priority == AMD_SCHED_PRIORITY_NORMAL)
+ return;
+
+ mutex_lock(&ring->priority_mutex);
+ /* something higher prio is executing, no need to decay */
+ if (ring->priority > priority)
+ goto out_unlock;
+
+ /* decay priority to the next level with a job available */
+ for (i = priority; i >= AMD_SCHED_PRIORITY_MIN; i--) {
+ if (i == AMD_SCHED_PRIORITY_NORMAL
+ || atomic_read(&ring->num_jobs[i])) {
+ ring->priority = i;
+ ring->funcs->set_priority(ring, i);
+ break;
+ }
+ }
+
+out_unlock:
+ mutex_unlock(&ring->priority_mutex);
+}
+
+/**
+ * amdgpu_ring_priority_get - change the ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Request a ring's priority to be raised to @priority (refcounted).
+ */
+void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
+ enum amd_sched_priority priority)
+{
+ if (!ring->funcs->set_priority)
+ return;
+
+ atomic_inc(&ring->num_jobs[priority]);
+
+ mutex_lock(&ring->priority_mutex);
+ if (priority <= ring->priority)
+ goto out_unlock;
+
+ ring->priority = priority;
+ ring->funcs->set_priority(ring, priority);
+
+out_unlock:
+ mutex_unlock(&ring->priority_mutex);
+}
+
+/**
* amdgpu_ring_init - init driver ring struct.
*
* @adev: amdgpu_device pointer
@@ -169,7 +239,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
unsigned max_dw, struct amdgpu_irq_src *irq_src,
unsigned irq_type)
{
- int r;
+ int r, i;
int sched_hw_submission = amdgpu_sched_hw_submission;
/* Set the hw submission limit higher for KIQ because
@@ -247,9 +317,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
}
ring->max_dw = max_dw;
+ ring->priority = AMD_SCHED_PRIORITY_NORMAL;
+ mutex_init(&ring->priority_mutex);
INIT_LIST_HEAD(&ring->lru_list);
amdgpu_ring_lru_touch(adev, ring);
+ for (i = 0; i < AMD_SCHED_PRIORITY_MAX; ++i)
+ atomic_set(&ring->num_jobs[i], 0);
+
if (amdgpu_debugfs_ring_init(adev, ring)) {
DRM_ERROR("Failed to register debugfs file for rings !\n");
}
@@ -315,14 +390,16 @@ static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
* @type: amdgpu_ring_type enum
* @blacklist: blacklisted ring ids array
* @num_blacklist: number of entries in @blacklist
+ * @lru_pipe_order: find a ring from the least recently used pipe
* @ring: output ring
*
* Retrieve the amdgpu_ring structure for the least recently used ring of
* a specific IP block (all asics).
* Returns 0 on success, error on failure.
*/
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
- int num_blacklist, struct amdgpu_ring **ring)
+int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
+ int *blacklist, int num_blacklist,
+ bool lru_pipe_order, struct amdgpu_ring **ring)
{
struct amdgpu_ring *entry;
@@ -337,10 +414,23 @@ int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
continue;
- *ring = entry;
- amdgpu_ring_lru_touch_locked(adev, *ring);
- break;
+ if (!*ring) {
+ *ring = entry;
+
+ /* We are done for ring LRU */
+ if (!lru_pipe_order)
+ break;
+ }
+
+ /* Move all rings on the same pipe to the end of the list */
+ if (entry->pipe == (*ring)->pipe)
+ amdgpu_ring_lru_touch_locked(adev, entry);
}
+
+ /* Move the ring we found to the end of the list */
+ if (*ring)
+ amdgpu_ring_lru_touch_locked(adev, *ring);
+
spin_unlock(&adev->ring_lru_list_lock);
if (!*ring) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 322d25299a00..b18c2b96691f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -24,6 +24,7 @@
#ifndef __AMDGPU_RING_H__
#define __AMDGPU_RING_H__
+#include <drm/amdgpu_drm.h>
#include "gpu_scheduler.h"
/* max number of rings */
@@ -56,6 +57,7 @@ struct amdgpu_device;
struct amdgpu_ring;
struct amdgpu_ib;
struct amdgpu_cs_parser;
+struct amdgpu_job;
/*
* Fences.
@@ -88,8 +90,12 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence);
+int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s);
void amdgpu_fence_process(struct amdgpu_ring *ring);
int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
+signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
+ uint32_t wait_seq,
+ signed long timeout);
unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
/*
@@ -147,6 +153,9 @@ struct amdgpu_ring_funcs {
void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
void (*emit_tmz)(struct amdgpu_ring *ring, bool start);
+ /* priority functions */
+ void (*set_priority) (struct amdgpu_ring *ring,
+ enum amd_sched_priority priority);
};
struct amdgpu_ring {
@@ -187,6 +196,12 @@ struct amdgpu_ring {
volatile u32 *cond_exe_cpu_addr;
unsigned vm_inv_eng;
bool has_compute_vm_bug;
+
+ atomic_t num_jobs[AMD_SCHED_PRIORITY_MAX];
+ struct mutex priority_mutex;
+ /* protected by priority_mutex */
+ int priority;
+
#if defined(CONFIG_DEBUG_FS)
struct dentry *ent;
#endif
@@ -197,12 +212,17 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
void amdgpu_ring_commit(struct amdgpu_ring *ring);
void amdgpu_ring_undo(struct amdgpu_ring *ring);
+void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
+ enum amd_sched_priority priority);
+void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
+ enum amd_sched_priority priority);
int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
unsigned ring_size, struct amdgpu_irq_src *irq_src,
unsigned irq_type);
void amdgpu_ring_fini(struct amdgpu_ring *ring);
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
- int num_blacklist, struct amdgpu_ring **ring);
+int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
+ int *blacklist, int num_blacklist,
+ bool lru_pipe_order, struct amdgpu_ring **ring);
void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring);
static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
new file mode 100644
index 000000000000..290cc3f9c433
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2017 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Andres Rodriguez <andresx7@gmail.com>
+ */
+
+#include <linux/fdtable.h>
+#include <linux/pid.h>
+#include <drm/amdgpu_drm.h>
+#include "amdgpu.h"
+
+#include "amdgpu_vm.h"
+
+enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority)
+{
+ switch (amdgpu_priority) {
+ case AMDGPU_CTX_PRIORITY_VERY_HIGH:
+ return AMD_SCHED_PRIORITY_HIGH_HW;
+ case AMDGPU_CTX_PRIORITY_HIGH:
+ return AMD_SCHED_PRIORITY_HIGH_SW;
+ case AMDGPU_CTX_PRIORITY_NORMAL:
+ return AMD_SCHED_PRIORITY_NORMAL;
+ case AMDGPU_CTX_PRIORITY_LOW:
+ case AMDGPU_CTX_PRIORITY_VERY_LOW:
+ return AMD_SCHED_PRIORITY_LOW;
+ case AMDGPU_CTX_PRIORITY_UNSET:
+ return AMD_SCHED_PRIORITY_UNSET;
+ default:
+ WARN(1, "Invalid context priority %d\n", amdgpu_priority);
+ return AMD_SCHED_PRIORITY_INVALID;
+ }
+}
+
+static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
+ int fd,
+ enum amd_sched_priority priority)
+{
+ struct file *filp = fcheck(fd);
+ struct drm_file *file;
+ struct pid *pid;
+ struct amdgpu_fpriv *fpriv;
+ struct amdgpu_ctx *ctx;
+ uint32_t id;
+
+ if (!filp)
+ return -EINVAL;
+
+ pid = get_pid(((struct drm_file *)filp->private_data)->pid);
+
+ mutex_lock(&adev->ddev->filelist_mutex);
+ list_for_each_entry(file, &adev->ddev->filelist, lhead) {
+ if (file->pid != pid)
+ continue;
+
+ fpriv = file->driver_priv;
+ idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id)
+ amdgpu_ctx_priority_override(ctx, priority);
+ }
+ mutex_unlock(&adev->ddev->filelist_mutex);
+
+ put_pid(pid);
+
+ return 0;
+}
+
+int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ union drm_amdgpu_sched *args = data;
+ struct amdgpu_device *adev = dev->dev_private;
+ enum amd_sched_priority priority;
+ int r;
+
+ priority = amdgpu_to_sched_priority(args->in.priority);
+ if (args->in.flags || priority == AMD_SCHED_PRIORITY_INVALID)
+ return -EINVAL;
+
+ switch (args->in.op) {
+ case AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE:
+ r = amdgpu_sched_process_priority_override(adev,
+ args->in.fd,
+ priority);
+ break;
+ default:
+ DRM_ERROR("Invalid sched op specified: %d\n", args->in.op);
+ r = -EINVAL;
+ break;
+ }
+
+ return r;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
index 13c8dbbccaf2..b28c067d3822 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
@@ -1,5 +1,5 @@
/*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2017 Valve Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -19,22 +19,16 @@
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
+ * Authors: Andres Rodriguez <andresx7@gmail.com>
*/
-#ifndef _ICELAND_SMC_H
-#define _ICELAND_SMC_H
-#include "smumgr.h"
+#ifndef __AMDGPU_SCHED_H__
+#define __AMDGPU_SCHED_H__
+#include <drm/drmP.h>
-int iceland_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
-int iceland_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
-int iceland_init_smc_table(struct pp_hwmgr *hwmgr);
-int iceland_thermal_setup_fan_table(struct pp_hwmgr *hwmgr);
-int iceland_update_sclk_threshold(struct pp_hwmgr *hwmgr);
-uint32_t iceland_get_offsetof(uint32_t type, uint32_t member);
-uint32_t iceland_get_mac_definition(uint32_t value);
-int iceland_process_firmware_header(struct pp_hwmgr *hwmgr);
-int iceland_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
-bool iceland_is_dpm_running(struct pp_hwmgr *hwmgr);
-#endif
+enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority);
+int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+#endif // __AMDGPU_SCHED_H__
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c586f44312f9..a4bf21f8f1c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -169,14 +169,14 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
*
* @sync: sync object to add fences from reservation object to
* @resv: reservation object with embedded fence
- * @shared: true if we should only sync to the exclusive fence
+ * @explicit_sync: true if we should only sync to the exclusive fence
*
* Sync to the fence
*/
int amdgpu_sync_resv(struct amdgpu_device *adev,
struct amdgpu_sync *sync,
struct reservation_object *resv,
- void *owner)
+ void *owner, bool explicit_sync)
{
struct reservation_object_list *flist;
struct dma_fence *f;
@@ -191,6 +191,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
f = reservation_object_get_excl(resv);
r = amdgpu_sync_fence(adev, sync, f);
+ if (explicit_sync)
+ return r;
+
flist = reservation_object_get_list(resv);
if (!flist || r)
return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index dc7687993317..70d7e3a279a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -45,7 +45,8 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
int amdgpu_sync_resv(struct amdgpu_device *adev,
struct amdgpu_sync *sync,
struct reservation_object *resv,
- void *owner);
+ void *owner,
+ bool explicit_sync);
struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
struct amdgpu_ring *ring);
struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 15a28578d458..1f036af85ba6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -44,6 +44,7 @@
#include <linux/debugfs.h>
#include <linux/iommu.h>
#include "amdgpu.h"
+#include "amdgpu_object.h"
#include "amdgpu_trace.h"
#include "bif/bif_4_1_d.h"
@@ -209,7 +210,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
placement->num_busy_placement = 1;
return;
}
- abo = container_of(bo, struct amdgpu_bo, tbo);
+ abo = ttm_to_amdgpu_bo(bo);
switch (bo->mem.mem_type) {
case TTM_PL_VRAM:
if (adev->mman.buffer_funcs &&
@@ -257,7 +258,7 @@ gtt:
static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
- struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+ struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
if (amdgpu_ttm_tt_get_usermm(bo->ttm))
return -EPERM;
@@ -289,97 +290,177 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
return addr;
}
-static int amdgpu_move_blit(struct ttm_buffer_object *bo,
- bool evict, bool no_wait_gpu,
- struct ttm_mem_reg *new_mem,
- struct ttm_mem_reg *old_mem)
+/**
+ * amdgpu_find_mm_node - Helper function finds the drm_mm_node
+ * corresponding to @offset. It also modifies the offset to be
+ * within the drm_mm_node returned
+ */
+static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
+ unsigned long *offset)
{
- struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
- struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+ struct drm_mm_node *mm_node = mem->mm_node;
- struct drm_mm_node *old_mm, *new_mm;
- uint64_t old_start, old_size, new_start, new_size;
- unsigned long num_pages;
- struct dma_fence *fence = NULL;
- int r;
+ while (*offset >= (mm_node->size << PAGE_SHIFT)) {
+ *offset -= (mm_node->size << PAGE_SHIFT);
+ ++mm_node;
+ }
+ return mm_node;
+}
- BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);
+/**
+ * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
+ *
+ * The function copies @size bytes from {src->mem + src->offset} to
+ * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
+ * move and different for a BO to BO copy.
+ *
+ * @f: Returns the last fence if multiple jobs are submitted.
+ */
+int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
+ struct amdgpu_copy_mem *src,
+ struct amdgpu_copy_mem *dst,
+ uint64_t size,
+ struct reservation_object *resv,
+ struct dma_fence **f)
+{
+ struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+ struct drm_mm_node *src_mm, *dst_mm;
+ uint64_t src_node_start, dst_node_start, src_node_size,
+ dst_node_size, src_page_offset, dst_page_offset;
+ struct dma_fence *fence = NULL;
+ int r = 0;
+ const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
+ AMDGPU_GPU_PAGE_SIZE);
if (!ring->ready) {
DRM_ERROR("Trying to move memory with ring turned off.\n");
return -EINVAL;
}
- old_mm = old_mem->mm_node;
- old_size = old_mm->size;
- old_start = amdgpu_mm_node_addr(bo, old_mm, old_mem);
+ src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
+ src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
+ src->offset;
+ src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
+ src_page_offset = src_node_start & (PAGE_SIZE - 1);
- new_mm = new_mem->mm_node;
- new_size = new_mm->size;
- new_start = amdgpu_mm_node_addr(bo, new_mm, new_mem);
+ dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
+ dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
+ dst->offset;
+ dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
+ dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
- num_pages = new_mem->num_pages;
mutex_lock(&adev->mman.gtt_window_lock);
- while (num_pages) {
- unsigned long cur_pages = min(min(old_size, new_size),
- (u64)AMDGPU_GTT_MAX_TRANSFER_SIZE);
- uint64_t from = old_start, to = new_start;
+
+ while (size) {
+ unsigned long cur_size;
+ uint64_t from = src_node_start, to = dst_node_start;
struct dma_fence *next;
- if (old_mem->mem_type == TTM_PL_TT &&
- !amdgpu_gtt_mgr_is_allocated(old_mem)) {
- r = amdgpu_map_buffer(bo, old_mem, cur_pages,
- old_start, 0, ring, &from);
+ /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
+ * begins at an offset, then adjust the size accordingly
+ */
+ cur_size = min3(min(src_node_size, dst_node_size), size,
+ GTT_MAX_BYTES);
+ if (cur_size + src_page_offset > GTT_MAX_BYTES ||
+ cur_size + dst_page_offset > GTT_MAX_BYTES)
+ cur_size -= max(src_page_offset, dst_page_offset);
+
+ /* Map only what needs to be accessed. Map src to window 0 and
+ * dst to window 1
+ */
+ if (src->mem->mem_type == TTM_PL_TT &&
+ !amdgpu_gtt_mgr_is_allocated(src->mem)) {
+ r = amdgpu_map_buffer(src->bo, src->mem,
+ PFN_UP(cur_size + src_page_offset),
+ src_node_start, 0, ring,
+ &from);
if (r)
goto error;
+ /* Adjust the offset because amdgpu_map_buffer returns
+ * start of mapped page
+ */
+ from += src_page_offset;
}
- if (new_mem->mem_type == TTM_PL_TT &&
- !amdgpu_gtt_mgr_is_allocated(new_mem)) {
- r = amdgpu_map_buffer(bo, new_mem, cur_pages,
- new_start, 1, ring, &to);
+ if (dst->mem->mem_type == TTM_PL_TT &&
+ !amdgpu_gtt_mgr_is_allocated(dst->mem)) {
+ r = amdgpu_map_buffer(dst->bo, dst->mem,
+ PFN_UP(cur_size + dst_page_offset),
+ dst_node_start, 1, ring,
+ &to);
if (r)
goto error;
+ to += dst_page_offset;
}
- r = amdgpu_copy_buffer(ring, from, to,
- cur_pages * PAGE_SIZE,
- bo->resv, &next, false, true);
+ r = amdgpu_copy_buffer(ring, from, to, cur_size,
+ resv, &next, false, true);
if (r)
goto error;
dma_fence_put(fence);
fence = next;
- num_pages -= cur_pages;
- if (!num_pages)
+ size -= cur_size;
+ if (!size)
break;
- old_size -= cur_pages;
- if (!old_size) {
- old_start = amdgpu_mm_node_addr(bo, ++old_mm, old_mem);
- old_size = old_mm->size;
+ src_node_size -= cur_size;
+ if (!src_node_size) {
+ src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
+ src->mem);
+ src_node_size = (src_mm->size << PAGE_SHIFT);
} else {
- old_start += cur_pages * PAGE_SIZE;
+ src_node_start += cur_size;
+ src_page_offset = src_node_start & (PAGE_SIZE - 1);
}
-
- new_size -= cur_pages;
- if (!new_size) {
- new_start = amdgpu_mm_node_addr(bo, ++new_mm, new_mem);
- new_size = new_mm->size;
+ dst_node_size -= cur_size;
+ if (!dst_node_size) {
+ dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
+ dst->mem);
+ dst_node_size = (dst_mm->size << PAGE_SHIFT);
} else {
- new_start += cur_pages * PAGE_SIZE;
+ dst_node_start += cur_size;
+ dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
}
}
+error:
mutex_unlock(&adev->mman.gtt_window_lock);
+ if (f)
+ *f = dma_fence_get(fence);
+ dma_fence_put(fence);
+ return r;
+}
+
+
+static int amdgpu_move_blit(struct ttm_buffer_object *bo,
+ bool evict, bool no_wait_gpu,
+ struct ttm_mem_reg *new_mem,
+ struct ttm_mem_reg *old_mem)
+{
+ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
+ struct amdgpu_copy_mem src, dst;
+ struct dma_fence *fence = NULL;
+ int r;
+
+ src.bo = bo;
+ dst.bo = bo;
+ src.mem = old_mem;
+ dst.mem = new_mem;
+ src.offset = 0;
+ dst.offset = 0;
+
+ r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
+ new_mem->num_pages << PAGE_SHIFT,
+ bo->resv, &fence);
+ if (r)
+ goto error;
r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
dma_fence_put(fence);
return r;
error:
- mutex_unlock(&adev->mman.gtt_window_lock);
-
if (fence)
dma_fence_wait(fence, false);
dma_fence_put(fence);
@@ -484,7 +565,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
int r;
/* Can't move a pinned BO */
- abo = container_of(bo, struct amdgpu_bo, tbo);
+ abo = ttm_to_amdgpu_bo(bo);
if (WARN_ON_ONCE(abo->pin_count > 0))
return -EINVAL;
@@ -582,13 +663,12 @@ static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re
static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
unsigned long page_offset)
{
- struct drm_mm_node *mm = bo->mem.mm_node;
- uint64_t size = mm->size;
- uint64_t offset = page_offset;
+ struct drm_mm_node *mm;
+ unsigned long offset = (page_offset << PAGE_SHIFT);
- page_offset = do_div(offset, size);
- mm += offset;
- return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + page_offset;
+ mm = amdgpu_find_mm_node(&bo->mem, &offset);
+ return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
+ (offset >> PAGE_SHIFT);
}
/*
@@ -829,7 +909,8 @@ int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
placement.busy_placement = &placements;
placements.fpfn = 0;
placements.lpfn = adev->mc.gart_size >> PAGE_SHIFT;
- placements.flags = bo->mem.placement | TTM_PL_FLAG_TT;
+ placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
+ TTM_PL_FLAG_TT;
r = ttm_bo_mem_space(bo, &placement, &tmp, true, false);
if (unlikely(r))
@@ -1112,9 +1193,6 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
unsigned long num_pages = bo->mem.num_pages;
struct drm_mm_node *node = bo->mem.mm_node;
- if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
- return ttm_bo_eviction_valuable(bo, place);
-
switch (bo->mem.mem_type) {
case TTM_PL_TT:
return true;
@@ -1129,7 +1207,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
num_pages -= node->size;
++node;
}
- break;
+ return false;
default:
break;
@@ -1142,9 +1220,9 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
unsigned long offset,
void *buf, int len, int write)
{
- struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+ struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
- struct drm_mm_node *nodes = abo->tbo.mem.mm_node;
+ struct drm_mm_node *nodes;
uint32_t value = 0;
int ret = 0;
uint64_t pos;
@@ -1153,10 +1231,7 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
if (bo->mem.mem_type != TTM_PL_VRAM)
return -EIO;
- while (offset >= (nodes->size << PAGE_SHIFT)) {
- offset -= nodes->size << PAGE_SHIFT;
- ++nodes;
- }
+ nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
pos = (nodes->start << PAGE_SHIFT) + offset;
while (len && pos < adev->mc.mc_vram_size) {
@@ -1255,6 +1330,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
/* Change the size here instead of the init above so only lpfn is affected */
amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
+ /*
+ *The reserved vram for firmware must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_fw_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
r = amdgpu_bo_create_kernel(adev, adev->mc.stolen_size, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM,
&adev->stolen_vga_memory,
@@ -1479,7 +1563,8 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
job->vm_needs_flush = vm_needs_flush;
if (resv) {
r = amdgpu_sync_resv(adev, &job->sync, resv,
- AMDGPU_FENCE_OWNER_UNDEFINED);
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ false);
if (r) {
DRM_ERROR("sync failed (%d).\n", r);
goto error_free;
@@ -1571,7 +1656,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
if (resv) {
r = amdgpu_sync_resv(adev, &job->sync, resv,
- AMDGPU_FENCE_OWNER_UNDEFINED);
+ AMDGPU_FENCE_OWNER_UNDEFINED, false);
if (r) {
DRM_ERROR("sync failed (%d).\n", r);
goto error_free;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 7abae6867339..abd4084982a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -58,6 +58,12 @@ struct amdgpu_mman {
struct amd_sched_entity entity;
};
+struct amdgpu_copy_mem {
+ struct ttm_buffer_object *bo;
+ struct ttm_mem_reg *mem;
+ unsigned long offset;
+};
+
extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
@@ -72,6 +78,12 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
struct reservation_object *resv,
struct dma_fence **fence, bool direct_submit,
bool vm_needs_flush);
+int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
+ struct amdgpu_copy_mem *src,
+ struct amdgpu_copy_mem *dst,
+ uint64_t size,
+ struct reservation_object *resv,
+ struct dma_fence **f);
int amdgpu_fill_buffer(struct amdgpu_bo *bo,
uint64_t src_data,
struct reservation_object *resv,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index b46280c1279f..2918de2f39ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -648,7 +648,7 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
uint32_t allocated = 0;
uint32_t tmp, handle = 0;
uint32_t *size = &tmp;
- int i, r, idx = 0;
+ int i, r = 0, idx = 0;
p->job->vm = NULL;
ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
index 45ac91861965..7f7097931c6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
@@ -25,30 +25,26 @@
#include "amdgpu_vf_error.h"
#include "mxgpu_ai.h"
-#define AMDGPU_VF_ERROR_ENTRY_SIZE 16
-
-/* struct error_entry - amdgpu VF error information. */
-struct amdgpu_vf_error_buffer {
- int read_count;
- int write_count;
- uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
- uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
- uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
-};
-
-struct amdgpu_vf_error_buffer admgpu_vf_errors;
-
-
-void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data)
+void amdgpu_vf_error_put(struct amdgpu_device *adev,
+ uint16_t sub_error_code,
+ uint16_t error_flags,
+ uint64_t error_data)
{
int index;
- uint16_t error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
+ uint16_t error_code;
- index = admgpu_vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
- admgpu_vf_errors.code [index] = error_code;
- admgpu_vf_errors.flags [index] = error_flags;
- admgpu_vf_errors.data [index] = error_data;
- admgpu_vf_errors.write_count ++;
+ if (!amdgpu_sriov_vf(adev))
+ return;
+
+ error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
+
+ mutex_lock(&adev->virt.vf_errors.lock);
+ index = adev->virt.vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+ adev->virt.vf_errors.code [index] = error_code;
+ adev->virt.vf_errors.flags [index] = error_flags;
+ adev->virt.vf_errors.data [index] = error_data;
+ adev->virt.vf_errors.write_count ++;
+ mutex_unlock(&adev->virt.vf_errors.lock);
}
@@ -58,7 +54,8 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
u32 data1, data2, data3;
int index;
- if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) || (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
+ if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) ||
+ (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
return;
}
/*
@@ -68,18 +65,22 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
return;
}
*/
+
+ mutex_lock(&adev->virt.vf_errors.lock);
/* The errors are overlay of array, correct read_count as full. */
- if (admgpu_vf_errors.write_count - admgpu_vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
- admgpu_vf_errors.read_count = admgpu_vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
+ if (adev->virt.vf_errors.write_count - adev->virt.vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
+ adev->virt.vf_errors.read_count = adev->virt.vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
}
- while (admgpu_vf_errors.read_count < admgpu_vf_errors.write_count) {
- index =admgpu_vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
- data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX (admgpu_vf_errors.code[index], admgpu_vf_errors.flags[index]);
- data2 = admgpu_vf_errors.data[index] & 0xFFFFFFFF;
- data3 = (admgpu_vf_errors.data[index] >> 32) & 0xFFFFFFFF;
+ while (adev->virt.vf_errors.read_count < adev->virt.vf_errors.write_count) {
+ index =adev->virt.vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+ data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(adev->virt.vf_errors.code[index],
+ adev->virt.vf_errors.flags[index]);
+ data2 = adev->virt.vf_errors.data[index] & 0xFFFFFFFF;
+ data3 = (adev->virt.vf_errors.data[index] >> 32) & 0xFFFFFFFF;
adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3);
- admgpu_vf_errors.read_count ++;
+ adev->virt.vf_errors.read_count ++;
}
+ mutex_unlock(&adev->virt.vf_errors.lock);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
index 2a3278ec76ba..6436bd053325 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
@@ -56,7 +56,10 @@ enum AMDGIM_ERROR_CATEGORY {
AMDGIM_ERROR_CATEGORY_MAX
};
-void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data);
+void amdgpu_vf_error_put(struct amdgpu_device *adev,
+ uint16_t sub_error_code,
+ uint16_t error_flags,
+ uint64_t error_data);
void amdgpu_vf_error_trans_all (struct amdgpu_device *adev);
#endif /* __VF_ERROR_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index ab05121b9272..6738df836a70 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -22,7 +22,7 @@
*/
#include "amdgpu.h"
-#define MAX_KIQ_REG_WAIT 100000
+#define MAX_KIQ_REG_WAIT 100000000 /* in usecs */
int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
{
@@ -114,27 +114,25 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
{
signed long r;
- uint32_t val;
- struct dma_fence *f;
+ unsigned long flags;
+ uint32_t val, seq;
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
struct amdgpu_ring *ring = &kiq->ring;
BUG_ON(!ring->funcs->emit_rreg);
- mutex_lock(&kiq->ring_mutex);
+ spin_lock_irqsave(&kiq->ring_lock, flags);
amdgpu_ring_alloc(ring, 32);
amdgpu_ring_emit_rreg(ring, reg);
- amdgpu_fence_emit(ring, &f);
+ amdgpu_fence_emit_polling(ring, &seq);
amdgpu_ring_commit(ring);
- mutex_unlock(&kiq->ring_mutex);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
- r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
- dma_fence_put(f);
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
if (r < 1) {
- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+ DRM_ERROR("wait for kiq fence error: %ld\n", r);
return ~0;
}
-
val = adev->wb.wb[adev->virt.reg_val_offs];
return val;
@@ -143,23 +141,23 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
{
signed long r;
- struct dma_fence *f;
+ unsigned long flags;
+ uint32_t seq;
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
struct amdgpu_ring *ring = &kiq->ring;
BUG_ON(!ring->funcs->emit_wreg);
- mutex_lock(&kiq->ring_mutex);
+ spin_lock_irqsave(&kiq->ring_lock, flags);
amdgpu_ring_alloc(ring, 32);
amdgpu_ring_emit_wreg(ring, reg, v);
- amdgpu_fence_emit(ring, &f);
+ amdgpu_fence_emit_polling(ring, &seq);
amdgpu_ring_commit(ring);
- mutex_unlock(&kiq->ring_mutex);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
- r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
if (r < 1)
- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
- dma_fence_put(f);
+ DRM_ERROR("wait for kiq fence error: %ld\n", r);
}
/**
@@ -274,3 +272,80 @@ void amdgpu_virt_free_mm_table(struct amdgpu_device *adev)
(void *)&adev->virt.mm_table.cpu_addr);
adev->virt.mm_table.gpu_addr = 0;
}
+
+
+int amdgpu_virt_fw_reserve_get_checksum(void *obj,
+ unsigned long obj_size,
+ unsigned int key,
+ unsigned int chksum)
+{
+ unsigned int ret = key;
+ unsigned long i = 0;
+ unsigned char *pos;
+
+ pos = (char *)obj;
+ /* calculate checksum */
+ for (i = 0; i < obj_size; ++i)
+ ret += *(pos + i);
+ /* minus the chksum itself */
+ pos = (char *)&chksum;
+ for (i = 0; i < sizeof(chksum); ++i)
+ ret -= *(pos + i);
+ return ret;
+}
+
+void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
+{
+ uint32_t pf2vf_ver = 0;
+ uint32_t pf2vf_size = 0;
+ uint32_t checksum = 0;
+ uint32_t checkval;
+ char *str;
+
+ adev->virt.fw_reserve.p_pf2vf = NULL;
+ adev->virt.fw_reserve.p_vf2pf = NULL;
+
+ if (adev->fw_vram_usage.va != NULL) {
+ adev->virt.fw_reserve.p_pf2vf =
+ (struct amdgim_pf2vf_info_header *)(
+ adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET);
+ pf2vf_ver = adev->virt.fw_reserve.p_pf2vf->version;
+ AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size);
+ AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum);
+
+ /* pf2vf message must be in 4K */
+ if (pf2vf_size > 0 && pf2vf_size < 4096) {
+ checkval = amdgpu_virt_fw_reserve_get_checksum(
+ adev->virt.fw_reserve.p_pf2vf, pf2vf_size,
+ adev->virt.fw_reserve.checksum_key, checksum);
+ if (checkval == checksum) {
+ adev->virt.fw_reserve.p_vf2pf =
+ ((void *)adev->virt.fw_reserve.p_pf2vf +
+ pf2vf_size);
+ memset((void *)adev->virt.fw_reserve.p_vf2pf, 0,
+ sizeof(amdgim_vf2pf_info));
+ AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.version,
+ AMDGPU_FW_VRAM_VF2PF_VER);
+ AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.size,
+ sizeof(amdgim_vf2pf_info));
+ AMDGPU_FW_VRAM_VF2PF_READ(adev, driver_version,
+ &str);
+#ifdef MODULE
+ if (THIS_MODULE->version != NULL)
+ strcpy(str, THIS_MODULE->version);
+ else
+#endif
+ strcpy(str, "N/A");
+ AMDGPU_FW_VRAM_VF2PF_WRITE(adev, driver_cert,
+ 0);
+ AMDGPU_FW_VRAM_VF2PF_WRITE(adev, checksum,
+ amdgpu_virt_fw_reserve_get_checksum(
+ adev->virt.fw_reserve.p_vf2pf,
+ pf2vf_size,
+ adev->virt.fw_reserve.checksum_key, 0));
+ }
+ }
+ }
+}
+
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index afcfb8bcfb65..b89d37fc406f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -36,6 +36,18 @@ struct amdgpu_mm_table {
uint64_t gpu_addr;
};
+#define AMDGPU_VF_ERROR_ENTRY_SIZE 16
+
+/* struct error_entry - amdgpu VF error information. */
+struct amdgpu_vf_error_buffer {
+ struct mutex lock;
+ int read_count;
+ int write_count;
+ uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
+ uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
+ uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
+};
+
/**
* struct amdgpu_virt_ops - amdgpu device virt operations
*/
@@ -46,6 +58,179 @@ struct amdgpu_virt_ops {
void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
};
+/*
+ * Firmware Reserve Frame buffer
+ */
+struct amdgpu_virt_fw_reserve {
+ struct amdgim_pf2vf_info_header *p_pf2vf;
+ struct amdgim_vf2pf_info_header *p_vf2pf;
+ unsigned int checksum_key;
+};
+/*
+ * Defination between PF and VF
+ * Structures forcibly aligned to 4 to keep the same style as PF.
+ */
+#define AMDGIM_DATAEXCHANGE_OFFSET (64 * 1024)
+
+#define AMDGIM_GET_STRUCTURE_RESERVED_SIZE(total, u8, u16, u32, u64) \
+ (total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2))
+
+enum AMDGIM_FEATURE_FLAG {
+ /* GIM supports feature of Error log collecting */
+ AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1,
+ /* GIM supports feature of loading uCodes */
+ AMDGIM_FEATURE_GIM_LOAD_UCODES = 0x2,
+};
+
+struct amdgim_pf2vf_info_header {
+ /* the total structure size in byte. */
+ uint32_t size;
+ /* version of this structure, written by the GIM */
+ uint32_t version;
+} __aligned(4);
+struct amdgim_pf2vf_info_v1 {
+ /* header contains size and version */
+ struct amdgim_pf2vf_info_header header;
+ /* max_width * max_height */
+ unsigned int uvd_enc_max_pixels_count;
+ /* 16x16 pixels/sec, codec independent */
+ unsigned int uvd_enc_max_bandwidth;
+ /* max_width * max_height */
+ unsigned int vce_enc_max_pixels_count;
+ /* 16x16 pixels/sec, codec independent */
+ unsigned int vce_enc_max_bandwidth;
+ /* MEC FW position in kb from the start of visible frame buffer */
+ unsigned int mecfw_kboffset;
+ /* The features flags of the GIM driver supports. */
+ unsigned int feature_flags;
+ /* use private key from mailbox 2 to create chueksum */
+ unsigned int checksum;
+} __aligned(4);
+
+struct amdgim_pf2vf_info_v2 {
+ /* header contains size and version */
+ struct amdgim_pf2vf_info_header header;
+ /* use private key from mailbox 2 to create chueksum */
+ uint32_t checksum;
+ /* The features flags of the GIM driver supports. */
+ uint32_t feature_flags;
+ /* max_width * max_height */
+ uint32_t uvd_enc_max_pixels_count;
+ /* 16x16 pixels/sec, codec independent */
+ uint32_t uvd_enc_max_bandwidth;
+ /* max_width * max_height */
+ uint32_t vce_enc_max_pixels_count;
+ /* 16x16 pixels/sec, codec independent */
+ uint32_t vce_enc_max_bandwidth;
+ /* MEC FW position in kb from the start of VF visible frame buffer */
+ uint64_t mecfw_kboffset;
+ /* MEC FW size in KB */
+ uint32_t mecfw_ksize;
+ /* UVD FW position in kb from the start of VF visible frame buffer */
+ uint64_t uvdfw_kboffset;
+ /* UVD FW size in KB */
+ uint32_t uvdfw_ksize;
+ /* VCE FW position in kb from the start of VF visible frame buffer */
+ uint64_t vcefw_kboffset;
+ /* VCE FW size in KB */
+ uint32_t vcefw_ksize;
+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 + sizeof(struct amdgim_pf2vf_info_header)/sizeof(uint32_t)), 3)];
+} __aligned(4);
+
+
+struct amdgim_vf2pf_info_header {
+ /* the total structure size in byte. */
+ uint32_t size;
+ /*version of this structure, written by the guest */
+ uint32_t version;
+} __aligned(4);
+
+struct amdgim_vf2pf_info_v1 {
+ /* header contains size and version */
+ struct amdgim_vf2pf_info_header header;
+ /* driver version */
+ char driver_version[64];
+ /* driver certification, 1=WHQL, 0=None */
+ unsigned int driver_cert;
+ /* guest OS type and version: need a define */
+ unsigned int os_info;
+ /* in the unit of 1M */
+ unsigned int fb_usage;
+ /* guest gfx engine usage percentage */
+ unsigned int gfx_usage;
+ /* guest gfx engine health percentage */
+ unsigned int gfx_health;
+ /* guest compute engine usage percentage */
+ unsigned int compute_usage;
+ /* guest compute engine health percentage */
+ unsigned int compute_health;
+ /* guest vce engine usage percentage. 0xffff means N/A. */
+ unsigned int vce_enc_usage;
+ /* guest vce engine health percentage. 0xffff means N/A. */
+ unsigned int vce_enc_health;
+ /* guest uvd engine usage percentage. 0xffff means N/A. */
+ unsigned int uvd_enc_usage;
+ /* guest uvd engine usage percentage. 0xffff means N/A. */
+ unsigned int uvd_enc_health;
+ unsigned int checksum;
+} __aligned(4);
+
+struct amdgim_vf2pf_info_v2 {
+ /* header contains size and version */
+ struct amdgim_vf2pf_info_header header;
+ uint32_t checksum;
+ /* driver version */
+ uint8_t driver_version[64];
+ /* driver certification, 1=WHQL, 0=None */
+ uint32_t driver_cert;
+ /* guest OS type and version: need a define */
+ uint32_t os_info;
+ /* in the unit of 1M */
+ uint32_t fb_usage;
+ /* guest gfx engine usage percentage */
+ uint32_t gfx_usage;
+ /* guest gfx engine health percentage */
+ uint32_t gfx_health;
+ /* guest compute engine usage percentage */
+ uint32_t compute_usage;
+ /* guest compute engine health percentage */
+ uint32_t compute_health;
+ /* guest vce engine usage percentage. 0xffff means N/A. */
+ uint32_t vce_enc_usage;
+ /* guest vce engine health percentage. 0xffff means N/A. */
+ uint32_t vce_enc_health;
+ /* guest uvd engine usage percentage. 0xffff means N/A. */
+ uint32_t uvd_enc_usage;
+ /* guest uvd engine usage percentage. 0xffff means N/A. */
+ uint32_t uvd_enc_health;
+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 64, 0, (12 + sizeof(struct amdgim_vf2pf_info_header)/sizeof(uint32_t)), 0)];
+} __aligned(4);
+
+#define AMDGPU_FW_VRAM_VF2PF_VER 2
+typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
+
+#define AMDGPU_FW_VRAM_VF2PF_WRITE(adev, field, val) \
+ do { \
+ ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field = (val); \
+ } while (0)
+
+#define AMDGPU_FW_VRAM_VF2PF_READ(adev, field, val) \
+ do { \
+ (*val) = ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field; \
+ } while (0)
+
+#define AMDGPU_FW_VRAM_PF2VF_READ(adev, field, val) \
+ do { \
+ if (!adev->virt.fw_reserve.p_pf2vf) \
+ *(val) = 0; \
+ else { \
+ if (adev->virt.fw_reserve.p_pf2vf->version == 1) \
+ *(val) = ((struct amdgim_pf2vf_info_v1 *)adev->virt.fw_reserve.p_pf2vf)->field; \
+ if (adev->virt.fw_reserve.p_pf2vf->version == 2) \
+ *(val) = ((struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf)->field; \
+ } \
+ } while (0)
+
/* GPU virtualization */
struct amdgpu_virt {
uint32_t caps;
@@ -59,6 +244,8 @@ struct amdgpu_virt {
struct work_struct flr_work;
struct amdgpu_mm_table mm_table;
const struct amdgpu_virt_ops *ops;
+ struct amdgpu_vf_error_buffer vf_errors;
+ struct amdgpu_virt_fw_reserve fw_reserve;
};
#define AMDGPU_CSA_SIZE (8 * 1024)
@@ -101,5 +288,9 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job);
int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
+int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
+ unsigned int key,
+ unsigned int chksum);
+void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev);
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bbcc67038203..c8c26f21993c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -328,9 +328,10 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
AMDGPU_GEM_CREATE_SHADOW);
if (vm->pte_support_ats) {
- init_value = AMDGPU_PTE_SYSTEM;
+ init_value = AMDGPU_PTE_DEFAULT_ATC;
if (level != adev->vm_manager.num_level - 1)
init_value |= AMDGPU_PDE_PTE;
+
}
/* walk over the address space and allocate the page tables */
@@ -1034,7 +1035,7 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int r;
amdgpu_sync_create(&sync);
- amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner);
+ amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
r = amdgpu_sync_wait(&sync, true);
amdgpu_sync_free(&sync);
@@ -1175,11 +1176,11 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
amdgpu_ring_pad_ib(ring, params.ib);
amdgpu_sync_resv(adev, &job->sync,
parent->base.bo->tbo.resv,
- AMDGPU_FENCE_OWNER_VM);
+ AMDGPU_FENCE_OWNER_VM, false);
if (shadow)
amdgpu_sync_resv(adev, &job->sync,
shadow->tbo.resv,
- AMDGPU_FENCE_OWNER_VM);
+ AMDGPU_FENCE_OWNER_VM, false);
WARN_ON(params.ib->length_dw > ndw);
r = amdgpu_job_submit(job, ring, &vm->entity,
@@ -1243,7 +1244,7 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
int amdgpu_vm_update_directories(struct amdgpu_device *adev,
struct amdgpu_vm *vm)
{
- int r;
+ int r = 0;
spin_lock(&vm->status_lock);
while (!list_empty(&vm->relocated)) {
@@ -1643,7 +1644,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
goto error_free;
r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
- owner);
+ owner, false);
if (r)
goto error_free;
@@ -1698,6 +1699,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
struct drm_mm_node *nodes,
struct dma_fence **fence)
{
+ unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
uint64_t pfn, start = mapping->start;
int r;
@@ -1732,6 +1734,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
}
do {
+ dma_addr_t *dma_addr = NULL;
uint64_t max_entries;
uint64_t addr, last;
@@ -1745,15 +1748,32 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
}
if (pages_addr) {
+ uint64_t count;
+
max_entries = min(max_entries, 16ull * 1024ull);
- addr = 0;
+ for (count = 1; count < max_entries; ++count) {
+ uint64_t idx = pfn + count;
+
+ if (pages_addr[idx] !=
+ (pages_addr[idx - 1] + PAGE_SIZE))
+ break;
+ }
+
+ if (count < min_linear_pages) {
+ addr = pfn << PAGE_SHIFT;
+ dma_addr = pages_addr;
+ } else {
+ addr = pages_addr[pfn];
+ max_entries = count;
+ }
+
} else if (flags & AMDGPU_PTE_VALID) {
addr += adev->vm_manager.vram_base_offset;
+ addr += pfn << PAGE_SHIFT;
}
- addr += pfn << PAGE_SHIFT;
last = min((uint64_t)mapping->last, start + max_entries - 1);
- r = amdgpu_vm_bo_update_mapping(adev, exclusive, pages_addr, vm,
+ r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
start, last, flags, addr,
fence);
if (r)
@@ -2017,7 +2037,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
list_del(&mapping->list);
if (vm->pte_support_ats)
- init_pte_value = AMDGPU_PTE_SYSTEM;
+ init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
mapping->start, mapping->last,
@@ -2541,7 +2561,8 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
* @adev: amdgpu_device pointer
* @fragment_size_default: the default fragment size if it's set auto
*/
-void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
+ uint32_t fragment_size_default)
{
if (amdgpu_vm_fragment_size == -1)
adev->vm_manager.fragment_size = fragment_size_default;
@@ -2555,7 +2576,8 @@ void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_s
* @adev: amdgpu_device pointer
* @vm_size: the default vm size if it's set auto
*/
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
+ uint32_t fragment_size_default)
{
/* adjust vm size firstly */
if (amdgpu_vm_size == -1)
@@ -2627,7 +2649,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
if (adev->asic_type == CHIP_RAVEN) {
vm->pte_support_ats = true;
- init_pde_value = AMDGPU_PTE_SYSTEM | AMDGPU_PDE_PTE;
+ init_pde_value = AMDGPU_PTE_DEFAULT_ATC
+ | AMDGPU_PDE_PTE;
+
}
} else
vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
@@ -2682,6 +2706,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
}
INIT_KFIFO(vm->faults);
+ vm->fault_credit = 16;
return 0;
@@ -2734,8 +2759,9 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
struct amdgpu_bo_va_mapping *mapping, *tmp;
bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
+ struct amdgpu_bo *root;
u64 fault;
- int i;
+ int i, r;
/* Clear pending page faults from IH when the VM is destroyed */
while (kfifo_get(&vm->faults, &fault))
@@ -2770,13 +2796,51 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
}
- amdgpu_vm_free_levels(&vm->root);
+ root = amdgpu_bo_ref(vm->root.base.bo);
+ r = amdgpu_bo_reserve(root, true);
+ if (r) {
+ dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
+ } else {
+ amdgpu_vm_free_levels(&vm->root);
+ amdgpu_bo_unreserve(root);
+ }
+ amdgpu_bo_unref(&root);
dma_fence_put(vm->last_update);
for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
amdgpu_vm_free_reserved_vmid(adev, vm, i);
}
/**
+ * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
+ *
+ * @adev: amdgpu_device pointer
+ * @pasid: PASID do identify the VM
+ *
+ * This function is expected to be called in interrupt context. Returns
+ * true if there was fault credit, false otherwise
+ */
+bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
+ unsigned int pasid)
+{
+ struct amdgpu_vm *vm;
+
+ spin_lock(&adev->vm_manager.pasid_lock);
+ vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
+ spin_unlock(&adev->vm_manager.pasid_lock);
+ if (!vm)
+ /* VM not found, can't track fault credit */
+ return true;
+
+ /* No lock needed. only accessed by IRQ handler */
+ if (!vm->fault_credit)
+ /* Too many faults in this VM */
+ return false;
+
+ vm->fault_credit--;
+ return true;
+}
+
+/**
* amdgpu_vm_manager_init - init the VM manager
*
* @adev: amdgpu_device pointer
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 0af090667dfc..aa914256b4bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -73,6 +73,16 @@ struct amdgpu_bo_list_entry;
#define AMDGPU_PTE_MTYPE(a) ((uint64_t)a << 57)
#define AMDGPU_PTE_MTYPE_MASK AMDGPU_PTE_MTYPE(3ULL)
+/* For Raven */
+#define AMDGPU_MTYPE_CC 2
+
+#define AMDGPU_PTE_DEFAULT_ATC (AMDGPU_PTE_SYSTEM \
+ | AMDGPU_PTE_SNOOPED \
+ | AMDGPU_PTE_EXECUTABLE \
+ | AMDGPU_PTE_READABLE \
+ | AMDGPU_PTE_WRITEABLE \
+ | AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_CC))
+
/* How to programm VM fault handling */
#define AMDGPU_VM_FAULT_STOP_NEVER 0
#define AMDGPU_VM_FAULT_STOP_FIRST 1
@@ -165,8 +175,11 @@ struct amdgpu_vm {
/* Flag to indicate ATS support from PTE for GFX9 */
bool pte_support_ats;
- /* Up to 128 pending page faults */
+ /* Up to 128 pending retry page faults */
DECLARE_KFIFO(faults, u64, 128);
+
+ /* Limit non-retry fault storms */
+ unsigned int fault_credit;
};
struct amdgpu_vm_id {
@@ -244,6 +257,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int vm_context, unsigned int pasid);
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
+ unsigned int pasid);
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
struct list_head *validated,
struct amdgpu_bo_list_entry *entry);
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 68ce1bdaf2fc..68b505c768ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -6365,7 +6365,6 @@ static int ci_dpm_sw_fini(void *handle)
flush_work(&adev->pm.dpm.thermal.work);
mutex_lock(&adev->pm.mutex);
- amdgpu_pm_sysfs_fini(adev);
ci_dpm_fini(adev);
mutex_unlock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 07d3d895da10..a870b354e3f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -237,8 +237,23 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
*/
static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
{
- /* Process all interrupts */
- return true;
+ u32 ring_index = adev->irq.ih.rptr >> 2;
+ u16 pasid;
+
+ switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+ case 146:
+ case 147:
+ pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+ if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+ return true;
+ break;
+ default:
+ /* Not a VM fault */
+ return true;
+ }
+
+ adev->irq.ih.rptr += 16;
+ return false;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index b6cdf4afaf46..fa61d649bb44 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -216,8 +216,23 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
*/
static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
{
- /* Process all interrupts */
- return true;
+ u32 ring_index = adev->irq.ih.rptr >> 2;
+ u16 pasid;
+
+ switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+ case 146:
+ case 147:
+ pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+ if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+ return true;
+ break;
+ default:
+ /* Not a VM fault */
+ return true;
+ }
+
+ adev->irq.ih.rptr += 16;
+ return false;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index b9ee9073cb0d..a8829af120c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -288,7 +288,7 @@ dce_virtual_encoder(struct drm_connector *connector)
if (connector->encoder_ids[i] == 0)
break;
- encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+ encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
if (!encoder)
continue;
@@ -298,7 +298,7 @@ dce_virtual_encoder(struct drm_connector *connector)
/* pick the first one */
if (enc_id)
- return drm_encoder_find(connector->dev, enc_id);
+ return drm_encoder_find(connector->dev, NULL, enc_id);
return NULL;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index dfc10b1baea0..b8002ac3e536 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -20,6 +20,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+#include <linux/kernel.h>
#include <linux/firmware.h>
#include <drm/drmP.h>
#include "amdgpu.h"
@@ -3952,10 +3953,10 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
adev->gfx.rlc.reg_list_format_size_bytes >> 2,
unique_indices,
&indices_count,
- sizeof(unique_indices) / sizeof(int),
+ ARRAY_SIZE(unique_indices),
indirect_start_offsets,
&offset_count,
- sizeof(indirect_start_offsets)/sizeof(int));
+ ARRAY_SIZE(indirect_start_offsets));
/* save and restore list */
WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
@@ -3977,14 +3978,14 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
/* starting offsets starts */
WREG32(mmRLC_GPM_SCRATCH_ADDR,
adev->gfx.rlc.starting_offsets_start);
- for (i = 0; i < sizeof(indirect_start_offsets)/sizeof(int); i++)
+ for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
WREG32(mmRLC_GPM_SCRATCH_DATA,
indirect_start_offsets[i]);
/* unique indices */
temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
data = mmRLC_SRM_INDEX_CNTL_DATA_0;
- for (i = 0; i < sizeof(unique_indices) / sizeof(int); i++) {
+ for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
if (unique_indices[i] != 0) {
WREG32(temp + i, unique_indices[i] & 0x3FFFF);
WREG32(data + i, unique_indices[i] >> 20);
@@ -4132,18 +4133,12 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
gfx_v8_0_rlc_reset(adev);
gfx_v8_0_init_pg(adev);
- if (!adev->pp_enabled) {
- if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
- /* legacy rlc firmware loading */
- r = gfx_v8_0_rlc_load_microcode(adev);
- if (r)
- return r;
- } else {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_RLC_G);
- if (r)
- return -EINVAL;
- }
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ /* legacy rlc firmware loading */
+ r = gfx_v8_0_rlc_load_microcode(adev);
+ if (r)
+ return r;
}
gfx_v8_0_rlc_start(adev);
@@ -4959,43 +4954,15 @@ static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
if (!(adev->flags & AMD_IS_APU))
gfx_v8_0_enable_gui_idle_interrupt(adev, false);
- if (!adev->pp_enabled) {
- if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
/* legacy firmware loading */
- r = gfx_v8_0_cp_gfx_load_microcode(adev);
- if (r)
- return r;
+ r = gfx_v8_0_cp_gfx_load_microcode(adev);
+ if (r)
+ return r;
- r = gfx_v8_0_cp_compute_load_microcode(adev);
- if (r)
- return r;
- } else {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_CP_CE);
- if (r)
- return -EINVAL;
-
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_CP_PFP);
- if (r)
- return -EINVAL;
-
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_CP_ME);
- if (r)
- return -EINVAL;
-
- if (adev->asic_type == CHIP_TOPAZ) {
- r = gfx_v8_0_cp_compute_load_microcode(adev);
- if (r)
- return r;
- } else {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_CP_MEC1);
- if (r)
- return -EINVAL;
- }
- }
+ r = gfx_v8_0_cp_compute_load_microcode(adev);
+ if (r)
+ return r;
}
r = gfx_v8_0_cp_gfx_resume(adev);
@@ -6018,7 +5985,6 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
{
uint32_t msg_id, pp_state = 0;
uint32_t pp_support_state = 0;
- void *pp_handle = adev->powerplay.pp_handle;
if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -6036,7 +6002,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_CG,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -6057,7 +6024,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_MG,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
return 0;
@@ -6069,7 +6037,6 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
uint32_t msg_id, pp_state = 0;
uint32_t pp_support_state = 0;
- void *pp_handle = adev->powerplay.pp_handle;
if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -6087,7 +6054,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_CG,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
@@ -6106,7 +6074,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_3D,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -6127,7 +6096,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_MG,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
@@ -6142,7 +6112,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_RLC,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
@@ -6156,7 +6127,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
PP_BLOCK_GFX_CP,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
return 0;
@@ -6423,6 +6395,104 @@ static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
}
+static void gfx_v8_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
+ bool acquire)
+{
+ struct amdgpu_device *adev = ring->adev;
+ int pipe_num, tmp, reg;
+ int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
+
+ pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
+
+ /* first me only has 2 entries, GFX and HP3D */
+ if (ring->me > 0)
+ pipe_num -= 2;
+
+ reg = mmSPI_WCL_PIPE_PERCENT_GFX + pipe_num;
+ tmp = RREG32(reg);
+ tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
+ WREG32(reg, tmp);
+}
+
+static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring,
+ bool acquire)
+{
+ int i, pipe;
+ bool reserve;
+ struct amdgpu_ring *iring;
+
+ mutex_lock(&adev->gfx.pipe_reserve_mutex);
+ pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
+ if (acquire)
+ set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+ else
+ clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+
+ if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
+ /* Clear all reservations - everyone reacquires all resources */
+ for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
+ gfx_v8_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
+ true);
+
+ for (i = 0; i < adev->gfx.num_compute_rings; ++i)
+ gfx_v8_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
+ true);
+ } else {
+ /* Lower all pipes without a current reservation */
+ for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
+ iring = &adev->gfx.gfx_ring[i];
+ pipe = amdgpu_gfx_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
+ reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+ gfx_v8_0_ring_set_pipe_percent(iring, reserve);
+ }
+
+ for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
+ iring = &adev->gfx.compute_ring[i];
+ pipe = amdgpu_gfx_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
+ reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+ gfx_v8_0_ring_set_pipe_percent(iring, reserve);
+ }
+ }
+
+ mutex_unlock(&adev->gfx.pipe_reserve_mutex);
+}
+
+static void gfx_v8_0_hqd_set_priority(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring,
+ bool acquire)
+{
+ uint32_t pipe_priority = acquire ? 0x2 : 0x0;
+ uint32_t queue_priority = acquire ? 0xf : 0x0;
+
+ mutex_lock(&adev->srbm_mutex);
+ vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+
+ WREG32(mmCP_HQD_PIPE_PRIORITY, pipe_priority);
+ WREG32(mmCP_HQD_QUEUE_PRIORITY, queue_priority);
+
+ vi_srbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+static void gfx_v8_0_ring_set_priority_compute(struct amdgpu_ring *ring,
+ enum amd_sched_priority priority)
+{
+ struct amdgpu_device *adev = ring->adev;
+ bool acquire = priority == AMD_SCHED_PRIORITY_HIGH_HW;
+
+ if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
+ return;
+
+ gfx_v8_0_hqd_set_priority(adev, ring, acquire);
+ gfx_v8_0_pipe_reserve_resources(adev, ring, acquire);
+}
+
static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
u64 addr, u64 seq,
unsigned flags)
@@ -6868,6 +6938,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
.test_ib = gfx_v8_0_ring_test_ib,
.insert_nop = amdgpu_ring_insert_nop,
.pad_ib = amdgpu_ring_generic_pad_ib,
+ .set_priority = gfx_v8_0_ring_set_priority_compute,
};
static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
@@ -7076,7 +7147,7 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
{
uint64_t ce_payload_addr;
int cnt_ce;
- static union {
+ union {
struct vi_ce_ib_state regular;
struct vi_ce_ib_state_chained_ib chained;
} ce_payload = {};
@@ -7105,7 +7176,7 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
{
uint64_t de_payload_addr, gds_addr, csa_addr;
int cnt_de;
- static union {
+ union {
struct vi_de_ib_state regular;
struct vi_de_ib_state_chained_ib chained;
} de_payload = {};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index deeaee1457ef..7f15bb2c5233 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -20,6 +20,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+#include <linux/kernel.h>
#include <linux/firmware.h>
#include <drm/drmP.h>
#include "amdgpu.h"
@@ -1730,10 +1731,10 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
adev->gfx.rlc.reg_list_format_size_bytes >> 2,
unique_indirect_regs,
&unique_indirect_reg_count,
- sizeof(unique_indirect_regs)/sizeof(int),
+ ARRAY_SIZE(unique_indirect_regs),
indirect_start_offsets,
&indirect_start_offsets_count,
- sizeof(indirect_start_offsets)/sizeof(int));
+ ARRAY_SIZE(indirect_start_offsets));
/* enable auto inc in case it is disabled */
tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
@@ -1770,12 +1771,12 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
/* write the starting offsets to RLC scratch ram */
WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
adev->gfx.rlc.starting_offsets_start);
- for (i = 0; i < sizeof(indirect_start_offsets)/sizeof(int); i++)
+ for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
indirect_start_offsets[i]);
/* load unique indirect regs*/
- for (i = 0; i < sizeof(unique_indirect_regs)/sizeof(int); i++) {
+ for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) + i,
unique_indirect_regs[i] & 0x3FFFF);
WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) + i,
@@ -3583,7 +3584,7 @@ static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
{
u32 ref_and_mask, reg_mem_engine;
- struct nbio_hdp_flush_reg *nbio_hf_reg;
+ const struct nbio_hdp_flush_reg *nbio_hf_reg;
if (ring->adev->flags & AMD_IS_APU)
nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
@@ -3806,7 +3807,7 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
{
- static struct v9_ce_ib_state ce_payload = {0};
+ struct v9_ce_ib_state ce_payload = {0};
uint64_t csa_addr;
int cnt;
@@ -3825,7 +3826,7 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
{
- static struct v9_de_ib_state de_payload = {0};
+ struct v9_de_ib_state de_payload = {0};
uint64_t csa_addr, gds_addr;
int cnt;
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 65ed6d3a8f05..bd592cb39f37 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -216,8 +216,23 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
*/
static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
{
- /* Process all interrupts */
- return true;
+ u32 ring_index = adev->irq.ih.rptr >> 2;
+ u16 pasid;
+
+ switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+ case 146:
+ case 147:
+ pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+ if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+ return true;
+ break;
+ default:
+ /* Not a VM fault */
+ return true;
+ }
+
+ adev->irq.ih.rptr += 16;
+ return false;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index b57399a462c2..f33d1ffdb20b 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2969,16 +2969,10 @@ static int kv_dpm_late_init(void *handle)
{
/* powerdown unused blocks for now */
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- int ret;
if (!amdgpu_dpm)
return 0;
- /* init the sysfs and debugfs files late */
- ret = amdgpu_pm_sysfs_init(adev);
- if (ret)
- return ret;
-
kv_dpm_powergate_acp(adev, true);
kv_dpm_powergate_samu(adev, true);
@@ -3040,7 +3034,6 @@ static int kv_dpm_sw_fini(void *handle)
flush_work(&adev->pm.dpm.thermal.work);
mutex_lock(&adev->pm.mutex);
- amdgpu_pm_sysfs_fini(adev);
kv_dpm_fini(adev);
mutex_unlock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 2812d88a8bdd..b4906d2f30d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -183,6 +183,12 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
pr_err("Doesn't get READY_TO_ACCESS_GPU from pf, give up\n");
return r;
}
+ /* Retrieve checksum from mailbox2 */
+ if (req == IDH_REQ_GPU_INIT_ACCESS) {
+ adev->virt.fw_reserve.checksum_key =
+ RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
+ mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2));
+ }
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 045988b18bc3..904a1bab9b9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -215,31 +215,27 @@ void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
*flags |= AMD_CG_SUPPORT_BIF_LS;
}
-struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
-struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
+const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
+ .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ),
+ .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE),
+ .ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
+ .ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
+ .ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
+ .ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK,
+ .ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK,
+ .ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK,
+ .ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK,
+ .ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK,
+ .ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK,
+ .ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK,
+ .ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+ .ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK
+};
-int nbio_v6_1_init(struct amdgpu_device *adev)
-{
- nbio_v6_1_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
- nbio_v6_1_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK;
- nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK;
-
- nbio_v6_1_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX);
- nbio_v6_1_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA);
-
- return 0;
-}
+const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data = {
+ .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX),
+ .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA),
+};
void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
index 686e4b4d296a..14ca8d45a46c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
@@ -26,8 +26,8 @@
#include "soc15_common.h"
-extern struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
-extern struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
+extern const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
+extern const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
int nbio_v6_1_init(struct amdgpu_device *adev);
u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev,
uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index 11b70d601922..f802b973410a 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -185,28 +185,24 @@ void nbio_v7_0_ih_control(struct amdgpu_device *adev)
WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
}
-struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
-struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
+const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
+ .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ),
+ .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE),
+ .ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK,
+ .ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK,
+ .ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK,
+ .ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK,
+ .ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK,
+ .ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK,
+ .ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK,
+ .ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK,
+ .ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK,
+ .ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK,
+ .ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+ .ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+};
-int nbio_v7_0_init(struct amdgpu_device *adev)
-{
- nbio_v7_0_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ);
- nbio_v7_0_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE);
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
- nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
-
- nbio_v7_0_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
- nbio_v7_0_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
-
- return 0;
-}
+const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data = {
+ .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2),
+ .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2)
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
index 054ff49427e6..df8fa90f40d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
@@ -26,8 +26,8 @@
#include "soc15_common.h"
-extern struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
-extern struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
+extern const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
+extern const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
int nbio_v7_0_init(struct amdgpu_device *adev);
u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev,
uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index dea7c909ca5f..4e20d91d5d50 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -257,6 +257,9 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
unsigned int psp_write_ptr_reg = 0;
struct psp_gfx_rb_frame * write_frame = psp->km_ring.ring_mem;
struct psp_ring *ring = &psp->km_ring;
+ struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
+ struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
+ ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
struct amdgpu_device *adev = psp->adev;
uint32_t ring_size_dw = ring->ring_size / 4;
uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
@@ -266,9 +269,16 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
/* Update KM RB frame pointer to new frame */
if ((psp_write_ptr_reg % ring_size_dw) == 0)
- write_frame = ring->ring_mem;
+ write_frame = ring_buffer_start;
else
- write_frame = ring->ring_mem + (psp_write_ptr_reg / rb_frame_size_dw);
+ write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
+ /* Check invalid write_frame ptr address */
+ if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
+ DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
+ ring_buffer_start, ring_buffer_end, write_frame);
+ DRM_ERROR("write_frame is pointing to address out of bounds\n");
+ return -EINVAL;
+ }
/* Initialize KM RB frame */
memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index cee5c396b277..c7bcfe8e286c 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -367,6 +367,9 @@ int psp_v3_1_cmd_submit(struct psp_context *psp,
unsigned int psp_write_ptr_reg = 0;
struct psp_gfx_rb_frame * write_frame = psp->km_ring.ring_mem;
struct psp_ring *ring = &psp->km_ring;
+ struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
+ struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
+ ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
struct amdgpu_device *adev = psp->adev;
uint32_t ring_size_dw = ring->ring_size / 4;
uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
@@ -378,9 +381,16 @@ int psp_v3_1_cmd_submit(struct psp_context *psp,
/* write_frame ptr increments by size of rb_frame in bytes */
/* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
if ((psp_write_ptr_reg % ring_size_dw) == 0)
- write_frame = ring->ring_mem;
+ write_frame = ring_buffer_start;
else
- write_frame = ring->ring_mem + (psp_write_ptr_reg / rb_frame_size_dw);
+ write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
+ /* Check invalid write_frame ptr address */
+ if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
+ DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
+ ring_buffer_start, ring_buffer_end, write_frame);
+ DRM_ERROR("write_frame is pointing to address out of bounds\n");
+ return -EINVAL;
+ }
/* Initialize KM RB frame */
memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index acdee3a4602c..67f375bfe452 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -561,21 +561,11 @@ static int sdma_v2_4_start(struct amdgpu_device *adev)
{
int r;
- if (!adev->pp_enabled) {
- if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
- r = sdma_v2_4_load_microcode(adev);
- if (r)
- return r;
- } else {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_SDMA0);
- if (r)
- return -EINVAL;
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- AMDGPU_UCODE_ID_SDMA1);
- if (r)
- return -EINVAL;
- }
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ r = sdma_v2_4_load_microcode(adev);
+ if (r)
+ return r;
}
/* halt the engine before programing */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 72f31cc7df00..6d06f8eb659f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -819,23 +819,12 @@ static int sdma_v3_0_load_microcode(struct amdgpu_device *adev)
*/
static int sdma_v3_0_start(struct amdgpu_device *adev)
{
- int r, i;
+ int r;
- if (!adev->pp_enabled) {
- if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
- r = sdma_v3_0_load_microcode(adev);
- if (r)
- return r;
- } else {
- for (i = 0; i < adev->sdma.num_instances; i++) {
- r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
- (i == 0) ?
- AMDGPU_UCODE_ID_SDMA0 :
- AMDGPU_UCODE_ID_SDMA1);
- if (r)
- return -EINVAL;
- }
- }
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ r = sdma_v3_0_load_microcode(adev);
+ if (r)
+ return r;
}
/* disable sdma engine before programing it */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index c26d205ff3bf..46009db3d195 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -371,7 +371,7 @@ static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
{
u32 ref_and_mask = 0;
- struct nbio_hdp_flush_reg *nbio_hf_reg;
+ const struct nbio_hdp_flush_reg *nbio_hf_reg;
if (ring->adev->flags & AMD_IS_APU)
nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 9b8db6046271..51fd0c9a20a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -7604,11 +7604,6 @@ static int si_dpm_late_init(void *handle)
if (!amdgpu_dpm)
return 0;
- /* init the sysfs and debugfs files late */
- ret = amdgpu_pm_sysfs_init(adev);
- if (ret)
- return ret;
-
ret = si_set_temperature_range(adev);
if (ret)
return ret;
@@ -7764,7 +7759,6 @@ static int si_dpm_sw_fini(void *handle)
flush_work(&adev->pm.dpm.thermal.work);
mutex_lock(&adev->pm.mutex);
- amdgpu_pm_sysfs_fini(adev);
si_dpm_fini(adev);
mutex_unlock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 245a18aeb389..3ca9d114f630 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -101,7 +101,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
{
unsigned long flags, address, data;
u32 r;
- struct nbio_pcie_index_data *nbio_pcie_id;
+ const struct nbio_pcie_index_data *nbio_pcie_id;
if (adev->flags & AMD_IS_APU)
nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -122,7 +122,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
{
unsigned long flags, address, data;
- struct nbio_pcie_index_data *nbio_pcie_id;
+ const struct nbio_pcie_index_data *nbio_pcie_id;
if (adev->flags & AMD_IS_APU)
nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -279,10 +279,7 @@ static void soc15_init_golden_registers(struct amdgpu_device *adev)
}
static u32 soc15_get_xclk(struct amdgpu_device *adev)
{
- if (adev->asic_type == CHIP_VEGA10)
- return adev->clock.spll.reference_freq/4;
- else
- return adev->clock.spll.reference_freq;
+ return adev->clock.spll.reference_freq;
}
@@ -604,21 +601,6 @@ static int soc15_common_early_init(void *handle)
(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
psp_enabled = true;
- /*
- * nbio need be used for both sdma and gfx9, but only
- * initializes once
- */
- switch(adev->asic_type) {
- case CHIP_VEGA10:
- nbio_v6_1_init(adev);
- break;
- case CHIP_RAVEN:
- nbio_v7_0_init(adev);
- break;
- default:
- return -EINVAL;
- }
-
adev->rev_id = soc15_get_rev_id(adev);
adev->external_rev_id = 0xFF;
switch (adev->asic_type) {
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 5ed00692618e..aa4e320e31f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -227,8 +227,23 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
*/
static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
{
- /* Process all interrupts */
- return true;
+ u32 ring_index = adev->irq.ih.rptr >> 2;
+ u16 pasid;
+
+ switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+ case 146:
+ case 147:
+ pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+ if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+ return true;
+ break;
+ default:
+ /* Not a VM fault */
+ return true;
+ }
+
+ adev->irq.ih.rptr += 16;
+ return false;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 31db356476f8..2581543b35a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -38,6 +38,8 @@
#include "vi.h"
static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
+static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev);
+
static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev);
static int uvd_v6_0_start(struct amdgpu_device *adev);
static void uvd_v6_0_stop(struct amdgpu_device *adev);
@@ -48,6 +50,18 @@ static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
bool enable);
/**
+* uvd_v6_0_enc_support - get encode support status
+*
+* @adev: amdgpu_device pointer
+*
+* Returns the current hardware encode support status
+*/
+static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
+{
+ return ((adev->asic_type >= CHIP_POLARIS10) && (adev->asic_type <= CHIP_POLARIS12));
+}
+
+/**
* uvd_v6_0_ring_get_rptr - get read pointer
*
* @ring: amdgpu_ring pointer
@@ -62,6 +76,22 @@ static uint64_t uvd_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
}
/**
+ * uvd_v6_0_enc_ring_get_rptr - get enc read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc read pointer
+ */
+static uint64_t uvd_v6_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->uvd.ring_enc[0])
+ return RREG32(mmUVD_RB_RPTR);
+ else
+ return RREG32(mmUVD_RB_RPTR2);
+}
+/**
* uvd_v6_0_ring_get_wptr - get write pointer
*
* @ring: amdgpu_ring pointer
@@ -76,6 +106,23 @@ static uint64_t uvd_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
}
/**
+ * uvd_v6_0_enc_ring_get_wptr - get enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc write pointer
+ */
+static uint64_t uvd_v6_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->uvd.ring_enc[0])
+ return RREG32(mmUVD_RB_WPTR);
+ else
+ return RREG32(mmUVD_RB_WPTR2);
+}
+
+/**
* uvd_v6_0_ring_set_wptr - set write pointer
*
* @ring: amdgpu_ring pointer
@@ -89,11 +136,248 @@ static void uvd_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
}
+/**
+ * uvd_v6_0_enc_ring_set_wptr - set enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the enc write pointer to the hardware
+ */
+static void uvd_v6_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->uvd.ring_enc[0])
+ WREG32(mmUVD_RB_WPTR,
+ lower_32_bits(ring->wptr));
+ else
+ WREG32(mmUVD_RB_WPTR2,
+ lower_32_bits(ring->wptr));
+}
+
+/**
+ * uvd_v6_0_enc_ring_test_ring - test if UVD ENC ring is working
+ *
+ * @ring: the engine to test on
+ *
+ */
+static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ uint32_t rptr = amdgpu_ring_get_rptr(ring);
+ unsigned i;
+ int r;
+
+ r = amdgpu_ring_alloc(ring, 16);
+ if (r) {
+ DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
+ ring->idx, r);
+ return r;
+ }
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
+ amdgpu_ring_commit(ring);
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (amdgpu_ring_get_rptr(ring) != rptr)
+ break;
+ DRM_UDELAY(1);
+ }
+
+ if (i < adev->usec_timeout) {
+ DRM_INFO("ring test on %d succeeded in %d usecs\n",
+ ring->idx, i);
+ } else {
+ DRM_ERROR("amdgpu: ring %d test failed\n",
+ ring->idx);
+ r = -ETIMEDOUT;
+ }
+
+ return r;
+}
+
+/**
+ * uvd_v6_0_enc_get_create_msg - generate a UVD ENC create msg
+ *
+ * @adev: amdgpu_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: session handle to use
+ * @fence: optional fence to return
+ *
+ * Open up a stream for HW test
+ */
+static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
+ struct dma_fence **fence)
+{
+ const unsigned ib_size_dw = 16;
+ struct amdgpu_job *job;
+ struct amdgpu_ib *ib;
+ struct dma_fence *f = NULL;
+ uint64_t dummy;
+ int i, r;
+
+ r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
+ if (r)
+ return r;
+
+ ib = &job->ibs[0];
+ dummy = ib->gpu_addr + 1024;
+
+ ib->length_dw = 0;
+ ib->ptr[ib->length_dw++] = 0x00000018;
+ ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
+ ib->ptr[ib->length_dw++] = handle;
+ ib->ptr[ib->length_dw++] = 0x00010000;
+ ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+ ib->ptr[ib->length_dw++] = dummy;
+
+ ib->ptr[ib->length_dw++] = 0x00000014;
+ ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
+ ib->ptr[ib->length_dw++] = 0x0000001c;
+ ib->ptr[ib->length_dw++] = 0x00000001;
+ ib->ptr[ib->length_dw++] = 0x00000000;
+
+ ib->ptr[ib->length_dw++] = 0x00000008;
+ ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
+
+ for (i = ib->length_dw; i < ib_size_dw; ++i)
+ ib->ptr[i] = 0x0;
+
+ r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
+ job->fence = dma_fence_get(f);
+ if (r)
+ goto err;
+
+ amdgpu_job_free(job);
+ if (fence)
+ *fence = dma_fence_get(f);
+ dma_fence_put(f);
+ return 0;
+
+err:
+ amdgpu_job_free(job);
+ return r;
+}
+
+/**
+ * uvd_v6_0_enc_get_destroy_msg - generate a UVD ENC destroy msg
+ *
+ * @adev: amdgpu_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: session handle to use
+ * @fence: optional fence to return
+ *
+ * Close up a stream for HW test or if userspace failed to do so
+ */
+static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
+ uint32_t handle,
+ bool direct, struct dma_fence **fence)
+{
+ const unsigned ib_size_dw = 16;
+ struct amdgpu_job *job;
+ struct amdgpu_ib *ib;
+ struct dma_fence *f = NULL;
+ uint64_t dummy;
+ int i, r;
+
+ r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
+ if (r)
+ return r;
+
+ ib = &job->ibs[0];
+ dummy = ib->gpu_addr + 1024;
+
+ ib->length_dw = 0;
+ ib->ptr[ib->length_dw++] = 0x00000018;
+ ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
+ ib->ptr[ib->length_dw++] = handle;
+ ib->ptr[ib->length_dw++] = 0x00010000;
+ ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+ ib->ptr[ib->length_dw++] = dummy;
+
+ ib->ptr[ib->length_dw++] = 0x00000014;
+ ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
+ ib->ptr[ib->length_dw++] = 0x0000001c;
+ ib->ptr[ib->length_dw++] = 0x00000001;
+ ib->ptr[ib->length_dw++] = 0x00000000;
+
+ ib->ptr[ib->length_dw++] = 0x00000008;
+ ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
+
+ for (i = ib->length_dw; i < ib_size_dw; ++i)
+ ib->ptr[i] = 0x0;
+
+ if (direct) {
+ r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
+ job->fence = dma_fence_get(f);
+ if (r)
+ goto err;
+
+ amdgpu_job_free(job);
+ } else {
+ r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
+ AMDGPU_FENCE_OWNER_UNDEFINED, &f);
+ if (r)
+ goto err;
+ }
+
+ if (fence)
+ *fence = dma_fence_get(f);
+ dma_fence_put(f);
+ return 0;
+
+err:
+ amdgpu_job_free(job);
+ return r;
+}
+
+/**
+ * uvd_v6_0_enc_ring_test_ib - test if UVD ENC IBs are working
+ *
+ * @ring: the engine to test on
+ *
+ */
+static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+ struct dma_fence *fence = NULL;
+ long r;
+
+ r = uvd_v6_0_enc_get_create_msg(ring, 1, NULL);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
+ goto error;
+ }
+
+ r = uvd_v6_0_enc_get_destroy_msg(ring, 1, true, &fence);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
+ goto error;
+ }
+
+ r = dma_fence_wait_timeout(fence, false, timeout);
+ if (r == 0) {
+ DRM_ERROR("amdgpu: IB test timed out.\n");
+ r = -ETIMEDOUT;
+ } else if (r < 0) {
+ DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+ } else {
+ DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+ r = 0;
+ }
+error:
+ dma_fence_put(fence);
+ return r;
+}
static int uvd_v6_0_early_init(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
uvd_v6_0_set_ring_funcs(adev);
+
+ if (uvd_v6_0_enc_support(adev)) {
+ adev->uvd.num_enc_rings = 2;
+ uvd_v6_0_set_enc_ring_funcs(adev);
+ }
+
uvd_v6_0_set_irq_funcs(adev);
return 0;
@@ -102,7 +386,7 @@ static int uvd_v6_0_early_init(void *handle)
static int uvd_v6_0_sw_init(void *handle)
{
struct amdgpu_ring *ring;
- int r;
+ int i, r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
/* UVD TRAP */
@@ -110,10 +394,31 @@ static int uvd_v6_0_sw_init(void *handle)
if (r)
return r;
+ /* UVD ENC TRAP */
+ if (uvd_v6_0_enc_support(adev)) {
+ for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+ r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.irq);
+ if (r)
+ return r;
+ }
+ }
+
r = amdgpu_uvd_sw_init(adev);
if (r)
return r;
+ if (uvd_v6_0_enc_support(adev)) {
+ struct amd_sched_rq *rq;
+ ring = &adev->uvd.ring_enc[0];
+ rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
+ r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
+ rq, amdgpu_sched_jobs);
+ if (r) {
+ DRM_ERROR("Failed setting up UVD ENC run queue.\n");
+ return r;
+ }
+ }
+
r = amdgpu_uvd_resume(adev);
if (r)
return r;
@@ -121,19 +426,38 @@ static int uvd_v6_0_sw_init(void *handle)
ring = &adev->uvd.ring;
sprintf(ring->name, "uvd");
r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+ if (r)
+ return r;
+
+ if (uvd_v6_0_enc_support(adev)) {
+ for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+ ring = &adev->uvd.ring_enc[i];
+ sprintf(ring->name, "uvd_enc%d", i);
+ r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+ if (r)
+ return r;
+ }
+ }
return r;
}
static int uvd_v6_0_sw_fini(void *handle)
{
- int r;
+ int i, r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
r = amdgpu_uvd_suspend(adev);
if (r)
return r;
+ if (uvd_v6_0_enc_support(adev)) {
+ amd_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
+
+ for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+ amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+ }
+
return amdgpu_uvd_sw_fini(adev);
}
@@ -149,7 +473,7 @@ static int uvd_v6_0_hw_init(void *handle)
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
struct amdgpu_ring *ring = &adev->uvd.ring;
uint32_t tmp;
- int r;
+ int i, r;
amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
@@ -189,9 +513,25 @@ static int uvd_v6_0_hw_init(void *handle)
amdgpu_ring_commit(ring);
+ if (uvd_v6_0_enc_support(adev)) {
+ for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+ ring = &adev->uvd.ring_enc[i];
+ ring->ready = true;
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->ready = false;
+ goto done;
+ }
+ }
+ }
+
done:
- if (!r)
- DRM_INFO("UVD initialized successfully.\n");
+ if (!r) {
+ if (uvd_v6_0_enc_support(adev))
+ DRM_INFO("UVD and UVD ENC initialized successfully.\n");
+ else
+ DRM_INFO("UVD initialized successfully.\n");
+ }
return r;
}
@@ -225,11 +565,7 @@ static int uvd_v6_0_suspend(void *handle)
if (r)
return r;
- /* Skip this for APU for now */
- if (!(adev->flags & AMD_IS_APU))
- r = amdgpu_uvd_suspend(adev);
-
- return r;
+ return amdgpu_uvd_suspend(adev);
}
static int uvd_v6_0_resume(void *handle)
@@ -237,12 +573,10 @@ static int uvd_v6_0_resume(void *handle)
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- /* Skip this for APU for now */
- if (!(adev->flags & AMD_IS_APU)) {
- r = amdgpu_uvd_resume(adev);
- if (r)
- return r;
- }
+ r = amdgpu_uvd_resume(adev);
+ if (r)
+ return r;
+
return uvd_v6_0_hw_init(adev);
}
@@ -514,6 +848,22 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
+ if (uvd_v6_0_enc_support(adev)) {
+ ring = &adev->uvd.ring_enc[0];
+ WREG32(mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+ WREG32(mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+ WREG32(mmUVD_RB_BASE_LO, ring->gpu_addr);
+ WREG32(mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+ WREG32(mmUVD_RB_SIZE, ring->ring_size / 4);
+
+ ring = &adev->uvd.ring_enc[1];
+ WREG32(mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+ WREG32(mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+ WREG32(mmUVD_RB_BASE_LO2, ring->gpu_addr);
+ WREG32(mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+ WREG32(mmUVD_RB_SIZE2, ring->ring_size / 4);
+ }
+
return 0;
}
@@ -577,6 +927,26 @@ static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
}
/**
+ * uvd_v6_0_enc_ring_emit_fence - emit an enc fence & trap command
+ *
+ * @ring: amdgpu_ring pointer
+ * @fence: fence to emit
+ *
+ * Write enc a fence and a trap command to the ring.
+ */
+static void uvd_v6_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ u64 seq, unsigned flags)
+{
+ WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
+ amdgpu_ring_write(ring, addr);
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, seq);
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
+}
+
+/**
* uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush
*
* @ring: amdgpu_ring pointer
@@ -667,6 +1037,24 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
amdgpu_ring_write(ring, ib->length_dw);
}
+/**
+ * uvd_v6_0_enc_ring_emit_ib - enc execute indirect buffer
+ *
+ * @ring: amdgpu_ring pointer
+ * @ib: indirect buffer to execute
+ *
+ * Write enc ring commands to execute the indirect buffer
+ */
+static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
+ struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+{
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
+ amdgpu_ring_write(ring, vm_id);
+ amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, ib->length_dw);
+}
+
static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
unsigned vm_id, uint64_t pd_addr)
{
@@ -718,6 +1106,33 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
amdgpu_ring_write(ring, 0xE);
}
+static void uvd_v6_0_enc_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+ uint32_t seq = ring->fence_drv.sync_seq;
+ uint64_t addr = ring->fence_drv.gpu_addr;
+
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_WAIT_GE);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, seq);
+}
+
+static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
+}
+
+static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned int vm_id, uint64_t pd_addr)
+{
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
+ amdgpu_ring_write(ring, vm_id);
+ amdgpu_ring_write(ring, pd_addr >> 12);
+
+ amdgpu_ring_write(ring, HEVC_ENC_CMD_FLUSH_TLB);
+ amdgpu_ring_write(ring, vm_id);
+}
+
static bool uvd_v6_0_is_idle(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -825,8 +1240,31 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
struct amdgpu_irq_src *source,
struct amdgpu_iv_entry *entry)
{
+ bool int_handled = true;
DRM_DEBUG("IH: UVD TRAP\n");
- amdgpu_fence_process(&adev->uvd.ring);
+
+ switch (entry->src_id) {
+ case 124:
+ amdgpu_fence_process(&adev->uvd.ring);
+ break;
+ case 119:
+ if (likely(uvd_v6_0_enc_support(adev)))
+ amdgpu_fence_process(&adev->uvd.ring_enc[0]);
+ else
+ int_handled = false;
+ break;
+ case 120:
+ if (likely(uvd_v6_0_enc_support(adev)))
+ amdgpu_fence_process(&adev->uvd.ring_enc[1]);
+ else
+ int_handled = false;
+ break;
+ }
+
+ if (false == int_handled)
+ DRM_ERROR("Unhandled interrupt: %d %d\n",
+ entry->src_id, entry->src_data[0]);
+
return 0;
}
@@ -1153,6 +1591,33 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
.end_use = amdgpu_uvd_ring_end_use,
};
+static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
+ .type = AMDGPU_RING_TYPE_UVD_ENC,
+ .align_mask = 0x3f,
+ .nop = HEVC_ENC_CMD_NO_OP,
+ .support_64bit_ptrs = false,
+ .get_rptr = uvd_v6_0_enc_ring_get_rptr,
+ .get_wptr = uvd_v6_0_enc_ring_get_wptr,
+ .set_wptr = uvd_v6_0_enc_ring_set_wptr,
+ .emit_frame_size =
+ 4 + /* uvd_v6_0_enc_ring_emit_pipeline_sync */
+ 6 + /* uvd_v6_0_enc_ring_emit_vm_flush */
+ 5 + 5 + /* uvd_v6_0_enc_ring_emit_fence x2 vm fence */
+ 1, /* uvd_v6_0_enc_ring_insert_end */
+ .emit_ib_size = 5, /* uvd_v6_0_enc_ring_emit_ib */
+ .emit_ib = uvd_v6_0_enc_ring_emit_ib,
+ .emit_fence = uvd_v6_0_enc_ring_emit_fence,
+ .emit_vm_flush = uvd_v6_0_enc_ring_emit_vm_flush,
+ .emit_pipeline_sync = uvd_v6_0_enc_ring_emit_pipeline_sync,
+ .test_ring = uvd_v6_0_enc_ring_test_ring,
+ .test_ib = uvd_v6_0_enc_ring_test_ib,
+ .insert_nop = amdgpu_ring_insert_nop,
+ .insert_end = uvd_v6_0_enc_ring_insert_end,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .begin_use = amdgpu_uvd_ring_begin_use,
+ .end_use = amdgpu_uvd_ring_end_use,
+};
+
static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
{
if (adev->asic_type >= CHIP_POLARIS10) {
@@ -1164,6 +1629,16 @@ static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
}
}
+static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev)
+{
+ int i;
+
+ for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+ adev->uvd.ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
+
+ DRM_INFO("UVD ENC is enabled in VM mode\n");
+}
+
static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
.set = uvd_v6_0_set_interrupt_state,
.process = uvd_v6_0_process_interrupt,
@@ -1171,7 +1646,11 @@ static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
{
- adev->uvd.irq.num_types = 1;
+ if (uvd_v6_0_enc_support(adev))
+ adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
+ else
+ adev->uvd.irq.num_types = 1;
+
adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index b8ed8faf2003..6634545060fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -592,11 +592,7 @@ static int uvd_v7_0_suspend(void *handle)
if (r)
return r;
- /* Skip this for APU for now */
- if (!(adev->flags & AMD_IS_APU))
- r = amdgpu_uvd_suspend(adev);
-
- return r;
+ return amdgpu_uvd_suspend(adev);
}
static int uvd_v7_0_resume(void *handle)
@@ -604,12 +600,10 @@ static int uvd_v7_0_resume(void *handle)
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- /* Skip this for APU for now */
- if (!(adev->flags & AMD_IS_APU)) {
- r = amdgpu_uvd_resume(adev);
- if (r)
- return r;
- }
+ r = amdgpu_uvd_resume(adev);
+ if (r)
+ return r;
+
return uvd_v7_0_hw_init(adev);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index a3b30d84dbb3..697325737ba8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -260,15 +260,18 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
return true;
}
- /* Not a retry fault */
- if (!(dw5 & 0x80))
- return true;
-
pasid = dw3 & 0xffff;
/* No PASID, can't identify faulting process */
if (!pasid)
return true;
+ /* Not a retry fault, check fault credit */
+ if (!(dw5 & 0x80)) {
+ if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
+ goto ignore_iv;
+ return true;
+ }
+
addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
key = AMDGPU_VM_FAULT(pasid, addr);
r = amdgpu_ih_add_fault(adev, key);
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 9ff69b90df36..f3cfef48aa99 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1254,7 +1254,6 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
uint32_t msg_id, pp_state = 0;
uint32_t pp_support_state = 0;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- void *pp_handle = adev->powerplay.pp_handle;
if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) {
if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) {
@@ -1271,7 +1270,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_MC,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) {
@@ -1289,7 +1289,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_SDMA,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) {
@@ -1307,7 +1308,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_HDP,
pp_support_state,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
@@ -1321,7 +1323,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_BIF,
PP_STATE_SUPPORT_LS,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) {
if (state == AMD_CG_STATE_UNGATE)
@@ -1333,7 +1336,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_BIF,
PP_STATE_SUPPORT_CG,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) {
@@ -1347,7 +1351,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_DRM,
PP_STATE_SUPPORT_LS,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) {
@@ -1361,7 +1366,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
PP_BLOCK_SYS_ROM,
PP_STATE_SUPPORT_CG,
pp_state);
- amd_set_clockgating_by_smu(pp_handle, msg_id);
+ if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+ amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
}
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index a6485254a169..dbf3703cbd1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -465,6 +465,16 @@
#define VCE_CMD_UPDATE_PTB 0x00000107
#define VCE_CMD_FLUSH_TLB 0x00000108
+/* HEVC ENC */
+#define HEVC_ENC_CMD_NO_OP 0x00000000
+#define HEVC_ENC_CMD_END 0x00000001
+#define HEVC_ENC_CMD_FENCE 0x00000003
+#define HEVC_ENC_CMD_TRAP 0x00000004
+#define HEVC_ENC_CMD_IB_VM 0x00000102
+#define HEVC_ENC_CMD_WAIT_GE 0x00000106
+#define HEVC_ENC_CMD_UPDATE_PTB 0x00000107
+#define HEVC_ENC_CMD_FLUSH_TLB 0x00000108
+
/* mmPA_SC_RASTER_CONFIG mask */
#define RB_MAP_PKR0(x) ((x) << 0)
#define RB_MAP_PKR0_MASK (0x3 << 0)
diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index e13c67c8d2c0..bc5a2945bd2b 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -4,6 +4,6 @@
config HSA_AMD
tristate "HSA kernel driver for AMD GPU devices"
- depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
+ depends on DRM_AMDGPU && AMD_IOMMU_V2 && X86_64
help
Enable this if you want to use HSA features on AMD GPU devices.
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
index 211fc48697fa..3d5ccb3755d4 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
@@ -36,6 +36,7 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev,
/* Do not process in ISR, just request it to be forwarded to WQ. */
return (pasid != 0) &&
(ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
+ ihre->source_id == CIK_INTSRC_SDMA_TRAP ||
ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE);
}
@@ -46,6 +47,7 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev,
unsigned int pasid;
const struct cik_ih_ring_entry *ihre =
(const struct cik_ih_ring_entry *)ih_ring_entry;
+ uint32_t context_id = ihre->data & 0xfffffff;
pasid = (ihre->ring_id & 0xffff0000) >> 16;
@@ -53,9 +55,11 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev,
return;
if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE)
- kfd_signal_event_interrupt(pasid, 0, 0);
+ kfd_signal_event_interrupt(pasid, context_id, 28);
+ else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP)
+ kfd_signal_event_interrupt(pasid, context_id, 28);
else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG)
- kfd_signal_event_interrupt(pasid, ihre->data & 0xFF, 8);
+ kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);
else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
kfd_signal_hw_exception_event(pasid);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_int.h b/drivers/gpu/drm/amd/amdkfd/cik_int.h
index 79a16d24c1b8..109298b9d507 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_int.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_int.h
@@ -32,9 +32,10 @@ struct cik_ih_ring_entry {
uint32_t reserved;
};
-#define CIK_INTSRC_DEQUEUE_COMPLETE 0xC6
#define CIK_INTSRC_CP_END_OF_PIPE 0xB5
#define CIK_INTSRC_CP_BAD_OPCODE 0xB7
+#define CIK_INTSRC_DEQUEUE_COMPLETE 0xC6
+#define CIK_INTSRC_SDMA_TRAP 0xE0
#define CIK_INTSRC_SQ_INTERRUPT_MSG 0xEF
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 660b3fbade41..505d39156acd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -282,8 +282,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
p->pasid,
dev->id);
- err = pqm_create_queue(&p->pqm, dev, filep, &q_properties,
- 0, q_properties.type, &queue_id);
+ err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id);
if (err != 0)
goto err_create_queue;
@@ -451,8 +450,8 @@ static int kfd_ioctl_dbg_register(struct file *filep,
return -EINVAL;
}
- mutex_lock(kfd_get_dbgmgr_mutex());
mutex_lock(&p->mutex);
+ mutex_lock(kfd_get_dbgmgr_mutex());
/*
* make sure that we have pdd, if this the first queue created for
@@ -480,8 +479,8 @@ static int kfd_ioctl_dbg_register(struct file *filep,
}
out:
- mutex_unlock(&p->mutex);
mutex_unlock(kfd_get_dbgmgr_mutex());
+ mutex_unlock(&p->mutex);
return status;
}
@@ -836,15 +835,12 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
void *data)
{
struct kfd_ioctl_wait_events_args *args = data;
- enum kfd_event_wait_result wait_result;
int err;
err = kfd_wait_on_events(p, args->num_events,
(void __user *)args->events_ptr,
(args->wait_for_all != 0),
- args->timeout, &wait_result);
-
- args->wait_result = wait_result;
+ args->timeout, &args->wait_result);
return err;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index 0aa021aa0aa1..c407f6bd9956 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -184,9 +184,10 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
struct kernel_queue *kq = NULL;
int status;
+ properties.type = KFD_QUEUE_TYPE_DIQ;
+
status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
- &properties, 0, KFD_QUEUE_TYPE_DIQ,
- &qid);
+ &properties, &qid);
if (status) {
pr_err("Failed to create DIQ\n");
@@ -769,13 +770,8 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
union GRBM_GFX_INDEX_BITS reg_gfx_index;
struct kfd_process_device *pdd;
struct dbg_wave_control_info wac_info;
- int temp;
- int first_vmid_to_scan = 8;
- int last_vmid_to_scan = 15;
-
- first_vmid_to_scan = ffs(dev->shared_resources.compute_vmid_bitmap) - 1;
- temp = dev->shared_resources.compute_vmid_bitmap >> first_vmid_to_scan;
- last_vmid_to_scan = first_vmid_to_scan + ffz(temp);
+ int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
+ int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
reg_sq_cmd.u32All = 0;
status = 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 5df12b287201..621a3b53a038 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -92,6 +92,8 @@ static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
unsigned int chunk_size);
static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
+static int kfd_resume(struct kfd_dev *kfd);
+
static const struct kfd_device_info *lookup_device_info(unsigned short did)
{
size_t i;
@@ -169,15 +171,8 @@ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
(unsigned int)(1 << kfd->device_info->max_pasid_bits),
iommu_info.max_pasids);
- err = amd_iommu_init_device(kfd->pdev, pasid_limit);
- if (err < 0) {
- dev_err(kfd_device, "error initializing iommu device\n");
- return false;
- }
-
if (!kfd_set_pasid_limit(pasid_limit)) {
dev_err(kfd_device, "error setting pasid limit\n");
- amd_iommu_free_device(kfd->pdev);
return false;
}
@@ -189,7 +184,7 @@ static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
if (dev)
- kfd_unbind_process_from_device(dev, pasid);
+ kfd_process_iommu_unbind_callback(dev, pasid);
}
/*
@@ -224,6 +219,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
kfd->shared_resources = *gpu_resources;
+ kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
+ kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
+ kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
+ - kfd->vm_info.first_vmid_kfd + 1;
+
/* calculate max size of mqds needed for queues */
size = max_num_of_queues_per_device *
kfd->device_info->mqd_size_aligned;
@@ -273,29 +273,22 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
goto kfd_interrupt_error;
}
- if (!device_iommu_pasid_init(kfd)) {
- dev_err(kfd_device,
- "Error initializing iommuv2 for device %x:%x\n",
- kfd->pdev->vendor, kfd->pdev->device);
- goto device_iommu_pasid_error;
- }
- amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
- iommu_pasid_shutdown_callback);
- amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
-
kfd->dqm = device_queue_manager_init(kfd);
if (!kfd->dqm) {
dev_err(kfd_device, "Error initializing queue manager\n");
goto device_queue_manager_error;
}
- if (kfd->dqm->ops.start(kfd->dqm)) {
+ if (!device_iommu_pasid_init(kfd)) {
dev_err(kfd_device,
- "Error starting queue manager for device %x:%x\n",
+ "Error initializing iommuv2 for device %x:%x\n",
kfd->pdev->vendor, kfd->pdev->device);
- goto dqm_start_error;
+ goto device_iommu_pasid_error;
}
+ if (kfd_resume(kfd))
+ goto kfd_resume_error;
+
kfd->dbgmgr = NULL;
kfd->init_complete = true;
@@ -307,11 +300,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
goto out;
-dqm_start_error:
+kfd_resume_error:
+device_iommu_pasid_error:
device_queue_manager_uninit(kfd->dqm);
device_queue_manager_error:
- amd_iommu_free_device(kfd->pdev);
-device_iommu_pasid_error:
kfd_interrupt_exit(kfd);
kfd_interrupt_error:
kfd_topology_remove_device(kfd);
@@ -331,8 +323,8 @@ out:
void kgd2kfd_device_exit(struct kfd_dev *kfd)
{
if (kfd->init_complete) {
+ kgd2kfd_suspend(kfd);
device_queue_manager_uninit(kfd->dqm);
- amd_iommu_free_device(kfd->pdev);
kfd_interrupt_exit(kfd);
kfd_topology_remove_device(kfd);
kfd_doorbell_fini(kfd);
@@ -345,35 +337,59 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
void kgd2kfd_suspend(struct kfd_dev *kfd)
{
- if (kfd->init_complete) {
- kfd->dqm->ops.stop(kfd->dqm);
- amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
- amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
- amd_iommu_free_device(kfd->pdev);
- }
+ if (!kfd->init_complete)
+ return;
+
+ kfd->dqm->ops.stop(kfd->dqm);
+
+ kfd_unbind_processes_from_device(kfd);
+
+ amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
+ amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
+ amd_iommu_free_device(kfd->pdev);
}
int kgd2kfd_resume(struct kfd_dev *kfd)
{
- unsigned int pasid_limit;
- int err;
+ if (!kfd->init_complete)
+ return 0;
- pasid_limit = kfd_get_pasid_limit();
+ return kfd_resume(kfd);
- if (kfd->init_complete) {
- err = amd_iommu_init_device(kfd->pdev, pasid_limit);
- if (err < 0) {
- dev_err(kfd_device, "failed to initialize iommu\n");
- return -ENXIO;
- }
+}
+
+static int kfd_resume(struct kfd_dev *kfd)
+{
+ int err = 0;
+ unsigned int pasid_limit = kfd_get_pasid_limit();
- amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
- iommu_pasid_shutdown_callback);
- amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
- kfd->dqm->ops.start(kfd->dqm);
+ err = amd_iommu_init_device(kfd->pdev, pasid_limit);
+ if (err)
+ return -ENXIO;
+ amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
+ iommu_pasid_shutdown_callback);
+ amd_iommu_set_invalid_ppr_cb(kfd->pdev,
+ iommu_invalid_ppr_cb);
+
+ err = kfd_bind_processes_to_device(kfd);
+ if (err)
+ goto processes_bind_error;
+
+ err = kfd->dqm->ops.start(kfd->dqm);
+ if (err) {
+ dev_err(kfd_device,
+ "Error starting queue manager for device %x:%x\n",
+ kfd->pdev->vendor, kfd->pdev->device);
+ goto dqm_start_error;
}
- return 0;
+ return err;
+
+dqm_start_error:
+processes_bind_error:
+ amd_iommu_free_device(kfd->pdev);
+
+ return err;
}
/* This is called directly from KGD at ISR. */
@@ -387,7 +403,7 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
if (kfd->interrupts_active
&& interrupt_is_wanted(kfd, ih_ring_entry)
&& enqueue_ih_ring_entry(kfd, ih_ring_entry))
- schedule_work(&kfd->interrupt_work);
+ queue_work(kfd->ih_wq, &kfd->interrupt_work);
spin_unlock(&kfd->interrupt_lock);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 53a66e821624..e202921c150e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -44,9 +44,14 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd);
-static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
-static int destroy_queues_cpsch(struct device_queue_manager *dqm,
- bool preempt_static_queues, bool lock);
+static int execute_queues_cpsch(struct device_queue_manager *dqm,
+ enum kfd_unmap_queues_filter filter,
+ uint32_t filter_param);
+static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+ enum kfd_unmap_queues_filter filter,
+ uint32_t filter_param);
+
+static int map_queues_cpsch(struct device_queue_manager *dqm);
static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
@@ -113,11 +118,11 @@ static int allocate_vmid(struct device_queue_manager *dqm,
if (dqm->vmid_bitmap == 0)
return -ENOMEM;
- bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
+ bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap,
+ dqm->dev->vm_info.vmid_num_kfd);
clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
- /* Kaveri kfd vmid's starts from vmid 8 */
- allocated_vmid = bit + KFD_VMID_START_OFFSET;
+ allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
pr_debug("vmid allocation %d\n", allocated_vmid);
qpd->vmid = allocated_vmid;
q->properties.vmid = allocated_vmid;
@@ -132,7 +137,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
struct queue *q)
{
- int bit = qpd->vmid - KFD_VMID_START_OFFSET;
+ int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
/* Release the vmid mapping */
set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
@@ -184,6 +189,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
}
list_add(&q->list, &qpd->queues_list);
+ qpd->queue_count++;
if (q->properties.is_active)
dqm->queue_count++;
@@ -273,6 +279,9 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
dqm->dev->kfd2kgd->set_scratch_backing_va(
dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
+ if (!q->properties.is_active)
+ return 0;
+
retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
q->process->mm);
if (retval)
@@ -288,65 +297,74 @@ out_deallocate_hqd:
return retval;
}
-static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
+/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
+ * to avoid asynchronized access
+ */
+static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
struct queue *q)
{
int retval;
struct mqd_manager *mqd;
- retval = 0;
-
- mutex_lock(&dqm->lock);
+ mqd = dqm->ops.get_mqd_manager(dqm,
+ get_mqd_type_from_queue_type(q->properties.type));
+ if (!mqd)
+ return -ENOMEM;
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
- mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
- if (mqd == NULL) {
- retval = -ENOMEM;
- goto out;
- }
deallocate_hqd(dqm, q);
} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
- mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
- if (mqd == NULL) {
- retval = -ENOMEM;
- goto out;
- }
dqm->sdma_queue_count--;
deallocate_sdma_queue(dqm, q->sdma_id);
} else {
pr_debug("q->properties.type %d is invalid\n",
q->properties.type);
- retval = -EINVAL;
- goto out;
+ return -EINVAL;
}
+ dqm->total_queue_count--;
retval = mqd->destroy_mqd(mqd, q->mqd,
KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
- QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
+ KFD_UNMAP_LATENCY_MS,
q->pipe, q->queue);
-
- if (retval)
- goto out;
+ if (retval == -ETIME)
+ qpd->reset_wavefronts = true;
mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
list_del(&q->list);
- if (list_empty(&qpd->queues_list))
+ if (list_empty(&qpd->queues_list)) {
+ if (qpd->reset_wavefronts) {
+ pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
+ dqm->dev);
+ /* dbgdev_wave_reset_wavefronts has to be called before
+ * deallocate_vmid(), i.e. when vmid is still in use.
+ */
+ dbgdev_wave_reset_wavefronts(dqm->dev,
+ qpd->pqm->process);
+ qpd->reset_wavefronts = false;
+ }
+
deallocate_vmid(dqm, qpd, q);
+ }
+ qpd->queue_count--;
if (q->properties.is_active)
dqm->queue_count--;
- /*
- * Unconditionally decrement this counter, regardless of the queue's
- * type
- */
- dqm->total_queue_count--;
- pr_debug("Total of %d queues are accountable so far\n",
- dqm->total_queue_count);
+ return retval;
+}
-out:
+static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ struct queue *q)
+{
+ int retval;
+
+ mutex_lock(&dqm->lock);
+ retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
mutex_unlock(&dqm->lock);
+
return retval;
}
@@ -364,29 +382,56 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
goto out_unlock;
}
- if (q->properties.is_active)
- prev_active = true;
+ /* Save previous activity state for counters */
+ prev_active = q->properties.is_active;
+
+ /* Make sure the queue is unmapped before updating the MQD */
+ if (sched_policy != KFD_SCHED_POLICY_NO_HWS) {
+ retval = unmap_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+ if (retval) {
+ pr_err("unmap queue failed\n");
+ goto out_unlock;
+ }
+ } else if (prev_active &&
+ (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
+ retval = mqd->destroy_mqd(mqd, q->mqd,
+ KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
+ KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
+ if (retval) {
+ pr_err("destroy mqd failed\n");
+ goto out_unlock;
+ }
+ }
+
+ retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
/*
- *
- * check active state vs. the previous state
- * and modify counter accordingly
+ * check active state vs. the previous state and modify
+ * counter accordingly. map_queues_cpsch uses the
+ * dqm->queue_count to determine whether a new runlist must be
+ * uploaded.
*/
- retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
- if ((q->properties.is_active) && (!prev_active))
+ if (q->properties.is_active && !prev_active)
dqm->queue_count++;
else if (!q->properties.is_active && prev_active)
dqm->queue_count--;
if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
- retval = execute_queues_cpsch(dqm, false);
+ retval = map_queues_cpsch(dqm);
+ else if (q->properties.is_active &&
+ (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA))
+ retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
+ &q->properties, q->process->mm);
out_unlock:
mutex_unlock(&dqm->lock);
return retval;
}
-static struct mqd_manager *get_mqd_manager_nocpsch(
+static struct mqd_manager *get_mqd_manager(
struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
{
struct mqd_manager *mqd;
@@ -407,7 +452,7 @@ static struct mqd_manager *get_mqd_manager_nocpsch(
return mqd;
}
-static int register_process_nocpsch(struct device_queue_manager *dqm,
+static int register_process(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
struct device_process_node *n;
@@ -422,7 +467,7 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
mutex_lock(&dqm->lock);
list_add(&n->list, &dqm->queues);
- retval = dqm->ops_asic_specific.register_process(dqm, qpd);
+ retval = dqm->asic_ops.update_qpd(dqm, qpd);
dqm->processes_count++;
@@ -431,7 +476,7 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
return retval;
}
-static int unregister_process_nocpsch(struct device_queue_manager *dqm,
+static int unregister_process(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
int retval;
@@ -507,13 +552,13 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
dqm->allocated_queues[pipe] |= 1 << queue;
}
- dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
+ dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
return 0;
}
-static void uninitialize_nocpsch(struct device_queue_manager *dqm)
+static void uninitialize(struct device_queue_manager *dqm)
{
int i;
@@ -577,14 +622,14 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
if (retval)
return retval;
- q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
- q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
+ q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+ q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
pr_debug("SDMA id is: %d\n", q->sdma_id);
pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
- dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
+ dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
if (retval)
@@ -613,8 +658,7 @@ static int set_sched_resources(struct device_queue_manager *dqm)
int i, mec;
struct scheduling_resources res;
- res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
- res.vmid_mask <<= KFD_VMID_START_OFFSET;
+ res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
res.queue_mask = 0;
for (i = 0; i < KGD_MAX_QUEUES; ++i) {
@@ -652,8 +696,6 @@ static int set_sched_resources(struct device_queue_manager *dqm)
static int initialize_cpsch(struct device_queue_manager *dqm)
{
- int retval;
-
pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
mutex_init(&dqm->lock);
@@ -661,16 +703,13 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
dqm->queue_count = dqm->processes_count = 0;
dqm->sdma_queue_count = 0;
dqm->active_runlist = false;
- retval = dqm->ops_asic_specific.initialize(dqm);
- if (retval)
- mutex_destroy(&dqm->lock);
+ dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
- return retval;
+ return 0;
}
static int start_cpsch(struct device_queue_manager *dqm)
{
- struct device_process_node *node;
int retval;
retval = 0;
@@ -697,12 +736,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
init_interrupts(dqm);
- list_for_each_entry(node, &dqm->queues, list)
- if (node->qpd->pqm->process && dqm->dev)
- kfd_bind_process_to_device(dqm->dev,
- node->qpd->pqm->process);
-
- execute_queues_cpsch(dqm, true);
+ mutex_lock(&dqm->lock);
+ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+ mutex_unlock(&dqm->lock);
return 0;
fail_allocate_vidmem:
@@ -714,15 +750,10 @@ fail_packet_manager_init:
static int stop_cpsch(struct device_queue_manager *dqm)
{
- struct device_process_node *node;
- struct kfd_process_device *pdd;
-
- destroy_queues_cpsch(dqm, true, true);
+ mutex_lock(&dqm->lock);
+ unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+ mutex_unlock(&dqm->lock);
- list_for_each_entry(node, &dqm->queues, list) {
- pdd = qpd_to_pdd(node->qpd);
- pdd->bound = false;
- }
kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
pm_uninit(&dqm->packets);
@@ -752,7 +783,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
list_add(&kq->list, &qpd->priv_queue_list);
dqm->queue_count++;
qpd->is_debug = true;
- execute_queues_cpsch(dqm, false);
+ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
mutex_unlock(&dqm->lock);
return 0;
@@ -763,12 +794,10 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
mutex_lock(&dqm->lock);
- /* here we actually preempt the DIQ */
- destroy_queues_cpsch(dqm, true, false);
list_del(&kq->list);
dqm->queue_count--;
qpd->is_debug = false;
- execute_queues_cpsch(dqm, false);
+ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
/*
* Unconditionally decrement this counter, regardless of the queue's
* type.
@@ -779,14 +808,6 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
mutex_unlock(&dqm->lock);
}
-static void select_sdma_engine_id(struct queue *q)
-{
- static int sdma_id;
-
- q->sdma_id = sdma_id;
- sdma_id = (sdma_id + 1) % 2;
-}
-
static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd, int *allocate_vmid)
{
@@ -807,9 +828,15 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
goto out;
}
- if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
- select_sdma_engine_id(q);
-
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+ retval = allocate_sdma_queue(dqm, &q->sdma_id);
+ if (retval)
+ goto out;
+ q->properties.sdma_queue_id =
+ q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+ q->properties.sdma_engine_id =
+ q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+ }
mqd = dqm->ops.get_mqd_manager(dqm,
get_mqd_type_from_queue_type(q->properties.type));
@@ -818,16 +845,18 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
goto out;
}
- dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
+ dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
if (retval)
goto out;
list_add(&q->list, &qpd->queues_list);
+ qpd->queue_count++;
if (q->properties.is_active) {
dqm->queue_count++;
- retval = execute_queues_cpsch(dqm, false);
+ retval = execute_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
}
if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
@@ -848,12 +877,12 @@ out:
int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
unsigned int fence_value,
- unsigned long timeout)
+ unsigned int timeout_ms)
{
- timeout += jiffies;
+ unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
while (*fence_addr != fence_value) {
- if (time_after(jiffies, timeout)) {
+ if (time_after(jiffies, end_jiffies)) {
pr_err("qcm fence wait loop timeout expired\n");
return -ETIME;
}
@@ -863,44 +892,57 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
return 0;
}
-static int destroy_sdma_queues(struct device_queue_manager *dqm,
+static int unmap_sdma_queues(struct device_queue_manager *dqm,
unsigned int sdma_engine)
{
return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
- KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES, 0, false,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
sdma_engine);
}
-static int destroy_queues_cpsch(struct device_queue_manager *dqm,
- bool preempt_static_queues, bool lock)
+/* dqm->lock mutex has to be locked before calling this function */
+static int map_queues_cpsch(struct device_queue_manager *dqm)
{
int retval;
- enum kfd_preempt_type_filter preempt_type;
- struct kfd_process_device *pdd;
- retval = 0;
+ if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
+ return 0;
+
+ if (dqm->active_runlist)
+ return 0;
+
+ retval = pm_send_runlist(&dqm->packets, &dqm->queues);
+ if (retval) {
+ pr_err("failed to execute runlist\n");
+ return retval;
+ }
+ dqm->active_runlist = true;
+
+ return retval;
+}
+
+/* dqm->lock mutex has to be locked before calling this function */
+static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+ enum kfd_unmap_queues_filter filter,
+ uint32_t filter_param)
+{
+ int retval = 0;
- if (lock)
- mutex_lock(&dqm->lock);
if (!dqm->active_runlist)
- goto out;
+ return retval;
pr_debug("Before destroying queues, sdma queue count is : %u\n",
dqm->sdma_queue_count);
if (dqm->sdma_queue_count > 0) {
- destroy_sdma_queues(dqm, 0);
- destroy_sdma_queues(dqm, 1);
+ unmap_sdma_queues(dqm, 0);
+ unmap_sdma_queues(dqm, 1);
}
- preempt_type = preempt_static_queues ?
- KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES :
- KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES;
-
retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
- preempt_type, 0, false, 0);
+ filter, filter_param, false, 0);
if (retval)
- goto out;
+ return retval;
*dqm->fence_addr = KFD_FENCE_INIT;
pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
@@ -908,55 +950,29 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
/* should be timed out */
retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
- if (retval) {
- pdd = kfd_get_process_device_data(dqm->dev,
- kfd_get_process(current));
- pdd->reset_wavefronts = true;
- goto out;
- }
+ if (retval)
+ return retval;
+
pm_release_ib(&dqm->packets);
dqm->active_runlist = false;
-out:
- if (lock)
- mutex_unlock(&dqm->lock);
return retval;
}
-static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
+/* dqm->lock mutex has to be locked before calling this function */
+static int execute_queues_cpsch(struct device_queue_manager *dqm,
+ enum kfd_unmap_queues_filter filter,
+ uint32_t filter_param)
{
int retval;
- if (lock)
- mutex_lock(&dqm->lock);
-
- retval = destroy_queues_cpsch(dqm, false, false);
- if (retval) {
- pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption");
- goto out;
- }
-
- if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
- retval = 0;
- goto out;
- }
-
- if (dqm->active_runlist) {
- retval = 0;
- goto out;
- }
-
- retval = pm_send_runlist(&dqm->packets, &dqm->queues);
+ retval = unmap_queues_cpsch(dqm, filter, filter_param);
if (retval) {
- pr_err("failed to execute runlist");
- goto out;
+ pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
+ return retval;
}
- dqm->active_runlist = true;
-out:
- if (lock)
- mutex_unlock(&dqm->lock);
- return retval;
+ return map_queues_cpsch(dqm);
}
static int destroy_queue_cpsch(struct device_queue_manager *dqm,
@@ -991,14 +1007,20 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
goto failed;
}
- if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
dqm->sdma_queue_count--;
+ deallocate_sdma_queue(dqm, q->sdma_id);
+ }
list_del(&q->list);
+ qpd->queue_count--;
if (q->properties.is_active)
dqm->queue_count--;
- execute_queues_cpsch(dqm, false);
+ retval = execute_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+ if (retval == -ETIME)
+ qpd->reset_wavefronts = true;
mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
@@ -1068,7 +1090,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
qpd->sh_mem_ape1_limit = limit >> 16;
}
- retval = dqm->ops_asic_specific.set_cache_memory_policy(
+ retval = dqm->asic_ops.set_cache_memory_policy(
dqm,
qpd,
default_policy,
@@ -1088,6 +1110,109 @@ out:
return retval;
}
+static int process_termination_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+{
+ struct queue *q, *next;
+ struct device_process_node *cur, *next_dpn;
+ int retval = 0;
+
+ mutex_lock(&dqm->lock);
+
+ /* Clear all user mode queues */
+ list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+ int ret;
+
+ ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
+ if (ret)
+ retval = ret;
+ }
+
+ /* Unregister process */
+ list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+ if (qpd == cur->qpd) {
+ list_del(&cur->list);
+ kfree(cur);
+ dqm->processes_count--;
+ break;
+ }
+ }
+
+ mutex_unlock(&dqm->lock);
+ return retval;
+}
+
+
+static int process_termination_cpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+{
+ int retval;
+ struct queue *q, *next;
+ struct kernel_queue *kq, *kq_next;
+ struct mqd_manager *mqd;
+ struct device_process_node *cur, *next_dpn;
+ enum kfd_unmap_queues_filter filter =
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
+
+ retval = 0;
+
+ mutex_lock(&dqm->lock);
+
+ /* Clean all kernel queues */
+ list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
+ list_del(&kq->list);
+ dqm->queue_count--;
+ qpd->is_debug = false;
+ dqm->total_queue_count--;
+ filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
+ }
+
+ /* Clear all user mode queues */
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+ dqm->sdma_queue_count--;
+
+ if (q->properties.is_active)
+ dqm->queue_count--;
+
+ dqm->total_queue_count--;
+ }
+
+ /* Unregister process */
+ list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+ if (qpd == cur->qpd) {
+ list_del(&cur->list);
+ kfree(cur);
+ dqm->processes_count--;
+ break;
+ }
+ }
+
+ retval = execute_queues_cpsch(dqm, filter, 0);
+ if (retval || qpd->reset_wavefronts) {
+ pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
+ dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
+ qpd->reset_wavefronts = false;
+ }
+
+ /* lastly, free mqd resources */
+ list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+ mqd = dqm->ops.get_mqd_manager(dqm,
+ get_mqd_type_from_queue_type(q->properties.type));
+ if (!mqd) {
+ retval = -ENOMEM;
+ goto out;
+ }
+ list_del(&q->list);
+ qpd->queue_count--;
+ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+ }
+
+out:
+ mutex_unlock(&dqm->lock);
+ return retval;
+}
+
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
{
struct device_queue_manager *dqm;
@@ -1109,13 +1234,14 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.stop = stop_cpsch;
dqm->ops.destroy_queue = destroy_queue_cpsch;
dqm->ops.update_queue = update_queue;
- dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
- dqm->ops.register_process = register_process_nocpsch;
- dqm->ops.unregister_process = unregister_process_nocpsch;
- dqm->ops.uninitialize = uninitialize_nocpsch;
+ dqm->ops.get_mqd_manager = get_mqd_manager;
+ dqm->ops.register_process = register_process;
+ dqm->ops.unregister_process = unregister_process;
+ dqm->ops.uninitialize = uninitialize;
dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+ dqm->ops.process_termination = process_termination_cpsch;
break;
case KFD_SCHED_POLICY_NO_HWS:
/* initialize dqm for no cp scheduling */
@@ -1124,12 +1250,13 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.create_queue = create_queue_nocpsch;
dqm->ops.destroy_queue = destroy_queue_nocpsch;
dqm->ops.update_queue = update_queue;
- dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
- dqm->ops.register_process = register_process_nocpsch;
- dqm->ops.unregister_process = unregister_process_nocpsch;
+ dqm->ops.get_mqd_manager = get_mqd_manager;
+ dqm->ops.register_process = register_process;
+ dqm->ops.unregister_process = unregister_process;
dqm->ops.initialize = initialize_nocpsch;
- dqm->ops.uninitialize = uninitialize_nocpsch;
+ dqm->ops.uninitialize = uninitialize;
dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+ dqm->ops.process_termination = process_termination_nocpsch;
break;
default:
pr_err("Invalid scheduling policy %d\n", sched_policy);
@@ -1138,12 +1265,16 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
switch (dev->device_info->asic_family) {
case CHIP_CARRIZO:
- device_queue_manager_init_vi(&dqm->ops_asic_specific);
+ device_queue_manager_init_vi(&dqm->asic_ops);
break;
case CHIP_KAVERI:
- device_queue_manager_init_cik(&dqm->ops_asic_specific);
+ device_queue_manager_init_cik(&dqm->asic_ops);
break;
+ default:
+ WARN(1, "Unexpected ASIC family %u",
+ dev->device_info->asic_family);
+ goto out_free;
}
if (!dqm->ops.initialize(dqm))
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index faf820a06400..5b77cb69f732 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -29,11 +29,9 @@
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
-#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500)
-#define CIK_VMID_NUM (8)
-#define KFD_VMID_START_OFFSET (8)
-#define VMID_PER_DEVICE CIK_VMID_NUM
-#define KFD_DQM_FIRST_PIPE (0)
+#define KFD_UNMAP_LATENCY_MS (4000)
+#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
+
#define CIK_SDMA_QUEUES (4)
#define CIK_SDMA_QUEUES_PER_ENGINE (2)
#define CIK_SDMA_ENGINE_NUM (2)
@@ -79,6 +77,8 @@ struct device_process_node {
* @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
* memory apertures.
*
+ * @process_termination: Clears all process queues belongs to that device.
+ *
*/
struct device_queue_manager_ops {
@@ -122,12 +122,14 @@ struct device_queue_manager_ops {
enum cache_policy alternate_policy,
void __user *alternate_aperture_base,
uint64_t alternate_aperture_size);
+
+ int (*process_termination)(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
};
struct device_queue_manager_asic_ops {
- int (*register_process)(struct device_queue_manager *dqm,
+ int (*update_qpd)(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
- int (*initialize)(struct device_queue_manager *dqm);
bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
enum cache_policy default_policy,
@@ -153,7 +155,7 @@ struct device_queue_manager_asic_ops {
struct device_queue_manager {
struct device_queue_manager_ops ops;
- struct device_queue_manager_asic_ops ops_asic_specific;
+ struct device_queue_manager_asic_ops asic_ops;
struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
struct packet_manager packets;
@@ -176,8 +178,10 @@ struct device_queue_manager {
bool active_runlist;
};
-void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops);
-void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops);
+void device_queue_manager_init_cik(
+ struct device_queue_manager_asic_ops *asic_ops);
+void device_queue_manager_init_vi(
+ struct device_queue_manager_asic_ops *asic_ops);
void program_sh_mem_settings(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
unsigned int get_queues_num(struct device_queue_manager *dqm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index 72c3cbabc0a7..28e48c90c596 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -32,18 +32,17 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
enum cache_policy alternate_policy,
void __user *alternate_aperture_base,
uint64_t alternate_aperture_size);
-static int register_process_cik(struct device_queue_manager *dqm,
+static int update_qpd_cik(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
-static int initialize_cpsch_cik(struct device_queue_manager *dqm);
static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd);
-void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops)
+void device_queue_manager_init_cik(
+ struct device_queue_manager_asic_ops *asic_ops)
{
- ops->set_cache_memory_policy = set_cache_memory_policy_cik;
- ops->register_process = register_process_cik;
- ops->initialize = initialize_cpsch_cik;
- ops->init_sdma_vm = init_sdma_vm;
+ asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
+ asic_ops->update_qpd = update_qpd_cik;
+ asic_ops->init_sdma_vm = init_sdma_vm;
}
static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
@@ -99,7 +98,7 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
return true;
}
-static int register_process_cik(struct device_queue_manager *dqm,
+static int update_qpd_cik(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
struct kfd_process_device *pdd;
@@ -148,8 +147,3 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
q->properties.sdma_vm_addr = value;
}
-
-static int initialize_cpsch_cik(struct device_queue_manager *dqm)
-{
- return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index 40e9ddd096cd..2fbce57a2f21 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -33,18 +33,17 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
enum cache_policy alternate_policy,
void __user *alternate_aperture_base,
uint64_t alternate_aperture_size);
-static int register_process_vi(struct device_queue_manager *dqm,
+static int update_qpd_vi(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
-static int initialize_cpsch_vi(struct device_queue_manager *dqm);
static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd);
-void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops)
+void device_queue_manager_init_vi(
+ struct device_queue_manager_asic_ops *asic_ops)
{
- ops->set_cache_memory_policy = set_cache_memory_policy_vi;
- ops->register_process = register_process_vi;
- ops->initialize = initialize_cpsch_vi;
- ops->init_sdma_vm = init_sdma_vm;
+ asic_ops->set_cache_memory_policy = set_cache_memory_policy_vi;
+ asic_ops->update_qpd = update_qpd_vi;
+ asic_ops->init_sdma_vm = init_sdma_vm;
}
static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
@@ -104,7 +103,7 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
return true;
}
-static int register_process_vi(struct device_queue_manager *dqm,
+static int update_qpd_vi(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
struct kfd_process_device *pdd;
@@ -160,8 +159,3 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
q->properties.sdma_vm_addr = value;
}
-
-static int initialize_cpsch_vi(struct device_queue_manager *dqm)
-{
- return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 944abfad39c1..cb92d4b72400 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -24,8 +24,8 @@
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/sched/signal.h>
+#include <linux/sched/mm.h>
#include <linux/uaccess.h>
-#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/memory.h>
#include "kfd_priv.h"
@@ -33,185 +33,89 @@
#include <linux/device.h>
/*
- * A task can only be on a single wait_queue at a time, but we need to support
- * waiting on multiple events (any/all).
- * Instead of each event simply having a wait_queue with sleeping tasks, it
- * has a singly-linked list of tasks.
- * A thread that wants to sleep creates an array of these, one for each event
- * and adds one to each event's waiter chain.
+ * Wrapper around wait_queue_entry_t
*/
struct kfd_event_waiter {
- struct list_head waiters;
- struct task_struct *sleeping_task;
-
- /* Transitions to true when the event this belongs to is signaled. */
- bool activated;
-
- /* Event */
- struct kfd_event *event;
- uint32_t input_index;
+ wait_queue_entry_t wait;
+ struct kfd_event *event; /* Event to wait for */
+ bool activated; /* Becomes true when event is signaled */
};
/*
- * Over-complicated pooled allocator for event notification slots.
- *
* Each signal event needs a 64-bit signal slot where the signaler will write
- * a 1 before sending an interrupt.l (This is needed because some interrupts
+ * a 1 before sending an interrupt. (This is needed because some interrupts
* do not contain enough spare data bits to identify an event.)
- * We get whole pages from vmalloc and map them to the process VA.
- * Individual signal events are then allocated a slot in a page.
+ * We get whole pages and map them to the process VA.
+ * Individual signal events use their event_id as slot index.
*/
-
-struct signal_page {
- struct list_head event_pages; /* kfd_process.signal_event_pages */
+struct kfd_signal_page {
uint64_t *kernel_address;
uint64_t __user *user_address;
- uint32_t page_index; /* Index into the mmap aperture. */
- unsigned int free_slots;
- unsigned long used_slot_bitmap[0];
};
-#define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT
-#define SLOT_BITMAP_SIZE BITS_TO_LONGS(SLOTS_PER_PAGE)
-#define BITS_PER_PAGE (ilog2(SLOTS_PER_PAGE)+1)
-#define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + \
- SLOT_BITMAP_SIZE * sizeof(long))
-
-/*
- * For signal events, the event ID is used as the interrupt user data.
- * For SQ s_sendmsg interrupts, this is limited to 8 bits.
- */
-
-#define INTERRUPT_DATA_BITS 8
-#define SIGNAL_EVENT_ID_SLOT_SHIFT 0
-static uint64_t *page_slots(struct signal_page *page)
+static uint64_t *page_slots(struct kfd_signal_page *page)
{
return page->kernel_address;
}
-static bool allocate_free_slot(struct kfd_process *process,
- struct signal_page **out_page,
- unsigned int *out_slot_index)
-{
- struct signal_page *page;
-
- list_for_each_entry(page, &process->signal_event_pages, event_pages) {
- if (page->free_slots > 0) {
- unsigned int slot =
- find_first_zero_bit(page->used_slot_bitmap,
- SLOTS_PER_PAGE);
-
- __set_bit(slot, page->used_slot_bitmap);
- page->free_slots--;
-
- page_slots(page)[slot] = UNSIGNALED_EVENT_SLOT;
-
- *out_page = page;
- *out_slot_index = slot;
-
- pr_debug("Allocated event signal slot in page %p, slot %d\n",
- page, slot);
-
- return true;
- }
- }
-
- pr_debug("No free event signal slots were found for process %p\n",
- process);
-
- return false;
-}
-
-#define list_tail_entry(head, type, member) \
- list_entry((head)->prev, type, member)
-
-static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
+static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
{
void *backing_store;
- struct signal_page *page;
+ struct kfd_signal_page *page;
- page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL);
+ page = kzalloc(sizeof(*page), GFP_KERNEL);
if (!page)
- goto fail_alloc_signal_page;
+ return NULL;
- page->free_slots = SLOTS_PER_PAGE;
-
- backing_store = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
+ backing_store = (void *) __get_free_pages(GFP_KERNEL,
get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
if (!backing_store)
goto fail_alloc_signal_store;
- /* prevent user-mode info leaks */
+ /* Initialize all events to unsignaled */
memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
- KFD_SIGNAL_EVENT_LIMIT * 8);
+ KFD_SIGNAL_EVENT_LIMIT * 8);
page->kernel_address = backing_store;
-
- if (list_empty(&p->signal_event_pages))
- page->page_index = 0;
- else
- page->page_index = list_tail_entry(&p->signal_event_pages,
- struct signal_page,
- event_pages)->page_index + 1;
-
pr_debug("Allocated new event signal page at %p, for process %p\n",
page, p);
- pr_debug("Page index is %d\n", page->page_index);
- list_add(&page->event_pages, &p->signal_event_pages);
-
- return true;
+ return page;
fail_alloc_signal_store:
kfree(page);
-fail_alloc_signal_page:
- return false;
+ return NULL;
}
-static bool allocate_event_notification_slot(struct file *devkfd,
- struct kfd_process *p,
- struct signal_page **page,
- unsigned int *signal_slot_index)
+static int allocate_event_notification_slot(struct kfd_process *p,
+ struct kfd_event *ev)
{
- bool ret;
+ int id;
- ret = allocate_free_slot(p, page, signal_slot_index);
- if (!ret) {
- ret = allocate_signal_page(devkfd, p);
- if (ret)
- ret = allocate_free_slot(p, page, signal_slot_index);
+ if (!p->signal_page) {
+ p->signal_page = allocate_signal_page(p);
+ if (!p->signal_page)
+ return -ENOMEM;
+ /* Oldest user mode expects 256 event slots */
+ p->signal_mapped_size = 256*8;
}
- return ret;
-}
-
-/* Assumes that the process's event_mutex is locked. */
-static void release_event_notification_slot(struct signal_page *page,
- size_t slot_index)
-{
- __clear_bit(slot_index, page->used_slot_bitmap);
- page->free_slots++;
-
- /* We don't free signal pages, they are retained by the process
- * and reused until it exits.
- */
-}
-
-static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
- unsigned int page_index)
-{
- struct signal_page *page;
-
/*
- * This is safe because we don't delete signal pages until the
- * process exits.
+ * Compatibility with old user mode: Only use signal slots
+ * user mode has mapped, may be less than
+ * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
+ * of the event limit without breaking user mode.
*/
- list_for_each_entry(page, &p->signal_event_pages, event_pages)
- if (page->page_index == page_index)
- return page;
+ id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
+ GFP_KERNEL);
+ if (id < 0)
+ return id;
- return NULL;
+ ev->event_id = id;
+ page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;
+
+ return 0;
}
/*
@@ -220,99 +124,81 @@ static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
*/
static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
{
- struct kfd_event *ev;
-
- hash_for_each_possible(p->events, ev, events, id)
- if (ev->event_id == id)
- return ev;
-
- return NULL;
+ return idr_find(&p->event_idr, id);
}
-static u32 make_signal_event_id(struct signal_page *page,
- unsigned int signal_slot_index)
-{
- return page->page_index |
- (signal_slot_index << SIGNAL_EVENT_ID_SLOT_SHIFT);
-}
-
-/*
- * Produce a kfd event id for a nonsignal event.
- * These are arbitrary numbers, so we do a sequential search through
- * the hash table for an unused number.
+/**
+ * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID
+ * @p: Pointer to struct kfd_process
+ * @id: ID to look up
+ * @bits: Number of valid bits in @id
+ *
+ * Finds the first signaled event with a matching partial ID. If no
+ * matching signaled event is found, returns NULL. In that case the
+ * caller should assume that the partial ID is invalid and do an
+ * exhaustive search of all siglaned events.
+ *
+ * If multiple events with the same partial ID signal at the same
+ * time, they will be found one interrupt at a time, not necessarily
+ * in the same order the interrupts occurred. As long as the number of
+ * interrupts is correct, all signaled events will be seen by the
+ * driver.
*/
-static u32 make_nonsignal_event_id(struct kfd_process *p)
+static struct kfd_event *lookup_signaled_event_by_partial_id(
+ struct kfd_process *p, uint32_t id, uint32_t bits)
{
- u32 id;
-
- for (id = p->next_nonsignal_event_id;
- id < KFD_LAST_NONSIGNAL_EVENT_ID &&
- lookup_event_by_id(p, id);
- id++)
- ;
+ struct kfd_event *ev;
- if (id < KFD_LAST_NONSIGNAL_EVENT_ID) {
+ if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT)
+ return NULL;
- /*
- * What if id == LAST_NONSIGNAL_EVENT_ID - 1?
- * Then next_nonsignal_event_id = LAST_NONSIGNAL_EVENT_ID so
- * the first loop fails immediately and we proceed with the
- * wraparound loop below.
- */
- p->next_nonsignal_event_id = id + 1;
+ /* Fast path for the common case that @id is not a partial ID
+ * and we only need a single lookup.
+ */
+ if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
+ if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
+ return NULL;
- return id;
+ return idr_find(&p->event_idr, id);
}
- for (id = KFD_FIRST_NONSIGNAL_EVENT_ID;
- id < KFD_LAST_NONSIGNAL_EVENT_ID &&
- lookup_event_by_id(p, id);
- id++)
- ;
-
+ /* General case for partial IDs: Iterate over all matching IDs
+ * and find the first one that has signaled.
+ */
+ for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {
+ if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
+ continue;
- if (id < KFD_LAST_NONSIGNAL_EVENT_ID) {
- p->next_nonsignal_event_id = id + 1;
- return id;
+ ev = idr_find(&p->event_idr, id);
}
- p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;
- return 0;
-}
-
-static struct kfd_event *lookup_event_by_page_slot(struct kfd_process *p,
- struct signal_page *page,
- unsigned int signal_slot)
-{
- return lookup_event_by_id(p, make_signal_event_id(page, signal_slot));
+ return ev;
}
static int create_signal_event(struct file *devkfd,
struct kfd_process *p,
struct kfd_event *ev)
{
- if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) {
+ int ret;
+
+ if (p->signal_mapped_size &&
+ p->signal_event_count == p->signal_mapped_size / 8) {
if (!p->signal_event_limit_reached) {
pr_warn("Signal event wasn't created because limit was reached\n");
p->signal_event_limit_reached = true;
}
- return -ENOMEM;
+ return -ENOSPC;
}
- if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page,
- &ev->signal_slot_index)) {
+ ret = allocate_event_notification_slot(p, ev);
+ if (ret) {
pr_warn("Signal event wasn't created because out of kernel memory\n");
- return -ENOMEM;
+ return ret;
}
p->signal_event_count++;
- ev->user_signal_address =
- &ev->signal_page->user_address[ev->signal_slot_index];
-
- ev->event_id = make_signal_event_id(ev->signal_page,
- ev->signal_slot_index);
-
+ ev->user_signal_address = &p->signal_page->user_address[ev->event_id];
pr_debug("Signal event number %zu created with id %d, address %p\n",
p->signal_event_count, ev->event_id,
ev->user_signal_address);
@@ -320,16 +206,20 @@ static int create_signal_event(struct file *devkfd,
return 0;
}
-/*
- * No non-signal events are supported yet.
- * We create them as events that never signal.
- * Set event calls from user-mode are failed.
- */
static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
{
- ev->event_id = make_nonsignal_event_id(p);
- if (ev->event_id == 0)
- return -ENOMEM;
+ /* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
+ * intentional integer overflow to -1 without a compiler
+ * warning. idr_alloc treats a negative value as "maximum
+ * signed integer".
+ */
+ int id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
+ (uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
+ GFP_KERNEL);
+
+ if (id < 0)
+ return id;
+ ev->event_id = id;
return 0;
}
@@ -337,50 +227,47 @@ static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
void kfd_event_init_process(struct kfd_process *p)
{
mutex_init(&p->event_mutex);
- hash_init(p->events);
- INIT_LIST_HEAD(&p->signal_event_pages);
- p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+ idr_init(&p->event_idr);
+ p->signal_page = NULL;
p->signal_event_count = 0;
}
static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
{
- if (ev->signal_page) {
- release_event_notification_slot(ev->signal_page,
- ev->signal_slot_index);
- p->signal_event_count--;
- }
+ struct kfd_event_waiter *waiter;
- /*
- * Abandon the list of waiters. Individual waiting threads will
- * clean up their own data.
- */
- list_del(&ev->waiters);
+ /* Wake up pending waiters. They will return failure */
+ list_for_each_entry(waiter, &ev->wq.head, wait.entry)
+ waiter->event = NULL;
+ wake_up_all(&ev->wq);
+
+ if (ev->type == KFD_EVENT_TYPE_SIGNAL ||
+ ev->type == KFD_EVENT_TYPE_DEBUG)
+ p->signal_event_count--;
- hash_del(&ev->events);
+ idr_remove(&p->event_idr, ev->event_id);
kfree(ev);
}
static void destroy_events(struct kfd_process *p)
{
struct kfd_event *ev;
- struct hlist_node *tmp;
- unsigned int hash_bkt;
+ uint32_t id;
- hash_for_each_safe(p->events, hash_bkt, tmp, ev, events)
+ idr_for_each_entry(&p->event_idr, ev, id)
destroy_event(p, ev);
+ idr_destroy(&p->event_idr);
}
/*
* We assume that the process is being destroyed and there is no need to
* unmap the pages or keep bookkeeping data in order.
*/
-static void shutdown_signal_pages(struct kfd_process *p)
+static void shutdown_signal_page(struct kfd_process *p)
{
- struct signal_page *page, *tmp;
+ struct kfd_signal_page *page = p->signal_page;
- list_for_each_entry_safe(page, tmp, &p->signal_event_pages,
- event_pages) {
+ if (page) {
free_pages((unsigned long)page->kernel_address,
get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
kfree(page);
@@ -390,7 +277,7 @@ static void shutdown_signal_pages(struct kfd_process *p)
void kfd_event_free_process(struct kfd_process *p)
{
destroy_events(p);
- shutdown_signal_pages(p);
+ shutdown_signal_page(p);
}
static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
@@ -419,7 +306,7 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
ev->auto_reset = auto_reset;
ev->signaled = false;
- INIT_LIST_HEAD(&ev->waiters);
+ init_waitqueue_head(&ev->wq);
*event_page_offset = 0;
@@ -430,10 +317,9 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
case KFD_EVENT_TYPE_DEBUG:
ret = create_signal_event(devkfd, p, ev);
if (!ret) {
- *event_page_offset = (ev->signal_page->page_index |
- KFD_MMAP_EVENTS_MASK);
+ *event_page_offset = KFD_MMAP_EVENTS_MASK;
*event_page_offset <<= PAGE_SHIFT;
- *event_slot_index = ev->signal_slot_index;
+ *event_slot_index = ev->event_id;
}
break;
default:
@@ -442,8 +328,6 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
}
if (!ret) {
- hash_add(p->events, &ev->events, ev->event_id);
-
*event_id = ev->event_id;
*event_trigger_data = ev->event_id;
} else {
@@ -477,19 +361,18 @@ int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
static void set_event(struct kfd_event *ev)
{
struct kfd_event_waiter *waiter;
- struct kfd_event_waiter *next;
- /* Auto reset if the list is non-empty and we're waking someone. */
- ev->signaled = !ev->auto_reset || list_empty(&ev->waiters);
+ /* Auto reset if the list is non-empty and we're waking
+ * someone. waitqueue_active is safe here because we're
+ * protected by the p->event_mutex, which is also held when
+ * updating the wait queues in kfd_wait_on_events.
+ */
+ ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);
- list_for_each_entry_safe(waiter, next, &ev->waiters, waiters) {
+ list_for_each_entry(waiter, &ev->wq.head, wait.entry)
waiter->activated = true;
- /* _init because free_waiters will call list_del */
- list_del_init(&waiter->waiters);
-
- wake_up_process(waiter->sleeping_task);
- }
+ wake_up_all(&ev->wq);
}
/* Assumes that p is current. */
@@ -538,13 +421,7 @@ int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
{
- page_slots(ev->signal_page)[ev->signal_slot_index] =
- UNSIGNALED_EVENT_SLOT;
-}
-
-static bool is_slot_signaled(struct signal_page *page, unsigned int index)
-{
- return page_slots(page)[index] != UNSIGNALED_EVENT_SLOT;
+ page_slots(p->signal_page)[ev->event_id] = UNSIGNALED_EVENT_SLOT;
}
static void set_event_from_interrupt(struct kfd_process *p,
@@ -559,7 +436,7 @@ static void set_event_from_interrupt(struct kfd_process *p,
void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
uint32_t valid_id_bits)
{
- struct kfd_event *ev;
+ struct kfd_event *ev = NULL;
/*
* Because we are called from arbitrary context (workqueue) as opposed
@@ -573,26 +450,46 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
mutex_lock(&p->event_mutex);
- if (valid_id_bits >= INTERRUPT_DATA_BITS) {
- /* Partial ID is a full ID. */
- ev = lookup_event_by_id(p, partial_id);
+ if (valid_id_bits)
+ ev = lookup_signaled_event_by_partial_id(p, partial_id,
+ valid_id_bits);
+ if (ev) {
set_event_from_interrupt(p, ev);
- } else {
+ } else if (p->signal_page) {
/*
- * Partial ID is in fact partial. For now we completely
- * ignore it, but we could use any bits we did receive to
- * search faster.
+ * Partial ID lookup failed. Assume that the event ID
+ * in the interrupt payload was invalid and do an
+ * exhaustive search of signaled events.
*/
- struct signal_page *page;
- unsigned int i;
-
- list_for_each_entry(page, &p->signal_event_pages, event_pages)
- for (i = 0; i < SLOTS_PER_PAGE; i++)
- if (is_slot_signaled(page, i)) {
- ev = lookup_event_by_page_slot(p,
- page, i);
+ uint64_t *slots = page_slots(p->signal_page);
+ uint32_t id;
+
+ if (valid_id_bits)
+ pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n",
+ partial_id, valid_id_bits);
+
+ if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT/2) {
+ /* With relatively few events, it's faster to
+ * iterate over the event IDR
+ */
+ idr_for_each_entry(&p->event_idr, ev, id) {
+ if (id >= KFD_SIGNAL_EVENT_LIMIT)
+ break;
+
+ if (slots[id] != UNSIGNALED_EVENT_SLOT)
+ set_event_from_interrupt(p, ev);
+ }
+ } else {
+ /* With relatively many events, it's faster to
+ * iterate over the signal slots and lookup
+ * only signaled events from the IDR.
+ */
+ for (id = 0; id < KFD_SIGNAL_EVENT_LIMIT; id++)
+ if (slots[id] != UNSIGNALED_EVENT_SLOT) {
+ ev = lookup_event_by_id(p, id);
set_event_from_interrupt(p, ev);
}
+ }
}
mutex_unlock(&p->event_mutex);
@@ -609,18 +506,16 @@ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
GFP_KERNEL);
for (i = 0; (event_waiters) && (i < num_events) ; i++) {
- INIT_LIST_HEAD(&event_waiters[i].waiters);
- event_waiters[i].sleeping_task = current;
+ init_wait(&event_waiters[i].wait);
event_waiters[i].activated = false;
}
return event_waiters;
}
-static int init_event_waiter(struct kfd_process *p,
+static int init_event_waiter_get_status(struct kfd_process *p,
struct kfd_event_waiter *waiter,
- uint32_t event_id,
- uint32_t input_index)
+ uint32_t event_id)
{
struct kfd_event *ev = lookup_event_by_id(p, event_id);
@@ -628,38 +523,60 @@ static int init_event_waiter(struct kfd_process *p,
return -EINVAL;
waiter->event = ev;
- waiter->input_index = input_index;
waiter->activated = ev->signaled;
ev->signaled = ev->signaled && !ev->auto_reset;
- list_add(&waiter->waiters, &ev->waiters);
-
return 0;
}
-static bool test_event_condition(bool all, uint32_t num_events,
+static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter)
+{
+ struct kfd_event *ev = waiter->event;
+
+ /* Only add to the wait list if we actually need to
+ * wait on this event.
+ */
+ if (!waiter->activated)
+ add_wait_queue(&ev->wq, &waiter->wait);
+}
+
+/* test_event_condition - Test condition of events being waited for
+ * @all: Return completion only if all events have signaled
+ * @num_events: Number of events to wait for
+ * @event_waiters: Array of event waiters, one per event
+ *
+ * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have
+ * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all)
+ * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of
+ * the events have been destroyed.
+ */
+static uint32_t test_event_condition(bool all, uint32_t num_events,
struct kfd_event_waiter *event_waiters)
{
uint32_t i;
uint32_t activated_count = 0;
for (i = 0; i < num_events; i++) {
+ if (!event_waiters[i].event)
+ return KFD_IOC_WAIT_RESULT_FAIL;
+
if (event_waiters[i].activated) {
if (!all)
- return true;
+ return KFD_IOC_WAIT_RESULT_COMPLETE;
activated_count++;
}
}
- return activated_count == num_events;
+ return activated_count == num_events ?
+ KFD_IOC_WAIT_RESULT_COMPLETE : KFD_IOC_WAIT_RESULT_TIMEOUT;
}
/*
* Copy event specific data, if defined.
* Currently only memory exception events have additional data to copy to user
*/
-static bool copy_signaled_event_data(uint32_t num_events,
+static int copy_signaled_event_data(uint32_t num_events,
struct kfd_event_waiter *event_waiters,
struct kfd_event_data __user *data)
{
@@ -673,15 +590,15 @@ static bool copy_signaled_event_data(uint32_t num_events,
waiter = &event_waiters[i];
event = waiter->event;
if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) {
- dst = &data[waiter->input_index].memory_exception_data;
+ dst = &data[i].memory_exception_data;
src = &event->memory_exception_data;
if (copy_to_user(dst, src,
sizeof(struct kfd_hsa_memory_exception_data)))
- return false;
+ return -EFAULT;
}
}
- return true;
+ return 0;
}
@@ -710,7 +627,9 @@ static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters)
uint32_t i;
for (i = 0; i < num_events; i++)
- list_del(&waiters[i].waiters);
+ if (waiters[i].event)
+ remove_wait_queue(&waiters[i].event->wq,
+ &waiters[i].wait);
kfree(waiters);
}
@@ -718,38 +637,56 @@ static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters)
int kfd_wait_on_events(struct kfd_process *p,
uint32_t num_events, void __user *data,
bool all, uint32_t user_timeout_ms,
- enum kfd_event_wait_result *wait_result)
+ uint32_t *wait_result)
{
struct kfd_event_data __user *events =
(struct kfd_event_data __user *) data;
uint32_t i;
int ret = 0;
+
struct kfd_event_waiter *event_waiters = NULL;
long timeout = user_timeout_to_jiffies(user_timeout_ms);
- mutex_lock(&p->event_mutex);
-
event_waiters = alloc_event_waiters(num_events);
if (!event_waiters) {
ret = -ENOMEM;
- goto fail;
+ goto out;
}
+ mutex_lock(&p->event_mutex);
+
for (i = 0; i < num_events; i++) {
struct kfd_event_data event_data;
if (copy_from_user(&event_data, &events[i],
sizeof(struct kfd_event_data))) {
ret = -EFAULT;
- goto fail;
+ goto out_unlock;
}
- ret = init_event_waiter(p, &event_waiters[i],
- event_data.event_id, i);
+ ret = init_event_waiter_get_status(p, &event_waiters[i],
+ event_data.event_id);
if (ret)
- goto fail;
+ goto out_unlock;
}
+ /* Check condition once. */
+ *wait_result = test_event_condition(all, num_events, event_waiters);
+ if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) {
+ ret = copy_signaled_event_data(num_events,
+ event_waiters, events);
+ goto out_unlock;
+ } else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) {
+ /* This should not happen. Events shouldn't be
+ * destroyed while we're holding the event_mutex
+ */
+ goto out_unlock;
+ }
+
+ /* Add to wait lists if we need to wait. */
+ for (i = 0; i < num_events; i++)
+ init_event_waiter_add_to_waitlist(&event_waiters[i]);
+
mutex_unlock(&p->event_mutex);
while (true) {
@@ -771,62 +708,66 @@ int kfd_wait_on_events(struct kfd_process *p,
break;
}
- if (test_event_condition(all, num_events, event_waiters)) {
- if (copy_signaled_event_data(num_events,
- event_waiters, events))
- *wait_result = KFD_WAIT_COMPLETE;
- else
- *wait_result = KFD_WAIT_ERROR;
+ /* Set task state to interruptible sleep before
+ * checking wake-up conditions. A concurrent wake-up
+ * will put the task back into runnable state. In that
+ * case schedule_timeout will not put the task to
+ * sleep and we'll get a chance to re-check the
+ * updated conditions almost immediately. Otherwise,
+ * this race condition would lead to a soft hang or a
+ * very long sleep.
+ */
+ set_current_state(TASK_INTERRUPTIBLE);
+
+ *wait_result = test_event_condition(all, num_events,
+ event_waiters);
+ if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT)
break;
- }
- if (timeout <= 0) {
- *wait_result = KFD_WAIT_TIMEOUT;
+ if (timeout <= 0)
break;
- }
- timeout = schedule_timeout_interruptible(timeout);
+ timeout = schedule_timeout(timeout);
}
__set_current_state(TASK_RUNNING);
+ /* copy_signaled_event_data may sleep. So this has to happen
+ * after the task state is set back to RUNNING.
+ */
+ if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE)
+ ret = copy_signaled_event_data(num_events,
+ event_waiters, events);
+
mutex_lock(&p->event_mutex);
+out_unlock:
free_waiters(num_events, event_waiters);
mutex_unlock(&p->event_mutex);
-
- return ret;
-
-fail:
- if (event_waiters)
- free_waiters(num_events, event_waiters);
-
- mutex_unlock(&p->event_mutex);
-
- *wait_result = KFD_WAIT_ERROR;
+out:
+ if (ret)
+ *wait_result = KFD_IOC_WAIT_RESULT_FAIL;
+ else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL)
+ ret = -EIO;
return ret;
}
int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
{
-
- unsigned int page_index;
unsigned long pfn;
- struct signal_page *page;
+ struct kfd_signal_page *page;
+ int ret;
- /* check required size is logical */
- if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) !=
+ /* check required size doesn't exceed the allocated size */
+ if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) <
get_order(vma->vm_end - vma->vm_start)) {
pr_err("Event page mmap requested illegal size\n");
return -EINVAL;
}
- page_index = vma->vm_pgoff;
-
- page = lookup_signal_page_by_index(p, page_index);
+ page = p->signal_page;
if (!page) {
/* Probably KFD bug, but mmap is user-accessible. */
- pr_debug("Signal page could not be found for page_index %u\n",
- page_index);
+ pr_debug("Signal page could not be found\n");
return -EINVAL;
}
@@ -847,8 +788,12 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
page->user_address = (uint64_t __user *)vma->vm_start;
/* mapping the page to user process */
- return remap_pfn_range(vma, vma->vm_start, pfn,
+ ret = remap_pfn_range(vma, vma->vm_start, pfn,
vma->vm_end - vma->vm_start, vma->vm_page_prot);
+ if (!ret)
+ p->signal_mapped_size = vma->vm_end - vma->vm_start;
+
+ return ret;
}
/*
@@ -860,12 +805,13 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
{
struct kfd_hsa_memory_exception_data *ev_data;
struct kfd_event *ev;
- int bkt;
+ uint32_t id;
bool send_signal = true;
ev_data = (struct kfd_hsa_memory_exception_data *) event_data;
- hash_for_each(p->events, bkt, ev, events)
+ id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+ idr_for_each_entry_continue(&p->event_idr, ev, id)
if (ev->type == type) {
send_signal = false;
dev_dbg(kfd_device,
@@ -904,14 +850,24 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
* running so the lookup function returns a locked process.
*/
struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+ struct mm_struct *mm;
if (!p)
return; /* Presumably process exited. */
+ /* Take a safe reference to the mm_struct, which may otherwise
+ * disappear even while the kfd_process is still referenced.
+ */
+ mm = get_task_mm(p->lead_thread);
+ if (!mm) {
+ mutex_unlock(&p->mutex);
+ return; /* Process is exiting */
+ }
+
memset(&memory_exception_data, 0, sizeof(memory_exception_data));
- down_read(&p->mm->mmap_sem);
- vma = find_vma(p->mm, address);
+ down_read(&mm->mmap_sem);
+ vma = find_vma(mm, address);
memory_exception_data.gpu_id = dev->id;
memory_exception_data.va = address;
@@ -937,7 +893,8 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
}
}
- up_read(&p->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
+ mmput(mm);
mutex_lock(&p->event_mutex);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index 28f6838b1f4c..abca5bfebbff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -27,12 +27,17 @@
#include <linux/hashtable.h>
#include <linux/types.h>
#include <linux/list.h>
+#include <linux/wait.h>
#include "kfd_priv.h"
#include <uapi/linux/kfd_ioctl.h>
-#define KFD_EVENT_ID_NONSIGNAL_MASK 0x80000000U
-#define KFD_FIRST_NONSIGNAL_EVENT_ID KFD_EVENT_ID_NONSIGNAL_MASK
-#define KFD_LAST_NONSIGNAL_EVENT_ID UINT_MAX
+/*
+ * IDR supports non-negative integer IDs. Small IDs are used for
+ * signal events to match their signal slot. Use the upper half of the
+ * ID space for non-signal events.
+ */
+#define KFD_FIRST_NONSIGNAL_EVENT_ID ((INT_MAX >> 1) + 1)
+#define KFD_LAST_NONSIGNAL_EVENT_ID INT_MAX
/*
* Written into kfd_signal_slot_t to indicate that the event is not signaled.
@@ -46,9 +51,6 @@ struct kfd_event_waiter;
struct signal_page;
struct kfd_event {
- /* All events in process, rooted at kfd_process.events. */
- struct hlist_node events;
-
u32 event_id;
bool signaled;
@@ -56,11 +58,9 @@ struct kfd_event {
int type;
- struct list_head waiters; /* List of kfd_event_waiter by waiters. */
+ wait_queue_head_t wq; /* List of event waiters. */
/* Only for signal events. */
- struct signal_page *signal_page;
- unsigned int signal_slot_index;
uint64_t __user *user_signal_address;
/* type specific data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index 70b3a99cffc2..035c351f47c5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -42,26 +42,26 @@
#include <linux/slab.h>
#include <linux/device.h>
+#include <linux/kfifo.h>
#include "kfd_priv.h"
-#define KFD_INTERRUPT_RING_SIZE 1024
+#define KFD_IH_NUM_ENTRIES 8192
static void interrupt_wq(struct work_struct *);
int kfd_interrupt_init(struct kfd_dev *kfd)
{
- void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE,
- kfd->device_info->ih_ring_entry_size,
- GFP_KERNEL);
- if (!interrupt_ring)
- return -ENOMEM;
-
- kfd->interrupt_ring = interrupt_ring;
- kfd->interrupt_ring_size =
- KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size;
- atomic_set(&kfd->interrupt_ring_wptr, 0);
- atomic_set(&kfd->interrupt_ring_rptr, 0);
+ int r;
+
+ r = kfifo_alloc(&kfd->ih_fifo,
+ KFD_IH_NUM_ENTRIES * kfd->device_info->ih_ring_entry_size,
+ GFP_KERNEL);
+ if (r) {
+ dev_err(kfd_chardev(), "Failed to allocate IH fifo\n");
+ return r;
+ }
+ kfd->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
spin_lock_init(&kfd->interrupt_lock);
INIT_WORK(&kfd->interrupt_work, interrupt_wq);
@@ -92,74 +92,47 @@ void kfd_interrupt_exit(struct kfd_dev *kfd)
spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
/*
- * Flush_scheduled_work ensures that there are no outstanding
+ * flush_work ensures that there are no outstanding
* work-queue items that will access interrupt_ring. New work items
* can't be created because we stopped interrupt handling above.
*/
- flush_scheduled_work();
+ flush_workqueue(kfd->ih_wq);
- kfree(kfd->interrupt_ring);
+ kfifo_free(&kfd->ih_fifo);
}
/*
- * This assumes that it can't be called concurrently with itself
- * but only with dequeue_ih_ring_entry.
+ * Assumption: single reader/writer. This function is not re-entrant
*/
bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
{
- unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
- unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
+ int count;
- if ((rptr - wptr) % kfd->interrupt_ring_size ==
- kfd->device_info->ih_ring_entry_size) {
- /* This is very bad, the system is likely to hang. */
+ count = kfifo_in(&kfd->ih_fifo, ih_ring_entry,
+ kfd->device_info->ih_ring_entry_size);
+ if (count != kfd->device_info->ih_ring_entry_size) {
dev_err_ratelimited(kfd_chardev(),
- "Interrupt ring overflow, dropping interrupt.\n");
+ "Interrupt ring overflow, dropping interrupt %d\n",
+ count);
return false;
}
- memcpy(kfd->interrupt_ring + wptr, ih_ring_entry,
- kfd->device_info->ih_ring_entry_size);
-
- wptr = (wptr + kfd->device_info->ih_ring_entry_size) %
- kfd->interrupt_ring_size;
- smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */
- atomic_set(&kfd->interrupt_ring_wptr, wptr);
-
return true;
}
/*
- * This assumes that it can't be called concurrently with itself
- * but only with enqueue_ih_ring_entry.
+ * Assumption: single reader/writer. This function is not re-entrant
*/
static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
{
- /*
- * Assume that wait queues have an implicit barrier, i.e. anything that
- * happened in the ISR before it queued work is visible.
- */
-
- unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
- unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
+ int count;
- if (rptr == wptr)
- return false;
-
- memcpy(ih_ring_entry, kfd->interrupt_ring + rptr,
- kfd->device_info->ih_ring_entry_size);
-
- rptr = (rptr + kfd->device_info->ih_ring_entry_size) %
- kfd->interrupt_ring_size;
+ count = kfifo_out(&kfd->ih_fifo, ih_ring_entry,
+ kfd->device_info->ih_ring_entry_size);
- /*
- * Ensure the rptr write update is not visible until
- * memcpy has finished reading.
- */
- smp_mb();
- atomic_set(&kfd->interrupt_ring_rptr, rptr);
+ WARN_ON(count && count != kfd->device_info->ih_ring_entry_size);
- return true;
+ return count == kfd->device_info->ih_ring_entry_size;
}
static void interrupt_wq(struct work_struct *work)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index ed71ad40e8f7..8b0c0645d7c0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -185,7 +185,7 @@ static void uninitialize(struct kernel_queue *kq)
kq->mqd->destroy_mqd(kq->mqd,
kq->queue->mqd,
KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
- QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
+ KFD_UNMAP_LATENCY_MS,
kq->queue->pipe,
kq->queue->queue);
else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
@@ -303,14 +303,20 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
case CHIP_KAVERI:
kernel_queue_init_cik(&kq->ops_asic_specific);
break;
+ default:
+ WARN(1, "Unexpected ASIC family %u",
+ dev->device_info->asic_family);
+ goto out_free;
}
- if (!kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE)) {
- pr_err("Failed to init kernel queue\n");
- kfree(kq);
- return NULL;
- }
- return kq;
+ if (kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE))
+ return kq;
+
+ pr_err("Failed to init kernel queue\n");
+
+out_free:
+ kfree(kq);
+ return NULL;
}
void kernel_queue_uninit(struct kernel_queue *kq)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index b1ef1368c3bb..dfd260ef81ff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -31,6 +31,9 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
return mqd_manager_init_cik(type, dev);
case CHIP_CARRIZO:
return mqd_manager_init_vi(type, dev);
+ default:
+ WARN(1, "Unexpected ASIC family %u",
+ dev->device_info->asic_family);
}
return NULL;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 44ffd23348fc..4859d263fa2a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -189,12 +189,9 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
- q->is_active = false;
- if (q->queue_size > 0 &&
+ q->is_active = (q->queue_size > 0 &&
q->queue_address != 0 &&
- q->queue_percent > 0) {
- q->is_active = true;
- }
+ q->queue_percent > 0);
return 0;
}
@@ -215,24 +212,17 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
- m->sdma_rlc_doorbell = q->doorbell_off <<
- SDMA0_RLC0_DOORBELL__OFFSET__SHIFT |
- 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT;
+ m->sdma_rlc_doorbell =
+ q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
m->sdma_engine_id = q->sdma_engine_id;
m->sdma_queue_id = q->sdma_queue_id;
- q->is_active = false;
- if (q->queue_size > 0 &&
+ q->is_active = (q->queue_size > 0 &&
q->queue_address != 0 &&
- q->queue_percent > 0) {
- m->sdma_rlc_rb_cntl |=
- 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT;
-
- q->is_active = true;
- }
+ q->queue_percent > 0);
return 0;
}
@@ -359,19 +349,13 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
- m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
- DOORBELL_OFFSET(q->doorbell_off);
+ m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
m->cp_hqd_vmid = q->vmid;
- m->cp_hqd_active = 0;
- q->is_active = false;
- if (q->queue_size > 0 &&
+ q->is_active = (q->queue_size > 0 &&
q->queue_address != 0 &&
- q->queue_percent > 0) {
- m->cp_hqd_active = 1;
- q->is_active = true;
- }
+ q->queue_percent > 0);
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 73cbfe186dd2..4ea854f9007b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -163,12 +163,9 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
}
- q->is_active = false;
- if (q->queue_size > 0 &&
+ q->is_active = (q->queue_size > 0 &&
q->queue_address != 0 &&
- q->queue_percent > 0) {
- q->is_active = true;
- }
+ q->queue_percent > 0);
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 1d312603de9f..16da8ad02d8b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -140,8 +140,6 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
struct qcm_process_device *qpd)
{
struct pm4_mes_map_process *packet;
- struct queue *cur;
- uint32_t num_queues;
packet = (struct pm4_mes_map_process *)buffer;
@@ -156,10 +154,7 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
packet->bitfields10.gds_size = qpd->gds_size;
packet->bitfields10.num_gws = qpd->num_gws;
packet->bitfields10.num_oac = qpd->num_oac;
- num_queues = 0;
- list_for_each_entry(cur, &qpd->queues_list, list)
- num_queues++;
- packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : num_queues;
+ packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
packet->sh_mem_config = qpd->sh_mem_config;
packet->sh_mem_bases = qpd->sh_mem_bases;
@@ -208,7 +203,7 @@ static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
queue_type__mes_map_queues__debug_interface_queue_vi;
break;
case KFD_QUEUE_TYPE_SDMA:
- packet->bitfields2.engine_sel =
+ packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
engine_sel__mes_map_queues__sdma0_vi;
use_static = false; /* no static queues under SDMA */
break;
@@ -376,7 +371,7 @@ int pm_send_set_resources(struct packet_manager *pm,
packet->bitfields2.queue_type =
queue_type__mes_set_resources__hsa_interface_queue_hiq;
packet->bitfields2.vmid_mask = res->vmid_mask;
- packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY;
+ packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100;
packet->bitfields7.oac_mask = res->oac_mask;
packet->bitfields8.gds_heap_base = res->gds_heap_base;
packet->bitfields8.gds_heap_size = res->gds_heap_size;
@@ -476,7 +471,7 @@ fail_acquire_packet_buffer:
}
int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
- enum kfd_preempt_type_filter mode,
+ enum kfd_unmap_queues_filter filter,
uint32_t filter_param, bool reset,
unsigned int sdma_engine)
{
@@ -494,8 +489,8 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
packet = (struct pm4_mes_unmap_queues *)buffer;
memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
- pr_debug("static_queue: unmapping queues: mode is %d , reset is %d , type is %d\n",
- mode, reset, type);
+ pr_debug("static_queue: unmapping queues: filter is %d , reset is %d , type is %d\n",
+ filter, reset, type);
packet->header.u32All = build_pm4_header(IT_UNMAP_QUEUES,
sizeof(struct pm4_mes_unmap_queues));
switch (type) {
@@ -521,29 +516,29 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
packet->bitfields2.action =
action__mes_unmap_queues__preempt_queues;
- switch (mode) {
- case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE:
+ switch (filter) {
+ case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
packet->bitfields2.queue_sel =
queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
packet->bitfields2.num_queues = 1;
packet->bitfields3b.doorbell_offset0 = filter_param;
break;
- case KFD_PREEMPT_TYPE_FILTER_BY_PASID:
+ case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
packet->bitfields2.queue_sel =
queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
packet->bitfields3a.pasid = filter_param;
break;
- case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
+ case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
packet->bitfields2.queue_sel =
queue_sel__mes_unmap_queues__unmap_all_queues;
break;
- case KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES:
+ case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
/* in this case, we do not preempt static queues */
packet->bitfields2.queue_sel =
queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
break;
default:
- WARN(1, "filter %d", mode);
+ WARN(1, "filter %d", filter);
retval = -EINVAL;
goto err_invalid;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 634083e340d1..9e4134c5b481 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -31,8 +31,12 @@
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#include <linux/kfd_ioctl.h>
+#include <linux/idr.h>
+#include <linux/kfifo.h>
#include <kgd_kfd_interface.h>
+#include "amd_shared.h"
+
#define KFD_SYSFS_FILE_MODE 0444
#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
@@ -112,11 +116,6 @@ enum cache_policy {
cache_policy_noncoherent
};
-enum asic_family_type {
- CHIP_KAVERI = 0,
- CHIP_CARRIZO
-};
-
struct kfd_event_interrupt_class {
bool (*interrupt_isr)(struct kfd_dev *dev,
const uint32_t *ih_ring_entry);
@@ -125,7 +124,7 @@ struct kfd_event_interrupt_class {
};
struct kfd_device_info {
- unsigned int asic_family;
+ enum amd_asic_type asic_family;
const struct kfd_event_interrupt_class *event_interrupt_class;
unsigned int max_pasid_bits;
unsigned int max_no_of_hqd;
@@ -141,6 +140,12 @@ struct kfd_mem_obj {
uint32_t *cpu_ptr;
};
+struct kfd_vmid_info {
+ uint32_t first_vmid_kfd;
+ uint32_t last_vmid_kfd;
+ uint32_t vmid_num_kfd;
+};
+
struct kfd_dev {
struct kgd_dev *kgd;
@@ -162,6 +167,7 @@ struct kfd_dev {
*/
struct kgd2kfd_shared_resources shared_resources;
+ struct kfd_vmid_info vm_info;
const struct kfd2kgd_calls *kfd2kgd;
struct mutex doorbell_mutex;
@@ -177,10 +183,8 @@ struct kfd_dev {
unsigned int gtt_sa_num_of_chunks;
/* Interrupts */
- void *interrupt_ring;
- size_t interrupt_ring_size;
- atomic_t interrupt_ring_rptr;
- atomic_t interrupt_ring_wptr;
+ struct kfifo ih_fifo;
+ struct workqueue_struct *ih_wq;
struct work_struct interrupt_work;
spinlock_t interrupt_lock;
@@ -218,22 +222,22 @@ void kfd_chardev_exit(void);
struct device *kfd_chardev(void);
/**
- * enum kfd_preempt_type_filter
+ * enum kfd_unmap_queues_filter
*
- * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue.
+ * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
*
- * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the
+ * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
* running queues list.
*
- * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to
+ * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
* specific process.
*
*/
-enum kfd_preempt_type_filter {
- KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
- KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
- KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES,
- KFD_PREEMPT_TYPE_FILTER_BY_PASID
+enum kfd_unmap_queues_filter {
+ KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
+ KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
+ KFD_UNMAP_QUEUES_FILTER_BY_PASID
};
/**
@@ -401,7 +405,6 @@ struct scheduling_resources {
struct process_queue_manager {
/* data */
struct kfd_process *process;
- unsigned int num_concurrent_processes;
struct list_head queues;
unsigned long *queue_slot_bitmap;
};
@@ -417,6 +420,12 @@ struct qcm_process_device {
unsigned int queue_count;
unsigned int vmid;
bool is_debug;
+
+ /* This flag tells if we should reset all wavefronts on
+ * process termination
+ */
+ bool reset_wavefronts;
+
/*
* All the memory management data should be here too
*/
@@ -432,6 +441,13 @@ struct qcm_process_device {
uint32_t sh_hidden_private_base;
};
+
+enum kfd_pdd_bound {
+ PDD_UNBOUND = 0,
+ PDD_BOUND,
+ PDD_BOUND_SUSPENDED,
+};
+
/* Data that is per-process-per device. */
struct kfd_process_device {
/*
@@ -443,6 +459,8 @@ struct kfd_process_device {
/* The device that owns this data. */
struct kfd_dev *dev;
+ /* The process that owns this kfd_process_device. */
+ struct kfd_process *process;
/* per-process-per device QCM data structure */
struct qcm_process_device qpd;
@@ -456,12 +474,14 @@ struct kfd_process_device {
uint64_t scratch_limit;
/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
- bool bound;
+ enum kfd_pdd_bound bound;
- /* This flag tells if we should reset all
- * wavefronts on process termination
+ /* Flag used to tell the pdd has dequeued from the dqm.
+ * This is used to prevent dev->dqm->ops.process_termination() from
+ * being called twice when it is already called in IOMMU callback
+ * function.
*/
- bool reset_wavefronts;
+ bool already_dequeued;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
@@ -474,7 +494,12 @@ struct kfd_process {
*/
struct hlist_node kfd_processes;
- struct mm_struct *mm;
+ /*
+ * Opaque pointer to mm_struct. We don't hold a reference to
+ * it so it should never be dereferenced from here. This is
+ * only used for looking up processes by their mm.
+ */
+ void *mm;
struct mutex mutex;
@@ -482,6 +507,8 @@ struct kfd_process {
* In any process, the thread that started main() is the lead
* thread and outlives the rest.
* It is here because amd_iommu_bind_pasid wants a task_struct.
+ * It can also be used for safely getting a reference to the
+ * mm_struct of the process.
*/
struct task_struct *lead_thread;
@@ -502,22 +529,16 @@ struct kfd_process {
struct process_queue_manager pqm;
- /* The process's queues. */
- size_t queue_array_size;
-
- /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
- struct kfd_queue **queues;
-
/*Is the user space process 32 bit?*/
bool is_32bit_user_mode;
/* Event-related data */
struct mutex event_mutex;
- /* All events in process hashed by ID, linked on kfd_event.events. */
- DECLARE_HASHTABLE(events, 4);
- /* struct slot_page_header.event_pages */
- struct list_head signal_event_pages;
- u32 next_nonsignal_event_id;
+ /* Event ID allocator and lookup */
+ struct idr event_idr;
+ /* Event page */
+ struct kfd_signal_page *signal_page;
+ size_t signal_mapped_size;
size_t signal_event_count;
bool signal_event_limit_reached;
};
@@ -547,8 +568,10 @@ struct kfd_process *kfd_get_process(const struct task_struct *);
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
- struct kfd_process *p);
-void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
+ struct kfd_process *p);
+int kfd_bind_processes_to_device(struct kfd_dev *dev);
+void kfd_unbind_processes_from_device(struct kfd_dev *dev);
+void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid);
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
struct kfd_process *p);
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
@@ -646,14 +669,14 @@ struct process_queue_node {
struct list_head process_queue_list;
};
+void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
+void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
struct kfd_dev *dev,
struct file *f,
struct queue_properties *properties,
- unsigned int flags,
- enum kfd_queue_type type,
unsigned int *qid);
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
@@ -663,15 +686,12 @@ struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
unsigned int fence_value,
- unsigned long timeout);
+ unsigned int timeout_ms);
/* Packet Manager */
-#define KFD_HIQ_TIMEOUT (500)
-
#define KFD_FENCE_COMPLETED (100)
#define KFD_FENCE_INIT (10)
-#define KFD_UNMAP_LATENCY (150)
struct packet_manager {
struct device_queue_manager *dqm;
@@ -690,7 +710,7 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
uint32_t fence_value);
int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
- enum kfd_preempt_type_filter mode,
+ enum kfd_unmap_queues_filter mode,
uint32_t filter_param, bool reset,
unsigned int sdma_engine);
@@ -702,19 +722,13 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
extern const struct kfd_device_global_init_class device_global_init_class_cik;
-enum kfd_event_wait_result {
- KFD_WAIT_COMPLETE,
- KFD_WAIT_TIMEOUT,
- KFD_WAIT_ERROR
-};
-
void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
uint32_t num_events, void __user *data,
bool all, uint32_t user_timeout_ms,
- enum kfd_event_wait_result *wait_result);
+ uint32_t *wait_result);
void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
uint32_t valid_id_bits);
void kfd_signal_iommu_event(struct kfd_dev *dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 9e65ce3c1967..1f5ccd28bd41 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -35,13 +35,6 @@ struct mm_struct;
#include "kfd_dbgmgr.h"
/*
- * Initial size for the array of queues.
- * The allocated size is doubled each time
- * it is exceeded up to MAX_PROCESS_QUEUES.
- */
-#define INITIAL_QUEUE_ARRAY_SIZE 16
-
-/*
* List of struct kfd_process (field kfd_process).
* Unique/indexed by mm_struct*
*/
@@ -171,12 +164,10 @@ static void kfd_process_wq_release(struct work_struct *work)
pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
pdd->dev->id, p->pasid);
- if (pdd->reset_wavefronts)
- dbgdev_wave_reset_wavefronts(pdd->dev, p);
+ if (pdd->bound == PDD_BOUND)
+ amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
- amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
list_del(&pdd->per_device_list);
-
kfree(pdd);
}
@@ -189,8 +180,6 @@ static void kfd_process_wq_release(struct work_struct *work)
mutex_destroy(&p->mutex);
- kfree(p->queues);
-
kfree(p);
kfree(work);
@@ -202,7 +191,6 @@ static void kfd_process_destroy_delayed(struct rcu_head *rcu)
struct kfd_process *p;
p = container_of(rcu, struct kfd_process, rcu);
- WARN_ON(atomic_read(&p->mm->mm_count) <= 0);
mmdrop(p->mm);
@@ -236,24 +224,26 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
mutex_lock(&p->mutex);
- /* In case our notifier is called before IOMMU notifier */
- pqm_uninit(&p->pqm);
-
- /* Iterate over all process device data structure and check
- * if we should delete debug managers and reset all wavefronts
+ /* Iterate over all process device data structures and if the
+ * pdd is in debug mode, we should first force unregistration,
+ * then we will be able to destroy the queues
*/
list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
- if ((pdd->dev->dbgmgr) &&
- (pdd->dev->dbgmgr->pasid == p->pasid))
- kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
-
- if (pdd->reset_wavefronts) {
- pr_warn("Resetting all wave fronts\n");
- dbgdev_wave_reset_wavefronts(pdd->dev, p);
- pdd->reset_wavefronts = false;
+ struct kfd_dev *dev = pdd->dev;
+
+ mutex_lock(kfd_get_dbgmgr_mutex());
+ if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
+ if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
+ kfd_dbgmgr_destroy(dev->dbgmgr);
+ dev->dbgmgr = NULL;
+ }
}
+ mutex_unlock(kfd_get_dbgmgr_mutex());
}
+ kfd_process_dequeue_from_all_devices(p);
+ pqm_uninit(&p->pqm);
+
mutex_unlock(&p->mutex);
/*
@@ -280,11 +270,6 @@ static struct kfd_process *create_process(const struct task_struct *thread)
if (!process)
goto err_alloc_process;
- process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
- sizeof(process->queues[0]), GFP_KERNEL);
- if (!process->queues)
- goto err_alloc_queues;
-
process->pasid = kfd_pasid_alloc();
if (process->pasid == 0)
goto err_alloc_pasid;
@@ -307,8 +292,6 @@ static struct kfd_process *create_process(const struct task_struct *thread)
process->lead_thread = thread->group_leader;
- process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
-
INIT_LIST_HEAD(&process->per_device_data);
kfd_event_init_process(process);
@@ -337,8 +320,6 @@ err_mmu_notifier:
err_alloc_doorbells:
kfd_pasid_free(process->pasid);
err_alloc_pasid:
- kfree(process->queues);
-err_alloc_queues:
kfree(process);
err_alloc_process:
return ERR_PTR(err);
@@ -351,9 +332,9 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
list_for_each_entry(pdd, &p->per_device_data, per_device_list)
if (pdd->dev == dev)
- break;
+ return pdd;
- return pdd;
+ return NULL;
}
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
@@ -367,7 +348,9 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
INIT_LIST_HEAD(&pdd->qpd.queues_list);
INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
pdd->qpd.dqm = dev->dqm;
- pdd->reset_wavefronts = false;
+ pdd->process = p;
+ pdd->bound = PDD_UNBOUND;
+ pdd->already_dequeued = false;
list_add(&pdd->per_device_list, &p->per_device_data);
}
@@ -393,19 +376,87 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
return ERR_PTR(-ENOMEM);
}
- if (pdd->bound)
+ if (pdd->bound == PDD_BOUND) {
return pdd;
+ } else if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
+ pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
+ return ERR_PTR(-EINVAL);
+ }
err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
if (err < 0)
return ERR_PTR(err);
- pdd->bound = true;
+ pdd->bound = PDD_BOUND;
return pdd;
}
-void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
+/*
+ * Bind processes do the device that have been temporarily unbound
+ * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
+ */
+int kfd_bind_processes_to_device(struct kfd_dev *dev)
+{
+ struct kfd_process_device *pdd;
+ struct kfd_process *p;
+ unsigned int temp;
+ int err = 0;
+
+ int idx = srcu_read_lock(&kfd_processes_srcu);
+
+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+ mutex_lock(&p->mutex);
+ pdd = kfd_get_process_device_data(dev, p);
+ if (pdd->bound != PDD_BOUND_SUSPENDED) {
+ mutex_unlock(&p->mutex);
+ continue;
+ }
+
+ err = amd_iommu_bind_pasid(dev->pdev, p->pasid,
+ p->lead_thread);
+ if (err < 0) {
+ pr_err("Unexpected pasid %d binding failure\n",
+ p->pasid);
+ mutex_unlock(&p->mutex);
+ break;
+ }
+
+ pdd->bound = PDD_BOUND;
+ mutex_unlock(&p->mutex);
+ }
+
+ srcu_read_unlock(&kfd_processes_srcu, idx);
+
+ return err;
+}
+
+/*
+ * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
+ * processes will be restored to PDD_BOUND state in
+ * kfd_bind_processes_to_device.
+ */
+void kfd_unbind_processes_from_device(struct kfd_dev *dev)
+{
+ struct kfd_process_device *pdd;
+ struct kfd_process *p;
+ unsigned int temp;
+
+ int idx = srcu_read_lock(&kfd_processes_srcu);
+
+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+ mutex_lock(&p->mutex);
+ pdd = kfd_get_process_device_data(dev, p);
+
+ if (pdd->bound == PDD_BOUND)
+ pdd->bound = PDD_BOUND_SUSPENDED;
+ mutex_unlock(&p->mutex);
+ }
+
+ srcu_read_unlock(&kfd_processes_srcu, idx);
+}
+
+void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
{
struct kfd_process *p;
struct kfd_process_device *pdd;
@@ -421,31 +472,23 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
pr_debug("Unbinding process %d from IOMMU\n", pasid);
- if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
- kfd_dbgmgr_destroy(dev->dbgmgr);
-
- pqm_uninit(&p->pqm);
-
- pdd = kfd_get_process_device_data(dev, p);
+ mutex_lock(kfd_get_dbgmgr_mutex());
- if (!pdd) {
- mutex_unlock(&p->mutex);
- return;
+ if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
+ if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
+ kfd_dbgmgr_destroy(dev->dbgmgr);
+ dev->dbgmgr = NULL;
+ }
}
- if (pdd->reset_wavefronts) {
- dbgdev_wave_reset_wavefronts(pdd->dev, p);
- pdd->reset_wavefronts = false;
- }
+ mutex_unlock(kfd_get_dbgmgr_mutex());
- /*
- * Just mark pdd as unbound, because we still need it
- * to call amd_iommu_unbind_pasid() in when the
- * process exits.
- * We don't call amd_iommu_unbind_pasid() here
- * because the IOMMU called us.
- */
- pdd->bound = false;
+ pdd = kfd_get_process_device_data(dev, p);
+ if (pdd)
+ /* For GPU relying on IOMMU, we need to dequeue here
+ * when PASID is still bound.
+ */
+ kfd_process_dequeue_from_device(pdd);
mutex_unlock(&p->mutex);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 03bec765b03d..2bec902fc939 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -63,6 +63,25 @@ static int find_available_queue_slot(struct process_queue_manager *pqm,
return 0;
}
+void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
+{
+ struct kfd_dev *dev = pdd->dev;
+
+ if (pdd->already_dequeued)
+ return;
+
+ dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
+ pdd->already_dequeued = true;
+}
+
+void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
+{
+ struct kfd_process_device *pdd;
+
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list)
+ kfd_process_dequeue_from_device(pdd);
+}
+
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
{
INIT_LIST_HEAD(&pqm->queues);
@@ -78,21 +97,14 @@ int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
void pqm_uninit(struct process_queue_manager *pqm)
{
- int retval;
struct process_queue_node *pqn, *next;
list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
- retval = pqm_destroy_queue(
- pqm,
- (pqn->q != NULL) ?
- pqn->q->properties.queue_id :
- pqn->kq->queue->properties.queue_id);
-
- if (retval != 0) {
- pr_err("failed to destroy queue\n");
- return;
- }
+ uninit_queue(pqn->q);
+ list_del(&pqn->process_queue_list);
+ kfree(pqn);
}
+
kfree(pqm->queue_slot_bitmap);
pqm->queue_slot_bitmap = NULL;
}
@@ -130,20 +142,16 @@ int pqm_create_queue(struct process_queue_manager *pqm,
struct kfd_dev *dev,
struct file *f,
struct queue_properties *properties,
- unsigned int flags,
- enum kfd_queue_type type,
unsigned int *qid)
{
int retval;
struct kfd_process_device *pdd;
- struct queue_properties q_properties;
struct queue *q;
struct process_queue_node *pqn;
struct kernel_queue *kq;
- int num_queues = 0;
- struct queue *cur;
+ enum kfd_queue_type type = properties->type;
+ unsigned int max_queues = 127; /* HWS limit */
- memcpy(&q_properties, properties, sizeof(struct queue_properties));
q = NULL;
kq = NULL;
@@ -159,19 +167,18 @@ int pqm_create_queue(struct process_queue_manager *pqm,
* If we are just about to create DIQ, the is_debug flag is not set yet
* Hence we also check the type as well
*/
- if ((pdd->qpd.is_debug) ||
- (type == KFD_QUEUE_TYPE_DIQ)) {
- list_for_each_entry(cur, &pdd->qpd.queues_list, list)
- num_queues++;
- if (num_queues >= dev->device_info->max_no_of_hqd/2)
- return -ENOSPC;
- }
+ if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
+ max_queues = dev->device_info->max_no_of_hqd/2;
+
+ if (pdd->qpd.queue_count >= max_queues)
+ return -ENOSPC;
retval = find_available_queue_slot(pqm, qid);
if (retval != 0)
return retval;
- if (list_empty(&pqm->queues)) {
+ if (list_empty(&pdd->qpd.queues_list) &&
+ list_empty(&pdd->qpd.priv_queue_list)) {
pdd->qpd.pqm = pqm;
dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
}
@@ -187,14 +194,14 @@ int pqm_create_queue(struct process_queue_manager *pqm,
case KFD_QUEUE_TYPE_COMPUTE:
/* check if there is over subscription */
if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
- ((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
+ ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
retval = -EPERM;
goto err_create_queue;
}
- retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid);
+ retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
if (retval != 0)
goto err_create_queue;
pqn->q = q;
@@ -231,9 +238,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
list_add(&pqn->process_queue_list, &pqm->queues);
if (q) {
- *properties = q->properties;
pr_debug("PQM done creating queue\n");
- print_queue_properties(properties);
+ print_queue_properties(&q->properties);
}
return retval;
@@ -243,7 +249,8 @@ err_create_queue:
err_allocate_pqn:
/* check if queues list is empty unregister process from device */
clear_bit(*qid, pqm->queue_slot_bitmap);
- if (list_empty(&pqm->queues))
+ if (list_empty(&pdd->qpd.queues_list) &&
+ list_empty(&pdd->qpd.priv_queue_list))
dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
return retval;
}
@@ -290,9 +297,6 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
if (pqn->q) {
dqm = pqn->q->device->dqm;
retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
- if (retval != 0)
- return retval;
-
uninit_queue(pqn->q);
}
@@ -300,7 +304,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
kfree(pqn);
clear_bit(qid, pqm->queue_slot_bitmap);
- if (list_empty(&pqm->queues))
+ if (list_empty(&pdd->qpd.queues_list) &&
+ list_empty(&pdd->qpd.priv_queue_list))
dqm->ops.unregister_process(dqm, &pdd->qpd);
return retval;
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 20457bb5a906..b72f8a43d86b 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -23,36 +23,11 @@
#ifndef __AMD_SHARED_H__
#define __AMD_SHARED_H__
-#define AMD_MAX_USEC_TIMEOUT 200000 /* 200 ms */
+#include <drm/amd_asic_type.h>
struct seq_file;
-/*
- * Supported ASIC types
- */
-enum amd_asic_type {
- CHIP_TAHITI = 0,
- CHIP_PITCAIRN,
- CHIP_VERDE,
- CHIP_OLAND,
- CHIP_HAINAN,
- CHIP_BONAIRE,
- CHIP_KAVERI,
- CHIP_KABINI,
- CHIP_HAWAII,
- CHIP_MULLINS,
- CHIP_TOPAZ,
- CHIP_TONGA,
- CHIP_FIJI,
- CHIP_CARRIZO,
- CHIP_STONEY,
- CHIP_POLARIS10,
- CHIP_POLARIS11,
- CHIP_POLARIS12,
- CHIP_VEGA10,
- CHIP_RAVEN,
- CHIP_LAST,
-};
+#define AMD_MAX_USEC_TIMEOUT 200000 /* 200 ms */
/*
* Chip flags
@@ -257,43 +232,54 @@ struct amd_ip_funcs {
void (*get_clockgating_state)(void *handle, u32 *flags);
};
-enum amd_pp_task;
+enum amd_pp_task;
+enum amd_pp_clock_type;
struct pp_states_info;
+struct amd_pp_simple_clock_info;
+struct amd_pp_display_configuration;
+struct amd_pp_clock_info;
+struct pp_display_clock_request;
+struct pp_wm_sets_with_clock_ranges_soc15;
+struct pp_clock_levels_with_voltage;
+struct pp_clock_levels_with_latency;
+struct amd_pp_clocks;
struct amd_pm_funcs {
- int (*get_temperature)(void *handle);
+/* export for dpm on ci and si */
int (*pre_set_power_state)(void *handle);
int (*set_power_state)(void *handle);
void (*post_set_power_state)(void *handle);
void (*display_configuration_changed)(void *handle);
- u32 (*get_sclk)(void *handle, bool low);
- u32 (*get_mclk)(void *handle, bool low);
void (*print_power_state)(void *handle, void *ps);
- void (*debugfs_print_current_performance_level)(void *handle, struct seq_file *m);
- int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
bool (*vblank_too_short)(void *handle);
- void (*powergate_uvd)(void *handle, bool gate);
- void (*powergate_vce)(void *handle, bool gate);
void (*enable_bapm)(void *handle, bool enable);
+ int (*check_state_equal)(void *handle,
+ void *cps,
+ void *rps,
+ bool *equal);
+/* export for sysfs */
+ int (*get_temperature)(void *handle);
void (*set_fan_control_mode)(void *handle, u32 mode);
u32 (*get_fan_control_mode)(void *handle);
int (*set_fan_speed_percent)(void *handle, u32 speed);
int (*get_fan_speed_percent)(void *handle, u32 *speed);
int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask);
int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf);
+ int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
int (*get_sclk_od)(void *handle);
int (*set_sclk_od)(void *handle, uint32_t value);
int (*get_mclk_od)(void *handle);
int (*set_mclk_od)(void *handle, uint32_t value);
- int (*check_state_equal)(void *handle,
- void *cps,
- void *rps,
- bool *equal);
- int (*read_sensor)(void *handle, int idx, void *value,
- int *size);
+ int (*read_sensor)(void *handle, int idx, void *value, int *size);
+ enum amd_dpm_forced_level (*get_performance_level)(void *handle);
+ enum amd_pm_state_type (*get_current_power_state)(void *handle);
+ int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
+ int (*get_pp_num_states)(void *handle, struct pp_states_info *data);
+ int (*get_pp_table)(void *handle, char **table);
+ int (*set_pp_table)(void *handle, const char *buf, size_t size);
+ void (*debugfs_print_current_performance_level)(void *handle, struct seq_file *m);
- struct amd_vce_state* (*get_vce_clock_state)(void *handle, u32 idx);
int (*reset_power_profile_state)(void *handle,
struct amd_pp_profile *request);
int (*get_power_profile_state)(void *handle,
@@ -302,16 +288,39 @@ struct amd_pm_funcs {
struct amd_pp_profile *request);
int (*switch_power_profile)(void *handle,
enum amd_pp_profile_type type);
- int (*load_firmware)(void *handle);
- int (*wait_for_fw_loading_complete)(void *handle);
- enum amd_dpm_forced_level (*get_performance_level)(void *handle);
- enum amd_pm_state_type (*get_current_power_state)(void *handle);
+/* export to amdgpu */
+ void (*powergate_uvd)(void *handle, bool gate);
+ void (*powergate_vce)(void *handle, bool gate);
+ struct amd_vce_state* (*get_vce_clock_state)(void *handle, u32 idx);
int (*dispatch_tasks)(void *handle, enum amd_pp_task task_id,
void *input, void *output);
- int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
- int (*get_pp_num_states)(void *handle, struct pp_states_info *data);
- int (*get_pp_table)(void *handle, char **table);
- int (*set_pp_table)(void *handle, const char *buf, size_t size);
+ int (*load_firmware)(void *handle);
+ int (*wait_for_fw_loading_complete)(void *handle);
+ int (*set_clockgating_by_smu)(void *handle, uint32_t msg_id);
+/* export to DC */
+ u32 (*get_sclk)(void *handle, bool low);
+ u32 (*get_mclk)(void *handle, bool low);
+ int (*display_configuration_change)(void *handle,
+ const struct amd_pp_display_configuration *input);
+ int (*get_display_power_level)(void *handle,
+ struct amd_pp_simple_clock_info *output);
+ int (*get_current_clocks)(void *handle,
+ struct amd_pp_clock_info *clocks);
+ int (*get_clock_by_type)(void *handle,
+ enum amd_pp_clock_type type,
+ struct amd_pp_clocks *clocks);
+ int (*get_clock_by_type_with_latency)(void *handle,
+ enum amd_pp_clock_type type,
+ struct pp_clock_levels_with_latency *clocks);
+ int (*get_clock_by_type_with_voltage)(void *handle,
+ enum amd_pp_clock_type type,
+ struct pp_clock_levels_with_voltage *clocks);
+ int (*set_watermarks_for_clocks_ranges)(void *handle,
+ struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+ int (*display_clock_voltage_request)(void *handle,
+ struct pp_display_clock_request *clock);
+ int (*get_display_mode_validation_clocks)(void *handle,
+ struct amd_pp_simple_clock_info *clocks);
};
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
index 378f4b6b43da..344237256d02 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
@@ -36,6 +36,16 @@
#define mmUVD_UDEC_DBW_ADDR_CONFIG 0x3bd5
#define mmUVD_POWER_STATUS_U 0x3bfd
#define mmUVD_NO_OP 0x3bff
+#define mmUVD_RB_BASE_LO2 0x3c21
+#define mmUVD_RB_BASE_HI2 0x3c22
+#define mmUVD_RB_SIZE2 0x3c23
+#define mmUVD_RB_RPTR2 0x3c24
+#define mmUVD_RB_WPTR2 0x3c25
+#define mmUVD_RB_BASE_LO 0x3c26
+#define mmUVD_RB_BASE_HI 0x3c27
+#define mmUVD_RB_SIZE 0x3c28
+#define mmUVD_RB_RPTR 0x3c29
+#define mmUVD_RB_WPTR 0x3c2a
#define mmUVD_LMI_RBC_RB_64BIT_BAR_LOW 0x3c69
#define mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH 0x3c68
#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW 0x3c67
@@ -43,6 +53,11 @@
#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW 0x3c5f
#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH 0x3c5e
#define mmUVD_SEMA_CNTL 0x3d00
+#define mmUVD_RB_WPTR3 0x3d1c
+#define mmUVD_RB_RPTR3 0x3d1b
+#define mmUVD_RB_BASE_LO3 0x3d1d
+#define mmUVD_RB_BASE_HI3 0x3d1e
+#define mmUVD_RB_SIZE3 0x3d1f
#define mmUVD_LMI_EXT40_ADDR 0x3d26
#define mmUVD_CTX_INDEX 0x3d28
#define mmUVD_CTX_DATA 0x3d29
diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h
index 181a2c3c6362..f696bbb643ef 100644
--- a/drivers/gpu/drm/amd/include/atombios.h
+++ b/drivers/gpu/drm/amd/include/atombios.h
@@ -4292,6 +4292,7 @@ typedef struct _ATOM_DPCD_INFO
#define ATOM_VRAM_OPERATION_FLAGS_SHIFT 30
#define ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION 0x1
#define ATOM_VRAM_BLOCK_NEEDS_RESERVATION 0x0
+#define ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION 0x2
/***********************************************************************************/
// Structure used in VRAM_UsageByFirmwareTable
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index 030b14649c4e..675988d56392 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -423,6 +423,10 @@ typedef int (*cgs_enter_safe_mode)(struct cgs_device *cgs_device, bool en);
typedef void (*cgs_lock_grbm_idx)(struct cgs_device *cgs_device, bool lock);
+struct amd_pp_init;
+typedef void* (*cgs_register_pp_handle)(struct cgs_device *cgs_device,
+ int (*call_back_func)(struct amd_pp_init *, void **));
+
struct cgs_ops {
/* memory management calls (similar to KFD interface) */
cgs_alloc_gpu_mem_t alloc_gpu_mem;
@@ -459,6 +463,7 @@ struct cgs_ops {
cgs_is_virtualization_enabled_t is_virtualization_enabled;
cgs_enter_safe_mode enter_safe_mode;
cgs_lock_grbm_idx lock_grbm_idx;
+ cgs_register_pp_handle register_pp_handle;
};
struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -537,4 +542,7 @@ struct cgs_device
#define cgs_lock_grbm_idx(cgs_device, lock) \
CGS_CALL(lock_grbm_idx, cgs_device, lock)
+#define cgs_register_pp_handle(cgs_device, call_back_func) \
+ CGS_CALL(register_pp_handle, cgs_device, call_back_func)
+
#endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/lib/Kconfig b/drivers/gpu/drm/amd/lib/Kconfig
index 03ee7ad21ac3..776ef3434c10 100644
--- a/drivers/gpu/drm/amd/lib/Kconfig
+++ b/drivers/gpu/drm/amd/lib/Kconfig
@@ -4,7 +4,8 @@ menu "AMD Library routines"
# Closed hash table
#
config CHASH
- tristate "Closed hash table"
+ tristate
+ default DRM_AMDGPU
help
Statically sized closed hash table implementation with low
memory and CPU overhead.
diff --git a/drivers/gpu/drm/amd/lib/chash.c b/drivers/gpu/drm/amd/lib/chash.c
index e07e6f3030d6..b8e45f356a1c 100644
--- a/drivers/gpu/drm/amd/lib/chash.c
+++ b/drivers/gpu/drm/amd/lib/chash.c
@@ -223,8 +223,8 @@ static int chash_table_check(struct __chash_table *table)
static void chash_iter_relocate(struct chash_iter dst, struct chash_iter src)
{
BUG_ON(src.table == dst.table && src.slot == dst.slot);
- BUG_ON(src.table->key_size != src.table->key_size);
- BUG_ON(src.table->value_size != src.table->value_size);
+ BUG_ON(src.table->key_size != dst.table->key_size);
+ BUG_ON(src.table->value_size != dst.table->value_size);
if (dst.table->key_size == 4)
dst.table->keys32[dst.slot] = src.table->keys32[src.slot];
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index 9f3f3b8cf64f..c7e34128cbde 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -30,9 +30,14 @@
#include "pp_instance.h"
#include "power_state.h"
+#define PP_DPM_DISABLED 0xCCCC
+
+static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_task task_id,
+ void *input, void *output);
+
static inline int pp_check(struct pp_instance *handle)
{
- if (handle == NULL || handle->pp_valid != PP_VALID)
+ if (handle == NULL)
return -EINVAL;
if (handle->hwmgr == NULL || handle->hwmgr->smumgr_funcs == NULL)
@@ -47,19 +52,57 @@ static inline int pp_check(struct pp_instance *handle)
return 0;
}
+static int amd_powerplay_create(struct amd_pp_init *pp_init,
+ void **handle)
+{
+ struct pp_instance *instance;
+
+ if (pp_init == NULL || handle == NULL)
+ return -EINVAL;
+
+ instance = kzalloc(sizeof(struct pp_instance), GFP_KERNEL);
+ if (instance == NULL)
+ return -ENOMEM;
+
+ instance->chip_family = pp_init->chip_family;
+ instance->chip_id = pp_init->chip_id;
+ instance->pm_en = pp_init->pm_en;
+ instance->feature_mask = pp_init->feature_mask;
+ instance->device = pp_init->device;
+ mutex_init(&instance->pp_lock);
+ *handle = instance;
+ return 0;
+}
+
+static int amd_powerplay_destroy(void *handle)
+{
+ struct pp_instance *instance = (struct pp_instance *)handle;
+
+ kfree(instance->hwmgr->hardcode_pp_table);
+ instance->hwmgr->hardcode_pp_table = NULL;
+
+ kfree(instance->hwmgr);
+ instance->hwmgr = NULL;
+
+ kfree(instance);
+ instance = NULL;
+ return 0;
+}
+
static int pp_early_init(void *handle)
{
int ret;
- struct pp_instance *pp_handle = (struct pp_instance *)handle;
+ struct pp_instance *pp_handle = NULL;
+
+ pp_handle = cgs_register_pp_handle(handle, amd_powerplay_create);
+
+ if (!pp_handle)
+ return -EINVAL;
ret = hwmgr_early_init(pp_handle);
if (ret)
return -EINVAL;
- if ((pp_handle->pm_en == 0)
- || cgs_is_virtualization_enabled(pp_handle->device))
- return PP_DPM_DISABLED;
-
return 0;
}
@@ -71,7 +114,7 @@ static int pp_sw_init(void *handle)
ret = pp_check(pp_handle);
- if (ret == 0 || ret == PP_DPM_DISABLED) {
+ if (ret >= 0) {
hwmgr = pp_handle->hwmgr;
if (hwmgr->smumgr_funcs->smu_init == NULL)
@@ -91,7 +134,7 @@ static int pp_sw_fini(void *handle)
struct pp_instance *pp_handle = (struct pp_instance *)handle;
ret = pp_check(pp_handle);
- if (ret == 0 || ret == PP_DPM_DISABLED) {
+ if (ret >= 0) {
hwmgr = pp_handle->hwmgr;
if (hwmgr->smumgr_funcs->smu_fini == NULL)
@@ -110,7 +153,7 @@ static int pp_hw_init(void *handle)
ret = pp_check(pp_handle);
- if (ret == 0 || ret == PP_DPM_DISABLED) {
+ if (ret >= 0) {
hwmgr = pp_handle->hwmgr;
if (hwmgr->smumgr_funcs->start_smu == NULL)
@@ -122,16 +165,17 @@ static int pp_hw_init(void *handle)
return -EINVAL;;
}
if (ret == PP_DPM_DISABLED)
- return PP_DPM_DISABLED;
+ goto exit;
+ ret = hwmgr_hw_init(pp_handle);
+ if (ret)
+ goto exit;
}
-
- ret = hwmgr_hw_init(pp_handle);
- if (ret)
- goto err;
- return 0;
-err:
+ return ret;
+exit:
pp_handle->pm_en = 0;
- return PP_DPM_DISABLED;
+ cgs_notify_dpm_enabled(hwmgr->device, false);
+ return 0;
+
}
static int pp_hw_fini(void *handle)
@@ -146,6 +190,25 @@ static int pp_hw_fini(void *handle)
return 0;
}
+static int pp_late_init(void *handle)
+{
+ struct pp_instance *pp_handle = (struct pp_instance *)handle;
+ int ret = 0;
+
+ ret = pp_check(pp_handle);
+ if (ret == 0)
+ pp_dpm_dispatch_tasks(pp_handle,
+ AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
+
+ return 0;
+}
+
+static void pp_late_fini(void *handle)
+{
+ amd_powerplay_destroy(handle);
+}
+
+
static bool pp_is_idle(void *handle)
{
return false;
@@ -161,28 +224,6 @@ static int pp_sw_reset(void *handle)
return 0;
}
-
-int amd_set_clockgating_by_smu(void *handle, uint32_t msg_id)
-{
- struct pp_hwmgr *hwmgr;
- struct pp_instance *pp_handle = (struct pp_instance *)handle;
- int ret = 0;
-
- ret = pp_check(pp_handle);
-
- if (ret != 0)
- return ret;
-
- hwmgr = pp_handle->hwmgr;
-
- if (hwmgr->hwmgr_func->update_clock_gatings == NULL) {
- pr_info("%s was not implemented.\n", __func__);
- return 0;
- }
-
- return hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-}
-
static int pp_set_powergating_state(void *handle,
enum amd_powergating_state state)
{
@@ -192,7 +233,7 @@ static int pp_set_powergating_state(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -213,39 +254,34 @@ static int pp_suspend(void *handle)
int ret = 0;
ret = pp_check(pp_handle);
-
- if (ret == PP_DPM_DISABLED)
- return 0;
- else if (ret != 0)
- return ret;
-
- return hwmgr_hw_suspend(pp_handle);
+ if (ret == 0)
+ hwmgr_hw_suspend(pp_handle);
+ return 0;
}
static int pp_resume(void *handle)
{
struct pp_hwmgr *hwmgr;
- int ret, ret1;
+ int ret;
struct pp_instance *pp_handle = (struct pp_instance *)handle;
- ret1 = pp_check(pp_handle);
+ ret = pp_check(pp_handle);
- if (ret1 != 0 && ret1 != PP_DPM_DISABLED)
- return ret1;
+ if (ret < 0)
+ return ret;
hwmgr = pp_handle->hwmgr;
if (hwmgr->smumgr_funcs->start_smu == NULL)
return -EINVAL;
- ret = hwmgr->smumgr_funcs->start_smu(pp_handle->hwmgr);
- if (ret) {
+ if (hwmgr->smumgr_funcs->start_smu(pp_handle->hwmgr)) {
pr_err("smc start failed\n");
hwmgr->smumgr_funcs->smu_fini(pp_handle->hwmgr);
- return ret;
+ return -EINVAL;
}
- if (ret1 == PP_DPM_DISABLED)
+ if (ret == PP_DPM_DISABLED)
return 0;
return hwmgr_hw_resume(pp_handle);
@@ -254,11 +290,12 @@ static int pp_resume(void *handle)
const struct amd_ip_funcs pp_ip_funcs = {
.name = "powerplay",
.early_init = pp_early_init,
- .late_init = NULL,
+ .late_init = pp_late_init,
.sw_init = pp_sw_init,
.sw_fini = pp_sw_fini,
.hw_init = pp_hw_init,
.hw_fini = pp_hw_fini,
+ .late_fini = pp_late_fini,
.suspend = pp_suspend,
.resume = pp_resume,
.is_idle = pp_is_idle,
@@ -278,6 +315,27 @@ static int pp_dpm_fw_loading_complete(void *handle)
return 0;
}
+static int pp_set_clockgating_by_smu(void *handle, uint32_t msg_id)
+{
+ struct pp_hwmgr *hwmgr;
+ struct pp_instance *pp_handle = (struct pp_instance *)handle;
+ int ret = 0;
+
+ ret = pp_check(pp_handle);
+
+ if (ret)
+ return ret;
+
+ hwmgr = pp_handle->hwmgr;
+
+ if (hwmgr->hwmgr_func->update_clock_gatings == NULL) {
+ pr_info("%s was not implemented.\n", __func__);
+ return 0;
+ }
+
+ return hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
+}
+
static void pp_dpm_en_umd_pstate(struct pp_hwmgr *hwmgr,
enum amd_dpm_forced_level *level)
{
@@ -323,7 +381,7 @@ static int pp_dpm_force_performance_level(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -358,7 +416,7 @@ static enum amd_dpm_forced_level pp_dpm_get_performance_level(
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -377,7 +435,7 @@ static uint32_t pp_dpm_get_sclk(void *handle, bool low)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -401,7 +459,7 @@ static uint32_t pp_dpm_get_mclk(void *handle, bool low)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -424,7 +482,7 @@ static void pp_dpm_powergate_vce(void *handle, bool gate)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return;
hwmgr = pp_handle->hwmgr;
@@ -446,7 +504,7 @@ static void pp_dpm_powergate_uvd(void *handle, bool gate)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return;
hwmgr = pp_handle->hwmgr;
@@ -468,7 +526,7 @@ static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_task task_id,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
mutex_lock(&pp_handle->pp_lock);
@@ -488,7 +546,7 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -530,7 +588,7 @@ static void pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return;
hwmgr = pp_handle->hwmgr;
@@ -553,7 +611,7 @@ static uint32_t pp_dpm_get_fan_control_mode(void *handle)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -576,7 +634,7 @@ static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -599,7 +657,7 @@ static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -623,7 +681,7 @@ static int pp_dpm_get_fan_speed_rpm(void *handle, uint32_t *rpm)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -645,7 +703,7 @@ static int pp_dpm_get_temperature(void *handle)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -670,7 +728,7 @@ static int pp_dpm_get_pp_num_states(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -715,7 +773,7 @@ static int pp_dpm_get_pp_table(void *handle, char **table)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -738,7 +796,7 @@ static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -780,7 +838,7 @@ static int pp_dpm_force_clock_level(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -804,7 +862,7 @@ static int pp_dpm_print_clock_levels(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -827,7 +885,7 @@ static int pp_dpm_get_sclk_od(void *handle)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -850,7 +908,7 @@ static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -874,7 +932,7 @@ static int pp_dpm_get_mclk_od(void *handle)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -897,7 +955,7 @@ static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -921,7 +979,7 @@ static int pp_dpm_read_sensor(void *handle, int idx,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -947,7 +1005,7 @@ pp_dpm_get_vce_clock_state(void *handle, unsigned idx)
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return NULL;
hwmgr = pp_handle->hwmgr;
@@ -1120,53 +1178,16 @@ const struct amd_pm_funcs pp_dpm_funcs = {
.get_power_profile_state = pp_dpm_get_power_profile_state,
.set_power_profile_state = pp_dpm_set_power_profile_state,
.switch_power_profile = pp_dpm_switch_power_profile,
+ .set_clockgating_by_smu = pp_set_clockgating_by_smu,
};
-int amd_powerplay_create(struct amd_pp_init *pp_init,
- void **handle)
-{
- struct pp_instance *instance;
-
- if (pp_init == NULL || handle == NULL)
- return -EINVAL;
-
- instance = kzalloc(sizeof(struct pp_instance), GFP_KERNEL);
- if (instance == NULL)
- return -ENOMEM;
-
- instance->pp_valid = PP_VALID;
- instance->chip_family = pp_init->chip_family;
- instance->chip_id = pp_init->chip_id;
- instance->pm_en = pp_init->pm_en;
- instance->feature_mask = pp_init->feature_mask;
- instance->device = pp_init->device;
- mutex_init(&instance->pp_lock);
- *handle = instance;
- return 0;
-}
-
-int amd_powerplay_destroy(void *handle)
-{
- struct pp_instance *instance = (struct pp_instance *)handle;
-
- kfree(instance->hwmgr);
- instance->hwmgr = NULL;
-
- kfree(instance);
- instance = NULL;
- return 0;
-}
-
int amd_powerplay_reset(void *handle)
{
struct pp_instance *instance = (struct pp_instance *)handle;
int ret;
- if (cgs_is_virtualization_enabled(instance->hwmgr->device))
- return PP_DPM_DISABLED;
-
ret = pp_check(instance);
- if (ret != 0)
+ if (ret)
return ret;
ret = pp_hw_fini(instance);
@@ -1175,7 +1196,7 @@ int amd_powerplay_reset(void *handle)
ret = hwmgr_hw_init(instance);
if (ret)
- return PP_DPM_DISABLED;
+ return ret;
return hwmgr_handle_task(instance, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
}
@@ -1191,7 +1212,7 @@ int amd_powerplay_display_configuration_change(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -1210,7 +1231,7 @@ int amd_powerplay_get_display_power_level(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -1235,7 +1256,7 @@ int amd_powerplay_get_current_clocks(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -1252,7 +1273,7 @@ int amd_powerplay_get_current_clocks(void *handle,
ret = phm_get_clock_info(hwmgr, &hwmgr->current_ps->hardware,
&hw_clocks, PHM_PerformanceLevelDesignation_Activity);
- if (ret != 0) {
+ if (ret) {
pr_info("Error in phm_get_clock_info \n");
mutex_unlock(&pp_handle->pp_lock);
return -EINVAL;
@@ -1286,7 +1307,7 @@ int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, s
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
@@ -1309,7 +1330,7 @@ int amd_powerplay_get_clock_by_type_with_latency(void *handle,
int ret = 0;
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
if (!clocks)
@@ -1331,7 +1352,7 @@ int amd_powerplay_get_clock_by_type_with_voltage(void *handle,
int ret = 0;
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
if (!clocks)
@@ -1355,7 +1376,7 @@ int amd_powerplay_set_watermarks_for_clocks_ranges(void *handle,
int ret = 0;
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
if (!wm_with_clock_ranges)
@@ -1379,7 +1400,7 @@ int amd_powerplay_display_clock_voltage_request(void *handle,
int ret = 0;
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
if (!clock)
@@ -1403,7 +1424,7 @@ int amd_powerplay_get_display_mode_validation_clocks(void *handle,
ret = pp_check(pp_handle);
- if (ret != 0)
+ if (ret)
return ret;
hwmgr = pp_handle->hwmgr;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 73bb99d62a44..ad1f6b57884b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -704,7 +704,7 @@ static int cz_update_sclk_limit(struct pp_hwmgr *hwmgr)
clock = hwmgr->display_config.min_core_set_clock;
if (clock == 0)
- pr_info("min_core_set_clock not set\n");
+ pr_debug("min_core_set_clock not set\n");
if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) {
cz_hwmgr->sclk_dpm.hard_min_clk = clock;
@@ -961,18 +961,13 @@ static void cz_clear_voting_clients(struct pp_hwmgr *hwmgr)
static int cz_start_dpm(struct pp_hwmgr *hwmgr)
{
- int ret = 0;
struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
- unsigned long dpm_features = 0;
cz_hwmgr->dpm_flags |= DPMFlags_SCLK_Enabled;
- dpm_features |= SCLK_DPM_MASK;
- ret = smum_send_msg_to_smc_with_parameter(hwmgr,
+ return smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_EnableAllSmuFeatures,
- dpm_features);
-
- return ret;
+ SCLK_DPM_MASK);
}
static int cz_stop_dpm(struct pp_hwmgr *hwmgr)
@@ -1279,27 +1274,18 @@ static int cz_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
int cz_dpm_powerdown_uvd(struct pp_hwmgr *hwmgr)
{
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_UVDPowerGating))
- return smum_send_msg_to_smc(hwmgr,
- PPSMC_MSG_UVDPowerOFF);
+ if (PP_CAP(PHM_PlatformCaps_UVDPowerGating))
+ return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_UVDPowerOFF);
return 0;
}
int cz_dpm_powerup_uvd(struct pp_hwmgr *hwmgr)
{
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_UVDPowerGating)) {
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_UVDDynamicPowerGating)) {
- return smum_send_msg_to_smc_with_parameter(
- hwmgr,
- PPSMC_MSG_UVDPowerON, 1);
- } else {
- return smum_send_msg_to_smc_with_parameter(
- hwmgr,
- PPSMC_MSG_UVDPowerON, 0);
- }
+ if (PP_CAP(PHM_PlatformCaps_UVDPowerGating)) {
+ return smum_send_msg_to_smc_with_parameter(
+ hwmgr,
+ PPSMC_MSG_UVDPowerON,
+ PP_CAP(PHM_PlatformCaps_UVDDynamicPowerGating) ? 1 : 0);
}
return 0;
@@ -1313,17 +1299,16 @@ int cz_dpm_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
if (!bgate) {
/* Stable Pstate is enabled and we need to set the UVD DPM to highest level */
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_StablePState)
- || hwmgr->en_umd_pstate) {
+ if (PP_CAP(PHM_PlatformCaps_StablePState) ||
+ hwmgr->en_umd_pstate) {
cz_hwmgr->uvd_dpm.hard_min_clk =
ptable->entries[ptable->count - 1].vclk;
smum_send_msg_to_smc_with_parameter(hwmgr,
- PPSMC_MSG_SetUvdHardMin,
- cz_get_uvd_level(hwmgr,
- cz_hwmgr->uvd_dpm.hard_min_clk,
- PPSMC_MSG_SetUvdHardMin));
+ PPSMC_MSG_SetUvdHardMin,
+ cz_get_uvd_level(hwmgr,
+ cz_hwmgr->uvd_dpm.hard_min_clk,
+ PPSMC_MSG_SetUvdHardMin));
cz_enable_disable_uvd_dpm(hwmgr, true);
} else {
@@ -1343,17 +1328,16 @@ int cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr)
hwmgr->dyn_state.vce_clock_voltage_dependency_table;
/* Stable Pstate is enabled and we need to set the VCE DPM to highest level */
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_StablePState)
- || hwmgr->en_umd_pstate) {
+ if (PP_CAP(PHM_PlatformCaps_StablePState) ||
+ hwmgr->en_umd_pstate) {
cz_hwmgr->vce_dpm.hard_min_clk =
ptable->entries[ptable->count - 1].ecclk;
smum_send_msg_to_smc_with_parameter(hwmgr,
- PPSMC_MSG_SetEclkHardMin,
- cz_get_eclk_level(hwmgr,
- cz_hwmgr->vce_dpm.hard_min_clk,
- PPSMC_MSG_SetEclkHardMin));
+ PPSMC_MSG_SetEclkHardMin,
+ cz_get_eclk_level(hwmgr,
+ cz_hwmgr->vce_dpm.hard_min_clk,
+ PPSMC_MSG_SetEclkHardMin));
} else {
/*Program HardMin based on the vce_arbiter.ecclk */
if (hwmgr->vce_arbiter.ecclk == 0) {
@@ -1366,10 +1350,10 @@ int cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr)
} else {
cz_hwmgr->vce_dpm.hard_min_clk = hwmgr->vce_arbiter.ecclk;
smum_send_msg_to_smc_with_parameter(hwmgr,
- PPSMC_MSG_SetEclkHardMin,
- cz_get_eclk_level(hwmgr,
- cz_hwmgr->vce_dpm.hard_min_clk,
- PPSMC_MSG_SetEclkHardMin));
+ PPSMC_MSG_SetEclkHardMin,
+ cz_get_eclk_level(hwmgr,
+ cz_hwmgr->vce_dpm.hard_min_clk,
+ PPSMC_MSG_SetEclkHardMin));
}
}
return 0;
@@ -1377,8 +1361,7 @@ int cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr)
int cz_dpm_powerdown_vce(struct pp_hwmgr *hwmgr)
{
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_VCEPowerGating))
+ if (PP_CAP(PHM_PlatformCaps_VCEPowerGating))
return smum_send_msg_to_smc(hwmgr,
PPSMC_MSG_VCEPowerOFF);
return 0;
@@ -1386,8 +1369,7 @@ int cz_dpm_powerdown_vce(struct pp_hwmgr *hwmgr)
int cz_dpm_powerup_vce(struct pp_hwmgr *hwmgr)
{
- if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
- PHM_PlatformCaps_VCEPowerGating))
+ if (PP_CAP(PHM_PlatformCaps_VCEPowerGating))
return smum_send_msg_to_smc(hwmgr,
PPSMC_MSG_VCEPowerON);
return 0;
@@ -1871,6 +1853,33 @@ static int cz_read_sensor(struct pp_hwmgr *hwmgr, int idx,
}
}
+static int cz_notify_cac_buffer_info(struct pp_hwmgr *hwmgr,
+ uint32_t virtual_addr_low,
+ uint32_t virtual_addr_hi,
+ uint32_t mc_addr_low,
+ uint32_t mc_addr_hi,
+ uint32_t size)
+{
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_DramAddrHiVirtual,
+ mc_addr_hi);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_DramAddrLoVirtual,
+ mc_addr_low);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_DramAddrHiPhysical,
+ virtual_addr_hi);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_DramAddrLoPhysical,
+ virtual_addr_low);
+
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_DramBufferSize,
+ size);
+ return 0;
+}
+
+
static const struct pp_hwmgr_func cz_hwmgr_funcs = {
.backend_init = cz_hwmgr_backend_init,
.backend_fini = cz_hwmgr_backend_fini,
@@ -1894,12 +1903,14 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = {
.get_current_shallow_sleep_clocks = cz_get_current_shallow_sleep_clocks,
.get_clock_by_type = cz_get_clock_by_type,
.get_max_high_clocks = cz_get_max_high_clocks,
+ .get_temperature = cz_thermal_get_temperature,
.read_sensor = cz_read_sensor,
.power_off_asic = cz_power_off_asic,
.asic_setup = cz_setup_asic_task,
.dynamic_state_management_enable = cz_enable_dpm_tasks,
.power_state_set = cz_set_power_state_tasks,
.dynamic_state_management_disable = cz_disable_dpm_tasks,
+ .notify_cac_buffer_info = cz_notify_cac_buffer_info,
};
int cz_init_function_pointers(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index 35e80c969737..ce59e0e67cb2 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -292,7 +292,6 @@ int hwmgr_hw_fini(struct pp_instance *handle)
phm_stop_thermal_controller(hwmgr);
psm_set_boot_states(hwmgr);
- phm_display_configuration_changed(hwmgr);
psm_adjust_power_state_dynamic(hwmgr, false, NULL);
phm_disable_dynamic_state_management(hwmgr);
phm_disable_clock_power_gatings(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c
index d09f25485844..8ba75d43fba6 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.c
@@ -1,1263 +1,1252 @@
#include "pp_overdriver.h"
#include <linux/errno.h>
-struct phm_fuses_default vega10_fuses_default[] = {
- {"0000001000010011111010101001010011011110000011100100100101100100",0x00003C96,0xFFFFE226,0x00000656,0x00002203,0xFFFFF201,0x000003FF,0x00002203,0xFFFFF201,0x000003FF},
- {"0000001000010011111010101001010011011110000010100001100010000100",0x00003CC5,0xFFFFE23A,0x0000064E,0x00002258,0xFFFFF1F7,0x000003FC,0x00002258,0xFFFFF1F7,0x000003FC},
- {"0000001000010011111010101001010011011110000011100011000110100100",0x00003CAF,0xFFFFE36E,0x00000602,0x00001E98,0xFFFFF569,0x00000357,0x00001E98,0xFFFFF569,0x00000357},
- {"0000001000010011111010101001010011011110001011000001000101000100",0x0000391A,0xFFFFE548,0x000005C9,0x00001B98,0xFFFFF707,0x00000324,0x00001B98,0xFFFFF707,0x00000324},
- {"0000001000010011111010101001010011011110001011000001100011000100",0x00003821,0xFFFFE674,0x00000597,0x00002196,0xFFFFF361,0x000003C0,0x00002196,0xFFFFF361,0x000003C0},
- {"0000001000010011111010101001010011011110001001100011100010000100",0x000044A2,0xFFFFDCB7,0x00000738,0x0000325C,0xFFFFE6A7,0x000005E6,0x0000325C,0xFFFFE6A7,0x000005E6},
- {"0000001000010011111010101001010011011110000010000010100100100100",0x00004057,0xFFFFE1CF,0x0000063C,0x00002E2E,0xFFFFEB62,0x000004FD,0x00002E2E,0xFFFFEB62,0x000004FD},
- {"0000001000010011111010101001010011011110001010000100100100100100",0x00003FD0,0xFFFFDF0F,0x000006E5,0x0000267C,0xFFFFEE2D,0x000004AB,0x0000267C,0xFFFFEE2D,0x000004AB},
- {"0000001000010011111010101001010011011110001010000000100100000100",0x00003F13,0xFFFFE010,0x000006AD,0x000020E7,0xFFFFF266,0x000003EC,0x000020E7,0xFFFFF266,0x000003EC},
- {"0000001000010011111010101001010011011110000010000010000001000100",0x00004088,0xFFFFDFAB,0x000006B6,0x0000252B,0xFFFFEFDB,0x00000458,0x0000252B,0xFFFFEFDB,0x00000458},
- {"0000001000010011111010101001010011011110001010000011100010000100",0x00003EF6,0xFFFFE017,0x000006AA,0x00001F67,0xFFFFF369,0x000003BE,0x00001F67,0xFFFFF369,0x000003BE},
- {"0000001000010011111010101001010011011110001011000010000110000100",0x00003CDD,0xFFFFE2A7,0x0000063C,0x000026C6,0xFFFFEF38,0x00000478,0x000026C6,0xFFFFEF38,0x00000478},
- {"0000001000010011111010101001010011011110000100000101000100100100",0x00003FA8,0xFFFFDF02,0x000006F0,0x000027FE,0xFFFFECF6,0x000004EA,0x000027FE,0xFFFFECF6,0x000004EA},
- {"0000001000010011111010101001010011011110001001100011100011000100",0x00004670,0xFFFFDC40,0x00000742,0x00003A7A,0xFFFFE1A7,0x000006B6,0x00003A7A,0xFFFFE1A7,0x000006B6},
- {"0000001000010011111010101001010011011110001011000011000000100100",0x00003CDC,0xFFFFE18C,0x00000683,0x00002A69,0xFFFFEBE7,0x00000515,0x00002A69,0xFFFFEBE7,0x00000515},
- {"0000001000010011111010101001010011011110000011100011100011000100",0x00003CEC,0xFFFFE38E,0x00000601,0x00002752,0xFFFFEFA7,0x00000453,0x00002752,0xFFFFEFA7,0x00000453},
- {"0000001000010011111010101001010011011110001011000001000100100100",0x000037D0,0xFFFFE634,0x000005A7,0x00001CD2,0xFFFFF644,0x00000348,0x00001CD2,0xFFFFF644,0x00000348},
- {"0000001000010011111010101001010011011110001010000011100101100100",0x00003DF5,0xFFFFE0A5,0x00000698,0x00001FD5,0xFFFFF30E,0x000003D1,0x00001FD5,0xFFFFF30E,0x000003D1},
- {"0000001000010011111010101001010011011110000010000010100011000100",0x00004201,0xFFFFE03E,0x00000688,0x00003206,0xFFFFE852,0x0000058A,0x00003206,0xFFFFE852,0x0000058A},
- {"0000001000010011111010101001010011011110001011000001100001100100",0x00003BED,0xFFFFE2F5,0x00000638,0x0000270D,0xFFFFEED0,0x0000048E,0x0000270D,0xFFFFEED0,0x0000048E},
- {"0000001000010011111010101001010011011110000010100001100100000100",0x00003E82,0xFFFFE1BE,0x00000654,0x000025FB,0xFFFFEFFA,0x00000448,0x000025FB,0xFFFFEFFA,0x00000448},
- {"0000001000010011111010101001010011011110001011000100000011000100",0x00003962,0xFFFFE4B9,0x000005EF,0x00002385,0xFFFFF156,0x00000423,0x00002385,0xFFFFF156,0x00000423},
- {"0000001000010011111010101001010011011110001011000000100101000100",0x00003D88,0xFFFFE21A,0x00000655,0x0000295A,0xFFFFED68,0x000004C4,0x0000295A,0xFFFFED68,0x000004C4},
- {"0000001000010011111010101001010011011110001011000001000100000100",0x00003AA4,0xFFFFE4A3,0x000005E0,0x000022EF,0xFFFFF250,0x000003EB,0x000022EF,0xFFFFF250,0x000003EB},
- {"0000001000010011111010101001010011011110000011100010100110100100",0x00003D97,0xFFFFE30D,0x0000060D,0x0000205D,0xFFFFF45D,0x00000380,0x0000205D,0xFFFFF45D,0x00000380},
- {"0000001000010011111010101001010011011110001011000100000010100100",0x000039B6,0xFFFFE446,0x00000605,0x00002325,0xFFFFF16C,0x0000041F,0x00002325,0xFFFFF16C,0x0000041F},
- {"0000001000010011111010101001010011011110001001100011100100000100",0x0000457E,0xFFFFDCF6,0x00000722,0x00003972,0xFFFFE27B,0x0000068E,0x00003972,0xFFFFE27B,0x0000068E},
- {"0000001000010011111010101001010011011110000010100001100100100100",0x00003FB8,0xFFFFE101,0x00000670,0x00002787,0xFFFFEEF5,0x00000471,0x00002787,0xFFFFEEF5,0x00000471},
- {"0000001000010011111010101001010011011110000011100011100010100100",0x00003BB2,0xFFFFE430,0x000005EA,0x000024A5,0xFFFFF162,0x00000409,0x000024A5,0xFFFFF162,0x00000409},
- {"0000001000010011111010101001010011011110000010000010000101000100",0x00003EC5,0xFFFFE1BD,0x0000064F,0x000022F0,0xFFFFF227,0x000003E8,0x000022F0,0xFFFFF227,0x000003E8},
- {"0000001000010011111010101001010011011110001011000011000101100100",0x000038A7,0xFFFFE59F,0x000005C1,0x000021CC,0xFFFFF2DF,0x000003D9,0x000021CC,0xFFFFF2DF,0x000003D9},
- {"0000001000010011111010101001010011011110001100100100000110000100",0x00002995,0xFFFFEF7A,0x0000044C,0x00001552,0xFFFFFB5D,0x00000292,0x00001552,0xFFFFFB5D,0x00000292},
- {"0000001000010011111010101001010011011110001011000100000001100100",0x00003B26,0xFFFFE2D3,0x00000649,0x000023B4,0xFFFFF09B,0x00000449,0x000023B4,0xFFFFF09B,0x00000449},
- {"0000001000010011111010101001010011011110000010000001000100100100",0x000040D2,0xFFFFE00A,0x00000696,0x000022DA,0xFFFFF1E9,0x000003F2,0x000022DA,0xFFFFF1E9,0x000003F2},
- {"0000001000010011111010101001010011011110001011000011100100100100",0x00003C98,0xFFFFE365,0x00000618,0x00002D5D,0xFFFFEB3A,0x0000051D,0x00002D5D,0xFFFFEB3A,0x0000051D},
- {"0000001000010011111010101001010011011110001011000001000010100100",0x00003BBD,0xFFFFE37E,0x00000617,0x0000252E,0xFFFFF06E,0x00000441,0x0000252E,0xFFFFF06E,0x00000441},
- {"0000001000010011111010101001010011011110001001100010100100100100",0x00004363,0xFFFFDF7A,0x000006A0,0x000031F5,0xFFFFE880,0x0000057B,0x000031F5,0xFFFFE880,0x0000057B},
- {"0000001000010011111010101001010011011110000011100011100001000100",0x00003CFC,0xFFFFE2AF,0x0000062E,0x0000212A,0xFFFFF335,0x000003BF,0x0000212A,0xFFFFF335,0x000003BF},
- {"0000001000010011111010101001010011011110000111000100100100100100",0x0000252D,0xFFFFF31B,0x000003C3,0x00001A1A,0xFFFFF882,0x00000325,0x00001A1A,0xFFFFF882,0x00000325},
- {"0000001000010011111010101001010011011110000010100010100110100100",0x00003FE2,0xFFFFDFEF,0x000006AC,0x000025A2,0xFFFFEF84,0x00000462,0x000025A2,0xFFFFEF84,0x00000462},
- {"0000001000010011111010101001010011011110000010000010000011100100",0x000040A5,0xFFFFE13B,0x0000065B,0x00002C13,0xFFFFEC75,0x000004D7,0x00002C13,0xFFFFEC75,0x000004D7},
- {"0000001000010011111010101001010011011110000011100100100010100100",0x00003E42,0xFFFFE1B3,0x00000657,0x0000221D,0xFFFFF273,0x000003DE,0x0000221D,0xFFFFF273,0x000003DE},
- {"0000001000010011111010101001010011011110000010100010000011100100",0x00003E7F,0xFFFFE255,0x00000638,0x00002D30,0xFFFFEB8A,0x00000503,0x00002D30,0xFFFFEB8A,0x00000503},
- {"0000001000010011111010101001010011011110001011000010100111000100",0x00003E56,0xFFFFE16D,0x00000670,0x000028DC,0xFFFFEDA0,0x000004BA,0x000028DC,0xFFFFEDA0,0x000004BA},
- {"0000001000010011111010101001010011011110001001100011000010100100",0x000044AD,0xFFFFDE24,0x000006DD,0x000031AD,0xFFFFE850,0x00000585,0x000031AD,0xFFFFE850,0x00000585},
- {"0000001000010011111010101001010011011110001011000010000011100100",0x00003AF3,0xFFFFE5B0,0x000005A6,0x00002CF6,0xFFFFEC75,0x000004DD,0x00002CF6,0xFFFFEC75,0x000004DD},
- {"0000001000010011111010101001010011011110000010100010000010000100",0x00003E66,0xFFFFE19E,0x0000065B,0x00002332,0xFFFFF1B9,0x000003FD,0x00002332,0xFFFFF1B9,0x000003FD},
- {"0000001000010011111010101001010011011110000010000010100010000100",0x00003FB4,0xFFFFE0A5,0x00000686,0x0000253E,0xFFFFF02E,0x00000444,0x0000253E,0xFFFFF02E,0x00000444},
- {"0000001000010011111010101001010011011110001010000001100010100100",0x00003E28,0xFFFFE14D,0x0000066E,0x00001FE2,0xFFFFF39A,0x000003B1,0x00001FE2,0xFFFFF39A,0x000003B1},
- {"0000001000010011111010101001010011011110001011000000100100000100",0x000039E6,0xFFFFE44B,0x000005FE,0x0000210C,0xFFFFF2F4,0x000003DA,0x0000210C,0xFFFFF2F4,0x000003DA},
- {"0000001000010011111010101001010011011110001011000101000100000100",0x00003A4D,0xFFFFE252,0x0000067A,0x000027E2,0xFFFFECED,0x000004FA,0x000027E2,0xFFFFECED,0x000004FA},
- {"0000001000010011111010101001010011011110000010100010100101100100",0x00004065,0xFFFFE02F,0x0000069B,0x0000299D,0xFFFFED38,0x000004C2,0x0000299D,0xFFFFED38,0x000004C2},
- {"0000001000010011111010101001010011011110000011100010000010100100",0x000039EE,0xFFFFE603,0x00000594,0x0000214F,0xFFFFF429,0x0000038E,0x0000214F,0xFFFFF429,0x0000038E},
- {"0000001000010011111010101001010011011110000011100100100011100100",0x00003BD2,0xFFFFE351,0x00000618,0x000020B8,0xFFFFF377,0x000003B4,0x000020B8,0xFFFFF377,0x000003B4},
- {"0000001000010011111010101001010011011110000010100011000100100100",0x00003FAA,0xFFFFE183,0x0000065E,0x000032AE,0xFFFFE7C2,0x000005A6,0x000032AE,0xFFFFE7C2,0x000005A6},
- {"0000001000010011111010101001010011011110001011000010100110000100",0x00003AFB,0xFFFFE3E4,0x00000608,0x00002293,0xFFFFF21F,0x000003FA,0x00002293,0xFFFFF21F,0x000003FA},
- {"0000001000010011111010101001010011011110001001100010000001100100",0x0000448B,0xFFFFDD5D,0x0000070D,0x00002E4E,0xFFFFE9DF,0x00000551,0x00002E4E,0xFFFFE9DF,0x00000551},
- {"0000001000010011111010101001010011011110000011100010000110000100",0x00003D46,0xFFFFE39B,0x000005F3,0x0000218E,0xFFFFF3CD,0x00000398,0x0000218E,0xFFFFF3CD,0x00000398},
- {"0000001000010011111010101001010011011110000010000100100011100100",0x00003F01,0xFFFFDFD9,0x000006BF,0x000023AF,0xFFFFF04E,0x0000044C,0x000023AF,0xFFFFF04E,0x0000044C},
- {"0000001000010011111010101001010011011110000100000010100110100100",0x0000403D,0xFFFFDF6B,0x000006C9,0x0000270D,0xFFFFEE4B,0x0000049E,0x0000270D,0xFFFFEE4B,0x0000049E},
- {"0000001000010011111010101001010011011110000011100011100101100100",0x00003C11,0xFFFFE35C,0x00000613,0x000020F9,0xFFFFF365,0x000003B9,0x000020F9,0xFFFFF365,0x000003B9},
- {"0000001000010011111010101001010011011110001011000011100010000100",0x00003B58,0xFFFFE37D,0x0000061F,0x00002698,0xFFFFEF46,0x00000478,0x00002698,0xFFFFEF46,0x00000478},
- {"0000001000010011111010101001010011011110001010000100000110100100",0x00003EBC,0xFFFFDF7A,0x000006D6,0x0000212B,0xFFFFF195,0x0000041B,0x0000212B,0xFFFFF195,0x0000041B},
- {"0000001000010011111010101001010011011110000010000100100011000100",0x00004050,0xFFFFDEB3,0x000006FE,0x00002D6C,0xFFFFE961,0x00000582,0x00002D6C,0xFFFFE961,0x00000582},
- {"0000001000010011111010101001010011011110001001100010000001000100",0x000043F0,0xFFFFDD9C,0x00000702,0x00002B31,0xFFFFEBEA,0x000004F7,0x00002B31,0xFFFFEBEA,0x000004F7},
- {"0000001000010011111010101001010011011110000100000000100100100100",0x00003EFA,0xFFFFE093,0x00000696,0x000026DB,0xFFFFEEB3,0x00000489,0x000026DB,0xFFFFEEB3,0x00000489},
- {"0000001000010011111010101001010011011110000010000010000001100100",0x0000425D,0xFFFFDE8D,0x000006E6,0x00002CA4,0xFFFFEAD2,0x00000531,0x00002CA4,0xFFFFEAD2,0x00000531},
- {"0000001000010011111010101001010011011110001001100011100110100100",0x000043B0,0xFFFFDD03,0x00000728,0x00002946,0xFFFFECA6,0x000004DE,0x00002946,0xFFFFECA6,0x000004DE},
- {"0000001000010011111010101001010011011110001010000010100001100100",0x00003F6A,0xFFFFE03A,0x0000069D,0x00002208,0xFFFFF1F8,0x000003F6,0x00002208,0xFFFFF1F8,0x000003F6},
- {"0000001000010011111010101001010011011110001011000010100101100100",0x00003A94,0xFFFFE4A7,0x000005E2,0x000024D0,0xFFFFF100,0x00000426,0x000024D0,0xFFFFF100,0x00000426},
- {"0000001000010011111010101001010011011110001010000001000011000100",0x00003F2F,0xFFFFE0A3,0x00000688,0x00002198,0xFFFFF271,0x000003E2,0x00002198,0xFFFFF271,0x000003E2},
- {"0000001000010011111010101001010011011110000100000100100011100100",0x00003EA5,0xFFFFE032,0x000006AE,0x0000227C,0xFFFFF130,0x00000426,0x0000227C,0xFFFFF130,0x00000426},
- {"0000001000010011111010101001010011011110001001100100000101000100",0x0000442F,0xFFFFDBC4,0x0000078B,0x00003CD6,0xFFFFDE6C,0x0000076C,0x00003CD6,0xFFFFDE6C,0x0000076C},
- {"0000001000010011111010101001010011011110001010000010100010000100",0x00003DDE,0xFFFFE174,0x00000668,0x00001FF4,0xFFFFF38F,0x000003B1,0x00001FF4,0xFFFFF38F,0x000003B1},
- {"0000001000010011111010101001010011011110000010100011000101000100",0x000040B0,0xFFFFE016,0x000006A0,0x00002DBB,0xFFFFEA7F,0x00000537,0x00002DBB,0xFFFFEA7F,0x00000537},
- {"0000001000010011111010101001010011011110001011000011000100000100",0x00003429,0xFFFFEA97,0x000004DD,0x000024D5,0xFFFFF26F,0x000003DF,0x000024D5,0xFFFFF26F,0x000003DF},
- {"0000001000010011111010101001010011011110000011100001100100000100",0x00003AEB,0xFFFFE590,0x000005A3,0x000022CB,0xFFFFF347,0x000003B2,0x000022CB,0xFFFFF347,0x000003B2},
- {"0000001000010011111010101001010011011110001010000011100100000100",0x00003B8E,0xFFFFE2EF,0x00000636,0x00002351,0xFFFFF143,0x0000041C,0x00002351,0xFFFFF143,0x0000041C},
- {"0000001000010011111010101001010011011110001100100100000011000100",0x00002926,0xFFFFF0B0,0x00000410,0x0000194E,0xFFFFF94E,0x000002E9,0x0000194E,0xFFFFF94E,0x000002E9},
- {"0000001000010011111010101001010011011110001010000011000110000100",0x0000402B,0xFFFFDF78,0x000006C2,0x00002273,0xFFFFF16C,0x00000414,0x00002273,0xFFFFF16C,0x00000414},
- {"0000001000010011111010101001010011011110000010100001000010100100",0x00003D6A,0xFFFFE1D3,0x00000659,0x00002006,0xFFFFF394,0x000003B1,0x00002006,0xFFFFF394,0x000003B1},
- {"0000001000010011111010101001010011011110001010000100000001100100",0x00004042,0xFFFFDFD8,0x000006A8,0x00002135,0xFFFFF29F,0x000003D9,0x00002135,0xFFFFF29F,0x000003D9},
- {"0000001000010011111010101001010011011110000010000010000010100100",0x0000405B,0xFFFFE093,0x00000682,0x0000288F,0xFFFFEE3A,0x00000491,0x0000288F,0xFFFFEE3A,0x00000491},
- {"0000001000010011111010101001010011011110001011000100100010100100",0x00003A49,0xFFFFE30C,0x00000648,0x000023F9,0xFFFFF02D,0x00000460,0x000023F9,0xFFFFF02D,0x00000460},
- {"0000001000010011111010101001010011011110001010000010100101100100",0x00003D59,0xFFFFE1CC,0x0000065B,0x00002013,0xFFFFF37D,0x000003B6,0x00002013,0xFFFFF37D,0x000003B6},
- {"0000001000010011111010101001010011011110001011000011100110000100",0x000040C1,0xFFFFDF8C,0x000006CA,0x00003271,0xFFFFE6CA,0x000005EA,0x00003271,0xFFFFE6CA,0x000005EA},
- {"0000001000010011111010101001010011011110001001100010000011100100",0x000042E9,0xFFFFDFDC,0x0000068C,0x00002ED9,0xFFFFEAAF,0x0000051B,0x00002ED9,0xFFFFEAAF,0x0000051B},
- {"0000001000010011111010101001010011011110000010000011000010000100",0x000042ED,0xFFFFDE50,0x000006F0,0x00002FCF,0xFFFFE8BB,0x0000058C,0x00002FCF,0xFFFFE8BB,0x0000058C},
- {"0000001000010011111010101001010011011110000010100100000100000100",0x00003EBD,0xFFFFE099,0x00000698,0x00002709,0xFFFFEE7B,0x00000495,0x00002709,0xFFFFEE7B,0x00000495},
- {"0000001000010011111010101001010011011110001010000100100100000100",0x00003F71,0xFFFFDF82,0x000006C9,0x0000219B,0xFFFFF1AD,0x0000040F,0x0000219B,0xFFFFF1AD,0x0000040F},
- {"0000001000010011111010101001010011011110001010000000100011100100",0x00003E73,0xFFFFE080,0x0000069B,0x000020E7,0xFFFFF273,0x000003E9,0x000020E7,0xFFFFF273,0x000003E9},
- {"0000001000010011111010101001010011011110000011100011000110000100",0x00003E14,0xFFFFE278,0x0000062C,0x00002275,0xFFFFF2B3,0x000003CE,0x00002275,0xFFFFF2B3,0x000003CE},
- {"0000001000010011111010101001010011011110001011000010000110100100",0x00003ABB,0xFFFFE3B9,0x00000615,0x00002192,0xFFFFF28F,0x000003EB,0x00002192,0xFFFFF28F,0x000003EB},
- {"0000001000010011111010101001010011011110001010000011000100100100",0x00003D53,0xFFFFE255,0x00000643,0x0000275B,0xFFFFEEED,0x00000479,0x0000275B,0xFFFFEEED,0x00000479},
- {"0000001000010011111010101001010011011110001001100010100001100100",0x000043E3,0xFFFFDDC3,0x000006FB,0x00002B6B,0xFFFFEBD6,0x000004FA,0x00002B6B,0xFFFFEBD6,0x000004FA},
- {"0000001000010011111010101001010011011110000011100010000101000100",0x00003BDE,0xFFFFE507,0x000005B4,0x000022CE,0xFFFFF358,0x000003AB,0x000022CE,0xFFFFF358,0x000003AB},
- {"0000001000010011111010101001010011011110001100100011000101100100",0x00002460,0xFFFFF3B5,0x000003A2,0x000014E7,0xFFFFFC32,0x0000027C,0x000014E7,0xFFFFFC32,0x0000027C},
- {"0000001000010011111010101001010011011110001010000010000011000100",0x00003D20,0xFFFFE298,0x0000062F,0x00002080,0xFFFFF3AF,0x000003A8,0x00002080,0xFFFFF3AF,0x000003A8},
- {"0000001000010011111010101001010011011110000010000001100100000100",0x00003E14,0xFFFFE221,0x00000641,0x000021BB,0xFFFFF2EA,0x000003CA,0x000021BB,0xFFFFF2EA,0x000003CA},
- {"0000001000010011111010101001010011011110000010100100000011000100",0x00003DE1,0xFFFFE14E,0x00000677,0x00002468,0xFFFFF068,0x00000440,0x00002468,0xFFFFF068,0x00000440},
- {"0000001000010011111010101001010011011110001001100001000010000100",0x00004372,0xFFFFDDF8,0x000006F5,0x00002B3F,0xFFFFEBE8,0x000004F8,0x00002B3F,0xFFFFEBE8,0x000004F8},
- {"0000001000010011111010101001010011011110000010100010100011000100",0x00003E4F,0xFFFFE2A3,0x0000062B,0x00002F5A,0xFFFFEA37,0x0000053B,0x00002F5A,0xFFFFEA37,0x0000053B},
- {"0000001000010011111010101001010011011110001010000101000011100100",0x00003E07,0xFFFFE02F,0x000006B6,0x0000216B,0xFFFFF1A3,0x00000416,0x0000216B,0xFFFFF1A3,0x00000416},
- {"0000001000010011111010101001010011011110001010000011100010100100",0x00003DAB,0xFFFFE128,0x0000067F,0x0000216F,0xFFFFF236,0x000003F3,0x0000216F,0xFFFFF236,0x000003F3},
- {"0000001000010011111010101001010011011110001011000010100100100100",0x0000364B,0xFFFFE8CB,0x0000052A,0x00002568,0xFFFFF1B2,0x00000400,0x00002568,0xFFFFF1B2,0x00000400},
- {"0000001000010011111010101001010011011110001001100001000001100100",0x00004219,0xFFFFDE87,0x000006E8,0x00002C59,0xFFFFEAEE,0x00000529,0x00002C59,0xFFFFEAEE,0x00000529},
- {"0000001000010011111010101001010011011110000011100001100101000100",0x000039A8,0xFFFFE602,0x00000594,0x00001D06,0xFFFFF6F0,0x00000316,0x00001D06,0xFFFFF6F0,0x00000316},
- {"0000001000010011111010101001010011011110001001100001000011100100",0x00004052,0xFFFFE01C,0x00000698,0x00002310,0xFFFFF1A1,0x000003FE,0x00002310,0xFFFFF1A1,0x000003FE},
- {"0000001000010011111010101001010011011110000011100010100000100100",0x00003C1C,0xFFFFE3EB,0x000005F1,0x00002289,0xFFFFF2CF,0x000003C9,0x00002289,0xFFFFF2CF,0x000003C9},
- {"0000001000010011111010101001010011011110000011100101000100100100",0x00003F19,0xFFFFE085,0x0000069E,0x00002B94,0xFFFFEB72,0x0000051D,0x00002B94,0xFFFFEB72,0x0000051D},
- {"0000001000010011111010101001010011011110000011100100000110100100",0x00003C51,0xFFFFE2AD,0x00000638,0x0000206B,0xFFFFF361,0x000003BE,0x0000206B,0xFFFFF361,0x000003BE},
- {"0000001000010011111010101001010011011110001001100001000011000100",0x000040B9,0xFFFFDFBB,0x000006AB,0x0000241F,0xFFFFF0CC,0x00000425,0x0000241F,0xFFFFF0CC,0x00000425},
- {"0000001000010011111010101001010011011110000010100010000001100100",0x00003E62,0xFFFFE12C,0x00000678,0x00002445,0xFFFFF09E,0x00000435,0x00002445,0xFFFFF09E,0x00000435},
- {"0000001000010011111010101001010011011110000011100001100110000100",0x00003C97,0xFFFFE399,0x000005FB,0x0000209D,0xFFFFF41D,0x0000038F,0x0000209D,0xFFFFF41D,0x0000038F},
- {"0000001000010011111010101001010011011110000011100011000101000100",0x00003FF9,0xFFFFE1E9,0x0000063E,0x00002E96,0xFFFFEAF5,0x00000516,0x00002E96,0xFFFFEAF5,0x00000516},
- {"0000001000010011111010101001010011011110000010100011000010000100",0x00003F04,0xFFFFE109,0x0000067A,0x000026E1,0xFFFFEF0B,0x00000476,0x000026E1,0xFFFFEF0B,0x00000476},
- {"0000001000010011111010101001010011011110000100000001000100100100",0x00003E3E,0xFFFFE187,0x00000660,0x00002049,0xFFFFF38D,0x000003B0,0x00002049,0xFFFFF38D,0x000003B0},
- {"0000001000010011111010101001010011011110001010000010100101000100",0x00003D58,0xFFFFE253,0x0000063D,0x00002158,0xFFFFF308,0x000003C3,0x00002158,0xFFFFF308,0x000003C3},
- {"0000001000010011111010101001010011011110000010000100000011000100",0x00004074,0xFFFFDF8D,0x000006C0,0x00002799,0xFFFFEE19,0x000004A5,0x00002799,0xFFFFEE19,0x000004A5},
- {"0000001000010011111010101001010011011110001010000001100100100100",0x00003DAF,0xFFFFE1C9,0x00000659,0x000020E5,0xFFFFF313,0x000003C6,0x000020E5,0xFFFFF313,0x000003C6},
- {"0000001000010011111010101001010011011110000010100011100101100100",0x000041DD,0xFFFFDDFA,0x0000071B,0x0000348D,0xFFFFE4B4,0x0000064C,0x0000348D,0xFFFFE4B4,0x0000064C},
- {"0000001000010011111010101001010011011110001011000010100010000100",0x00003947,0xFFFFE5AE,0x000005B8,0x000024A6,0xFFFFF140,0x0000041D,0x000024A6,0xFFFFF140,0x0000041D},
- {"0000001000010011111010101001010011011110000100000001100001000100",0x00003D35,0xFFFFE197,0x0000066E,0x00002248,0xFFFFF1BC,0x00000408,0x00002248,0xFFFFF1BC,0x00000408},
- {"0000001000010011111010101001010011011110000010100001100011100100",0x00003F4F,0xFFFFE13E,0x0000066D,0x00002AF0,0xFFFFEC99,0x000004DB,0x00002AF0,0xFFFFEC99,0x000004DB},
- {"0000001000010011111010101001010011011110001001100011100101000100",0x0000430F,0xFFFFDDFB,0x000006FC,0x00002D4D,0xFFFFEA55,0x00000540,0x00002D4D,0xFFFFEA55,0x00000540},
- {"0000001000010011111010101001010011011110000011100010100101000100",0x00003B22,0xFFFFE543,0x000005B1,0x000022E1,0xFFFFF31B,0x000003B9,0x000022E1,0xFFFFF31B,0x000003B9},
- {"0000001000010011111010101001010011011110000011100010000010000100",0x00003978,0xFFFFE611,0x00000592,0x00001C36,0xFFFFF771,0x00000302,0x00001C36,0xFFFFF771,0x00000302},
- {"0000001000010011111010101001010011011110001001100010000101100100",0x000044DF,0xFFFFDDAB,0x000006F2,0x00002CEA,0xFFFFEB47,0x00000507,0x00002CEA,0xFFFFEB47,0x00000507},
- {"0000001000010011111010101001010011011110000010100011100011000100",0x00003E9B,0xFFFFE12C,0x0000067C,0x00002B79,0xFFFFEBD9,0x00000503,0x00002B79,0xFFFFEBD9,0x00000503},
- {"0000001000010011111010101001010011011110001001100011000001000100",0x00004464,0xFFFFDCD3,0x00000731,0x00002D14,0xFFFFEA2D,0x0000054E,0x00002D14,0xFFFFEA2D,0x0000054E},
- {"0000001000010011111010101001010011011110001010000001000100100100",0x00003FB3,0xFFFFE052,0x00000693,0x000020AC,0xFFFFF311,0x000003C6,0x000020AC,0xFFFFF311,0x000003C6},
- {"0000001000010011111010101001010011011110001011000001000010000100",0x00003BDA,0xFFFFE2FB,0x00000636,0x0000261E,0xFFFFEF72,0x00000471,0x0000261E,0xFFFFEF72,0x00000471},
- {"0000001000010011111010101001010011011110001011000001100101100100",0x00003D72,0xFFFFE28A,0x0000063E,0x000029D8,0xFFFFED54,0x000004C7,0x000029D8,0xFFFFED54,0x000004C7},
- {"0000001000010011111010101001010011011110001011000010100000100100",0x00003E26,0xFFFFE102,0x00000694,0x00002DD1,0xFFFFE9CA,0x0000056D,0x00002DD1,0xFFFFE9CA,0x0000056D},
- {"0000001000010011111010101001010011011110000100000100000100100100",0x000041CD,0xFFFFDE97,0x000006ED,0x00002DE5,0xFFFFE9B9,0x00000565,0x00002DE5,0xFFFFE9B9,0x00000565},
- {"0000001000010011111010101001010011011110000010100010100110000100",0x00003F30,0xFFFFE06E,0x00000698,0x000024FF,0xFFFFEFFC,0x0000044F,0x000024FF,0xFFFFEFFC,0x0000044F},
- {"0000001000010011111010101001010011011110001011000011100011000100",0x0000378B,0xFFFFE6B4,0x00000594,0x000023A7,0xFFFFF1DC,0x00000407,0x000023A7,0xFFFFF1DC,0x00000407},
- {"0000001000010011111010101001010011011110000011100100000101100100",0x00003CD7,0xFFFFE28D,0x00000636,0x00002036,0xFFFFF3B5,0x000003AA,0x00002036,0xFFFFF3B5,0x000003AA},
- {"0000001000010011111010101001010011011110000010100011100010000100",0x00003EF9,0xFFFFE0AA,0x0000068D,0x000024D3,0xFFFFF02F,0x00000445,0x000024D3,0xFFFFF02F,0x00000445},
- {"0000001000010011111010101001010011011110001010000011100101000100",0x00003D08,0xFFFFE1BB,0x00000665,0x00002159,0xFFFFF26F,0x000003E6,0x00002159,0xFFFFF26F,0x000003E6},
- {"0000001000010011111010101001010011011110001011000010000011000100",0x000038A9,0xFFFFE6CA,0x00000580,0x000025D3,0xFFFFF101,0x00000421,0x000025D3,0xFFFFF101,0x00000421},
- {"0000001000010011111010101001010011011110000010100010000010100100",0x00003E45,0xFFFFE1F8,0x0000064D,0x000027E3,0xFFFFEEBB,0x0000047F,0x000027E3,0xFFFFEEBB,0x0000047F},
- {"0000001000010011111010101001010011011110000011100011100001100100",0x00003F76,0xFFFFE128,0x0000066E,0x0000286B,0xFFFFEE4C,0x00000493,0x0000286B,0xFFFFEE4C,0x00000493},
- {"0000001000010011111010101001010011011110001001100100000100000100",0x0000440D,0xFFFFDCA2,0x0000074F,0x00003817,0xFFFFE256,0x000006AF,0x00003817,0xFFFFE256,0x000006AF},
- {"0000001000010011111010101001010011011110000100000101000100000100",0x00003EE1,0xFFFFDFA7,0x000006D4,0x000027EA,0xFFFFED2B,0x000004DE,0x000027EA,0xFFFFED2B,0x000004DE},
- {"0000001000010011111010101001010011011110001011000011100001100100",0x00003C62,0xFFFFE285,0x0000064A,0x00002520,0xFFFFF001,0x0000045C,0x00002520,0xFFFFF001,0x0000045C},
- {"0000001000010011111010101001010011011110001100100011100101100100",0x0000272E,0xFFFFF17A,0x000003FA,0x0000150B,0xFFFFFBD5,0x00000284,0x0000150B,0xFFFFFBD5,0x00000284},
- {"0000001000010011111010101001010011011110001001100001100100100100",0x00004275,0xFFFFDF69,0x000006A5,0x000025AA,0xFFFFF05C,0x0000042B,0x000025AA,0xFFFFF05C,0x0000042B},
- {"0000001000010011111010101001010011011110000011100100000011100100",0x00003CAA,0xFFFFE392,0x000005FF,0x000023A8,0xFFFFF20E,0x000003E9,0x000023A8,0xFFFFF20E,0x000003E9},
- {"0000001000010011111010101001010011011110001011000101000011000100",0x00003CF8,0xFFFFE0FB,0x000006A6,0x00002CA7,0xFFFFE9FF,0x0000056E,0x00002CA7,0xFFFFE9FF,0x0000056E},
- {"0000001000010011111010101001010011011110001010000010000100100100",0x00003D00,0xFFFFE296,0x00000633,0x000021C1,0xFFFFF2C8,0x000003CF,0x000021C1,0xFFFFF2C8,0x000003CF},
- {"0000001000010011111010101001010011011110001010000011100011100100",0x00003B46,0xFFFFE301,0x00000632,0x0000204C,0xFFFFF33B,0x000003C8,0x0000204C,0xFFFFF33B,0x000003C8},
- {"0000001000010011111010101001010011011110001000000100000101100100",0x00002026,0xFFFFF5CE,0x00000368,0x00001598,0xFFFFFB29,0x000002C3,0x00001598,0xFFFFFB29,0x000002C3},
- {"0000001000010011111010101001010011011110001010000011000101100100",0x00003DCA,0xFFFFE178,0x00000668,0x00001FDB,0xFFFFF39D,0x000003AF,0x00001FDB,0xFFFFF39D,0x000003AF},
- {"0000001000010011111010101001010011011110001011000100100011000100",0x00003A59,0xFFFFE327,0x00000642,0x000024B9,0xFFFFEFC4,0x00000471,0x000024B9,0xFFFFEFC4,0x00000471},
- {"0000001000010011111010101001010011011110001011000010100101000100",0x00003C26,0xFFFFE440,0x000005EB,0x00002C0F,0xFFFFEC88,0x000004E0,0x00002C0F,0xFFFFEC88,0x000004E0},
- {"0000001000010011111010101001010011011110000010000011100010000100",0x00004149,0xFFFFDEB8,0x000006E7,0x0000280A,0xFFFFED89,0x000004C2,0x0000280A,0xFFFFED89,0x000004C2},
- {"0000001000010011111010101001010011011110000011100100000100100100",0x00003EB4,0xFFFFE1E5,0x0000064D,0x0000299F,0xFFFFEDB3,0x000004A9,0x0000299F,0xFFFFEDB3,0x000004A9},
- {"0000001000010011111010101001010011011110001011000011100110100100",0x00003BBF,0xFFFFE268,0x0000065A,0x00002504,0xFFFFEFB0,0x00000470,0x00002504,0xFFFFEFB0,0x00000470},
- {"0000001000010011111010101001010011011110000010000100100100000100",0x00004203,0xFFFFDDC6,0x00000720,0x0000303B,0xFFFFE78F,0x000005D0,0x0000303B,0xFFFFE78F,0x000005D0},
- {"0000001000010011111010101001010011011110000011100011100110000100",0x00003DA3,0xFFFFE244,0x0000063E,0x000021B4,0xFFFFF2DA,0x000003CD,0x000021B4,0xFFFFF2DA,0x000003CD},
- {"0000001000010011111010101001010011011110000010100011100011100100",0x00004035,0xFFFFE065,0x0000069B,0x00003323,0xFFFFE6D6,0x000005D8,0x00003323,0xFFFFE6D6,0x000005D8},
- {"0000001000010011111010101001010011011110001011000001000101100100",0x00003944,0xFFFFE4E5,0x000005E2,0x00001F3C,0xFFFFF456,0x0000039D,0x00001F3C,0xFFFFF456,0x0000039D},
- {"0000001000010011111010101001010011011110000001100001100100000100",0x000032D8,0xFFFFEAE8,0x000004E6,0x00001812,0xFFFFFA1C,0x000002BC,0x00001812,0xFFFFFA1C,0x000002BC},
- {"0000001000010011111100001111110101000010110100100010100101000100",0x000041F6,0xFFFFE025,0x0000069A,0x0000241E,0xFFFFF1B4,0x00000402,0x0000241E,0xFFFFF1B4,0x00000402},
- {"0000001000010011111100001111111010011001000011000011000010100100",0x00003300,0xFFFFEB60,0x000004C1,0x00001E15,0xFFFFF6A6,0x0000033B,0x00001E15,0xFFFFF6A6,0x0000033B},
- {"0000001000010011111010101001010011011110000001000000100010100100",0x000037F0,0xFFFFE68F,0x0000059B,0x00001F8A,0xFFFFF467,0x000003A3,0x00001F8A,0xFFFFF467,0x000003A3},
- {"0000001000010011111100001111111010011001000110000010100110000100",0x000025D8,0xFFFFF2AA,0x000003C3,0x000018A8,0xFFFFF9BE,0x000002D2,0x000018A8,0xFFFFF9BE,0x000002D2},
- {"0000001000010011111100001111111010011001000001100010000011000100",0x0000364F,0xFFFFE988,0x000004FC,0x00001E51,0xFFFFF633,0x0000034F,0x00001E51,0xFFFFF633,0x0000034F},
- {"0000001000010011111010101001010011011110000001100001000101000100",0x00002288,0xFFFFF483,0x0000036C,0x0000280F,0xFFFFEF39,0x0000047B,0x0000280F,0xFFFFEF39,0x0000047B},
- {"0000001000010011111100001111111010011001000010000010000010000100",0x00003322,0xFFFFEA7E,0x000004ED,0x00001DAD,0xFFFFF62B,0x00000355,0x00001DAD,0xFFFFF62B,0x00000355},
- {"0000001000010011111010101001010011011110000000100101000011100100",0x00002B7B,0xFFFFEE4F,0x0000045B,0x00001AA2,0xFFFFF710,0x0000033E,0x00001AA2,0xFFFFF710,0x0000033E},
- {"0000001000010011111100001111111010011001000001000010000011000100",0x000034CC,0xFFFFEA79,0x000004E4,0x00001B05,0xFFFFF8B3,0x000002EC,0x00001B05,0xFFFFF8B3,0x000002EC},
- {"0000001000010011111100001111110101000010110111000010100001100100",0x00003837,0xFFFFE5ED,0x000005C3,0x00001ACB,0xFFFFF7B2,0x00000314,0x00001ACB,0xFFFFF7B2,0x00000314},
- {"0000001000010011111100001111111010011001000001000100000101100100",0x0000352D,0xFFFFE88F,0x00000548,0x000021E6,0xFFFFF3B5,0x000003AA,0x000021E6,0xFFFFF3B5,0x000003AA},
- {"0000001000010011111100001111111010011001000010100100100010000100",0x00003300,0xFFFFE835,0x0000057B,0x00001A85,0xFFFFF715,0x00000336,0x00001A85,0xFFFFF715,0x00000336},
- {"0000001000010011111010101001010011011110000001000100100010100100",0x000033FA,0xFFFFE851,0x00000565,0x00001A8E,0xFFFFF727,0x0000033B,0x00001A8E,0xFFFFF727,0x0000033B},
- {"0000001000010011111100001111110101000010110110100011100100100100",0x000039D3,0xFFFFE5D3,0x000005B0,0x00001888,0xFFFFF978,0x000002C8,0x00001888,0xFFFFF978,0x000002C8},
- {"0000001000010011111100001111111010011001000011100100100001100100",0x00002F6B,0xFFFFEC53,0x000004B9,0x00001C15,0xFFFFF71B,0x00000337,0x00001C15,0xFFFFF71B,0x00000337},
- {"0000001000010011111100001111111010011001000001100100000101000100",0x0000384D,0xFFFFE737,0x00000569,0x00001D2D,0xFFFFF673,0x00000343,0x00001D2D,0xFFFFF673,0x00000343},
- {"0000001000010011111100001111111010011001000001100010000010100100",0x00003A49,0xFFFFE70B,0x0000055F,0x00001A63,0xFFFFF8CD,0x000002E2,0x00001A63,0xFFFFF8CD,0x000002E2},
- {"0000001000010011111100001111111010011001000001000010100110000100",0x0000311E,0xFFFFEB97,0x000004C6,0x00001EAE,0xFFFFF5A9,0x00000367,0x00001EAE,0xFFFFF5A9,0x00000367},
- {"0000001000010011111100001111111010011001000011100001000100100100",0x000027D3,0xFFFFF075,0x00000417,0x00002001,0xFFFFF44A,0x000003A2,0x00002001,0xFFFFF44A,0x000003A2},
- {"0000001000010011111100001111111010011001000001100100100100000100",0x00003B72,0xFFFFE4BD,0x000005DC,0x00001D76,0xFFFFF606,0x0000035A,0x00001D76,0xFFFFF606,0x0000035A},
- {"0000001000010011111100001111111010011001000100000001000100100100",0x00002E0F,0xFFFFECA7,0x000004AE,0x00001DC6,0xFFFFF5BF,0x0000036A,0x00001DC6,0xFFFFF5BF,0x0000036A},
- {"0000001000010011111100001111111010011001000000100011100010100100",0x000032C7,0xFFFFEA7A,0x000004F0,0x00001A7B,0xFFFFF827,0x00000301,0x00001A7B,0xFFFFF827,0x00000301},
- {"0000001000010011111010101001010011011110000001000100100010000100",0x0000312D,0xFFFFEA39,0x00000515,0x00001948,0xFFFFF800,0x00000318,0x00001948,0xFFFFF800,0x00000318},
- {"0000001000010011111010101001010011011110000001100010000010000100",0x00003611,0xFFFFE8D7,0x00000533,0x00001929,0xFFFFF965,0x000002D2,0x00001929,0xFFFFF965,0x000002D2},
- {"0000001000010011111100001111111010011001001011000011000011100100",0x00002FE2,0xFFFFED89,0x00000470,0x00001A3C,0xFFFFF955,0x000002D5,0x00001A3C,0xFFFFF955,0x000002D5},
- {"0000001000010011111010101001010011011110000000100000100010100100",0x000035FF,0xFFFFE884,0x00000548,0x0000182A,0xFFFFF9AB,0x000002CF,0x0000182A,0xFFFFF9AB,0x000002CF},
- {"0000001000010011111100001111111010011001000000100010000011100100",0x00003597,0xFFFFE904,0x00000528,0x00001A94,0xFFFFF840,0x00000300,0x00001A94,0xFFFFF840,0x00000300},
- {"0000001000010011111100001111111010011001000110000001100101000100",0x000026CB,0xFFFFF1FB,0x000003E4,0x000017CC,0xFFFFFA25,0x000002C8,0x000017CC,0xFFFFFA25,0x000002C8},
- {"0000001000010011111010101001010011011110000001100000100011000100",0x00003274,0xFFFFEA39,0x0000050C,0x00001B20,0xFFFFF7C1,0x00000314,0x00001B20,0xFFFFF7C1,0x00000314},
- {"0000001000010011111100001111110101000010110110000010100100100100",0x0000280B,0xFFFFF283,0x000003B5,0x000018D0,0xFFFFF992,0x000002EC,0x000018D0,0xFFFFF992,0x000002EC},
- {"0000001000010011111100001111111010011001000001100010000100000100",0x000033AB,0xFFFFEB1B,0x000004C4,0x00001FEE,0xFFFFF53A,0x00000378,0x00001FEE,0xFFFFF53A,0x00000378},
- {"0000001000010011111100001111111010011001000010100011100101100100",0x00002F79,0xFFFFEB0C,0x000004FA,0x00001E57,0xFFFFF4BF,0x0000039B,0x00001E57,0xFFFFF4BF,0x0000039B},
- {"0000001000010011111100001111111010011001000001000100100011100100",0x00003487,0xFFFFE8F2,0x00000539,0x0000185B,0xFFFFF9AE,0x000002BA,0x0000185B,0xFFFFF9AE,0x000002BA},
- {"0000001000010011111100001111111010011001000010100001100010100100",0x00003500,0xFFFFE793,0x0000058A,0x00001AA2,0xFFFFF792,0x0000031D,0x00001AA2,0xFFFFF792,0x0000031D},
- {"0000001000010011111100001111111010011001000010000001000101100100",0x00003943,0xFFFFE54D,0x000005D9,0x00001BC8,0xFFFFF6E0,0x00000339,0x00001BC8,0xFFFFF6E0,0x00000339},
- {"0000001000010011111010101001010011011110000001000011000010100100",0x0000306D,0xFFFFEC5E,0x000004A5,0x00001A3A,0xFFFFF85F,0x00000304,0x00001A3A,0xFFFFF85F,0x00000304},
- {"0000001000010011111100001111110101000010110110000011000010000100",0x00002BA4,0xFFFFEE8D,0x0000046A,0x0000198C,0xFFFFF88E,0x00000307,0x0000198C,0xFFFFF88E,0x00000307},
- {"0000001000010011111100001111110101000010110100100001100011100100",0x00003D30,0xFFFFE2F6,0x0000062A,0x000025DC,0xFFFFF074,0x00000435,0x000025DC,0xFFFFF074,0x00000435},
- {"0000001000010011111100001111110101000010110110000011100101100100",0x00002CD6,0xFFFFED79,0x0000049B,0x000016D0,0xFFFFFA53,0x000002BB,0x000016D0,0xFFFFFA53,0x000002BB},
- {"0000001000010011111100001111111010011001000101100011000101100100",0x00002484,0xFFFFF3BD,0x000003A0,0x000015B8,0xFFFFFB6B,0x000002A4,0x000015B8,0xFFFFFB6B,0x000002A4},
- {"0000001000010011111100001111111010011001000011100011100101000100",0x000038AE,0xFFFFE6D1,0x00000587,0x00001A2A,0xFFFFF8F1,0x000002D4,0x00001A2A,0xFFFFF8F1,0x000002D4},
- {"0000001000010011111100001111111010011001000001000100100101000100",0x000036FD,0xFFFFE76C,0x00000576,0x00001EE4,0xFFFFF58D,0x00000361,0x00001EE4,0xFFFFF58D,0x00000361},
- {"0000001000010011111100001111110101000010110110000011000010100100",0x00002BCF,0xFFFFEF28,0x00000448,0x00001B93,0xFFFFF7BA,0x00000327,0x00001B93,0xFFFFF7BA,0x00000327},
- {"0000001000010011111100001111111010011001000001100010100010000100",0x00003834,0xFFFFE818,0x0000053B,0x00001AFE,0xFFFFF85C,0x000002F3,0x00001AFE,0xFFFFF85C,0x000002F3},
- {"0000001000010011111100001111111010011001001100100011000110100100",0x00002EF7,0xFFFFEBFC,0x000004CE,0x00001897,0xFFFFF8EF,0x000002EC,0x00001897,0xFFFFF8EF,0x000002EC},
- {"0000001000010011111100001111111010011001001011000001100011000100",0x000035BD,0xFFFFE8BB,0x0000053B,0x00001F22,0xFFFFF561,0x00000373,0x00001F22,0xFFFFF561,0x00000373},
- {"0000001000010011111100001111111010011001000110000011100110000100",0x00002D42,0xFFFFEE1D,0x00000478,0x000016F0,0xFFFFFAAE,0x000002B3,0x000016F0,0xFFFFFAAE,0x000002B3},
- {"0000001000010011111010101001010011011110000001000101000100100100",0x00002F98,0xFFFFEB3C,0x000004F0,0x00001903,0xFFFFF818,0x00000319,0x00001903,0xFFFFF818,0x00000319},
- {"0000001000010011111100001111110101000010110101000010000101000100",0x00004081,0xFFFFDF13,0x000006F3,0x00002A6D,0xFFFFEC1B,0x00000509,0x00002A6D,0xFFFFEC1B,0x00000509},
- {"0000001000010011111010101001010011011110000001000000100100000100",0x00002D68,0xFFFFED21,0x00000498,0x00001FF6,0xFFFFF427,0x000003B0,0x00001FF6,0xFFFFF427,0x000003B0},
- {"0000001000010011111100001111111010011001000000100011100010000100",0x00003243,0xFFFFEA5C,0x000004FD,0x000020FB,0xFFFFF39E,0x000003C0,0x000020FB,0xFFFFF39E,0x000003C0},
- {"0000001000010011111100001111110101000010110110000100100010100100",0x00002F20,0xFFFFEC19,0x000004C6,0x00001748,0xFFFFF99F,0x000002DA,0x00001748,0xFFFFF99F,0x000002DA},
- {"0000001000010011111100001111111010011001000100000011100110000100",0x00002D68,0xFFFFED21,0x00000498,0x00001A43,0xFFFFF843,0x000002F9,0x00001A43,0xFFFFF843,0x000002F9},
- {"0000001000010011111100001111111010011001000000100010000010100100",0x0000396E,0xFFFFE616,0x000005A9,0x00001A51,0xFFFFF850,0x000002FA,0x00001A51,0xFFFFF850,0x000002FA},
- {"0000001000010011111100001111111010011001000001000011000101000100",0x0000305C,0xFFFFED4B,0x0000046C,0x00001CF9,0xFFFFF7BA,0x00000304,0x00001CF9,0xFFFFF7BA,0x00000304},
- {"0000001000010011111100001111110101000010110110100100000101100100",0x0000343C,0xFFFFE869,0x00000559,0x00001CE2,0xFFFFF614,0x00000359,0x00001CE2,0xFFFFF614,0x00000359},
- {"0000001000010011111100001111111010011001000110000011100101100100",0x00002782,0xFFFFF1FE,0x000003D9,0x000015DC,0xFFFFFB8B,0x00000290,0x000015DC,0xFFFFFB8B,0x00000290},
- {"0000001000010011111100001111111010011001000110000001100011000100",0x00002B9C,0xFFFFEF63,0x00000443,0x00001369,0xFFFFFD51,0x00000244,0x00001369,0xFFFFFD51,0x00000244},
- {"0000001000010011111100001111111010011001000010100010000010000100",0x000035F8,0xFFFFE743,0x00000592,0x000018D8,0xFFFFF8EE,0x000002E4,0x000018D8,0xFFFFF8EE,0x000002E4},
- {"0000001000010011111010101001010011011110000001100010100001000100",0x00002B72,0xFFFFEF1E,0x0000043C,0x00002647,0xFFFFF092,0x0000043E,0x00002647,0xFFFFF092,0x0000043E},
- {"0000001000010011111100001111111010011001000100000010000110000100",0x00002EC9,0xFFFFEC5F,0x000004B8,0x000018B6,0xFFFFF936,0x000002D8,0x000018B6,0xFFFFF936,0x000002D8},
- {"0000001000010011111100001111111010011001000001100100000010000100",0x000038A7,0xFFFFE6AC,0x00000589,0x00001C42,0xFFFFF70B,0x00000329,0x00001C42,0xFFFFF70B,0x00000329},
- {"0000001000010011111100001111111010011001001100000000100010100100",0x00002F6B,0xFFFFEBF6,0x000004CF,0x000018AE,0xFFFFF928,0x000002E3,0x000018AE,0xFFFFF928,0x000002E3},
- {"0000001000010011111100001111110101000010110110100101000100000100",0x000029CD,0xFFFFEEE1,0x00000459,0x00001AB5,0xFFFFF76F,0x00000324,0x00001AB5,0xFFFFF76F,0x00000324},
- {"0000001000010011111010101001010011011110000001100011100011000100",0x00003921,0xFFFFE71D,0x00000577,0x00001646,0xFFFFFB24,0x00000293,0x00001646,0xFFFFFB24,0x00000293},
- {"0000001000010011111010101001010011011110000001000100000101100100",0x00003940,0xFFFFE521,0x000005E8,0x00001947,0xFFFFF839,0x0000030D,0x00001947,0xFFFFF839,0x0000030D},
- {"0000001000010011111100001111110101000010110100100100000101100100",0x00003DCA,0xFFFFE211,0x00000659,0x0000250E,0xFFFFF072,0x00000443,0x0000250E,0xFFFFF072,0x00000443},
- {"0000001000010011111100001111111010011001000011000000100100000100",0x00002E95,0xFFFFEC20,0x000004C9,0x000015B4,0xFFFFFAD3,0x0000029D,0x000015B4,0xFFFFFAD3,0x0000029D},
- {"0000001000010011111100001111111010011001000001000001000010000100",0x00002C11,0xFFFFEE6E,0x00000468,0x00001901,0xFFFFF924,0x000002E7,0x00001901,0xFFFFF924,0x000002E7},
- {"0000001000010011111010101001010011011110000001100010000100000100",0x0000293F,0xFFFFF158,0x000003E6,0x0000183F,0xFFFFF9F6,0x000002D2,0x0000183F,0xFFFFF9F6,0x000002D2},
- {"0000001000010011111100001111111010011001000011100001000100000100",0x00002A67,0xFFFFEF34,0x0000043E,0x00001C6F,0xFFFFF6F1,0x0000032B,0x00001C6F,0xFFFFF6F1,0x0000032B},
- {"0000001000010011111010101001010011011110000001100101000100100100",0x00002F8D,0xFFFFEB77,0x000004DA,0x00001C0D,0xFFFFF627,0x00000365,0x00001C0D,0xFFFFF627,0x00000365},
- {"0000001000010011111100001111111010011001000011000011100011000100",0x00003476,0xFFFFEA5B,0x000004E7,0x00001DBF,0xFFFFF6C7,0x00000333,0x00001DBF,0xFFFFF6C7,0x00000333},
- {"0000001000010011111100001111111010011001000011100000100101000100",0x00003336,0xFFFFE92F,0x00000546,0x00001614,0xFFFFFAE0,0x00000296,0x00001614,0xFFFFFAE0,0x00000296},
- {"0000001000010011111100001111111010011001000101100010000101100100",0x00002513,0xFFFFF323,0x000003BC,0x000016DB,0xFFFFFA79,0x000002CD,0x000016DB,0xFFFFFA79,0x000002CD},
- {"0000001000010011111100001111111010011001000010100010100101000100",0x000035A7,0xFFFFE78E,0x00000584,0x00001B0D,0xFFFFF77D,0x0000031F,0x00001B0D,0xFFFFF77D,0x0000031F},
- {"0000001000010011111100001111111010011001001100100011100011100100",0x00003171,0xFFFFEB98,0x000004C6,0x00001C76,0xFFFFF71F,0x0000032F,0x00001C76,0xFFFFF71F,0x0000032F},
- {"0000001000010011111100001111110101000010110110100001000010000100",0x00002C52,0xFFFFED2E,0x000004A7,0x00002182,0xFFFFF2F4,0x000003E4,0x00002182,0xFFFFF2F4,0x000003E4},
- {"0000001000010011111100001111111010011001000100000010100100100100",0x000032E1,0xFFFFEB39,0x000004D0,0x00001B55,0xFFFFF859,0x000002FA,0x00001B55,0xFFFFF859,0x000002FA},
- {"0000001000010011111100001111111010011001000110000100100010100100",0x000029B6,0xFFFFEFF7,0x00000430,0x0000151B,0xFFFFFBC6,0x0000027F,0x0000151B,0xFFFFFBC6,0x0000027F},
- {"0000001000010011111100001111110101000010110110100001100101100100",0x00002FF7,0xFFFFEB67,0x000004DA,0x000020E9,0xFFFFF363,0x000003CE,0x000020E9,0xFFFFF363,0x000003CE},
- {"0000001000010011111100001111110101000010110110100101000100100100",0x00003CDD,0xFFFFE2B2,0x00000649,0x00001B18,0xFFFFF739,0x00000329,0x00001B18,0xFFFFF739,0x00000329},
- {"0000001000010011111100001111111010011001000001100010100010100100",0x00003C82,0xFFFFE5C6,0x0000058E,0x00001F3F,0xFFFFF5AD,0x00000361,0x00001F3F,0xFFFFF5AD,0x00000361},
- {"0000001000010011111100001111110101000010110111000100000010000100",0x0000319B,0xFFFFEA15,0x0000051B,0x00001CC9,0xFFFFF62E,0x00000358,0x00001CC9,0xFFFFF62E,0x00000358},
- {"0000001000010011111010101001010011011110000001100011100011100100",0x000032B6,0xFFFFEB2B,0x000004D6,0x000018E0,0xFFFFF966,0x000002DE,0x000018E0,0xFFFFF966,0x000002DE},
- {"0000001000010011111010101001010011011110000000100011100110000100",0x0000300A,0xFFFFEBA6,0x000004D1,0x00001CFD,0xFFFFF5F6,0x0000036D,0x00001CFD,0xFFFFF5F6,0x0000036D},
- {"0000001000010011111100001111110101000010110110000010100110000100",0x000026A9,0xFFFFF15D,0x00000400,0x00001561,0xFFFFFB1F,0x000002A0,0x00001561,0xFFFFFB1F,0x000002A0},
- {"0000001000010011111100001111111010011001000011100101000100100100",0x00003123,0xFFFFEAD2,0x000004FA,0x000018CB,0xFFFFF8F5,0x000002EC,0x000018CB,0xFFFFF8F5,0x000002EC},
- {"0000001000010011111100001111111010011001000110000100000011000100",0x00003577,0xFFFFE935,0x00000533,0x000016CD,0xFFFFFB44,0x00000289,0x000016CD,0xFFFFFB44,0x00000289},
- {"0000001000010011111100001111111010011001001010000010000110000100",0x00002875,0xFFFFF170,0x000003F3,0x00001567,0xFFFFFBD5,0x00000289,0x00001567,0xFFFFFBD5,0x00000289},
- {"0000001000010011111100001111111010011001000010000100000010000100",0x00003AE2,0xFFFFE538,0x000005C1,0x00001CB4,0xFFFFF6A3,0x0000033C,0x00001CB4,0xFFFFF6A3,0x0000033C},
- {"0000001000010011111100001111111010011001000011000011100011100100",0x000031DF,0xFFFFEC2A,0x000004A3,0x00001EF0,0xFFFFF626,0x00000352,0x00001EF0,0xFFFFF626,0x00000352},
- {"0000001000010011111100001111110101000010110100100101000101000100",0x00004A6A,0xFFFFDB15,0x00000758,0x000027F3,0xFFFFEEEE,0x00000479,0x000027F3,0xFFFFEEEE,0x00000479},
- {"0000001000010011111010101001010011011110000001100011100100000100",0x00002BB9,0xFFFFEF5D,0x00000433,0x00001589,0xFFFFFB57,0x00000295,0x00001589,0xFFFFFB57,0x00000295},
- {"0000001000010011111100001111111010011001000001000010000101100100",0x000033A0,0xFFFFE98F,0x00000528,0x00001CB4,0xFFFFF706,0x0000032D,0x00001CB4,0xFFFFF706,0x0000032D},
- {"0000001000010011111100001111111010011001000101100011000001100100",0x0000248E,0xFFFFF380,0x000003AC,0x000016EA,0xFFFFFA6C,0x000002CE,0x000016EA,0xFFFFFA6C,0x000002CE},
- {"0000001000010011111100001111111010011001000000100010000110100100",0x00002FE2,0xFFFFEB2F,0x000004E9,0x00001D4E,0xFFFFF56B,0x00000380,0x00001D4E,0xFFFFF56B,0x00000380},
- {"0000001000010011111100001111111010011001000010100010100010000100",0x00003283,0xFFFFE9E7,0x0000051D,0x00000694,0xFFFFFD32,0x000003C3,0x00000694,0xFFFFFD32,0x000003C3},
- {"0000001000010011111100001111110101000010110110000101000011000100",0x00002EE4,0xFFFFEBFD,0x000004D3,0x0000151A,0xFFFFFAF6,0x000002A4,0x0000151A,0xFFFFFAF6,0x000002A4},
- {"0000001000010011111100001111110101000010110111000001100011100100",0x0000302D,0xFFFFEB7F,0x000004DA,0x00001E6D,0xFFFFF54B,0x00000380,0x00001E6D,0xFFFFF54B,0x00000380},
- {"0000001000010011111100001111110101000010110110100101000011000100",0x000033DA,0xFFFFE7FB,0x0000057F,0x00001DED,0xFFFFF50E,0x0000038D,0x00001DED,0xFFFFF50E,0x0000038D},
- {"0000001000010011111100001111111010011001001011000100000010000100",0x000030B5,0xFFFFEBB8,0x000004C4,0x00001C3F,0xFFFFF726,0x0000032A,0x00001C3F,0xFFFFF726,0x0000032A},
- {"0000001000010011111100001111111010011001000010000011000111000100",0x00003BBD,0xFFFFE55C,0x000005B8,0x000019DB,0xFFFFF8BB,0x000002EF,0x000019DB,0xFFFFF8BB,0x000002EF},
- {"0000001000010011111100001111111010011001000011100011100010000100",0x00002964,0xFFFFF051,0x0000040E,0x000025CD,0xFFFFF11B,0x0000041F,0x000025CD,0xFFFFF11B,0x0000041F},
- {"0000001000010011111100001111110101000010110111000100100010000100",0x000033F5,0xFFFFE863,0x00000560,0x00001BCE,0xFFFFF689,0x0000034B,0x00001BCE,0xFFFFF689,0x0000034B},
- {"0000001000010011111100001111111010011001000010100010100001100100",0x00003294,0xFFFFE924,0x00000548,0x00001D41,0xFFFFF580,0x0000037D,0x00001D41,0xFFFFF580,0x0000037D},
- {"0000001000010011111100001111110101000010110111000011100110100100",0x000034FB,0xFFFFE7FE,0x0000056D,0x00001CB1,0xFFFFF635,0x00000357,0x00001CB1,0xFFFFF635,0x00000357},
- {"0000001000010011111100001111111010011001000010100001000010100100",0x00002E28,0xFFFFEBB9,0x000004E0,0x00001B20,0xFFFFF6E3,0x0000033C,0x00001B20,0xFFFFF6E3,0x0000033C},
- {"0000001000010011111100001111110101000010110110100001100100000100",0x00002799,0xFFFFF0F4,0x000003FC,0x00001C9D,0xFFFFF6A1,0x00000345,0x00001C9D,0xFFFFF6A1,0x00000345},
- {"0000001000010011111100001111111010011001000001100100000100000100",0x00003AEA,0xFFFFE5DB,0x0000059D,0x00001B61,0xFFFFF7F0,0x00000301,0x00001B61,0xFFFFF7F0,0x00000301},
- {"0000001000010011111010101001010011011110000001000001100110000100",0x000031F6,0xFFFFEAB8,0x000004F3,0x00001D90,0xFFFFF622,0x00000359,0x00001D90,0xFFFFF622,0x00000359},
- {"0000001000010011111100001111111010011001000011000100000001100100",0x000031B8,0xFFFFEA61,0x0000050F,0x0000199D,0xFFFFF87C,0x000002FD,0x0000199D,0xFFFFF87C,0x000002FD},
- {"0000001000010011111100001111110101000010110100100011000101000100",0x00004514,0xFFFFDDFF,0x000006F6,0x000022CD,0xFFFFF29F,0x000003D9,0x000022CD,0xFFFFF29F,0x000003D9},
- {"0000001000010011111010101001010011011110000001000011000101100100",0x00002F30,0xFFFFECB8,0x000004A0,0x00001B07,0xFFFFF7E2,0x00000313,0x00001B07,0xFFFFF7E2,0x00000313},
- {"0000001000010011111100001111110101000010110111000011000010100100",0x0000383B,0xFFFFE702,0x00000581,0x00001A08,0xFFFFF8CA,0x000002E2,0x00001A08,0xFFFFF8CA,0x000002E2},
- {"0000001000010011111100001111111010011001000000100010000101100100",0x00002CC5,0xFFFFEDF8,0x00000465,0x00001F47,0xFFFFF4B2,0x00000393,0x00001F47,0xFFFFF4B2,0x00000393},
- {"0000001000010011111100001111111010011001000101100010000111000100",0x00002304,0xFFFFF453,0x00000384,0x0000170A,0xFFFFFA3F,0x000002CE,0x0000170A,0xFFFFFA3F,0x000002CE},
- {"0000001000010011111100001111111010011001000010100101000100100100",0x0000337E,0xFFFFE850,0x0000056E,0x00001BDD,0xFFFFF668,0x00000353,0x00001BDD,0xFFFFF668,0x00000353},
- {"0000001000010011111100001111111010011001000011100100100100100100",0x00002E2F,0xFFFFEC9B,0x000004AE,0x00001C4D,0xFFFFF6D3,0x00000338,0x00001C4D,0xFFFFF6D3,0x00000338},
- {"0000001000010011111010101001010011011110000001100001000100100100",0x00002DDD,0xFFFFEDA4,0x00000477,0x00002010,0xFFFFF4BB,0x00000390,0x00002010,0xFFFFF4BB,0x00000390},
- {"0000001000010011111100001111110101000010110110100100100011100100",0x0000290C,0xFFFFEF61,0x00000445,0x00002133,0xFFFFF324,0x000003D8,0x00002133,0xFFFFF324,0x000003D8},
- {"0000001000010011111100001111111010011001000001100010100100100100",0x0000371E,0xFFFFE8D5,0x00000524,0x00001C3A,0xFFFFF7AE,0x00000314,0x00001C3A,0xFFFFF7AE,0x00000314},
- {"0000001000010011111100001111110101000010110110000011100011100100",0x00002A58,0xFFFFF007,0x00000429,0x000018A6,0xFFFFF98F,0x000002E1,0x000018A6,0xFFFFF98F,0x000002E1},
- {"0000001000010011111100001111111010011001000000100011000010000100",0x00002FED,0xFFFFEC48,0x000004AA,0x00001E9D,0xFFFFF584,0x00000370,0x00001E9D,0xFFFFF584,0x00000370},
- {"0000001000010011111100001111111010011001000110000001100010000100",0x00002829,0xFFFFF15F,0x000003F7,0x0000157E,0xFFFFFBD4,0x00000282,0x0000157E,0xFFFFFBD4,0x00000282},
- {"0000001000010011111100001111111010011001000100000001100100100100",0x000030CF,0xFFFFEB8D,0x000004CE,0x00001A4C,0xFFFFF868,0x000002F7,0x00001A4C,0xFFFFF868,0x000002F7},
- {"0000001000010011111100001111110101000010110110100010000010000100",0x00002C8F,0xFFFFEDD2,0x0000047D,0x00001CCE,0xFFFFF6A1,0x00000343,0x00001CCE,0xFFFFF6A1,0x00000343},
- {"0000001000010011111100001111111010011001000110000010000101100100",0x00002A84,0xFFFFEFBA,0x0000043E,0x000015EF,0xFFFFFB4B,0x0000029E,0x000015EF,0xFFFFFB4B,0x0000029E},
- {"0000001000010011111100001111111010011001000011000010100010100100",0x000034CA,0xFFFFEA08,0x000004FF,0x00001C19,0xFFFFF7ED,0x00000309,0x00001C19,0xFFFFF7ED,0x00000309},
- {"0000001000010011111100001111111010011001000101100011100110100100",0x00002187,0xFFFFF4B0,0x0000037E,0x0000154A,0xFFFFFB0C,0x000002AE,0x0000154A,0xFFFFFB0C,0x000002AE},
- {"0000001000010011111100001111110101000010110110100011100001000100",0x00002F4F,0xFFFFEB3C,0x000004F8,0x0000181F,0xFFFFF92D,0x000002DF,0x0000181F,0xFFFFF92D,0x000002DF},
- {"0000001000010011111100001111111010011001000001000001000011100100",0x0000290C,0xFFFFF0B1,0x000003FC,0x00001DB0,0xFFFFF636,0x00000355,0x00001DB0,0xFFFFF636,0x00000355},
- {"0000001000010011111100001111111010011001000010100001000001100100",0x000034C1,0xFFFFE888,0x0000055A,0x000019BF,0xFFFFF881,0x000002FB,0x000019BF,0xFFFFF881,0x000002FB},
- {"0000001000010011111100001111110101000010110111000001100011000100",0x00003139,0xFFFFEA98,0x00000504,0x000019F2,0xFFFFF820,0x0000030B,0x000019F2,0xFFFFF820,0x0000030B},
- {"0000001000010011111100001111110101000010110110000011000101000100",0x00002CAC,0xFFFFEEB2,0x00000458,0x0000152C,0xFFFFFBEF,0x0000027B,0x0000152C,0xFFFFFBEF,0x0000027B},
- {"0000001000010011111100001111111010011001001011000011100011100100",0x00003577,0xFFFFE99C,0x0000050D,0x00001E64,0xFFFFF679,0x0000033F,0x00001E64,0xFFFFF679,0x0000033F},
- {"0000001000010011111100001111110101000010110110100100000100000100",0x0000263A,0xFFFFF1E4,0x000003D4,0x00001F68,0xFFFFF4ED,0x00000386,0x00001F68,0xFFFFF4ED,0x00000386},
- {"0000001000010011111100001111110101000010110110000001100110000100",0x00002CE9,0xFFFFED63,0x00000497,0x00001810,0xFFFFF94D,0x000002E3,0x00001810,0xFFFFF94D,0x000002E3},
- {"0000001000010011111010101001010011011110000001000100000100000100",0x0000318A,0xFFFFEAC8,0x000004F5,0x0000195C,0xFFFFF896,0x000002FB,0x0000195C,0xFFFFF896,0x000002FB},
- {"0000001000010011111100001111110101000010110110000011100100000100",0x00002C41,0xFFFFEEC6,0x0000045D,0x000017DD,0xFFFFFA16,0x000002CB,0x000017DD,0xFFFFFA16,0x000002CB},
- {"0000001000010011111100001111111010011001000000100011000110100100",0x00002DD4,0xFFFFEC98,0x000004AD,0x00001BD7,0xFFFFF69F,0x00000347,0x00001BD7,0xFFFFF69F,0x00000347},
- {"0000001000010011111100001111110101000010110110100011100101000100",0x00003351,0xFFFFE9B2,0x0000051A,0x00001CA1,0xFFFFF6A4,0x00000341,0x00001CA1,0xFFFFF6A4,0x00000341},
- {"0000001000010011111100001111111010011001000000100001000100000100",0x0000322D,0xFFFFE9BE,0x00000527,0x00001CF9,0xFFFFF5EB,0x00000366,0x00001CF9,0xFFFFF5EB,0x00000366},
- {"0000001000010011111100001111111010011001000011000010100011000100",0x00003678,0xFFFFE9A8,0x00000503,0x00001AD4,0xFFFFF8F6,0x000002E3,0x00001AD4,0xFFFFF8F6,0x000002E3},
- {"0000001000010011111100001111111010011001000101100001100100100100",0x0000260E,0xFFFFF2C1,0x000003CA,0x00001139,0xFFFFFE48,0x00000236,0x00001139,0xFFFFFE48,0x00000236},
- {"0000001000010011111100001111111010011001000010100010000101100100",0x000033D3,0xFFFFE872,0x00000565,0x00001B72,0xFFFFF713,0x00000332,0x00001B72,0xFFFFF713,0x00000332},
- {"0000001000010011111100001111111010011001001100100011100001000100",0x0000309B,0xFFFFEB42,0x000004E4,0x00001918,0xFFFFF8C8,0x000002F2,0x00001918,0xFFFFF8C8,0x000002F2},
- {"0000001000010011111100001111111010011001000110000010100001100100",0x000028B8,0xFFFFF105,0x00000402,0x000018BB,0xFFFFF9BC,0x000002D3,0x000018BB,0xFFFFF9BC,0x000002D3},
- {"0000001000010011111100001111111010011001000010100001100010000100",0x00003123,0xFFFFE9D1,0x00000534,0x00001B19,0xFFFFF6FE,0x0000033C,0x00001B19,0xFFFFF6FE,0x0000033C},
- {"0000001000010011111100001111111010011001000000100010000101000100",0x00003216,0xFFFFEA8E,0x000004F6,0x00001F72,0xFFFFF4CE,0x0000038B,0x00001F72,0xFFFFF4CE,0x0000038B},
- {"0000001000010011111100001111111010011001000101100010100101100100",0x00002564,0xFFFFF32D,0x000003B6,0x00001685,0xFFFFFADB,0x000002BB,0x00001685,0xFFFFFADB,0x000002BB},
- {"0000001000010011111100001111110101000010110110100010100100100100",0x00002E60,0xFFFFED13,0x00000497,0x00001CA5,0xFFFFF6B9,0x00000346,0x00001CA5,0xFFFFF6B9,0x00000346},
- {"0000001000010011111100001111111010011001000011100011100110100100",0x0000336D,0xFFFFE934,0x0000053B,0x00001B3E,0xFFFFF763,0x00000327,0x00001B3E,0xFFFFF763,0x00000327},
- {"0000001000010011111100001111111010011001000100000001000010000100",0x0000274A,0xFFFFF119,0x000003FA,0x00001D75,0xFFFFF5CD,0x0000036F,0x00001D75,0xFFFFF5CD,0x0000036F},
- {"0000001000010011111100001111110101000010110110100010000101100100",0x0000366B,0xFFFFE70A,0x0000059A,0x00001ED8,0xFFFFF501,0x00000389,0x00001ED8,0xFFFFF501,0x00000389},
- {"0000001000010011111100001111111010011001001000100011100101100100",0x00003164,0xFFFFEAB4,0x000004FA,0x00001C52,0xFFFFF6E0,0x00000336,0x00001C52,0xFFFFF6E0,0x00000336},
- {"0000001000010011111100001111110101000010110100100011000001100100",0x00004224,0xFFFFDF7F,0x000006C1,0x00002A52,0xFFFFED5E,0x000004BB,0x00002A52,0xFFFFED5E,0x000004BB},
- {"0000001000010011111100001111111010011001000100000010100001100100",0x000030E3,0xFFFFEB07,0x000004ED,0x00001FD3,0xFFFFF46D,0x000003A1,0x00001FD3,0xFFFFF46D,0x000003A1},
- {"0000001000010011111100001111110101000010110110000010100010000100",0x00002AEB,0xFFFFEF1B,0x00000454,0x00001829,0xFFFFF995,0x000002DD,0x00001829,0xFFFFF995,0x000002DD},
- {"0000001000010011111100001111110101000010110111000101000011100100",0x0000346B,0xFFFFE7A2,0x0000058B,0x000020C5,0xFFFFF2E8,0x000003EC,0x000020C5,0xFFFFF2E8,0x000003EC},
- {"0000001000010011111100001111110101000010110111000100000101100100",0x000039CF,0xFFFFE5D7,0x000005A9,0x00001D66,0xFFFFF5D6,0x00000366,0x00001D66,0xFFFFF5D6,0x00000366},
- {"0000001000010011111100001111111010011001000001000001100011100100",0x000034AC,0xFFFFE9AE,0x00000515,0x00001A28,0xFFFFF904,0x000002DC,0x00001A28,0xFFFFF904,0x000002DC},
- {"0000001000010011111100001111110101000010110111000010000010000100",0x00002D68,0xFFFFED21,0x00000498,0x00001C6F,0xFFFFF686,0x0000034C,0x00001C6F,0xFFFFF686,0x0000034C},
- {"0000001000010011111100001111111010011001000010000010000011000100",0x0000328B,0xFFFFEBA1,0x000004B4,0x00001DA3,0xFFFFF683,0x00000349,0x00001DA3,0xFFFFF683,0x00000349},
- {"0000001000010011111100001111111010011001000110000010100011000100",0x000027DC,0xFFFFF295,0x000003BF,0x000019C1,0xFFFFF98E,0x000002E8,0x000019C1,0xFFFFF98E,0x000002E8},
- {"0000001000010011111100001111111010011001000110000100000010000100",0x00002756,0xFFFFF1D7,0x000003DF,0x000015D9,0xFFFFFB51,0x00000298,0x000015D9,0xFFFFFB51,0x00000298},
- {"0000001000010011111100001111111010011001000010000011100010000100",0x00003526,0xFFFFE907,0x00000526,0x000017AB,0xFFFFFA12,0x000002AB,0x000017AB,0xFFFFFA12,0x000002AB},
- {"0000001000010011111100001111110101000010110110100001100011100100",0x0000351B,0xFFFFE8B7,0x00000540,0x00001A86,0xFFFFF821,0x00000303,0x00001A86,0xFFFFF821,0x00000303},
- {"0000001000010011111100001111111010011001000101100100000101000100",0x000024B2,0xFFFFF34E,0x000003B1,0x000018E2,0xFFFFF926,0x000002FC,0x000018E2,0xFFFFF926,0x000002FC},
- {"0000001000010011111100001111110101000010110110000010100010100100",0x00002F36,0xFFFFED5D,0x00000486,0x0000157A,0xFFFFFB85,0x00000293,0x0000157A,0xFFFFFB85,0x00000293},
- {"0000001000010011111100001111110101000010110111000101000011000100",0x00003A6E,0xFFFFE456,0x000005FD,0x00001F68,0xFFFFF3D1,0x000003C3,0x00001F68,0xFFFFF3D1,0x000003C3},
- {"0000001000010011111100001111111010011001000010100011000110100100",0x00002BC3,0xFFFFED2D,0x000004A7,0x00001C3F,0xFFFFF609,0x00000364,0x00001C3F,0xFFFFF609,0x00000364},
- {"0000001000010011111100001111111010011001000011100010000010000100",0x000032E1,0xFFFFEA83,0x000004F6,0x00001B37,0xFFFFF842,0x000002F5,0x00001B37,0xFFFFF842,0x000002F5},
- {"0000001000010011111100001111110101000010110110000011000110000100",0x000028E3,0xFFFFF07F,0x00000412,0x00001676,0xFFFFFA68,0x000002BE,0x00001676,0xFFFFFA68,0x000002BE},
- {"0000001000010011111100001111110101000010110100100001000100000100",0x0000444C,0xFFFFDDAD,0x00000712,0x00002634,0xFFFFEF89,0x0000046C,0x00002634,0xFFFFEF89,0x0000046C},
- {"0000001000010011111100001111111010011001000001000001100011000100",0x00003121,0xFFFFEBBB,0x000004C6,0x00001C98,0xFFFFF72B,0x0000032D,0x00001C98,0xFFFFF72B,0x0000032D},
- {"0000001000010011111100001111110101000010110110000100000010100100",0x00002C31,0xFFFFEDC4,0x00000490,0x0000162D,0xFFFFFA8E,0x000002B4,0x0000162D,0xFFFFFA8E,0x000002B4},
- {"0000001000010011111100001111110101000010110110100001100011000100",0x00002749,0xFFFFF112,0x000003FC,0x00001C85,0xFFFFF6B8,0x00000342,0x00001C85,0xFFFFF6B8,0x00000342},
- {"0000001000010011111100001111111010011001000001000100000100000100",0x00003159,0xFFFFEB99,0x000004C2,0x00001BD0,0xFFFFF7CA,0x00000307,0x00001BD0,0xFFFFF7CA,0x00000307},
- {"0000001000010011111100001111111010011001000101100100000101100100",0x00002610,0xFFFFF1FD,0x000003EC,0x000016BE,0xFFFFFA53,0x000002CB,0x000016BE,0xFFFFFA53,0x000002CB},
- {"0000001000010011111100001111111010011001000000100011000110000100",0x000037B5,0xFFFFE63D,0x000005B5,0x00002285,0xFFFFF25D,0x000003F7,0x00002285,0xFFFFF25D,0x000003F7},
- {"0000001000010011111100001111111010011001000010100010100010100100",0x00002FEE,0xFFFFEB47,0x000004EF,0x00001CBE,0xFFFFF64E,0x00000358,0x00001CBE,0xFFFFF64E,0x00000358},
- {"0000001000010011111100001111111010011001000100000101000100000100",0x00002E90,0xFFFFEC48,0x000004C0,0x00001A47,0xFFFFF7D1,0x0000031A,0x00001A47,0xFFFFF7D1,0x0000031A},
- {"0000001000010011111100001111110101000010110110100100000010000100",0x000034AB,0xFFFFE84A,0x00000559,0x00001A72,0xFFFFF79A,0x0000031C,0x00001A72,0xFFFFF79A,0x0000031C},
- {"0000001000010011111100001111111010011001000110000011100010000100",0x00002F7B,0xFFFFECFC,0x0000049C,0x00001814,0xFFFFFA22,0x000002C2,0x00001814,0xFFFFFA22,0x000002C2},
- {"0000001000010011111100001111111010011001000000100001100101100100",0x00003618,0xFFFFE709,0x00000596,0x00001EBF,0xFFFFF482,0x000003A5,0x00001EBF,0xFFFFF482,0x000003A5},
- {"0000001000010011111010101001010011011110000000100100100100000100",0x0000341B,0xFFFFE8B2,0x0000054F,0x00001D26,0xFFFFF578,0x00000388,0x00001D26,0xFFFFF578,0x00000388},
- {"0000001000010011111100001111111010011001000100000010000101000100",0x000030F6,0xFFFFEB89,0x000004CD,0x000019C0,0xFFFFF8CC,0x000002E6,0x000019C0,0xFFFFF8CC,0x000002E6},
- {"0000001000010011111100001111111010011001001010000100000110100100",0x00002B76,0xFFFFEF6C,0x00000444,0x00001563,0xFFFFFBBE,0x0000028D,0x00001563,0xFFFFFBBE,0x0000028D},
- {"0000001000010011111100001111110101000010110110000001100001100100",0x00002BA2,0xFFFFEE31,0x0000047F,0x00001A3D,0xFFFFF7F3,0x00000320,0x00001A3D,0xFFFFF7F3,0x00000320},
- {"0000001000010011111100001111111010011001001011000100100011100100",0x00003545,0xFFFFE87A,0x0000054A,0x00001B5A,0xFFFFF7B0,0x0000030C,0x00001B5A,0xFFFFF7B0,0x0000030C},
- {"0000001000010011111010101001010011011110000001000010100101000100",0x00003879,0xFFFFE73F,0x00000578,0x00001649,0xFFFFFB57,0x00000283,0x00001649,0xFFFFFB57,0x00000283},
- {"0000001000010011111100001111110101000010110110000100000011000100",0x00002772,0xFFFFF0F1,0x00000410,0x0000142F,0xFFFFFBCF,0x00000287,0x0000142F,0xFFFFFBCF,0x00000287},
- {"0000001000010011111100001111110101000010110110100011000110000100",0x00003228,0xFFFFE98E,0x00000535,0x00001F48,0xFFFFF495,0x00000399,0x00001F48,0xFFFFF495,0x00000399},
- {"0000001000010011111100001111111010011001000011100100000011100100",0x00002887,0xFFFFF119,0x000003E8,0x000021AA,0xFFFFF3F5,0x000003A5,0x000021AA,0xFFFFF3F5,0x000003A5},
- {"0000001000010011111100001111110101000010110110100010100010100100",0x0000301F,0xFFFFEBB2,0x000004D2,0x00001C02,0xFFFFF736,0x0000032B,0x00001C02,0xFFFFF736,0x0000032B},
- {"0000001000010011111100001111111010011001000110000010000010100100",0x00002E13,0xFFFFEE3F,0x00000468,0x000016AC,0xFFFFFB32,0x0000029E,0x000016AC,0xFFFFFB32,0x0000029E},
- {"0000001000010011111100001111111010011001000001000100100100100100",0x00003478,0xFFFFE8F9,0x00000538,0x00001DAB,0xFFFFF645,0x00000345,0x00001DAB,0xFFFFF645,0x00000345},
- {"0000001000010011111100001111111010011001000001100000100011000100",0x000030C6,0xFFFFEB6C,0x000004D4,0x0000184A,0xFFFFF934,0x000002E1,0x0000184A,0xFFFFF934,0x000002E1},
- {"0000001000010011111100001111111010011001000010100010000001000100",0x00002F1B,0xFFFFEBD3,0x000004D3,0x000019E7,0xFFFFF813,0x0000030D,0x000019E7,0xFFFFF813,0x0000030D},
- {"0000001000010011111100001111111010011001000000100011100100000100",0x00003214,0xFFFFEAE9,0x000004E0,0x0000178F,0xFFFFFA1C,0x000002B1,0x0000178F,0xFFFFFA1C,0x000002B1},
- {"0000001000010011111100001111110101000010110111000011000101000100",0x0000399C,0xFFFFE738,0x0000055E,0x00001EA1,0xFFFFF5E7,0x0000035A,0x00001EA1,0xFFFFF5E7,0x0000035A},
- {"0000001000010011111100001111111010011001000001100101000011000100",0x00003A01,0xFFFFE5B2,0x000005B6,0x00001D95,0xFFFFF5D2,0x0000036A,0x00001D95,0xFFFFF5D2,0x0000036A},
- {"0000001000010011111100001111111010011001000001000011100010000100",0x0000310D,0xFFFFEB78,0x000004D0,0x00001C06,0xFFFFF76E,0x0000031A,0x00001C06,0xFFFFF76E,0x0000031A},
- {"0000001000010011111100001111111010011001000001100011100001100100",0x00003CD1,0xFFFFE42F,0x000005EB,0x00001933,0xFFFFF91F,0x000002D4,0x00001933,0xFFFFF91F,0x000002D4},
- {"0000001000010011111100001111110101000010110110100011000101100100",0x00003119,0xFFFFEB1B,0x000004E1,0x00001FC7,0xFFFFF46A,0x000003A2,0x00001FC7,0xFFFFF46A,0x000003A2},
- {"0000001000010011111010101001010011011110000001100100100010100100",0x0000390D,0xFFFFE566,0x000005D8,0x00001EC6,0xFFFFF4DC,0x00000391,0x00001EC6,0xFFFFF4DC,0x00000391},
- {"0000001000010011111100001111110101000010110110100001000011000100",0x00003446,0xFFFFE858,0x00000561,0x00001FDB,0xFFFFF3FF,0x000003B9,0x00001FDB,0xFFFFF3FF,0x000003B9},
- {"0000001000010011111100001111111010011001000001000100100100000100",0x000032BA,0xFFFFEA07,0x00000511,0x00001B25,0xFFFFF7C9,0x0000030D,0x00001B25,0xFFFFF7C9,0x0000030D},
- {"0000001000010011111100001111111010011001000011100001100001100100",0x00002CCF,0xFFFFEDE5,0x00000478,0x00001BC8,0xFFFFF761,0x00000326,0x00001BC8,0xFFFFF761,0x00000326},
- {"0000001000010011111100001111111010011001000001100010100110000100",0x0000400E,0xFFFFE1CB,0x00000652,0x00001AF8,0xFFFFF7B9,0x00000312,0x00001AF8,0xFFFFF7B9,0x00000312},
- {"0000001000010011111100001111111010011001000001000000100011100100",0x00002F24,0xFFFFEC2A,0x000004C7,0x00001B94,0xFFFFF748,0x00000333,0x00001B94,0xFFFFF748,0x00000333},
- {"0000001000010011111100001111110101000010110100100001100100100100",0x00003FDA,0xFFFFE1C1,0x0000064B,0x00002427,0xFFFFF180,0x0000040C,0x00002427,0xFFFFF180,0x0000040C},
- {"0000001000010011111100001111111010011001000010100001100011000100",0x00002F6B,0xFFFFEBA7,0x000004DD,0x00001C25,0xFFFFF6C1,0x00000344,0x00001C25,0xFFFFF6C1,0x00000344},
- {"0000001000010011111100001111111010011001000110000010000100000100",0x00002A53,0xFFFFF0EE,0x00000402,0x000017C6,0xFFFFFAA0,0x000002BF,0x000017C6,0xFFFFFAA0,0x000002BF},
- {"0000001000010011111100001111111010011001000100000101000101000100",0x000031F4,0xFFFFEA34,0x00000517,0x000016FF,0xFFFFFA4E,0x000002AC,0x000016FF,0xFFFFFA4E,0x000002AC},
- {"0000001000010011111100001111111010011001001100100010000101000100",0x00002E24,0xFFFFED46,0x00000489,0x00001712,0xFFFFFA5D,0x000002AC,0x00001712,0xFFFFFA5D,0x000002AC},
- {"0000001000010011111100001111111010011001000110000010100000100100",0x000028CD,0xFFFFF0E3,0x0000040E,0x00001606,0xFFFFFB37,0x000002A4,0x00001606,0xFFFFFB37,0x000002A4},
- {"0000001000010011111100001111111010011001000000100010000011000100",0x00003184,0xFFFFEB88,0x000004C3,0x000018DA,0xFFFFF939,0x000002DB,0x000018DA,0xFFFFF939,0x000002DB},
- {"0000001000010011111100001111111010011001000101100010000100100100",0x0000239B,0xFFFFF470,0x00000386,0x00001714,0xFFFFFA9F,0x000002C8,0x00001714,0xFFFFFA9F,0x000002C8},
- {"0000001000010011111100001111110101000010110111000011100011100100",0x00003641,0xFFFFE92B,0x00000515,0x00001BE2,0xFFFFF795,0x0000031B,0x00001BE2,0xFFFFF795,0x0000031B},
- {"0000001000010011111100001111111010011001001011000001000101000100",0x00003278,0xFFFFEA17,0x00000510,0x00001B71,0xFFFFF778,0x0000031D,0x00001B71,0xFFFFF778,0x0000031D},
- {"0000001000010011111100001111111010011001000001100010100001000100",0x000035B9,0xFFFFE8DA,0x0000052D,0x00001A6A,0xFFFFF83B,0x000002FF,0x00001A6A,0xFFFFF83B,0x000002FF},
- {"0000001000010011111100001111111010011001000011100001100011000100",0x00002E5E,0xFFFFED32,0x0000048B,0x00001E7D,0xFFFFF60E,0x0000034E,0x00001E7D,0xFFFFF60E,0x0000034E},
- {"0000001000010011111100001111111010011001000100000001100110100100",0x00003178,0xFFFFEA52,0x00000513,0x00001AD0,0xFFFFF793,0x0000031F,0x00001AD0,0xFFFFF793,0x0000031F},
- {"0000001000010011111100001111110101000010110101000100000100000100",0x00003A2C,0xFFFFE346,0x00000641,0x000023D0,0xFFFFF0CE,0x00000433,0x000023D0,0xFFFFF0CE,0x00000433},
- {"0000001000010011111100001111110101000010110110000001100011000100",0x000028FD,0xFFFFF02A,0x0000042B,0x0000152B,0xFFFFFB90,0x00000289,0x0000152B,0xFFFFFB90,0x00000289},
- {"0000001000010011111100001111111010011001000011100011000010000100",0x000030DE,0xFFFFEBDF,0x000004BE,0x00001CDC,0xFFFFF747,0x0000031C,0x00001CDC,0xFFFFF747,0x0000031C},
- {"0000001000010011111100001111111010011001000000100001100101000100",0x000036CB,0xFFFFE6EE,0x00000596,0x00002096,0xFFFFF3C2,0x000003BB,0x00002096,0xFFFFF3C2,0x000003BB},
- {"0000001000010011111100001111111010011001000011000100100011000100",0x00003172,0xFFFFEAC1,0x000004F4,0x00001C87,0xFFFFF6CD,0x00000337,0x00001C87,0xFFFFF6CD,0x00000337},
- {"0000001000010011111100001111110101000010110100100100100001100100",0x00004A18,0xFFFFDB34,0x00000758,0x0000213C,0xFFFFF3A2,0x000003AC,0x0000213C,0xFFFFF3A2,0x000003AC},
- {"0000001000010011111100001111111010011001000000100010000100000100",0x000031F3,0xFFFFEB73,0x000004C6,0x00001B23,0xFFFFF7CB,0x0000031A,0x00001B23,0xFFFFF7CB,0x0000031A},
- {"0000001000010011111100001111111010011001000010100010100100100100",0x000031C0,0xFFFFEABA,0x000004F7,0x00001A5A,0xFFFFF845,0x000002FF,0x00001A5A,0xFFFFF845,0x000002FF},
- {"0000001000010011111100001111111010011001000100000100100101000100",0x00003B77,0xFFFFE3B3,0x00000623,0x00001BCA,0xFFFFF6F8,0x00000333,0x00001BCA,0xFFFFF6F8,0x00000333},
- {"0000001000010011111100001111111010011001000010100011100101000100",0x000035AF,0xFFFFE76D,0x00000588,0x00001C16,0xFFFFF6AB,0x00000341,0x00001C16,0xFFFFF6AB,0x00000341},
- {"0000001000010011111010101001010011011110000001000011100011000100",0x000032AD,0xFFFFEA8E,0x000004F8,0x00001A3A,0xFFFFF832,0x0000030E,0x00001A3A,0xFFFFF832,0x0000030E},
- {"0000001000010011111100001111111010011001000100000100100010000100",0x00002E92,0xFFFFEBD2,0x000004DA,0x00001E04,0xFFFFF51E,0x0000038A,0x00001E04,0xFFFFF51E,0x0000038A},
- {"0000001000010011111100001111110101000010110101000100000010100100",0x00003E57,0xFFFFE0F7,0x0000068F,0x000021F1,0xFFFFF1C6,0x00000411,0x000021F1,0xFFFFF1C6,0x00000411},
- {"0000001000010011111100001111111010011001000010000010000110100100",0x00003598,0xFFFFE8BB,0x00000535,0x00001B62,0xFFFFF764,0x00000326,0x00001B62,0xFFFFF764,0x00000326},
- {"0000001000010011111100001111111010011001000010100011100010000100",0x00002B15,0xFFFFEDEC,0x00000487,0x00001E8B,0xFFFFF4AB,0x0000039F,0x00001E8B,0xFFFFF4AB,0x0000039F},
- {"0000001000010011111010101001010011011110000001100000100100000100",0x0000267E,0xFFFFF1A7,0x000003E1,0x000021C1,0xFFFFF2E9,0x000003EA,0x000021C1,0xFFFFF2E9,0x000003EA},
- {"0000001000010011111010101001010011011110000000100011100110100100",0x00002ED7,0xFFFFEC88,0x000004A6,0x00001DEC,0xFFFFF57C,0x00000378,0x00001DEC,0xFFFFF57C,0x00000378},
- {"0000001000010011111010101001010011011110000001000100000110100100",0x00003365,0xFFFFE946,0x00000536,0x000019E9,0xFFFFF7E0,0x0000031D,0x000019E9,0xFFFFF7E0,0x0000031D},
- {"0000001000010011111100001111111010011001000110000001100011100100",0x000029A4,0xFFFFF0FD,0x000003FE,0x0000163F,0xFFFFFB68,0x00000299,0x0000163F,0xFFFFFB68,0x00000299},
- {"0000001000010011111010101001010011011110000000100001100100000100",0x0000348D,0xFFFFE9F7,0x00000509,0x000017A0,0xFFFFFA59,0x000002B6,0x000017A0,0xFFFFFA59,0x000002B6},
- {"0000001000010011111100001111111010011001000001100001000011000100",0x00003144,0xFFFFEB23,0x000004D9,0x00001C9B,0xFFFFF664,0x00000351,0x00001C9B,0xFFFFF664,0x00000351},
- {"0000001000010011111010101001010011011110000001100010000011100100",0x00002E95,0xFFFFEE1A,0x00000463,0x00001707,0xFFFFFAB7,0x000002B3,0x00001707,0xFFFFFAB7,0x000002B3},
- {"0000001000010011111100001111110101000010110101000001100001100100",0x0000489C,0xFFFFDA43,0x000007AC,0x00002866,0xFFFFED6B,0x000004D0,0x00002866,0xFFFFED6B,0x000004D0},
- {"0000001000010011111100001111111010011001000101100001100001000100",0x00002895,0xFFFFF10A,0x0000040A,0x000013E9,0xFFFFFC9F,0x0000026E,0x000013E9,0xFFFFFC9F,0x0000026E},
- {"0000001000010011111100001111111010011001000001100001100101100100",0x000033A0,0xFFFFE9B1,0x00000510,0x00001D96,0xFFFFF5AE,0x0000036F,0x00001D96,0xFFFFF5AE,0x0000036F},
- {"0000001000010011111100001111111010011001000010000011100110000100",0x0000327C,0xFFFFEAEA,0x000004DD,0x00001D45,0xFFFFF649,0x00000356,0x00001D45,0xFFFFF649,0x00000356},
- {"0000001000010011111010101001010011011110000000100100100010100100",0x000031DF,0xFFFFE9AB,0x0000052F,0x000019C8,0xFFFFF7B7,0x00000321,0x000019C8,0xFFFFF7B7,0x00000321},
- {"0000001000010011111100001111111010011001000101100100000010100100",0x00002BCC,0xFFFFEEF4,0x0000045C,0x000015CD,0xFFFFFB58,0x0000029E,0x000015CD,0xFFFFFB58,0x0000029E},
- {"0000001000010011111100001111111010011001000001100011100011100100",0x00003534,0xFFFFEA10,0x000004EB,0x00001BB6,0xFFFFF7B9,0x00000314,0x00001BB6,0xFFFFF7B9,0x00000314},
- {"0000001000010011111100001111111010011001000001000001100110000100",0x00002F4F,0xFFFFEC35,0x000004B9,0x0000205D,0xFFFFF47F,0x00000392,0x0000205D,0xFFFFF47F,0x00000392},
- {"0000001000010011111100001111111010011001000011000010000010100100",0x00003295,0xFFFFEB1C,0x000004D6,0x000019C1,0xFFFFF931,0x000002D5,0x000019C1,0xFFFFF931,0x000002D5},
- {"0000001000010011111100001111111010011001000000100100000101000100",0x00003557,0xFFFFE7F7,0x00000568,0x00002342,0xFFFFF1F9,0x00000405,0x00002342,0xFFFFF1F9,0x00000405},
- {"0000001000010011111100001111111010011001000001000101000011000100",0x00003487,0xFFFFE872,0x0000055D,0x000019D7,0xFFFFF823,0x0000030C,0x000019D7,0xFFFFF823,0x0000030C},
- {"0000001000010011111100001111111010011001001011000011100101000100",0x0000378F,0xFFFFE7A6,0x00000566,0x00001875,0xFFFFFA04,0x000002AF,0x00001875,0xFFFFFA04,0x000002AF},
- {"0000001000010011111010101001010011011110000000100011000011100100",0x00002A67,0xFFFFF157,0x000003DD,0x000017BD,0xFFFFFA53,0x000002D1,0x000017BD,0xFFFFFA53,0x000002D1},
- {"0000001000010011111100001111110101000010110100100010000011100100",0x000030B5,0xFFFFEB32,0x000004D9,0x00002129,0xFFFFF38A,0x000003BB,0x00002129,0xFFFFF38A,0x000003BB},
- {"0000001000010011111100001111111010011001000001100001000010100100",0x00003786,0xFFFFE703,0x00000584,0x00001D63,0xFFFFF5DC,0x00000367,0x00001D63,0xFFFFF5DC,0x00000367},
- {"0000001000010011111100001111110101000010110110100010000011000100",0x0000346A,0xFFFFE93E,0x0000052C,0x00001B27,0xFFFFF79D,0x0000031F,0x00001B27,0xFFFFF79D,0x0000031F},
- {"0000001000010011111100001111111010011001000011100011000000100100",0x0000294E,0xFFFFF0A5,0x00000409,0x00001928,0xFFFFF93B,0x000002E6,0x00001928,0xFFFFF93B,0x000002E6},
- {"0000001000010011111100001111110101000010110101000001000011000100",0x00003E09,0xFFFFE0FF,0x00000694,0x000025A0,0xFFFFEF0F,0x0000048F,0x000025A0,0xFFFFEF0F,0x0000048F},
- {"0000001000010011111100001111111010011001000010100010100101100100",0x00003197,0xFFFFEA06,0x00000520,0x00001B42,0xFFFFF73B,0x0000032A,0x00001B42,0xFFFFF73B,0x0000032A},
- {"0000001000010011111100001111111010011001000101100001100001100100",0x000022CB,0xFFFFF3FC,0x000003A3,0x00001449,0xFFFFFBD0,0x00000297,0x00001449,0xFFFFFBD0,0x00000297},
- {"0000001000010011111100001111110101000010110110000010100101000100",0x00002A79,0xFFFFEFD2,0x00000433,0x00001585,0xFFFFFB92,0x0000028E,0x00001585,0xFFFFFB92,0x0000028E},
- {"0000001000010011111100001111111010011001000011000100000110000100",0x00003249,0xFFFFEA92,0x000004F4,0x000019CB,0xFFFFF8CF,0x000002E1,0x000019CB,0xFFFFF8CF,0x000002E1},
- {"0000001000010011111010101001010011011110000000100001100010100100",0x00002CEA,0xFFFFEE46,0x00000463,0x00001A5E,0xFFFFF83C,0x0000030D,0x00001A5E,0xFFFFF83C,0x0000030D},
- {"0000001000010011111100001111110101000010110111000101000101000100",0x00003AE2,0xFFFFE422,0x00000600,0x00001C65,0xFFFFF62F,0x0000034B,0x00001C65,0xFFFFF62F,0x0000034B},
- {"0000001000010011111100001111111010011001000110000001000110000100",0x000026A0,0xFFFFF1C2,0x000003F8,0x000010E5,0xFFFFFE56,0x0000022A,0x000010E5,0xFFFFFE56,0x0000022A},
- {"0000001000010011111100001111111010011001001010000010100110100100",0x00002A7B,0xFFFFF063,0x00000417,0x000016FC,0xFFFFFAD7,0x000002B1,0x000016FC,0xFFFFFAD7,0x000002B1},
- {"0000001000010011111100001111111010011001001100100001000011000100",0x00003092,0xFFFFEAB9,0x00000507,0x00001AE3,0xFFFFF783,0x00000323,0x00001AE3,0xFFFFF783,0x00000323},
- {"0000001000010011111100001111111010011001000001000011100011100100",0x00003265,0xFFFFEBE8,0x000004AA,0x00001D65,0xFFFFF73F,0x00000321,0x00001D65,0xFFFFF73F,0x00000321},
- {"0000001000010011111010101001010011011110000000100011000010000100",0x00002F14,0xFFFFECC2,0x000004A4,0x00001A8D,0xFFFFF7F3,0x0000031D,0x00001A8D,0xFFFFF7F3,0x0000031D},
- {"0000001000010011111100001111110101000010110111000001000011100100",0x000035FB,0xFFFFE6D3,0x000005AC,0x00001B19,0xFFFFF712,0x00000338,0x00001B19,0xFFFFF712,0x00000338},
- {"0000001000010011111100001111110101000010110110100010000100100100",0x00003519,0xFFFFE8CC,0x0000053A,0x00001A0F,0xFFFFF86E,0x000002F5,0x00001A0F,0xFFFFF86E,0x000002F5},
- {"0000001000010011111100001111111010011001001011000010000101000100",0x0000364C,0xFFFFE879,0x00000541,0x00001A42,0xFFFFF8BA,0x000002E2,0x00001A42,0xFFFFF8BA,0x000002E2},
- {"0000001000010011111010101001010011011110000000100001100011000100",0x000029BA,0xFFFFF09A,0x00000408,0x00001986,0xFFFFF8D9,0x000002FE,0x00001986,0xFFFFF8D9,0x000002FE},
- {"0000001000010011111100001111110101000010110110100011100011100100",0x00003507,0xFFFFE961,0x00000518,0x00001B79,0xFFFFF775,0x00000325,0x00001B79,0xFFFFF775,0x00000325},
- {"0000001000010011111100001111110101000010110111000011000110000100",0x00003AD5,0xFFFFE415,0x00000613,0x00001CB4,0xFFFFF66D,0x00000348,0x00001CB4,0xFFFFF66D,0x00000348},
- {"0000001000010011111100001111111010011001000101100100000011100100",0x000023D1,0xFFFFF42B,0x0000038F,0x00001546,0xFFFFFBA0,0x0000029F,0x00001546,0xFFFFFBA0,0x0000029F},
- {"0000001000010011111100001111111010011001000010100001100100100100",0x0000399E,0xFFFFE518,0x000005E7,0x00001990,0xFFFFF871,0x000002FB,0x00001990,0xFFFFF871,0x000002FB},
- {"0000001000010011111100001111110101000010110110000010100101100100",0x00002EDE,0xFFFFEC93,0x000004B8,0x0000152C,0xFFFFFBB3,0x0000027E,0x0000152C,0xFFFFFBB3,0x0000027E},
- {"0000001000010011111010101001010011011110000001000010100101100100",0x00003140,0xFFFFEBC9,0x000004BB,0x000016BE,0xFFFFFB0A,0x00000288,0x000016BE,0xFFFFFB0A,0x00000288},
- {"0000001000010011111100001111111010011001000001100100000001100100",0x000030F6,0xFFFFEB89,0x000004CD,0x0000185D,0xFFFFF95A,0x000002D9,0x0000185D,0xFFFFF95A,0x000002D9},
- {"0000001000010011111100001111111010011001000000100011100001000100",0x0000389C,0xFFFFE65A,0x000005A2,0x0000195D,0xFFFFF8C8,0x000002E8,0x0000195D,0xFFFFF8C8,0x000002E8},
- {"0000001000010011111100001111111010011001000001000010000100000100",0x0000362B,0xFFFFE9EC,0x000004F6,0x00001605,0xFFFFFC1C,0x00000263,0x00001605,0xFFFFFC1C,0x00000263},
- {"0000001000010011111100001111111010011001001010100001100101100100",0x00002946,0xFFFFF04F,0x00000426,0x000015BA,0xFFFFFB2F,0x000002A3,0x000015BA,0xFFFFFB2F,0x000002A3},
- {"0000001000010011111100001111111010011001000010000010000110000100",0x0000368E,0xFFFFE837,0x0000054A,0x000017D7,0xFFFFF9EB,0x000002BA,0x000017D7,0xFFFFF9EB,0x000002BA},
- {"0000001000010011111100001111110101000010110110100010100001000100",0x00002E74,0xFFFFEBE8,0x000004DA,0x00001DD6,0xFFFFF57E,0x00000379,0x00001DD6,0xFFFFF57E,0x00000379},
- {"0000001000010011111100001111111010011001000001000001100101000100",0x0000322D,0xFFFFEAA8,0x000004F5,0x00001B55,0xFFFFF7DD,0x0000030B,0x00001B55,0xFFFFF7DD,0x0000030B},
- {"0000001000010011111100001111111010011001000110000001100100000100",0x00002A29,0xFFFFF07B,0x00000416,0x00001671,0xFFFFFB3E,0x0000029F,0x00001671,0xFFFFFB3E,0x0000029F},
- {"0000001000010011111100001111110101000010110110100010000100000100",0x000030F6,0xFFFFEB89,0x000004CD,0x00001815,0xFFFFF9AE,0x000002C9,0x00001815,0xFFFFF9AE,0x000002C9},
- {"0000001000010011111100001111111010011001000011100001000011100100",0x0000265F,0xFFFFF1CB,0x000003D5,0x00001ED2,0xFFFFF539,0x0000037A,0x00001ED2,0xFFFFF539,0x0000037A},
- {"0000001000010011111100001111111010011001000101100010000110000100",0x000027A8,0xFFFFF10D,0x00000413,0x000014B5,0xFFFFFBA1,0x00000299,0x000014B5,0xFFFFFBA1,0x00000299},
- {"0000001000010011111100001111111010011001000001000011000001100100",0x00002CEE,0xFFFFEDF6,0x00000476,0x00001A99,0xFFFFF83E,0x00000305,0x00001A99,0xFFFFF83E,0x00000305},
- {"0000001000010011111100001111111010011001000001100100000011000100",0x0000346C,0xFFFFEA17,0x000004EF,0x00001D38,0xFFFFF69F,0x0000033D,0x00001D38,0xFFFFF69F,0x0000033D},
- {"0000001000010011111100001111110101000010110110100010100101000100",0x00002DBB,0xFFFFED35,0x00000490,0x000018C1,0xFFFFF930,0x000002DA,0x000018C1,0xFFFFF930,0x000002DA},
- {"0000001000010011111100001111111010011001000001000010100100100100",0x000038DF,0xFFFFE8A7,0x0000051E,0x00001B59,0xFFFFF915,0x000002D3,0x00001B59,0xFFFFF915,0x000002D3},
- {"0000001000010011111100001111111010011001000010000000100101000100",0x00003384,0xFFFFE979,0x00000524,0x00001AF3,0xFFFFF74C,0x0000032F,0x00001AF3,0xFFFFF74C,0x0000032F},
- {"0000001000010011111100001111111010011001000110000001100001100100",0x0000258B,0xFFFFF2AE,0x000003CB,0x0000190C,0xFFFFF93E,0x000002EF,0x0000190C,0xFFFFF93E,0x000002EF},
- {"0000001000010011111100001111111010011001000100000011100010000100",0x000034F1,0xFFFFE84B,0x0000055E,0x00001CB8,0xFFFFF670,0x0000034A,0x00001CB8,0xFFFFF670,0x0000034A},
- {"0000001000010011111100001111111010011001000011000010000100000100",0x000030FB,0xFFFFECD2,0x00000488,0x00001BF4,0xFFFFF821,0x00000302,0x00001BF4,0xFFFFF821,0x00000302},
- {"0000001000010011111100001111111010011001000001100011000001000100",0x000036A6,0xFFFFE815,0x00000556,0x000018FD,0xFFFFF925,0x000002DF,0x000018FD,0xFFFFF925,0x000002DF},
- {"0000001000010011111010101001010011011110000000100011000001000100",0x0000302A,0xFFFFEB79,0x000004E0,0x00001C11,0xFFFFF694,0x00000358,0x00001C11,0xFFFFF694,0x00000358},
- {"0000001000010011111100001111111010011001000110000001000100100100",0x00002555,0xFFFFF2C4,0x000003CB,0x000017E3,0xFFFFFA1F,0x000002CB,0x000017E3,0xFFFFFA1F,0x000002CB},
- {"0000001000010011111100001111111010011001000010100011000101100100",0x000032A3,0xFFFFE933,0x00000544,0x000019D3,0xFFFFF81A,0x00000306,0x000019D3,0xFFFFF81A,0x00000306},
- {"0000001000010011111100001111110101000010110110000101000100000100",0x00002B91,0xFFFFED81,0x000004A9,0x0000158B,0xFFFFFAB9,0x000002AC,0x0000158B,0xFFFFFAB9,0x000002AC},
- {"0000001000010011111100001111111010011001000011100010000011000100",0x00003537,0xFFFFE912,0x0000052C,0x00001C8A,0xFFFFF754,0x0000031B,0x00001C8A,0xFFFFF754,0x0000031B},
- {"0000001000010011111010101001010011011110000001100011000110000100",0x000032E1,0xFFFFEA5A,0x000004F9,0x000017B4,0xFFFFF9D9,0x000002C2,0x000017B4,0xFFFFF9D9,0x000002C2},
- {"0000001000010011111100001111110101000010110100100001000011000100",0x00003B76,0xFFFFE330,0x00000636,0x000026FB,0xFFFFEF06,0x00000481,0x000026FB,0xFFFFEF06,0x00000481},
- {"0000001000010011111100001111111010011001000001000010000101000100",0x0000320C,0xFFFFEB84,0x000004C3,0x00001A3A,0xFFFFF8E9,0x000002DF,0x00001A3A,0xFFFFF8E9,0x000002DF},
- {"0000001000010011111100001111111010011001000000100011100110000100",0x0000317D,0xFFFFEA1F,0x00000515,0x00002100,0xFFFFF31B,0x000003DD,0x00002100,0xFFFFF31B,0x000003DD},
- {"0000001000010011111100001111110101000010110101000011000101100100",0x00003DCB,0xFFFFE0B4,0x000006B4,0x00002160,0xFFFFF269,0x000003F0,0x00002160,0xFFFFF269,0x000003F0},
- {"0000001000010011111100001111111010011001000101100001100011000100",0x00002737,0xFFFFF218,0x000003E1,0x000015B5,0xFFFFFB8F,0x0000029C,0x000015B5,0xFFFFFB8F,0x0000029C},
- {"0000001000010011111010101001010011011110000000100011000110000100",0x0000318F,0xFFFFEB3F,0x000004D8,0x00001938,0xFFFFF8E9,0x000002EB,0x00001938,0xFFFFF8E9,0x000002EB},
- {"0000001000010011111100001111111010011001000100000100100011000100",0x000031BD,0xFFFFE9DE,0x00000527,0x000018A7,0xFFFFF8CA,0x000002ED,0x000018A7,0xFFFFF8CA,0x000002ED},
- {"0000001000010011111100001111110101000010110110100011100010000100",0x00002F77,0xFFFFEC2F,0x000004B4,0x00001D25,0xFFFFF61B,0x0000035D,0x00001D25,0xFFFFF61B,0x0000035D},
- {"0000001000010011111100001111111010011001000011100100100100000100",0x00002CCA,0xFFFFEDB3,0x0000047C,0x00001FBD,0xFFFFF4A7,0x00000391,0x00001FBD,0xFFFFF4A7,0x00000391},
- {"0000001000010011111100001111110101000010110101000011100010100100",0x00003FF6,0xFFFFE058,0x000006A2,0x000024CD,0xFFFFF026,0x00000452,0x000024CD,0xFFFFF026,0x00000452},
- {"0000001000010011111100001111111010011001000010100011100011100100",0x00003161,0xFFFFEAC8,0x000004F3,0x00001BB6,0xFFFFF72A,0x0000032B,0x00001BB6,0xFFFFF72A,0x0000032B},
- {"0000001000010011111100001111110101000010110110000011100010100100",0x00002EA0,0xFFFFECA6,0x000004B7,0x000018C2,0xFFFFF94E,0x000002E1,0x000018C2,0xFFFFF94E,0x000002E1},
- {"0000001000010011111100001111111010011001000110000010000110000100",0x00002F62,0xFFFFEC9E,0x000004B8,0x00001531,0xFFFFFBCD,0x00000285,0x00001531,0xFFFFFBCD,0x00000285},
- {"0000001000010011111100001111111010011001000001000100000010100100",0x00003013,0xFFFFEBD6,0x000004C2,0x00001B01,0xFFFFF802,0x000002FF,0x00001B01,0xFFFFF802,0x000002FF},
- {"0000001000010011111100001111111010011001000110000011000001100100",0x00002972,0xFFFFF08D,0x00000417,0x00001A32,0xFFFFF8A4,0x00000305,0x00001A32,0xFFFFF8A4,0x00000305},
- {"0000001000010011111100001111110101000010110110000010000011100100",0x00002E95,0xFFFFED94,0x00000487,0x00001529,0xFFFFFC26,0x00000271,0x00001529,0xFFFFFC26,0x00000271},
- {"0000001000010011111100001111111010011001000010100001000010000100",0x00002D6A,0xFFFFEC79,0x000004C1,0x00001AE2,0xFFFFF725,0x00000337,0x00001AE2,0xFFFFF725,0x00000337},
- {"0000001000010011111100001111111010011001000000100001100010000100",0x000036B4,0xFFFFE704,0x00000591,0x00001E7E,0xFFFFF51C,0x00000383,0x00001E7E,0xFFFFF51C,0x00000383},
- {"0000001000010011111100001111111010011001000001000001100001000100",0x00002A6F,0xFFFFEF70,0x00000443,0x00001BAA,0xFFFFF752,0x00000336,0x00001BAA,0xFFFFF752,0x00000336},
- {"0000001000010011111100001111111010011001000110000011100101000100",0x00002C66,0xFFFFEF5F,0x0000043A,0x000019F7,0xFFFFF931,0x000002EC,0x000019F7,0xFFFFF931,0x000002EC},
- {"0000001000010011111010101001010011011110000001100011000111000100",0x00003852,0xFFFFE6AB,0x00000590,0x000019C1,0xFFFFF8B1,0x000002E5,0x000019C1,0xFFFFF8B1,0x000002E5},
- {"00000010000100111111000011