diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 187 |
1 files changed, 104 insertions, 83 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 12e49bd8fd2d..80054f36e487 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -43,7 +43,7 @@ #include "amdgpu_atombios.h" -static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev); +static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev); static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev); static int gmc_v7_0_wait_for_idle(void *handle); @@ -152,16 +152,16 @@ static int gmc_v7_0_init_microcode(struct amdgpu_device *adev) else snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); - err = request_firmware(&adev->mc.fw, fw_name, adev->dev); + err = request_firmware(&adev->gmc.fw, fw_name, adev->dev); if (err) goto out; - err = amdgpu_ucode_validate(adev->mc.fw); + err = amdgpu_ucode_validate(adev->gmc.fw); out: if (err) { pr_err("cik_mc: Failed to load firmware \"%s\"\n", fw_name); - release_firmware(adev->mc.fw); - adev->mc.fw = NULL; + release_firmware(adev->gmc.fw); + adev->gmc.fw = NULL; } return err; } @@ -182,19 +182,19 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev) u32 running; int i, ucode_size, regs_size; - if (!adev->mc.fw) + if (!adev->gmc.fw) return -EINVAL; - hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data; + hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data; amdgpu_ucode_print_mc_hdr(&hdr->header); - adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version); + adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version); regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); io_mc_regs = (const __le32 *) - (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); + (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; fw_data = (const __le32 *) - (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN); @@ -236,12 +236,12 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev) } static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev, - struct amdgpu_mc *mc) + struct amdgpu_gmc *mc) { u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; base <<= 24; - amdgpu_device_vram_location(adev, &adev->mc, base); + amdgpu_device_vram_location(adev, &adev->gmc, base); amdgpu_device_gart_location(adev, mc); } @@ -284,9 +284,9 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev) } /* Update configuration */ WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, - adev->mc.vram_start >> 12); + adev->gmc.vram_start >> 12); WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, - adev->mc.vram_end >> 12); + adev->gmc.vram_end >> 12); WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, adev->vram_scratch.gpu_addr >> 12); WREG32(mmMC_VM_AGP_BASE, 0); @@ -319,8 +319,8 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) { int r; - adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev); - if (!adev->mc.vram_width) { + adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev); + if (!adev->gmc.vram_width) { u32 tmp; int chansize, numchan; @@ -362,38 +362,38 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) numchan = 16; break; } - adev->mc.vram_width = numchan * chansize; + adev->gmc.vram_width = numchan * chansize; } /* size in MB on si */ - adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; - adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; + adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; + adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; if (!(adev->flags & AMD_IS_APU)) { r = amdgpu_device_resize_fb_bar(adev); if (r) return r; } - adev->mc.aper_base = pci_resource_start(adev->pdev, 0); - adev->mc.aper_size = pci_resource_len(adev->pdev, 0); + adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); + adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); #ifdef CONFIG_X86_64 if (adev->flags & AMD_IS_APU) { - adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; - adev->mc.aper_size = adev->mc.real_vram_size; + adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; + adev->gmc.aper_size = adev->gmc.real_vram_size; } #endif /* In case the PCI BAR is larger than the actual amount of vram */ - adev->mc.visible_vram_size = adev->mc.aper_size; - if (adev->mc.visible_vram_size > adev->mc.real_vram_size) - adev->mc.visible_vram_size = adev->mc.real_vram_size; + adev->gmc.visible_vram_size = adev->gmc.aper_size; + if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) + adev->gmc.visible_vram_size = adev->gmc.real_vram_size; /* set the gart size */ if (amdgpu_gart_size == -1) { switch (adev->asic_type) { case CHIP_TOPAZ: /* no MM engines */ default: - adev->mc.gart_size = 256ULL << 20; + adev->gmc.gart_size = 256ULL << 20; break; #ifdef CONFIG_DRM_AMDGPU_CIK case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */ @@ -401,15 +401,15 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */ case CHIP_KABINI: /* UVD, VCE do not support GPUVM */ case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */ - adev->mc.gart_size = 1024ULL << 20; + adev->gmc.gart_size = 1024ULL << 20; break; #endif } } else { - adev->mc.gart_size = (u64)amdgpu_gart_size << 20; + adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; } - gmc_v7_0_vram_gtt_location(adev, &adev->mc); + gmc_v7_0_vram_gtt_location(adev, &adev->gmc); return 0; } @@ -422,25 +422,44 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) */ /** - * gmc_v7_0_gart_flush_gpu_tlb - gart tlb flush callback + * gmc_v7_0_flush_gpu_tlb - gart tlb flush callback * * @adev: amdgpu_device pointer * @vmid: vm instance to flush * * Flush the TLB for the requested page table (CIK). */ -static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev, - uint32_t vmid) +static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid) { - /* flush hdp cache */ - WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0); - /* bits 0-15 are the VM contexts0-15 */ WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); } +static uint64_t gmc_v7_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, + unsigned vmid, uint64_t pd_addr) +{ + uint32_t reg; + + if (vmid < 8) + reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid; + else + reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8; + amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12); + + /* bits 0-15 are the VM contexts0-15 */ + amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid); + + return pd_addr; +} + +static void gmc_v7_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, + unsigned pasid) +{ + amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid); +} + /** - * gmc_v7_0_gart_set_pte_pde - update the page tables using MMIO + * gmc_v7_0_set_pte_pde - update the page tables using MMIO * * @adev: amdgpu_device pointer * @cpu_pt_addr: cpu address of the page table @@ -450,11 +469,9 @@ static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev, * * Update the page tables using the CPU. */ -static int gmc_v7_0_gart_set_pte_pde(struct amdgpu_device *adev, - void *cpu_pt_addr, - uint32_t gpu_page_idx, - uint64_t addr, - uint64_t flags) +static int gmc_v7_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, + uint32_t gpu_page_idx, uint64_t addr, + uint64_t flags) { void __iomem *ptr = (void *)cpu_pt_addr; uint64_t value; @@ -524,9 +541,9 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable) { uint32_t tmp; - if (enable && !adev->mc.prt_warning) { + if (enable && !adev->gmc.prt_warning) { dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n"); - adev->mc.prt_warning = true; + adev->gmc.prt_warning = true; } tmp = RREG32(mmVM_PRT_CNTL); @@ -548,7 +565,8 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable) if (enable) { uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT; - uint32_t high = adev->vm_manager.max_pfn; + uint32_t high = adev->vm_manager.max_pfn - + (AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT); WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low); WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low); @@ -622,11 +640,11 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev) tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field); WREG32(mmVM_L2_CNTL3, tmp); /* setup context0 */ - WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12); - WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12); + WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12); + WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12); WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12); WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, - (u32)(adev->dummy_page.addr >> 12)); + (u32)(adev->dummy_page_addr >> 12)); WREG32(mmVM_CONTEXT0_CNTL2, 0); tmp = RREG32(mmVM_CONTEXT0_CNTL); tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); @@ -656,7 +674,7 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev) /* enable context1-15 */ WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, - (u32)(adev->dummy_page.addr >> 12)); + (u32)(adev->dummy_page_addr >> 12)); WREG32(mmVM_CONTEXT1_CNTL2, 4); tmp = RREG32(mmVM_CONTEXT1_CNTL); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); @@ -675,9 +693,9 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev) WREG32(mmCHUB_CONTROL, tmp); } - gmc_v7_0_gart_flush_gpu_tlb(adev, 0); + gmc_v7_0_flush_gpu_tlb(adev, 0); DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", - (unsigned)(adev->mc.gart_size >> 20), + (unsigned)(adev->gmc.gart_size >> 20), (unsigned long long)adev->gart.table_addr); adev->gart.ready = true; return 0; @@ -750,21 +768,21 @@ static void gmc_v7_0_gart_fini(struct amdgpu_device *adev) * * Print human readable fault information (CIK). */ -static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev, - u32 status, u32 addr, u32 mc_client) +static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev, u32 status, + u32 addr, u32 mc_client, unsigned pasid) { - u32 mc_id; u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, PROTECTIONS); char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; + u32 mc_id; mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, MEMORY_CLIENT_ID); - dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", - protections, vmid, addr, + dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", + protections, vmid, pasid, addr, REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, MEMORY_CLIENT_RW) ? "write" : "read", block, mc_client, mc_id); @@ -922,16 +940,16 @@ static int gmc_v7_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - gmc_v7_0_set_gart_funcs(adev); + gmc_v7_0_set_gmc_funcs(adev); gmc_v7_0_set_irq_funcs(adev); - adev->mc.shared_aperture_start = 0x2000000000000000ULL; - adev->mc.shared_aperture_end = - adev->mc.shared_aperture_start + (4ULL << 30) - 1; - adev->mc.private_aperture_start = - adev->mc.shared_aperture_end + 1; - adev->mc.private_aperture_end = - adev->mc.private_aperture_start + (4ULL << 30) - 1; + adev->gmc.shared_aperture_start = 0x2000000000000000ULL; + adev->gmc.shared_aperture_end = + adev->gmc.shared_aperture_start + (4ULL << 30) - 1; + adev->gmc.private_aperture_start = + adev->gmc.shared_aperture_end + 1; + adev->gmc.private_aperture_end = + adev->gmc.private_aperture_start + (4ULL << 30) - 1; return 0; } @@ -941,7 +959,7 @@ static int gmc_v7_0_late_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS) - return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); + return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); else return 0; } @@ -953,18 +971,18 @@ static int gmc_v7_0_sw_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; if (adev->flags & AMD_IS_APU) { - adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; + adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; } else { u32 tmp = RREG32(mmMC_SEQ_MISC0); tmp &= MC_SEQ_MISC0__MT__MASK; - adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp); + adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp); } - r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault); + r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault); if (r) return r; - r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault); + r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault); if (r) return r; @@ -978,9 +996,9 @@ static int gmc_v7_0_sw_init(void *handle) * This is the max address of the GPU's * internal address space. */ - adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ + adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ - adev->mc.stolen_size = 256 * 1024; + adev->gmc.stolen_size = 256 * 1024; /* set DMA mask + need_dma32 flags. * PCIE - can handle 40-bits. @@ -1051,8 +1069,8 @@ static int gmc_v7_0_sw_fini(void *handle) amdgpu_vm_manager_fini(adev); gmc_v7_0_gart_fini(adev); amdgpu_bo_fini(adev); - release_firmware(adev->mc.fw); - adev->mc.fw = NULL; + release_firmware(adev->gmc.fw); + adev->gmc.fw = NULL; return 0; } @@ -1085,7 +1103,7 @@ static int gmc_v7_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - amdgpu_irq_put(adev, &adev->mc.vm_fault, 0); + amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); gmc_v7_0_gart_disable(adev); return 0; @@ -1259,7 +1277,8 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev, addr); dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", status); - gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client); + gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client, + entry->pasid); } return 0; @@ -1308,9 +1327,11 @@ static const struct amd_ip_funcs gmc_v7_0_ip_funcs = { .set_powergating_state = gmc_v7_0_set_powergating_state, }; -static const struct amdgpu_gart_funcs gmc_v7_0_gart_funcs = { - .flush_gpu_tlb = gmc_v7_0_gart_flush_gpu_tlb, - .set_pte_pde = gmc_v7_0_gart_set_pte_pde, +static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = { + .flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb, + .emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb, + .emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping, + .set_pte_pde = gmc_v7_0_set_pte_pde, .set_prt = gmc_v7_0_set_prt, .get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags, .get_vm_pde = gmc_v7_0_get_vm_pde @@ -1321,16 +1342,16 @@ static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = { .process = gmc_v7_0_process_interrupt, }; -static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev) +static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev) { - if (adev->gart.gart_funcs == NULL) - adev->gart.gart_funcs = &gmc_v7_0_gart_funcs; + if (adev->gmc.gmc_funcs == NULL) + adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs; } static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev) { - adev->mc.vm_fault.num_types = 1; - adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs; + adev->gmc.vm_fault.num_types = 1; + adev->gmc.vm_fault.funcs = &gmc_v7_0_irq_funcs; } const struct amdgpu_ip_block_version gmc_v7_0_ip_block = |