From 37708479608af877986b76302a9c92611d1e23d0 Mon Sep 17 00:00:00 2001 From: Vadim Girlin Date: Wed, 11 Jul 2012 02:19:51 +0400 Subject: r600g: improve flushed depth texture handling v2 Use r600_resource_texture::flished_depth_texture for GPU access, and allocate it in the VRAM. For transfers we'll allocate texture in the GTT and store it in the r600_transfer::staging. Improves performance when flushed depth texture is frequently used by the GPU, e.g. in Lightsmark (~30%) Signed-off-by: Vadim Girlin --- src/gallium/drivers/r600/evergreen_state.c | 5 +- src/gallium/drivers/r600/r600_blit.c | 21 +++--- src/gallium/drivers/r600/r600_pipe.h | 4 +- src/gallium/drivers/r600/r600_resource.h | 6 +- src/gallium/drivers/r600/r600_state.c | 2 +- src/gallium/drivers/r600/r600_texture.c | 106 ++++++++++++++++------------- 6 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c index bbdc5868a80..60825bc7244 100644 --- a/src/gallium/drivers/r600/evergreen_state.c +++ b/src/gallium/drivers/r600/evergreen_state.c @@ -990,7 +990,7 @@ static struct pipe_sampler_view *evergreen_create_sampler_view(struct pipe_conte } if (tmp->is_depth && !tmp->is_flushing_texture) { - r600_init_flushed_depth_texture(ctx, texture); + r600_init_flushed_depth_texture(ctx, texture, NULL); tmp = tmp->flushed_depth_texture; if (!tmp) { FREE(view); @@ -1318,7 +1318,8 @@ void evergreen_cb(struct r600_context *rctx, struct r600_pipe_state *rstate, rctx->have_depth_fb = TRUE; if (rtex->is_depth && !rtex->is_flushing_texture) { - r600_init_flushed_depth_texture(&rctx->context, state->cbufs[cb]->texture); + r600_init_flushed_depth_texture(&rctx->context, + state->cbufs[cb]->texture, NULL); rtex = rtex->flushed_depth_texture; assert(rtex); } diff --git a/src/gallium/drivers/r600/r600_blit.c b/src/gallium/drivers/r600/r600_blit.c index 4c72f16b701..d85324b6fbb 100644 --- a/src/gallium/drivers/r600/r600_blit.c +++ b/src/gallium/drivers/r600/r600_blit.c @@ -114,13 +114,17 @@ static unsigned u_num_layers(struct pipe_resource *r, unsigned level) } } -void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture) +void r600_blit_uncompress_depth(struct pipe_context *ctx, + struct r600_resource_texture *texture, + struct r600_resource_texture *staging) { struct r600_context *rctx = (struct r600_context *)ctx; unsigned layer, level; float depth = 1.0f; + struct r600_resource_texture *flushed_depth_texture = staging ? + staging : texture->flushed_depth_texture; - if (!texture->dirty_db) + if (!staging && !texture->dirty_db) return; if (rctx->family == CHIP_RV610 || rctx->family == CHIP_RV630 || @@ -141,10 +145,10 @@ void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_t zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl); - surf_tmpl.format = texture->flushed_depth_texture->real_format; + surf_tmpl.format = flushed_depth_texture->real_format; surf_tmpl.usage = PIPE_BIND_RENDER_TARGET; cbsurf = ctx->create_surface(ctx, - (struct pipe_resource*)texture->flushed_depth_texture, &surf_tmpl); + (struct pipe_resource*)flushed_depth_texture, &surf_tmpl); r600_blitter_begin(ctx, R600_DECOMPRESS); util_blitter_custom_depth_stencil(rctx->blitter, zsurf, cbsurf, rctx->custom_dsa_flush, depth); @@ -155,7 +159,8 @@ void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_t } } - texture->dirty_db = FALSE; + if (!staging) + texture->dirty_db = FALSE; } void r600_flush_depth_textures(struct r600_context *rctx) @@ -178,7 +183,7 @@ void r600_flush_depth_textures(struct r600_context *rctx) if (tex->is_flushing_texture) continue; - r600_blit_uncompress_depth(&rctx->context, tex); + r600_blit_uncompress_depth(&rctx->context, tex, NULL); } /* also check CB here */ @@ -192,7 +197,7 @@ void r600_flush_depth_textures(struct r600_context *rctx) if (tex->is_flushing_texture) continue; - r600_blit_uncompress_depth(&rctx->context, tex); + r600_blit_uncompress_depth(&rctx->context, tex, NULL); } } @@ -324,7 +329,7 @@ static void r600_resource_copy_region(struct pipe_context *ctx, } if (rsrc->is_depth && !rsrc->is_flushing_texture) - r600_texture_depth_flush(ctx, src); + r600_texture_depth_flush(ctx, src, NULL); restore_orig[0] = restore_orig[1] = FALSE; diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h index de43555ddbf..0785ade4162 100644 --- a/src/gallium/drivers/r600/r600_pipe.h +++ b/src/gallium/drivers/r600/r600_pipe.h @@ -433,7 +433,9 @@ void evergreen_update_dual_export_state(struct r600_context * rctx); /* r600_blit.c */ void r600_init_blit_functions(struct r600_context *rctx); -void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture); +void r600_blit_uncompress_depth(struct pipe_context *ctx, + struct r600_resource_texture *texture, + struct r600_resource_texture *staging); void r600_flush_depth_textures(struct r600_context *rctx); /* r600_buffer.c */ diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h index d1f3a4766ca..7327ba6c2d7 100644 --- a/src/gallium/drivers/r600/r600_resource.h +++ b/src/gallium/drivers/r600/r600_resource.h @@ -89,9 +89,11 @@ static INLINE struct r600_resource *r600_resource(struct pipe_resource *r) } void r600_init_flushed_depth_texture(struct pipe_context *ctx, - struct pipe_resource *texture); + struct pipe_resource *texture, + struct r600_resource_texture **staging); void r600_texture_depth_flush(struct pipe_context *ctx, - struct pipe_resource *texture); + struct pipe_resource *texture, + struct r600_resource_texture **staging); /* r600_texture.c texture transfer functions. */ struct pipe_transfer* r600_texture_get_transfer(struct pipe_context *ctx, diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c index d51c64bca5e..0ba7fade043 100644 --- a/src/gallium/drivers/r600/r600_state.c +++ b/src/gallium/drivers/r600/r600_state.c @@ -1002,7 +1002,7 @@ static struct pipe_sampler_view *r600_create_sampler_view(struct pipe_context *c } if (tmp->is_depth && !tmp->is_flushing_texture) { - r600_init_flushed_depth_texture(ctx, texture); + r600_init_flushed_depth_texture(ctx, texture, NULL); tmp = tmp->flushed_depth_texture; if (!tmp) { FREE(view); diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c index d8c5df3159c..da6634643a3 100644 --- a/src/gallium/drivers/r600/r600_texture.c +++ b/src/gallium/drivers/r600/r600_texture.c @@ -295,7 +295,8 @@ static int r600_init_surface(struct radeon_surface *surface, if (ptex->bind & PIPE_BIND_SCANOUT) { surface->flags |= RADEON_SURF_SCANOUT; } - if (util_format_is_depth_and_stencil(ptex->format) && !is_transfer) { + if ((ptex->bind & PIPE_BIND_DEPTH_STENCIL) && + util_format_is_depth_and_stencil(ptex->format) && !is_transfer) { surface->flags |= RADEON_SURF_ZBUFFER; surface->flags |= RADEON_SURF_SBUFFER; } @@ -514,7 +515,7 @@ r600_texture_create_object(struct pipe_screen *screen, rtex->real_format = base->format; /* We must split depth and stencil into two separate buffers on Evergreen. */ - if (!(base->flags & R600_RESOURCE_FLAG_TRANSFER) && + if ((base->bind & PIPE_BIND_DEPTH_STENCIL) && ((struct r600_screen*)screen)->chip_class >= EVERGREEN && util_format_is_depth_and_stencil(base->format) && !rscreen->use_surface_alloc) { @@ -558,7 +559,8 @@ r600_texture_create_object(struct pipe_screen *screen, } /* only mark depth textures the HW can hit as depth textures */ - if (util_format_is_depth_or_stencil(rtex->real_format) && permit_hardware_blit(screen, base)) + if (util_format_is_depth_or_stencil(rtex->real_format) && + permit_hardware_blit(screen, base)) rtex->is_depth = true; r600_setup_miptree(screen, rtex, array_mode); @@ -726,12 +728,15 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen, } void r600_init_flushed_depth_texture(struct pipe_context *ctx, - struct pipe_resource *texture) + struct pipe_resource *texture, + struct r600_resource_texture **staging) { struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture; struct pipe_resource resource; + struct r600_resource_texture **flushed_depth_texture = staging ? + staging : &rtex->flushed_depth_texture; - if (rtex->flushed_depth_texture) + if (!staging && rtex->flushed_depth_texture) return; /* it's ready */ resource.target = texture->target; @@ -742,32 +747,44 @@ void r600_init_flushed_depth_texture(struct pipe_context *ctx, resource.array_size = texture->array_size; resource.last_level = texture->last_level; resource.nr_samples = texture->nr_samples; - resource.usage = PIPE_USAGE_DYNAMIC; - resource.bind = texture->bind | PIPE_BIND_DEPTH_STENCIL; - resource.flags = R600_RESOURCE_FLAG_TRANSFER | texture->flags; + resource.usage = staging ? PIPE_USAGE_DYNAMIC : PIPE_USAGE_DEFAULT; + resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL; + resource.flags = texture->flags; + + if (staging) + resource.flags |= R600_RESOURCE_FLAG_TRANSFER; + else + rtex->dirty_db = TRUE; - rtex->flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource); - if (rtex->flushed_depth_texture == NULL) { - R600_ERR("failed to create temporary texture to hold untiled copy\n"); + *flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource); + if (*flushed_depth_texture == NULL) { + R600_ERR("failed to create temporary texture to hold flushed depth\n"); return; } - ((struct r600_resource_texture *)rtex->flushed_depth_texture)->is_flushing_texture = TRUE; + (*flushed_depth_texture)->is_flushing_texture = TRUE; + } void r600_texture_depth_flush(struct pipe_context *ctx, - struct pipe_resource *texture) + struct pipe_resource *texture, + struct r600_resource_texture **staging) { struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture; - r600_init_flushed_depth_texture(ctx, texture); + r600_init_flushed_depth_texture(ctx, texture, staging); - if (!rtex->flushed_depth_texture) - return; /* error */ + if (staging) { + if (!*staging) + return; /* error */ - /* XXX: only do this if the depth texture has actually changed: - */ - r600_blit_uncompress_depth(ctx, rtex); + r600_blit_uncompress_depth(ctx, rtex, *staging); + } else { + if (!rtex->flushed_depth_texture) + return; /* error */ + + r600_blit_uncompress_depth(ctx, rtex, NULL); + } } /* Needs adjustment for pixelformat: @@ -831,15 +848,18 @@ struct pipe_transfer* r600_texture_get_transfer(struct pipe_context *ctx, */ /* XXX: when discard is true, no need to read back from depth texture */ - r600_texture_depth_flush(ctx, texture); - if (!rtex->flushed_depth_texture) { + struct r600_resource_texture *staging_depth; + + r600_texture_depth_flush(ctx, texture, &staging_depth); + if (!staging_depth) { R600_ERR("failed to create temporary texture to hold untiled copy\n"); pipe_resource_reference(&trans->transfer.resource, NULL); FREE(trans); return NULL; } - trans->transfer.stride = rtex->flushed_depth_texture->pitch_in_bytes[level]; - trans->offset = r600_texture_get_offset(rtex->flushed_depth_texture, level, box->z); + trans->transfer.stride = staging_depth->pitch_in_bytes[level]; + trans->offset = r600_texture_get_offset(staging_depth, level, box->z); + trans->staging = (struct r600_resource*)staging_depth; return &trans->transfer; } else if (use_staging_texture) { resource.target = PIPE_TEXTURE_2D; @@ -894,25 +914,25 @@ void r600_texture_transfer_destroy(struct pipe_context *ctx, struct pipe_resource *texture = transfer->resource; struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture; - if (rtransfer->staging) { - if (transfer->usage & PIPE_TRANSFER_WRITE) { - r600_copy_from_staging_texture(ctx, rtransfer); - } - pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL); - } - - if (rtex->is_depth && !rtex->is_flushing_texture) { - if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtex->flushed_depth_texture) { + if (rtex->is_depth) { + if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) { struct pipe_box sbox; u_box_origin_2d(texture->width0, texture->height0, &sbox); ctx->resource_copy_region(ctx, texture, 0, 0, 0, 0, - &rtex->flushed_depth_texture->resource.b.b, 0, + &rtransfer->staging->b.b, 0, &sbox); } + } else if (rtransfer->staging) { + if (transfer->usage & PIPE_TRANSFER_WRITE) { + r600_copy_from_staging_texture(ctx, rtransfer); + } } + if (rtransfer->staging) + pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL); + pipe_resource_reference(&transfer->resource, NULL); FREE(transfer); } @@ -923,6 +943,8 @@ void* r600_texture_transfer_map(struct pipe_context *ctx, struct r600_context *rctx = (struct r600_context *)ctx; struct r600_transfer *rtransfer = (struct r600_transfer*)transfer; struct radeon_winsys_cs_handle *buf; + struct r600_resource_texture *rtex = + (struct r600_resource_texture*)transfer->resource; enum pipe_format format = transfer->resource->format; unsigned offset = 0; char *map; @@ -934,17 +956,13 @@ void* r600_texture_transfer_map(struct pipe_context *ctx, if (rtransfer->staging) { buf = ((struct r600_resource *)rtransfer->staging)->cs_buf; } else { - struct r600_resource_texture *rtex = (struct r600_resource_texture*)transfer->resource; - - if (rtex->flushed_depth_texture) - buf = ((struct r600_resource *)rtex->flushed_depth_texture)->cs_buf; - else - buf = ((struct r600_resource *)transfer->resource)->cs_buf; + buf = ((struct r600_resource *)transfer->resource)->cs_buf; + } + if (rtex->is_depth || !rtransfer->staging) offset = rtransfer->offset + transfer->box.y / util_format_get_blockheight(format) * transfer->stride + transfer->box.x / util_format_get_blockwidth(format) * util_format_get_blocksize(format); - } if (!(map = rctx->ws->buffer_map(buf, rctx->cs, transfer->usage))) { return NULL; @@ -967,13 +985,7 @@ void r600_texture_transfer_unmap(struct pipe_context *ctx, if (rtransfer->staging) { buf = ((struct r600_resource *)rtransfer->staging)->cs_buf; } else { - struct r600_resource_texture *rtex = (struct r600_resource_texture*)transfer->resource; - - if (rtex->flushed_depth_texture) { - buf = ((struct r600_resource *)rtex->flushed_depth_texture)->cs_buf; - } else { - buf = ((struct r600_resource *)transfer->resource)->cs_buf; - } + buf = ((struct r600_resource *)transfer->resource)->cs_buf; } rctx->ws->buffer_unmap(buf); } -- cgit v1.2.3