radeonsi: improve flushed depth texture handling

Use r600_resource_texture::flished_depth_texture for GPU access, and
allocate it in the VRAM. For transfers we'll allocate texture in the GTT
and store it in the r600_transfer::staging.

Improves performance when flushed depth texture is frequently used by the
GPU, e.g. in Lightsmark

[ Ported from r600g commit 37708479608af877986b76302a9c92611d1e23d0 ]

Signed-off-by: Michel Dänzer <michel.daenzer@amd.com>

5 files changed, 100 insertions, 81 deletions

diff --git a/src/gallium/drivers/radeonsi/r600_blit.c b/src/gallium/drivers/radeonsi/r600_blit.c
index f368292ccb2..d600962249f 100644
--- a/src/gallium/drivers/radeonsi/r600_blit.c
+++ b/src/gallium/drivers/radeonsi/r600_blit.c
@@ -114,13 +114,17 @@ static unsigned u_num_layers(struct pipe_resource *r, unsigned level)
 	}
 }
 
-void si_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture)
+void si_blit_uncompress_depth(struct pipe_context *ctx,
+		struct r600_resource_texture *texture,
+		struct r600_resource_texture *staging)
 {
 	struct r600_context *rctx = (struct r600_context *)ctx;
 	unsigned layer, level;
 	float depth = 1.0f;
+	struct r600_resource_texture *flushed_depth_texture = staging ?
+			staging : texture->flushed_depth_texture;
 
-	if (!texture->dirty_db)
+	if (!staging && !texture->dirty_db)
 		return;
 
 	for (level = 0; level <= texture->resource.b.b.last_level; level++) {
@@ -136,9 +140,9 @@ void si_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_tex
 
 			zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
 
-			surf_tmpl.format = texture->flushed_depth_texture->real_format;
+			surf_tmpl.format = flushed_depth_texture->real_format;
 			cbsurf = ctx->create_surface(ctx,
-					(struct pipe_resource*)texture->flushed_depth_texture, &surf_tmpl);
+					(struct pipe_resource*)flushed_depth_texture, &surf_tmpl);
 
 			r600_blitter_begin(ctx, R600_DECOMPRESS);
 			util_blitter_custom_depth_stencil(rctx->blitter, zsurf, cbsurf, ~0, rctx->custom_dsa_flush, depth);
@@ -149,7 +153,8 @@ void si_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_tex
 		}
 	}
 
-	texture->dirty_db = FALSE;
+	if (!staging)
+		texture->dirty_db = FALSE;
 }
 
 void si_flush_depth_textures(struct r600_context *rctx)
@@ -166,13 +171,13 @@ void si_flush_depth_textures(struct r600_context *rctx)
 		if (!view) continue;
 
 		tex = (struct r600_resource_texture *)view->base.texture;
-		if (!tex->depth)
+		if (!tex->is_depth)
 			continue;
 
 		if (tex->is_flushing_texture)
 			continue;
 
-		si_blit_uncompress_depth(&rctx->context, tex);
+		si_blit_uncompress_depth(&rctx->context, tex, NULL);
 	}
 
 	/* also check CB here */
@@ -180,13 +185,13 @@ void si_flush_depth_textures(struct r600_context *rctx)
 		struct r600_resource_texture *tex;
 		tex = (struct r600_resource_texture *)rctx->framebuffer.cbufs[i]->texture;
 
-		if (!tex->depth)
+		if (!tex->is_depth)
 			continue;
 
 		if (tex->is_flushing_texture)
 			continue;
 
-		si_blit_uncompress_depth(&rctx->context, tex);
+		si_blit_uncompress_depth(&rctx->context, tex, NULL);
 	}
 }
 
@@ -317,8 +322,8 @@ static void r600_resource_copy_region(struct pipe_context *ctx,
 		return;
 	}
 
-	if (rsrc->depth && !rsrc->is_flushing_texture)
-		r600_texture_depth_flush(ctx, src);
+	if (rsrc->is_depth && !rsrc->is_flushing_texture)
+		r600_texture_depth_flush(ctx, src, NULL);
 
 	restore_orig[0] = restore_orig[1] = FALSE;
 
@@ -371,8 +376,8 @@ static void si_blit(struct pipe_context *ctx,
 		return;
 	}
 
-	if (rsrc->depth && !rsrc->is_flushing_texture)
-		r600_texture_depth_flush(ctx, info->src.resource);
+	if (rsrc->is_depth && !rsrc->is_flushing_texture)
+		r600_texture_depth_flush(ctx, info->src.resource, NULL);
 
 	r600_blitter_begin(ctx, R600_BLIT);
 	util_blitter_blit(rctx->blitter, info);
diff --git a/src/gallium/drivers/radeonsi/r600_resource.h b/src/gallium/drivers/radeonsi/r600_resource.h
index 4cacf6274c7..8d809352e6d 100644
--- a/src/gallium/drivers/radeonsi/r600_resource.h
+++ b/src/gallium/drivers/radeonsi/r600_resource.h
@@ -36,7 +36,7 @@ struct r600_transfer {
 	/* Buffer transfer. */
 	struct pipe_transfer		*buffer_transfer;
 	unsigned			offset;
-	struct pipe_resource		*staging_texture;
+	struct pipe_resource		*staging;
 };
 
 struct r600_resource_texture {
@@ -48,7 +48,7 @@ struct r600_resource_texture {
 	enum pipe_format		real_format;
 
 	unsigned			pitch_override;
-	unsigned			depth;
+	unsigned			is_depth;
 	unsigned			dirty_db;
 	struct r600_resource_texture	*flushed_depth_texture;
 	boolean				is_flushing_texture;
@@ -69,9 +69,11 @@ struct pipe_resource *si_texture_from_handle(struct pipe_screen *screen,
 					     struct winsys_handle *whandle);
 
 void r600_init_flushed_depth_texture(struct pipe_context *ctx,
-				     struct pipe_resource *texture);
+				     struct pipe_resource *texture,
+				     struct r600_resource_texture **staging);
 void r600_texture_depth_flush(struct pipe_context *ctx,
-			      struct pipe_resource *texture);
+			      struct pipe_resource *texture,
+			      struct r600_resource_texture **staging);
 
 
 struct r600_context;
diff --git a/src/gallium/drivers/radeonsi/r600_texture.c b/src/gallium/drivers/radeonsi/r600_texture.c
index 4c107b26bea..b790d8d3719 100644
--- a/src/gallium/drivers/radeonsi/r600_texture.c
+++ b/src/gallium/drivers/radeonsi/r600_texture.c
@@ -42,7 +42,7 @@ static void r600_copy_to_staging_texture(struct pipe_context *ctx, struct r600_t
 	struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
 	struct pipe_resource *texture = transfer->resource;
 
-	ctx->resource_copy_region(ctx, rtransfer->staging_texture,
+	ctx->resource_copy_region(ctx, rtransfer->staging,
 				0, 0, 0, 0, texture, transfer->level,
 				&transfer->box);
 }
@@ -62,7 +62,7 @@ static void r600_copy_from_staging_texture(struct pipe_context *ctx, struct r600
 	sbox.depth = 1;
 	ctx->resource_copy_region(ctx, texture, transfer->level,
 				  transfer->box.x, transfer->box.y, transfer->box.z,
-				  rtransfer->staging_texture,
+				  rtransfer->staging,
 				  0, &sbox);
 }
 
@@ -153,7 +153,8 @@ static int r600_init_surface(struct r600_screen *rscreen,
 		surface->flags |= RADEON_SURF_SCANOUT;
 	}
 
-	if (!is_flushed_depth && is_depth) {
+	if ((ptex->bind & PIPE_BIND_DEPTH_STENCIL) &&
+	    !is_flushed_depth && is_depth) {
 		surface->flags |= RADEON_SURF_ZBUFFER;
 
 		if (is_stencil) {
@@ -323,20 +324,24 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 	trans->transfer.level = level;
 	trans->transfer.usage = usage;
 	trans->transfer.box = *box;
-	if (rtex->depth) {
+	if (rtex->is_depth) {
 		/* XXX: only readback the rectangle which is being mapped?
 		*/
 		/* XXX: when discard is true, no need to read back from depth texture
 		*/
-		r600_texture_depth_flush(ctx, texture);
-		if (!rtex->flushed_depth_texture) {
+		struct r600_resource_texture *staging_depth;
+
+		r600_texture_depth_flush(ctx, texture, &staging_depth);
+		if (!staging_depth) {
 			R600_ERR("failed to create temporary texture to hold untiled copy\n");
 			pipe_resource_reference(&trans->transfer.resource, NULL);
 			FREE(trans);
 			return NULL;
 		}
-		trans->transfer.stride = rtex->flushed_depth_texture->surface.level[level].pitch_bytes;
-		trans->offset = r600_texture_get_offset(rtex->flushed_depth_texture, level, box->z);
+		trans->transfer.stride = staging_depth->surface.level[level].pitch_bytes;
+		trans->offset = r600_texture_get_offset(staging_depth, level, box->z);
+
+		trans->staging = &staging_depth->resource.b.b;
 	} else if (use_staging_texture) {
 		resource.target = PIPE_TEXTURE_2D;
 		resource.format = texture->format;
@@ -360,15 +365,15 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 			resource.bind |= PIPE_BIND_SAMPLER_VIEW;
 		}
 		/* Create the temporary texture. */
-		trans->staging_texture = ctx->screen->resource_create(ctx->screen, &resource);
-		if (trans->staging_texture == NULL) {
+		trans->staging = ctx->screen->resource_create(ctx->screen, &resource);
+		if (trans->staging == NULL) {
 			R600_ERR("failed to create temporary texture to hold untiled copy\n");
 			pipe_resource_reference(&trans->transfer.resource, NULL);
 			FREE(trans);
 			return NULL;
 		}
 
-		trans->transfer.stride = ((struct r600_resource_texture *)trans->staging_texture)
+		trans->transfer.stride = ((struct r600_resource_texture *)trans->staging)
 					->surface.level[0].pitch_bytes;
 		if (usage & PIPE_TRANSFER_READ) {
 			r600_copy_to_staging_texture(ctx, trans);
@@ -381,23 +386,19 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 		trans->offset = r600_texture_get_offset(rtex, level, box->z);
 	}
 
-	if (trans->staging_texture) {
-		buf = si_resource(trans->staging_texture)->cs_buf;
+	if (trans->staging) {
+		buf = si_resource(trans->staging)->cs_buf;
 	} else {
-		struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
-
-		if (rtex->flushed_depth_texture)
-			buf = rtex->flushed_depth_texture->resource.cs_buf;
-		else
-			buf = si_resource(texture)->cs_buf;
+		buf = si_resource(trans->transfer.resource)->cs_buf;
+	}
 
+	if (rtex->is_depth || !trans->staging)
 		offset = trans->offset +
 			box->y / util_format_get_blockheight(format) * trans->transfer.stride +
 			box->x / util_format_get_blockwidth(format) * util_format_get_blocksize(format);
-	}
 
 	if (!(map = rctx->ws->buffer_map(buf, rctx->cs, usage))) {
-		pipe_resource_reference(&trans->staging_texture, NULL);
+		pipe_resource_reference(&trans->staging, NULL);
 		pipe_resource_reference(&trans->transfer.resource, NULL);
 		FREE(trans);
 		return NULL;
@@ -416,28 +417,15 @@ static void si_texture_transfer_unmap(struct pipe_context *ctx,
 	struct pipe_resource *texture = transfer->resource;
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 
-	if (rtransfer->staging_texture) {
-		buf = si_resource(rtransfer->staging_texture)->cs_buf;
+	if (rtransfer->staging) {
+		buf = si_resource(rtransfer->staging)->cs_buf;
 	} else {
-		struct r600_resource_texture *rtex = (struct r600_resource_texture*)transfer->resource;
-
-		if (rtex->flushed_depth_texture) {
-			buf = rtex->flushed_depth_texture->resource.cs_buf;
-		} else {
-			buf = si_resource(transfer->resource)->cs_buf;
-		}
+		buf = si_resource(transfer->resource)->cs_buf;
 	}
 	rctx->ws->buffer_unmap(buf);
 
-	if (rtransfer->staging_texture) {
-		if (transfer->usage & PIPE_TRANSFER_WRITE) {
-			r600_copy_from_staging_texture(ctx, rtransfer);
-		}
-		pipe_resource_reference(&rtransfer->staging_texture, NULL);
-	}
-
-	if (rtex->depth && !rtex->is_flushing_texture) {
-		if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtex->flushed_depth_texture) {
+	if (rtex->is_depth) {
+		if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
 			struct pipe_box sbox;
 
 			sbox.x = sbox.y = sbox.z = 0;
@@ -447,11 +435,18 @@ static void si_texture_transfer_unmap(struct pipe_context *ctx,
 			sbox.depth = 1;
 
 			ctx->resource_copy_region(ctx, texture, 0, 0, 0, 0,
-						  &rtex->flushed_depth_texture->resource.b.b, 0,
+						  &si_resource(rtransfer->staging)->b.b, 0,
 						  &sbox);
 		}
+	} else if (rtransfer->staging) {
+		if (transfer->usage & PIPE_TRANSFER_WRITE) {
+			r600_copy_from_staging_texture(ctx, rtransfer);
+		}
 	}
 
+	if (rtransfer->staging)
+		pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
+
 	pipe_resource_reference(&transfer->resource, NULL);
 	FREE(transfer);
 }
@@ -495,7 +490,7 @@ r600_texture_create_object(struct pipe_screen *screen,
 
 	/* only mark depth textures the HW can hit as depth textures */
 	if (util_format_is_depth_or_stencil(rtex->real_format) && permit_hardware_blit(screen, base))
-		rtex->depth = 1;
+		rtex->is_depth = 1;
 
 	rtex->surface = *surface;
 	r = r600_setup_surface(screen, rtex, array_mode, pitch_in_bytes_override);
@@ -628,12 +623,15 @@ struct pipe_resource *si_texture_from_handle(struct pipe_screen *screen,
 }
 
 void r600_init_flushed_depth_texture(struct pipe_context *ctx,
-				     struct pipe_resource *texture)
+				     struct pipe_resource *texture,
+				     struct r600_resource_texture **staging)
 {
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 	struct pipe_resource resource;
+	struct r600_resource_texture **flushed_depth_texture = staging ?
+			staging : &rtex->flushed_depth_texture;
 
-	if (rtex->flushed_depth_texture)
+	if (!staging && rtex->flushed_depth_texture)
 		return; /* it's ready */
 
 	resource.target = texture->target;
@@ -644,32 +642,43 @@ void r600_init_flushed_depth_texture(struct pipe_context *ctx,
 	resource.array_size = texture->array_size;
 	resource.last_level = texture->last_level;
 	resource.nr_samples = texture->nr_samples;
-	resource.usage = PIPE_USAGE_DYNAMIC;
-	resource.bind = texture->bind | PIPE_BIND_DEPTH_STENCIL;
-	resource.flags = R600_RESOURCE_FLAG_TRANSFER | R600_RESOURCE_FLAG_FLUSHED_DEPTH | texture->flags;
+	resource.usage = staging ? PIPE_USAGE_DYNAMIC : PIPE_USAGE_DEFAULT;
+	resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
+	resource.flags = texture->flags | R600_RESOURCE_FLAG_FLUSHED_DEPTH;
 
-	rtex->flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource);
-	if (rtex->flushed_depth_texture == NULL) {
-		R600_ERR("failed to create temporary texture to hold untiled copy\n");
+	if (staging)
+		resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
+	else
+		rtex->dirty_db = TRUE;
+
+	*flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource);
+	if (*flushed_depth_texture == NULL) {
+		R600_ERR("failed to create temporary texture to hold flushed depth\n");
 		return;
 	}
 
-	((struct r600_resource_texture *)rtex->flushed_depth_texture)->is_flushing_texture = TRUE;
+	(*flushed_depth_texture)->is_flushing_texture = TRUE;
 }
 
 void r600_texture_depth_flush(struct pipe_context *ctx,
-			      struct pipe_resource *texture)
+			      struct pipe_resource *texture,
+			      struct r600_resource_texture **staging)
 {
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 
-	r600_init_flushed_depth_texture(ctx, texture);
+	r600_init_flushed_depth_texture(ctx, texture, staging);
 
-	if (!rtex->flushed_depth_texture)
-		return; /* error */
+	if (staging) {
+		if (!*staging)
+			return; /* error */
 
-	/* XXX: only do this if the depth texture has actually changed:
-	 */
-	si_blit_uncompress_depth(ctx, rtex);
+		si_blit_uncompress_depth(ctx, rtex, *staging);
+	} else {
+		if (!rtex->flushed_depth_texture)
+			return; /* error */
+
+		si_blit_uncompress_depth(ctx, rtex, NULL);
+	}
 }
 
 void si_init_surface_functions(struct r600_context *r600)
diff --git a/src/gallium/drivers/radeonsi/radeonsi_pipe.h b/src/gallium/drivers/radeonsi/radeonsi_pipe.h
index 8df32416c1f..a0abdec9a2e 100644
--- a/src/gallium/drivers/radeonsi/radeonsi_pipe.h
+++ b/src/gallium/drivers/radeonsi/radeonsi_pipe.h
@@ -185,7 +185,9 @@ struct r600_context {
 
 /* r600_blit.c */
 void si_init_blit_functions(struct r600_context *rctx);
-void si_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_texture *texture);
+void si_blit_uncompress_depth(struct pipe_context *ctx,
+		struct r600_resource_texture *texture,
+		struct r600_resource_texture *staging);
 void si_flush_depth_textures(struct r600_context *rctx);
 
 /* r600_buffer.c */
diff --git a/src/gallium/drivers/radeonsi/si_state.c b/src/gallium/drivers/radeonsi/si_state.c
index fe3efb6adc9..3908b773b5c 100644
--- a/src/gallium/drivers/radeonsi/si_state.c
+++ b/src/gallium/drivers/radeonsi/si_state.c
@@ -1581,11 +1581,12 @@ static void si_cb(struct r600_context *rctx, struct si_pm4_state *pm4,
 	surf = (struct r600_surface *)state->cbufs[cb];
 	rtex = (struct r600_resource_texture*)state->cbufs[cb]->texture;
 
-	if (rtex->depth)
+	if (rtex->is_depth)
 		rctx->have_depth_fb = TRUE;
 
-	if (rtex->depth && !rtex->is_flushing_texture) {
-	        r600_init_flushed_depth_texture(&rctx->context, state->cbufs[cb]->texture);
+	if (rtex->is_depth && !rtex->is_flushing_texture) {
+		r600_init_flushed_depth_texture(&rctx->context,
+				state->cbufs[cb]->texture, NULL);
 		rtex = rtex->flushed_depth_texture;
 		assert(rtex);
 	}
@@ -2083,8 +2084,8 @@ static struct pipe_sampler_view *si_create_sampler_view(struct pipe_context *ctx
 		format = 0;
 	}
 
-	if (tmp->depth && !tmp->is_flushing_texture) {
-		r600_init_flushed_depth_texture(ctx, texture);
+	if (tmp->is_depth && !tmp->is_flushing_texture) {
+		r600_init_flushed_depth_texture(ctx, texture, NULL);
 		tmp = tmp->flushed_depth_texture;
 		if (!tmp) {
 			FREE(view);
@@ -2222,7 +2223,7 @@ static struct si_pm4_state *si_set_sampler_view(struct r600_context *rctx,
 		if (resource[i]) {
 			struct r600_resource_texture *rtex =
 				(struct r600_resource_texture *)views[i]->texture;
-			rctx->have_depth_texture |= rtex->depth && !rtex->is_flushing_texture;
+			rctx->have_depth_texture |= rtex->is_depth && !rtex->is_flushing_texture;
 			si_pm4_add_bo(pm4, resource[i]->resource, RADEON_USAGE_READ);
 		}