r600g: fix depth-stencil on evergreen

Such that it actually works in apps which use both.

A separate buffer is allocated for stencil. The only exception is
the window-system-provided depth-stencil buffer, where depth and stencil
share the same buffer.

This fixes:
- fbo-depthstencil-GL_DEPTH24_STENCIL8-clear
- fbo-depthstencil-GL_DEPTH24_STENCIL8-drawpixels-FLOAT-and-USHORT
- fbo-depthstencil-GL_DEPTH24_STENCIL8-readpixels-24_8
- fbo-depthstencil-GL_DEPTH24_STENCIL8-readpixels-FLOAT-and-USHORT

4 files changed, 99 insertions, 55 deletions

diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c
index f82e20306d..f2b3b8304b 100644
--- a/src/gallium/drivers/r600/evergreen_state.c
+++ b/src/gallium/drivers/r600/evergreen_state.c
@@ -262,7 +262,6 @@ static uint32_t r600_translate_dbformat(enum pipe_format format)
 	case PIPE_FORMAT_Z16_UNORM:
 		return V_028040_Z_16;
 	case PIPE_FORMAT_Z24X8_UNORM:
-		return V_028040_Z_24;
 	case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
 		return V_028040_Z_24;
 	default:
@@ -270,14 +269,6 @@ static uint32_t r600_translate_dbformat(enum pipe_format format)
 	}
 }
 
-static uint32_t r600_translate_stencilformat(enum pipe_format format)
-{
-	if (format == PIPE_FORMAT_Z24_UNORM_S8_USCALED)
-		return 1;
-	else
-		return 0;
-}
-
 static uint32_t r600_translate_colorswap(enum pipe_format format)
 {
 	switch (format) {
@@ -1381,55 +1372,51 @@ static void evergreen_cb(struct r600_pipe_context *rctx, struct r600_pipe_state
 }
 
 static void evergreen_db(struct r600_pipe_context *rctx, struct r600_pipe_state *rstate,
-			const struct pipe_framebuffer_state *state)
+			 const struct pipe_framebuffer_state *state)
 {
 	struct r600_resource_texture *rtex;
-	struct r600_resource *rbuffer;
 	struct r600_surface *surf;
-	unsigned level;
-	unsigned pitch, slice, format, stencil_format;
+	unsigned level, first_layer;
+	unsigned pitch, slice, format;
 	unsigned offset;
 
 	if (state->zsbuf == NULL)
 		return;
 
-	level = state->zsbuf->u.tex.level;
-
 	surf = (struct r600_surface *)state->zsbuf;
-	rtex = (struct r600_resource_texture*)state->zsbuf->texture;
+	rtex = (struct r600_resource_texture*)surf->base.texture;
 
-	rbuffer = &rtex->resource;
-
-	/* XXX quite sure for dx10+ hw don't need any offset hacks */
-	offset = r600_texture_get_offset((struct r600_resource_texture *)state->zsbuf->texture,
-					 level, state->zsbuf->u.tex.first_layer);
+	level = surf->base.u.tex.level;
+	first_layer = surf->base.u.tex.first_layer;
+	offset = r600_texture_get_offset(rtex, level, first_layer);
 	pitch = rtex->pitch_in_blocks[level] / 8 - 1;
 	slice = rtex->pitch_in_blocks[level] * surf->aligned_height / 64 - 1;
-	format = r600_translate_dbformat(state->zsbuf->texture->format);
-	stencil_format = r600_translate_stencilformat(state->zsbuf->texture->format);
+	format = r600_translate_dbformat(rtex->real_format);
 
 	r600_pipe_state_add_reg(rstate, R_028048_DB_Z_READ_BASE,
-				offset >> 8, 0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
+				offset >> 8, 0xFFFFFFFF, rtex->resource.bo, RADEON_USAGE_READWRITE);
 	r600_pipe_state_add_reg(rstate, R_028050_DB_Z_WRITE_BASE,
-				offset >> 8, 0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
+				offset >> 8, 0xFFFFFFFF, rtex->resource.bo, RADEON_USAGE_READWRITE);
+	r600_pipe_state_add_reg(rstate, R_028008_DB_DEPTH_VIEW, 0x00000000, 0xFFFFFFFF, NULL, 0);
 
-	if (stencil_format) {
-		uint32_t stencil_offset;
+	if (rtex->stencil) {
+		uint32_t stencil_offset =
+			r600_texture_get_offset(rtex->stencil, level, first_layer);
 
-		stencil_offset = ((surf->aligned_height * rtex->pitch_in_bytes[level]) + 255) & ~255;
 		r600_pipe_state_add_reg(rstate, R_02804C_DB_STENCIL_READ_BASE,
-					(offset + stencil_offset) >> 8, 0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
+					stencil_offset >> 8, 0xFFFFFFFF, rtex->stencil->resource.bo, RADEON_USAGE_READWRITE);
 		r600_pipe_state_add_reg(rstate, R_028054_DB_STENCIL_WRITE_BASE,
-					(offset + stencil_offset) >> 8, 0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
+					stencil_offset >> 8, 0xFFFFFFFF, rtex->stencil->resource.bo, RADEON_USAGE_READWRITE);
+		r600_pipe_state_add_reg(rstate, R_028044_DB_STENCIL_INFO,
+					1, 0xFFFFFFFF, rtex->stencil->resource.bo, RADEON_USAGE_READWRITE);
+	} else {
+		r600_pipe_state_add_reg(rstate, R_028044_DB_STENCIL_INFO,
+					0, 0xFFFFFFFF, NULL, RADEON_USAGE_READWRITE);
 	}
 
-	r600_pipe_state_add_reg(rstate, R_028008_DB_DEPTH_VIEW, 0x00000000, 0xFFFFFFFF, NULL, 0);
-	r600_pipe_state_add_reg(rstate, R_028044_DB_STENCIL_INFO,
-				S_028044_FORMAT(stencil_format), 0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
-
 	r600_pipe_state_add_reg(rstate, R_028040_DB_Z_INFO,
 				S_028040_ARRAY_MODE(rtex->array_mode[level]) | S_028040_FORMAT(format),
-				0xFFFFFFFF, rbuffer->bo, RADEON_USAGE_READWRITE);
+				0xFFFFFFFF, rtex->resource.bo, RADEON_USAGE_READWRITE);
 	r600_pipe_state_add_reg(rstate, R_028058_DB_DEPTH_SIZE,
 				S_028058_PITCH_TILE_MAX(pitch),
 				0xFFFFFFFF, NULL, 0);
diff --git a/src/gallium/drivers/r600/r600_blit.c b/src/gallium/drivers/r600/r600_blit.c
index e1cf585234..2f7e871448 100644
--- a/src/gallium/drivers/r600/r600_blit.c
+++ b/src/gallium/drivers/r600/r600_blit.c
@@ -111,7 +111,7 @@ void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_t
 	if (!texture->dirty_db)
 		return;
 
-	surf_tmpl.format = texture->resource.b.b.b.format;
+	surf_tmpl.format = texture->real_format;
 	surf_tmpl.u.tex.level = level;
 	surf_tmpl.u.tex.first_layer = 0;
 	surf_tmpl.u.tex.last_layer = 0;
@@ -119,7 +119,7 @@ void r600_blit_uncompress_depth(struct pipe_context *ctx, struct r600_resource_t
 
 	zsurf = ctx->create_surface(ctx, &texture->resource.b.b.b, &surf_tmpl);
 
-	surf_tmpl.format = ((struct pipe_resource*)texture->flushed_depth_texture)->format;
+	surf_tmpl.format = texture->flushed_depth_texture->real_format;
 	surf_tmpl.usage = PIPE_BIND_RENDER_TARGET;
 	cbsurf = ctx->create_surface(ctx,
 			(struct pipe_resource*)texture->flushed_depth_texture, &surf_tmpl);
@@ -249,7 +249,7 @@ static void r600_compressed_to_blittable(struct pipe_resource *tex,
 				   struct texture_orig_info *orig)
 {
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)tex;
-	unsigned pixsize = util_format_get_blocksize(tex->format);
+	unsigned pixsize = util_format_get_blocksize(rtex->real_format);
 	int new_format;
 	int new_height, new_width;
 
@@ -269,7 +269,6 @@ static void r600_compressed_to_blittable(struct pipe_resource *tex,
 	tex->width0 = new_width;
 	tex->height0 = new_height;
 	tex->format = new_format;
-
 }
 
 static void r600_reset_blittable_to_compressed(struct pipe_resource *tex,
diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index 836e7491f1..d9d29db796 100644
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -52,6 +52,12 @@ struct r600_resource {
 
 struct r600_resource_texture {
 	struct r600_resource		resource;
+
+	/* If this resource is a depth-stencil buffer on evergreen, this contains
+	 * the depth part of the format. There is a separate stencil resource
+	 * for the stencil buffer below. */
+	enum pipe_format		real_format;
+
 	unsigned			offset[PIPE_MAX_TEXTURE_LEVELS];
 	unsigned			pitch_in_bytes[PIPE_MAX_TEXTURE_LEVELS];  /* transfer */
 	unsigned			pitch_in_blocks[PIPE_MAX_TEXTURE_LEVELS]; /* texture resource */
@@ -62,6 +68,7 @@ struct r600_resource_texture {
 	unsigned			tile_type;
 	unsigned			depth;
 	unsigned			dirty_db;
+	struct r600_resource_texture    *stencil; /* Stencil is in a separate buffer on Evergreen. */
 	struct r600_resource_texture	*flushed_depth_texture;
 	boolean				is_flushing_texture;
 
diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c
index 7b5a3e74a2..1c6f39adee 100644
--- a/src/gallium/drivers/r600/r600_texture.c
+++ b/src/gallium/drivers/r600/r600_texture.c
@@ -173,15 +173,15 @@ static unsigned r600_texture_get_nblocksx(struct pipe_screen *screen,
 {
 	struct pipe_resource *ptex = &rtex->resource.b.b.b;
 	unsigned nblocksx, block_align, width;
-	unsigned blocksize = util_format_get_blocksize(ptex->format);
+	unsigned blocksize = util_format_get_blocksize(rtex->real_format);
 
 	if (rtex->pitch_override)
 		return rtex->pitch_override / blocksize;
 
 	width = mip_minify(ptex->width0, level);
-	nblocksx = util_format_get_nblocksx(ptex->format, width);
+	nblocksx = util_format_get_nblocksx(rtex->real_format, width);
 
-	block_align = r600_get_block_alignment(screen, ptex->format,
+	block_align = r600_get_block_alignment(screen, rtex->real_format,
 					      rtex->array_mode[level]);
 	nblocksx = align(nblocksx, block_align);
 	return nblocksx;
@@ -195,7 +195,7 @@ static unsigned r600_texture_get_nblocksy(struct pipe_screen *screen,
 	unsigned height, tile_height;
 
 	height = mip_minify(ptex->height0, level);
-	height = util_format_get_nblocksy(ptex->format, height);
+	height = util_format_get_nblocksy(rtex->real_format, height);
 	tile_height = r600_get_height_alignment(screen,
 						rtex->array_mode[level]);
 	height = align(height, tile_height);
@@ -220,7 +220,7 @@ static void r600_texture_set_array_mode(struct pipe_screen *screen,
 		unsigned w, h, tile_height, tile_width;
 
 		tile_height = r600_get_height_alignment(screen, array_mode);
-		tile_width = r600_get_block_alignment(screen, ptex->format, array_mode);
+		tile_width = r600_get_block_alignment(screen, rtex->real_format, array_mode);
 
 		w = mip_minify(ptex->width0, level);
 		h = mip_minify(ptex->height0, level);
@@ -241,11 +241,11 @@ static void r600_setup_miptree(struct pipe_screen *screen,
 	struct radeon *radeon = ((struct r600_screen*)screen)->radeon;
 	enum chip_class chipc = r600_get_family_class(radeon);
 	unsigned size, layer_size, i, offset;
-	unsigned nblocksx, nblocksy, extra_size = 0;
+	unsigned nblocksx, nblocksy;
 
 	for (i = 0, offset = 0; i <= ptex->last_level; i++) {
-		unsigned blocksize = util_format_get_blocksize(ptex->format);
-		unsigned base_align = r600_get_base_alignment(screen, ptex->format, array_mode);
+		unsigned blocksize = util_format_get_blocksize(rtex->real_format);
+		unsigned base_align = r600_get_base_alignment(screen, rtex->real_format, array_mode);
 
 		r600_texture_set_array_mode(screen, rtex, i, array_mode);
 
@@ -264,10 +264,6 @@ static void r600_setup_miptree(struct pipe_screen *screen,
 		else
 			size = layer_size * ptex->array_size;
 
-		/* evergreen stores depth and stencil separately */
-		if ((chipc >= EVERGREEN) && util_format_is_depth_or_stencil(ptex->format))
-			extra_size = align(extra_size + (nblocksx * nblocksy * 1), base_align);
-
 		/* align base image and start of miptree */
 		if ((i == 0) || (i == 1))
 			offset = align(offset, base_align);
@@ -278,7 +274,7 @@ static void r600_setup_miptree(struct pipe_screen *screen,
 
 		offset += size;
 	}
-	rtex->size = offset + extra_size;
+	rtex->size = offset;
 }
 
 /* Figure out whether u_blitter will fallback to a transfer operation.
@@ -384,20 +380,76 @@ r600_texture_create_object(struct pipe_screen *screen,
 	resource->b.b.b.screen = screen;
 	resource->bo = bo;
 	rtex->pitch_override = pitch_in_bytes_override;
+	rtex->real_format = base->format;
+
+	/* We must split depth and stencil into two separate buffers on Evergreen. */
+	if (r600_get_family_class(((struct r600_screen*)screen)->radeon) >= EVERGREEN &&
+	    util_format_is_depth_and_stencil(base->format)) {
+		struct pipe_resource stencil;
+		unsigned stencil_pitch_override = 0;
+
+		switch (base->format) {
+		case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
+			rtex->real_format = PIPE_FORMAT_Z24X8_UNORM;
+			break;
+		case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
+			rtex->real_format = PIPE_FORMAT_X8Z24_UNORM;
+			break;
+		case PIPE_FORMAT_Z32_FLOAT_S8X24_USCALED:
+			rtex->real_format = PIPE_FORMAT_Z32_FLOAT;
+			break;
+		default:
+			assert(0);
+			FREE(rtex);
+			return NULL;
+		}
+
+		/* Divide the pitch in bytes by 4 for stencil, because it has a smaller pixel size. */
+		if (pitch_in_bytes_override) {
+			assert(base->format == PIPE_FORMAT_Z24_UNORM_S8_USCALED ||
+			       base->format == PIPE_FORMAT_S8_USCALED_Z24_UNORM);
+			stencil_pitch_override = pitch_in_bytes_override / 4;
+		}
+
+		/* Allocate the stencil buffer. */
+		stencil = *base;
+		stencil.format = PIPE_FORMAT_S8_USCALED;
+		rtex->stencil = r600_texture_create_object(screen, &stencil, array_mode,
+							   stencil_pitch_override, max_buffer_size, bo);
+		if (!rtex->stencil) {
+			FREE(rtex);
+			return NULL;
+		}
+		/* Proceed in creating the depth buffer. */
+	}
+
 	/* only mark depth textures the HW can hit as depth textures */
-	if (util_format_is_depth_or_stencil(base->format) && permit_hardware_blit(screen, base))
+	if (util_format_is_depth_or_stencil(rtex->real_format) && permit_hardware_blit(screen, base))
 		rtex->depth = 1;
 
 	r600_setup_miptree(screen, rtex, array_mode);
 
 	resource->size = rtex->size;
 
+	/* If bo is not NULL, in which case depth and stencil must share the same buffer,
+	 * and we initialized separate stencil for Evergreen. place it after depth. */
+	if (bo && rtex->stencil) {
+		unsigned stencil_align, stencil_offset;
+
+		stencil_align = r600_get_base_alignment(screen, rtex->stencil->real_format, array_mode);
+		stencil_offset = align(rtex->size, stencil_align);
+
+		for (unsigned i = 0; i <= rtex->stencil->resource.b.b.b.last_level; i++)
+			rtex->stencil->offset[i] += stencil_offset;
+	}
+
 	if (!resource->bo) {
 		struct pipe_resource *ptex = &rtex->resource.b.b.b;
-		int base_align = r600_get_base_alignment(screen, ptex->format, array_mode);
+		unsigned base_align = r600_get_base_alignment(screen, ptex->format, array_mode);
 
 		resource->bo = r600_bo(radeon, rtex->size, base_align, base->bind, base->usage);
 		if (!resource->bo) {
+			pipe_resource_reference((struct pipe_resource**)&rtex->stencil, NULL);
 			FREE(rtex);
 			return NULL;
 		}
@@ -436,7 +488,6 @@ struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
 
 	return (struct pipe_resource *)r600_texture_create_object(screen, templ, array_mode,
 								  0, 0, NULL);
-
 }
 
 static struct pipe_surface *r600_create_surface(struct pipe_context *pipe,