Introduce COMMIT_RING() as in radeon DRM, stop using error prone writeback

    for ring read pointer (Paul Mackerras)
Get rid of some superfluous stuff, minor fixes

6 files changed, 130 insertions, 126 deletions

diff --git a/shared-core/r128_cce.c b/shared-core/r128_cce.c
index d0394a0c..13b44a5c 100644
--- a/shared-core/r128_cce.c
+++ b/shared-core/r128_cce.c
@@ -212,7 +212,7 @@ int r128_do_cce_idle( drm_r128_private_t *dev_priv )
 	int i;
 
 	for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
-		if ( GET_RING_HEAD( &dev_priv->ring ) == dev_priv->ring.tail ) {
+		if ( GET_RING_HEAD( dev_priv ) == dev_priv->ring.tail ) {
 			int pm4stat = R128_READ( R128_PM4_STAT );
 			if ( ( (pm4stat & R128_PM4_FIFOCNT_MASK) >=
 			       dev_priv->cce_fifo_size ) &&
@@ -238,7 +238,8 @@ static void r128_do_cce_start( drm_r128_private_t *dev_priv )
 	r128_do_wait_for_idle( dev_priv );
 
 	R128_WRITE( R128_PM4_BUFFER_CNTL,
-		    dev_priv->cce_mode | dev_priv->ring.size_l2qw );
+		    dev_priv->cce_mode | dev_priv->ring.size_l2qw
+		    | R128_PM4_BUFFER_CNTL_NOUPDATE );
 	R128_READ( R128_PM4_BUFFER_ADDR ); /* as per the sample code */
 	R128_WRITE( R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN );
 
@@ -253,7 +254,6 @@ static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
 {
 	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 	R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
-	SET_RING_HEAD( &dev_priv->ring, 0 );
 	dev_priv->ring.tail = 0;
 }
 
@@ -264,7 +264,8 @@ static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
 static void r128_do_cce_stop( drm_r128_private_t *dev_priv )
 {
 	R128_WRITE( R128_PM4_MICRO_CNTL, 0 );
-	R128_WRITE( R128_PM4_BUFFER_CNTL, R128_PM4_NONPM4 );
+	R128_WRITE( R128_PM4_BUFFER_CNTL,
+		    R128_PM4_NONPM4 | R128_PM4_BUFFER_CNTL_NOUPDATE );
 
 	dev_priv->cce_running = 0;
 }
@@ -333,26 +334,6 @@ static void r128_cce_init_ring_buffer( drm_device_t *dev,
 	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 	R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
 
-	/* DL_RPTR_ADDR is a physical address in AGP space. */
-	SET_RING_HEAD( &dev_priv->ring, 0 );
-
-	if ( !dev_priv->is_pci ) {
-		R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
-			    dev_priv->ring_rptr->offset );
-	} else {
-		drm_sg_mem_t *entry = dev->sg;
-		unsigned long tmp_ofs, page_ofs;
-
-		tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
-		page_ofs = tmp_ofs >> PAGE_SHIFT;
-
-		R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
-     			    entry->busaddr[page_ofs]);
-		DRM_DEBUG( "ring rptr: offset=0x%08lx handle=0x%08lx\n",
-			   (unsigned long) entry->busaddr[page_ofs],
-     			   entry->handle + tmp_ofs );
-	}
-
 	/* Set watermark control */
 	R128_WRITE( R128_PM4_BUFFER_WM_CNTL,
 		    ((R128_WATERMARK_L/4) << R128_WMA_SHIFT)
@@ -567,9 +548,6 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 #endif
 		dev_priv->cce_buffers_offset = dev->sg->handle;
 
-	dev_priv->ring.head = ((__volatile__ u32 *)
-			       dev_priv->ring_rptr->handle);
-
 	dev_priv->ring.start = (u32 *)dev_priv->cce_ring->handle;
 	dev_priv->ring.end = ((u32 *)dev_priv->cce_ring->handle
 			      + init->ring_size / sizeof(u32));
@@ -580,7 +558,6 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 		(dev_priv->ring.size / sizeof(u32)) - 1;
 
 	dev_priv->ring.high_mark = 128;
-	dev_priv->ring.ring_rptr = dev_priv->ring_rptr;
 
 	dev_priv->sarea_priv->last_frame = 0;
 	R128_WRITE( R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame );
@@ -589,8 +566,9 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 	R128_WRITE( R128_LAST_DISPATCH_REG,
 		    dev_priv->sarea_priv->last_dispatch );
 
-#if __REALLY_HAVE_SG
+#if __REALLY_HAVE_AGP
 	if ( dev_priv->is_pci ) {
+#endif
 		if (!DRM(ati_pcigart_init)( dev, &dev_priv->phys_pci_gart,
      					    &dev_priv->bus_pci_gart) ) {
 			DRM_ERROR( "failed to init PCI GART!\n" );
@@ -599,6 +577,7 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 			return DRM_ERR(ENOMEM);
 		}
 		R128_WRITE( R128_PCI_GART_PAGE, dev_priv->bus_pci_gart );
+#if __REALLY_HAVE_AGP
 	}
 #endif
 
@@ -901,7 +880,7 @@ int r128_wait_ring( drm_r128_private_t *dev_priv, int n )
 	int i;
 
 	for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
-		r128_update_ring_snapshot( ring );
+		r128_update_ring_snapshot( dev_priv );
 		if ( ring->space >= n )
 			return 0;
 		DRM_UDELAY( 1 );
diff --git a/shared-core/r128_drv.h b/shared-core/r128_drv.h
index 26f7cf86..10569f46 100644
--- a/shared-core/r128_drv.h
+++ b/shared-core/r128_drv.h
@@ -34,8 +34,7 @@
 #ifndef __R128_DRV_H__
 #define __R128_DRV_H__
 
-#define GET_RING_HEAD(ring)		DRM_READ32(  (ring)->ring_rptr, 0 ) /* (ring)->head */
-#define SET_RING_HEAD(ring,val)		DRM_WRITE32( (ring)->ring_rptr, 0, (val) ) /* (ring)->head */
+#define GET_RING_HEAD(dev_priv)		R128_READ( R128_PM4_BUFFER_DL_RPTR )
 
 typedef struct drm_r128_freelist {
    	unsigned int age;
@@ -50,13 +49,11 @@ typedef struct drm_r128_ring_buffer {
 	int size;
 	int size_l2qw;
 
-	volatile u32 *head;
 	u32 tail;
 	u32 tail_mask;
 	int space;
 
 	int high_mark;
-	drm_local_map_t *ring_rptr;
 } drm_r128_ring_buffer_t;
 
 typedef struct drm_r128_private {
@@ -132,14 +129,6 @@ extern drm_buf_t *r128_freelist_get( drm_device_t *dev );
 
 extern int r128_wait_ring( drm_r128_private_t *dev_priv, int n );
 
-static __inline__ void
-r128_update_ring_snapshot( drm_r128_ring_buffer_t *ring )
-{
-	ring->space = (GET_RING_HEAD( ring ) - ring->tail) * sizeof(u32);
-	if ( ring->space <= 0 )
-		ring->space += ring->size;
-}
-
 extern int r128_do_cce_idle( drm_r128_private_t *dev_priv );
 extern int r128_do_cleanup_cce( drm_device_t *dev );
 extern int r128_do_cleanup_pageflip( drm_device_t *dev );
@@ -279,6 +268,7 @@ extern int r128_cce_indirect( DRM_IOCTL_ARGS );
 #	define R128_PM4_64PIO_64VCBM_64INDBM	(7  << 28)
 #	define R128_PM4_64BM_64VCBM_64INDBM	(8  << 28)
 #	define R128_PM4_64PIO_64VCPIO_64INDPIO	(15 << 28)
+#	define R128_PM4_BUFFER_CNTL_NOUPDATE	(1  << 27)
 
 #define R128_PM4_BUFFER_WM_CNTL		0x0708
 #	define R128_WMA_SHIFT			0
@@ -403,6 +393,15 @@ extern int R128_READ_PLL(drm_device_t *dev, int addr);
 					 (pkt) | ((n) << 16))
 
 
+static __inline__ void
+r128_update_ring_snapshot( drm_r128_private_t *dev_priv )
+{
+	drm_r128_ring_buffer_t *ring = &dev_priv->ring;
+	ring->space = (GET_RING_HEAD( dev_priv ) - ring->tail) * sizeof(u32);
+	if ( ring->space <= 0 )
+		ring->space += ring->size;
+}
+
 /* ================================================================
  * Misc helper macros
  */
@@ -412,7 +411,7 @@ do {									\
 	drm_r128_ring_buffer_t *ring = &dev_priv->ring; int i;		\
 	if ( ring->space < ring->high_mark ) {				\
 		for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {	\
-			r128_update_ring_snapshot( ring );		\
+			r128_update_ring_snapshot( dev_priv );		\
 			if ( ring->space >= ring->high_mark )		\
 				goto __ring_space_done;			\
 			DRM_UDELAY(1);				\
@@ -445,17 +444,10 @@ do {									\
  * Ring control
  */
 
-#if defined(__powerpc__)
-#define r128_flush_write_combine()	(void) GET_RING_HEAD( &dev_priv->ring )
-#else
-#define r128_flush_write_combine()	DRM_WRITEMEMORYBARRIER()
-#endif
-
-
 #define R128_VERBOSE	0
 
 #define RING_LOCALS							\
-	int write; unsigned int tail_mask; volatile u32 *ring;
+	int write, _nr; unsigned int tail_mask; volatile u32 *ring;
 
 #define BEGIN_RING( n ) do {						\
 	if ( R128_VERBOSE ) {						\
@@ -463,9 +455,10 @@ do {									\
 			   (n), __FUNCTION__ );				\
 	}								\
 	if ( dev_priv->ring.space <= (n) * sizeof(u32) ) {		\
+		COMMIT_RING();						\
 		r128_wait_ring( dev_priv, (n) * sizeof(u32) );		\
 	}								\
-	dev_priv->ring.space -= (n) * sizeof(u32);			\
+	_nr = n; dev_priv->ring.space -= (n) * sizeof(u32);		\
 	ring = dev_priv->ring.start;					\
 	write = dev_priv->ring.tail;					\
 	tail_mask = dev_priv->ring.tail_mask;				\
@@ -488,9 +481,23 @@ do {									\
 			dev_priv->ring.start,				\
 			write * sizeof(u32) );				\
 	}								\
-	r128_flush_write_combine();					\
-	dev_priv->ring.tail = write;					\
-	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, write );			\
+	if (((dev_priv->ring.tail + _nr) & tail_mask) != write) {	\
+		DRM_ERROR( 						\
+			"ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n",	\
+			((dev_priv->ring.tail + _nr) & tail_mask),	\
+			write, __LINE__);				\
+	} else								\
+		dev_priv->ring.tail = write;				\
+} while (0)
+
+#define COMMIT_RING() do {						\
+	if ( R128_VERBOSE ) {						\
+		DRM_INFO( "COMMIT_RING() tail=0x%06x\n",		\
+			dev_priv->ring.tail );				\
+	}								\
+	DRM_MEMORYBARRIER();						\
+	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, dev_priv->ring.tail );	\
+	R128_READ( R128_PM4_BUFFER_DL_WPTR );				\
 } while (0)
 
 #define OUT_RING( x ) do {						\
diff --git a/shared-core/r128_state.c b/shared-core/r128_state.c
index 5416d35e..f411657f 100644
--- a/shared-core/r128_state.c
+++ b/shared-core/r128_state.c
@@ -45,7 +45,7 @@ static void r128_emit_clip_rects( drm_r128_private_t *dev_priv,
 	RING_LOCALS;
 	DRM_DEBUG( "    %s\n", __FUNCTION__ );
 
-	BEGIN_RING( 17 );
+	BEGIN_RING( (count < 3? count: 3) * 5 + 2 );
 
 	if ( count >= 1 ) {
 		OUT_RING( CCE_PACKET0( R128_AUX1_SC_LEFT, 3 ) );
@@ -1269,6 +1269,7 @@ int r128_cce_clear( DRM_IOCTL_ARGS )
 		sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
 
 	r128_cce_dispatch_clear( dev, &clear );
+	COMMIT_RING();
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -1304,8 +1305,10 @@ int r128_do_cleanup_pageflip( drm_device_t *dev )
 	R128_WRITE( R128_CRTC_OFFSET,      dev_priv->crtc_offset );
 	R128_WRITE( R128_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl );
 
-	if (dev_priv->current_page != 0)
+	if (dev_priv->current_page != 0) {
 		r128_cce_dispatch_flip( dev );
+		COMMIT_RING();
+	}
 
 	dev_priv->page_flipping = 0;
 	return 0;
@@ -1330,6 +1333,7 @@ int r128_cce_flip( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_flip( dev );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1351,6 +1355,7 @@ int r128_cce_swap( DRM_IOCTL_ARGS )
 	dev_priv->sarea_priv->dirty |= (R128_UPLOAD_CONTEXT |
 					R128_UPLOAD_MASKS);
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1410,6 +1415,7 @@ int r128_cce_vertex( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_vertex( dev, buf );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1481,6 +1487,7 @@ int r128_cce_indices( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_indices( dev, buf, elts.start, elts.end, count );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1490,6 +1497,7 @@ int r128_cce_blit( DRM_IOCTL_ARGS )
 	drm_device_dma_t *dma = dev->dma;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_blit_t blit;
+	int ret;
 
 	LOCK_TEST_WITH_RETURN( dev, filp );
 
@@ -1507,7 +1515,10 @@ int r128_cce_blit( DRM_IOCTL_ARGS )
 	RING_SPACE_TEST_WITH_RETURN( dev_priv );
 	VB_AGE_TEST_WITH_RETURN( dev_priv );
 
-	return r128_cce_dispatch_blit( filp, dev, &blit );
+	ret = r128_cce_dispatch_blit( filp, dev, &blit );
+
+	COMMIT_RING();
+	return ret;
 }
 
 int r128_cce_depth( DRM_IOCTL_ARGS )
@@ -1515,6 +1526,7 @@ int r128_cce_depth( DRM_IOCTL_ARGS )
 	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_depth_t depth;
+	int ret;
 
 	LOCK_TEST_WITH_RETURN( dev, filp );
 
@@ -1523,18 +1535,20 @@ int r128_cce_depth( DRM_IOCTL_ARGS )
 
 	RING_SPACE_TEST_WITH_RETURN( dev_priv );
 
+	ret = DRM_ERR(EINVAL);
 	switch ( depth.func ) {
 	case R128_WRITE_SPAN:
-		return r128_cce_dispatch_write_span( dev, &depth );
+		ret = r128_cce_dispatch_write_span( dev, &depth );
 	case R128_WRITE_PIXELS:
-		return r128_cce_dispatch_write_pixels( dev, &depth );
+		ret = r128_cce_dispatch_write_pixels( dev, &depth );
 	case R128_READ_SPAN:
-		return r128_cce_dispatch_read_span( dev, &depth );
+		ret = r128_cce_dispatch_read_span( dev, &depth );
 	case R128_READ_PIXELS:
-		return r128_cce_dispatch_read_pixels( dev, &depth );
+		ret = r128_cce_dispatch_read_pixels( dev, &depth );
 	}
 
-	return DRM_ERR(EINVAL);
+	COMMIT_RING();
+	return ret;
 }
 
 int r128_cce_stipple( DRM_IOCTL_ARGS )
@@ -1557,6 +1571,7 @@ int r128_cce_stipple( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_stipple( dev, mask );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1632,6 +1647,7 @@ int r128_cce_indirect( DRM_IOCTL_ARGS )
 	 */
 	r128_cce_dispatch_indirect( dev, buf, indirect.start, indirect.end );
 
+	COMMIT_RING();
 	return 0;
 }
 
diff --git a/shared/r128_cce.c b/shared/r128_cce.c
index d0394a0c..13b44a5c 100644
--- a/shared/r128_cce.c
+++ b/shared/r128_cce.c
@@ -212,7 +212,7 @@ int r128_do_cce_idle( drm_r128_private_t *dev_priv )
 	int i;
 
 	for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
-		if ( GET_RING_HEAD( &dev_priv->ring ) == dev_priv->ring.tail ) {
+		if ( GET_RING_HEAD( dev_priv ) == dev_priv->ring.tail ) {
 			int pm4stat = R128_READ( R128_PM4_STAT );
 			if ( ( (pm4stat & R128_PM4_FIFOCNT_MASK) >=
 			       dev_priv->cce_fifo_size ) &&
@@ -238,7 +238,8 @@ static void r128_do_cce_start( drm_r128_private_t *dev_priv )
 	r128_do_wait_for_idle( dev_priv );
 
 	R128_WRITE( R128_PM4_BUFFER_CNTL,
-		    dev_priv->cce_mode | dev_priv->ring.size_l2qw );
+		    dev_priv->cce_mode | dev_priv->ring.size_l2qw
+		    | R128_PM4_BUFFER_CNTL_NOUPDATE );
 	R128_READ( R128_PM4_BUFFER_ADDR ); /* as per the sample code */
 	R128_WRITE( R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN );
 
@@ -253,7 +254,6 @@ static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
 {
 	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 	R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
-	SET_RING_HEAD( &dev_priv->ring, 0 );
 	dev_priv->ring.tail = 0;
 }
 
@@ -264,7 +264,8 @@ static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
 static void r128_do_cce_stop( drm_r128_private_t *dev_priv )
 {
 	R128_WRITE( R128_PM4_MICRO_CNTL, 0 );
-	R128_WRITE( R128_PM4_BUFFER_CNTL, R128_PM4_NONPM4 );
+	R128_WRITE( R128_PM4_BUFFER_CNTL,
+		    R128_PM4_NONPM4 | R128_PM4_BUFFER_CNTL_NOUPDATE );
 
 	dev_priv->cce_running = 0;
 }
@@ -333,26 +334,6 @@ static void r128_cce_init_ring_buffer( drm_device_t *dev,
 	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 	R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
 
-	/* DL_RPTR_ADDR is a physical address in AGP space. */
-	SET_RING_HEAD( &dev_priv->ring, 0 );
-
-	if ( !dev_priv->is_pci ) {
-		R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
-			    dev_priv->ring_rptr->offset );
-	} else {
-		drm_sg_mem_t *entry = dev->sg;
-		unsigned long tmp_ofs, page_ofs;
-
-		tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
-		page_ofs = tmp_ofs >> PAGE_SHIFT;
-
-		R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
-     			    entry->busaddr[page_ofs]);
-		DRM_DEBUG( "ring rptr: offset=0x%08lx handle=0x%08lx\n",
-			   (unsigned long) entry->busaddr[page_ofs],
-     			   entry->handle + tmp_ofs );
-	}
-
 	/* Set watermark control */
 	R128_WRITE( R128_PM4_BUFFER_WM_CNTL,
 		    ((R128_WATERMARK_L/4) << R128_WMA_SHIFT)
@@ -567,9 +548,6 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 #endif
 		dev_priv->cce_buffers_offset = dev->sg->handle;
 
-	dev_priv->ring.head = ((__volatile__ u32 *)
-			       dev_priv->ring_rptr->handle);
-
 	dev_priv->ring.start = (u32 *)dev_priv->cce_ring->handle;
 	dev_priv->ring.end = ((u32 *)dev_priv->cce_ring->handle
 			      + init->ring_size / sizeof(u32));
@@ -580,7 +558,6 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 		(dev_priv->ring.size / sizeof(u32)) - 1;
 
 	dev_priv->ring.high_mark = 128;
-	dev_priv->ring.ring_rptr = dev_priv->ring_rptr;
 
 	dev_priv->sarea_priv->last_frame = 0;
 	R128_WRITE( R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame );
@@ -589,8 +566,9 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 	R128_WRITE( R128_LAST_DISPATCH_REG,
 		    dev_priv->sarea_priv->last_dispatch );
 
-#if __REALLY_HAVE_SG
+#if __REALLY_HAVE_AGP
 	if ( dev_priv->is_pci ) {
+#endif
 		if (!DRM(ati_pcigart_init)( dev, &dev_priv->phys_pci_gart,
      					    &dev_priv->bus_pci_gart) ) {
 			DRM_ERROR( "failed to init PCI GART!\n" );
@@ -599,6 +577,7 @@ static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 			return DRM_ERR(ENOMEM);
 		}
 		R128_WRITE( R128_PCI_GART_PAGE, dev_priv->bus_pci_gart );
+#if __REALLY_HAVE_AGP
 	}
 #endif
 
@@ -901,7 +880,7 @@ int r128_wait_ring( drm_r128_private_t *dev_priv, int n )
 	int i;
 
 	for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
-		r128_update_ring_snapshot( ring );
+		r128_update_ring_snapshot( dev_priv );
 		if ( ring->space >= n )
 			return 0;
 		DRM_UDELAY( 1 );
diff --git a/shared/r128_drv.h b/shared/r128_drv.h
index 26f7cf86..10569f46 100644
--- a/shared/r128_drv.h
+++ b/shared/r128_drv.h
@@ -34,8 +34,7 @@
 #ifndef __R128_DRV_H__
 #define __R128_DRV_H__
 
-#define GET_RING_HEAD(ring)		DRM_READ32(  (ring)->ring_rptr, 0 ) /* (ring)->head */
-#define SET_RING_HEAD(ring,val)		DRM_WRITE32( (ring)->ring_rptr, 0, (val) ) /* (ring)->head */
+#define GET_RING_HEAD(dev_priv)		R128_READ( R128_PM4_BUFFER_DL_RPTR )
 
 typedef struct drm_r128_freelist {
    	unsigned int age;
@@ -50,13 +49,11 @@ typedef struct drm_r128_ring_buffer {
 	int size;
 	int size_l2qw;
 
-	volatile u32 *head;
 	u32 tail;
 	u32 tail_mask;
 	int space;
 
 	int high_mark;
-	drm_local_map_t *ring_rptr;
 } drm_r128_ring_buffer_t;
 
 typedef struct drm_r128_private {
@@ -132,14 +129,6 @@ extern drm_buf_t *r128_freelist_get( drm_device_t *dev );
 
 extern int r128_wait_ring( drm_r128_private_t *dev_priv, int n );
 
-static __inline__ void
-r128_update_ring_snapshot( drm_r128_ring_buffer_t *ring )
-{
-	ring->space = (GET_RING_HEAD( ring ) - ring->tail) * sizeof(u32);
-	if ( ring->space <= 0 )
-		ring->space += ring->size;
-}
-
 extern int r128_do_cce_idle( drm_r128_private_t *dev_priv );
 extern int r128_do_cleanup_cce( drm_device_t *dev );
 extern int r128_do_cleanup_pageflip( drm_device_t *dev );
@@ -279,6 +268,7 @@ extern int r128_cce_indirect( DRM_IOCTL_ARGS );
 #	define R128_PM4_64PIO_64VCBM_64INDBM	(7  << 28)
 #	define R128_PM4_64BM_64VCBM_64INDBM	(8  << 28)
 #	define R128_PM4_64PIO_64VCPIO_64INDPIO	(15 << 28)
+#	define R128_PM4_BUFFER_CNTL_NOUPDATE	(1  << 27)
 
 #define R128_PM4_BUFFER_WM_CNTL		0x0708
 #	define R128_WMA_SHIFT			0
@@ -403,6 +393,15 @@ extern int R128_READ_PLL(drm_device_t *dev, int addr);
 					 (pkt) | ((n) << 16))
 
 
+static __inline__ void
+r128_update_ring_snapshot( drm_r128_private_t *dev_priv )
+{
+	drm_r128_ring_buffer_t *ring = &dev_priv->ring;
+	ring->space = (GET_RING_HEAD( dev_priv ) - ring->tail) * sizeof(u32);
+	if ( ring->space <= 0 )
+		ring->space += ring->size;
+}
+
 /* ================================================================
  * Misc helper macros
  */
@@ -412,7 +411,7 @@ do {									\
 	drm_r128_ring_buffer_t *ring = &dev_priv->ring; int i;		\
 	if ( ring->space < ring->high_mark ) {				\
 		for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {	\
-			r128_update_ring_snapshot( ring );		\
+			r128_update_ring_snapshot( dev_priv );		\
 			if ( ring->space >= ring->high_mark )		\
 				goto __ring_space_done;			\
 			DRM_UDELAY(1);				\
@@ -445,17 +444,10 @@ do {									\
  * Ring control
  */
 
-#if defined(__powerpc__)
-#define r128_flush_write_combine()	(void) GET_RING_HEAD( &dev_priv->ring )
-#else
-#define r128_flush_write_combine()	DRM_WRITEMEMORYBARRIER()
-#endif
-
-
 #define R128_VERBOSE	0
 
 #define RING_LOCALS							\
-	int write; unsigned int tail_mask; volatile u32 *ring;
+	int write, _nr; unsigned int tail_mask; volatile u32 *ring;
 
 #define BEGIN_RING( n ) do {						\
 	if ( R128_VERBOSE ) {						\
@@ -463,9 +455,10 @@ do {									\
 			   (n), __FUNCTION__ );				\
 	}								\
 	if ( dev_priv->ring.space <= (n) * sizeof(u32) ) {		\
+		COMMIT_RING();						\
 		r128_wait_ring( dev_priv, (n) * sizeof(u32) );		\
 	}								\
-	dev_priv->ring.space -= (n) * sizeof(u32);			\
+	_nr = n; dev_priv->ring.space -= (n) * sizeof(u32);		\
 	ring = dev_priv->ring.start;					\
 	write = dev_priv->ring.tail;					\
 	tail_mask = dev_priv->ring.tail_mask;				\
@@ -488,9 +481,23 @@ do {									\
 			dev_priv->ring.start,				\
 			write * sizeof(u32) );				\
 	}								\
-	r128_flush_write_combine();					\
-	dev_priv->ring.tail = write;					\
-	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, write );			\
+	if (((dev_priv->ring.tail + _nr) & tail_mask) != write) {	\
+		DRM_ERROR( 						\
+			"ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n",	\
+			((dev_priv->ring.tail + _nr) & tail_mask),	\
+			write, __LINE__);				\
+	} else								\
+		dev_priv->ring.tail = write;				\
+} while (0)
+
+#define COMMIT_RING() do {						\
+	if ( R128_VERBOSE ) {						\
+		DRM_INFO( "COMMIT_RING() tail=0x%06x\n",		\
+			dev_priv->ring.tail );				\
+	}								\
+	DRM_MEMORYBARRIER();						\
+	R128_WRITE( R128_PM4_BUFFER_DL_WPTR, dev_priv->ring.tail );	\
+	R128_READ( R128_PM4_BUFFER_DL_WPTR );				\
 } while (0)
 
 #define OUT_RING( x ) do {						\
diff --git a/shared/r128_state.c b/shared/r128_state.c
index 5416d35e..f411657f 100644
--- a/shared/r128_state.c
+++ b/shared/r128_state.c
@@ -45,7 +45,7 @@ static void r128_emit_clip_rects( drm_r128_private_t *dev_priv,
 	RING_LOCALS;
 	DRM_DEBUG( "    %s\n", __FUNCTION__ );
 
-	BEGIN_RING( 17 );
+	BEGIN_RING( (count < 3? count: 3) * 5 + 2 );
 
 	if ( count >= 1 ) {
 		OUT_RING( CCE_PACKET0( R128_AUX1_SC_LEFT, 3 ) );
@@ -1269,6 +1269,7 @@ int r128_cce_clear( DRM_IOCTL_ARGS )
 		sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
 
 	r128_cce_dispatch_clear( dev, &clear );
+	COMMIT_RING();
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -1304,8 +1305,10 @@ int r128_do_cleanup_pageflip( drm_device_t *dev )
 	R128_WRITE( R128_CRTC_OFFSET,      dev_priv->crtc_offset );
 	R128_WRITE( R128_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl );
 
-	if (dev_priv->current_page != 0)
+	if (dev_priv->current_page != 0) {
 		r128_cce_dispatch_flip( dev );
+		COMMIT_RING();
+	}
 
 	dev_priv->page_flipping = 0;
 	return 0;
@@ -1330,6 +1333,7 @@ int r128_cce_flip( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_flip( dev );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1351,6 +1355,7 @@ int r128_cce_swap( DRM_IOCTL_ARGS )
 	dev_priv->sarea_priv->dirty |= (R128_UPLOAD_CONTEXT |
 					R128_UPLOAD_MASKS);
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1410,6 +1415,7 @@ int r128_cce_vertex( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_vertex( dev, buf );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1481,6 +1487,7 @@ int r128_cce_indices( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_indices( dev, buf, elts.start, elts.end, count );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1490,6 +1497,7 @@ int r128_cce_blit( DRM_IOCTL_ARGS )
 	drm_device_dma_t *dma = dev->dma;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_blit_t blit;
+	int ret;
 
 	LOCK_TEST_WITH_RETURN( dev, filp );
 
@@ -1507,7 +1515,10 @@ int r128_cce_blit( DRM_IOCTL_ARGS )
 	RING_SPACE_TEST_WITH_RETURN( dev_priv );
 	VB_AGE_TEST_WITH_RETURN( dev_priv );
 
-	return r128_cce_dispatch_blit( filp, dev, &blit );
+	ret = r128_cce_dispatch_blit( filp, dev, &blit );
+
+	COMMIT_RING();
+	return ret;
 }
 
 int r128_cce_depth( DRM_IOCTL_ARGS )
@@ -1515,6 +1526,7 @@ int r128_cce_depth( DRM_IOCTL_ARGS )
 	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_depth_t depth;
+	int ret;
 
 	LOCK_TEST_WITH_RETURN( dev, filp );
 
@@ -1523,18 +1535,20 @@ int r128_cce_depth( DRM_IOCTL_ARGS )
 
 	RING_SPACE_TEST_WITH_RETURN( dev_priv );
 
+	ret = DRM_ERR(EINVAL);
 	switch ( depth.func ) {
 	case R128_WRITE_SPAN:
-		return r128_cce_dispatch_write_span( dev, &depth );
+		ret = r128_cce_dispatch_write_span( dev, &depth );
 	case R128_WRITE_PIXELS:
-		return r128_cce_dispatch_write_pixels( dev, &depth );
+		ret = r128_cce_dispatch_write_pixels( dev, &depth );
 	case R128_READ_SPAN:
-		return r128_cce_dispatch_read_span( dev, &depth );
+		ret = r128_cce_dispatch_read_span( dev, &depth );
 	case R128_READ_PIXELS:
-		return r128_cce_dispatch_read_pixels( dev, &depth );
+		ret = r128_cce_dispatch_read_pixels( dev, &depth );
 	}
 
-	return DRM_ERR(EINVAL);
+	COMMIT_RING();
+	return ret;
 }
 
 int r128_cce_stipple( DRM_IOCTL_ARGS )
@@ -1557,6 +1571,7 @@ int r128_cce_stipple( DRM_IOCTL_ARGS )
 
 	r128_cce_dispatch_stipple( dev, mask );
 
+	COMMIT_RING();
 	return 0;
 }
 
@@ -1632,6 +1647,7 @@ int r128_cce_indirect( DRM_IOCTL_ARGS )
 	 */
 	r128_cce_dispatch_indirect( dev, buf, indirect.start, indirect.end );
 
+	COMMIT_RING();
 	return 0;
 }