d3d9trace: Take bits per pixel in consideration.

Otherwise partial locks near the end of the texture might cause buffer
overflows when reading/writing.

But take extra care to not cause false warnings for old traces.

Fixes #361.

3 files changed, 267 insertions, 61 deletions

diff --git a/helpers/d3d8size.hpp b/helpers/d3d8size.hpp
index ba662f36..d105f8d5 100644
--- a/helpers/d3d8size.hpp
+++ b/helpers/d3d8size.hpp
@@ -46,6 +46,91 @@ _declCount(const DWORD *pDeclaration) {
 
 
 static inline void
+_getFormatSize(D3DFORMAT Format, size_t & BlockSize, UINT & BlockWidth, UINT & BlockHeight)
+{
+    BlockSize = 0;
+    BlockWidth = 1;
+    BlockHeight = 1;
+
+    switch ((DWORD)Format) {
+
+    case D3DFMT_P8:
+    case D3DFMT_L8:
+    case D3DFMT_A4L4:
+        BlockSize = 8;
+        break;
+
+    case D3DFMT_R5G6B5:
+    case D3DFMT_X1R5G5B5:
+    case D3DFMT_A1R5G5B5:
+    case D3DFMT_A4R4G4B4:
+    case D3DFMT_A8R3G3B2:
+    case D3DFMT_X4R4G4B4:
+    case D3DFMT_A8P8:
+    case D3DFMT_A8L8:
+    case D3DFMT_V8U8:
+    case D3DFMT_L6V5U5:
+    case D3DFMT_D16_LOCKABLE:
+    case D3DFMT_D15S1:
+    case D3DFMT_D16:
+    case D3DFMT_INDEX16:
+        BlockSize = 16;
+        break;
+
+    case D3DFMT_R8G8B8:
+        BlockSize = 24;
+        break;
+
+    case D3DFMT_A8R8G8B8:
+    case D3DFMT_X8R8G8B8:
+    case D3DFMT_A2B10G10R10:
+    case D3DFMT_G16R16:
+    case D3DFMT_X8L8V8U8:
+    case D3DFMT_Q8W8V8U8:
+    case D3DFMT_V16U16:
+    case D3DFMT_W11V11U10:
+    case D3DFMT_A2W10V10U10:
+    case D3DFMT_D32:
+    case D3DFMT_D24S8:
+    case D3DFMT_D24X8:
+    case D3DFMT_D24X4S4:
+    case D3DFMT_INDEX32:
+        BlockSize = 32;
+        break;
+
+    case D3DFMT_UYVY:
+    case D3DFMT_YUY2:
+        BlockWidth = 2;
+        BlockSize = 32;
+        break;
+
+    case D3DFMT_DXT1:
+        BlockHeight = BlockWidth = 4;
+        BlockSize = 64;
+        break;
+
+    case D3DFMT_DXT2:
+    case D3DFMT_DXT3:
+    case D3DFMT_DXT4:
+    case D3DFMT_DXT5:
+        BlockHeight = BlockWidth = 4;
+        BlockSize = 128;
+        break;
+
+    case D3DFMT_UNKNOWN:
+    case D3DFMT_VERTEXDATA:
+        os::log("apitrace: warning: %s: unexpected D3DFMT %u\n", __FUNCTION__, Format);
+        BlockSize = 0;
+        break;
+
+    default:
+        os::log("apitrace: warning: %s: unknown D3DFMT %u\n", __FUNCTION__, Format);
+        break;
+    }
+}
+
+
+static inline void
 _getMapInfo(IDirect3DVertexBuffer8 *pBuffer, UINT OffsetToLock, UINT SizeToLock, BYTE ** ppbData,
              void * & pLockedData, size_t & MappedSize) {
     pLockedData = *ppbData;
@@ -107,7 +192,7 @@ _getMapInfo(IDirect3DSurface8 *pSurface, const D3DLOCKED_RECT *pLockedRect, cons
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -135,7 +220,7 @@ _getMapInfo(IDirect3DTexture8 *pTexture, UINT Level, const D3DLOCKED_RECT *pLock
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -165,7 +250,7 @@ _getMapInfo(IDirect3DCubeTexture8 *pTexture, D3DCUBEMAP_FACES FaceType, UINT Lev
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -196,7 +281,7 @@ _getMapInfo(IDirect3DVolume8 *pVolume, const D3DLOCKED_BOX *pLockedVolume, const
         Depth  = Desc.Depth;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
+    MappedSize = _getLockSize(Desc.Format, pBox, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
 }
 
 
@@ -227,7 +312,7 @@ _getMapInfo(IDirect3DVolumeTexture8 *pTexture, UINT Level, const D3DLOCKED_BOX *
         Depth  = Desc.Depth;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
+    MappedSize = _getLockSize(Desc.Format, pBox, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
 }
 
 
diff --git a/helpers/d3d9size.hpp b/helpers/d3d9size.hpp
index 11ea14c0..3ae928b5 100644
--- a/helpers/d3d9size.hpp
+++ b/helpers/d3d9size.hpp
@@ -46,6 +46,144 @@ _declCount(const D3DVERTEXELEMENT9 *pVertexElements) {
     return count;
 }
 
+
+static inline void
+_getFormatSize(D3DFORMAT Format, size_t & BlockSize, UINT & BlockWidth, UINT & BlockHeight)
+{
+    BlockSize = 0;
+    BlockWidth = 1;
+    BlockHeight = 1;
+
+    switch ((DWORD)Format) {
+
+    case D3DFMT_A1:
+        BlockSize = 1;
+        break;
+
+    case D3DFMT_P8:
+    case D3DFMT_L8:
+    case D3DFMT_A4L4:
+    case D3DFMT_S8_LOCKABLE:
+    case D3DFMT_BINARYBUFFER:
+        BlockSize = 8;
+        break;
+
+    case D3DFMT_R5G6B5:
+    case D3DFMT_X1R5G5B5:
+    case D3DFMT_A1R5G5B5:
+    case D3DFMT_A4R4G4B4:
+    case D3DFMT_A8R3G3B2:
+    case D3DFMT_X4R4G4B4:
+    case D3DFMT_A8P8:
+    case D3DFMT_A8L8:
+    case D3DFMT_V8U8:
+    case D3DFMT_L6V5U5:
+    case D3DFMT_D16_LOCKABLE:
+    case D3DFMT_D15S1:
+    case D3DFMT_D16:
+    case D3DFMT_L16:
+    case D3DFMT_INDEX16:
+    case D3DFMT_R16F:
+    case D3DFMT_CxV8U8:
+    case D3DFMT_DF16:
+        BlockSize = 16;
+        break;
+
+    case D3DFMT_R8G8B8:
+        BlockSize = 24;
+        break;
+
+    case D3DFMT_A8R8G8B8:
+    case D3DFMT_X8R8G8B8:
+    case D3DFMT_A2B10G10R10:
+    case D3DFMT_A8B8G8R8:
+    case D3DFMT_X8B8G8R8:
+    case D3DFMT_G16R16:
+    case D3DFMT_A2R10G10B10:
+    case D3DFMT_X8L8V8U8:
+    case D3DFMT_Q8W8V8U8:
+    case D3DFMT_V16U16:
+    case D3DFMT_A2W10V10U10:
+    case D3DFMT_D32:
+    case D3DFMT_D24S8:
+    case D3DFMT_D24X8:
+    case D3DFMT_D24X4S4:
+    case D3DFMT_D32F_LOCKABLE:
+    case D3DFMT_D24FS8:
+    case D3DFMT_D32_LOCKABLE:
+    case D3DFMT_INDEX32:
+    case D3DFMT_G16R16F:
+    case D3DFMT_R32F:
+    case D3DFMT_A2B10G10R10_XR_BIAS:
+    case D3DFMT_DF24:
+    case D3DFMT_INTZ:
+    case D3DFMT_AYUV:
+    case D3DFMT_RAWZ:
+        BlockSize = 32;
+        break;
+
+    case D3DFMT_A16B16G16R16:
+    case D3DFMT_Q16W16V16U16:
+    case D3DFMT_A16B16G16R16F:
+    case D3DFMT_G32R32F:
+        BlockSize = 64;
+        break;
+
+    case D3DFMT_A32B32G32R32F:
+        BlockSize = 128;
+        break;
+
+    case D3DFMT_UYVY:
+    case D3DFMT_R8G8_B8G8:
+    case D3DFMT_YUY2:
+    case D3DFMT_G8R8_G8B8:
+        BlockWidth = 2;
+        BlockSize = 32;
+        break;
+
+    case D3DFMT_DXT1:
+        BlockHeight = BlockWidth = 4;
+        BlockSize = 64;
+        break;
+
+    case D3DFMT_DXT2:
+    case D3DFMT_DXT3:
+    case D3DFMT_DXT4:
+    case D3DFMT_DXT5:
+        BlockHeight = BlockWidth = 4;
+        BlockSize = 128;
+        break;
+
+    case D3DFMT_NV12:
+    case D3DFMT_YV12:
+        // Planar YUV
+    case D3DFMT_ATI1N:
+    case D3DFMT_ATI2N:
+        /*
+         * Because these are unsupported formats, RowPitch is not set to the
+         * number of bytes between row of blocks, but instead in such way that
+         * Height * RowPitch will match the expected size.
+         */
+        BlockWidth = 0;
+        BlockHeight = 0;
+        BlockSize = 0;
+        break;
+
+    case D3DFMT_UNKNOWN:
+    case D3DFMT_VERTEXDATA:
+    case D3DFMT_NULL:
+    case D3DFMT_MULTI2_ARGB8: // https://msdn.microsoft.com/en-us/library/bb147219.aspx
+        os::log("apitrace: warning: %s: unexpected D3DFMT %u\n", __FUNCTION__, Format);
+        BlockSize = 0;
+        break;
+
+    default:
+        os::log("apitrace: warning: %s: unknown D3DFMT %u\n", __FUNCTION__, Format);
+        break;
+    }
+}
+
+
 static inline void
 _getMapInfo(IDirect3DVertexBuffer9 *pBuffer, UINT OffsetToLock, UINT SizeToLock, void ** ppbData,
              void * & pLockedData, size_t & MappedSize) {
@@ -108,7 +246,7 @@ _getMapInfo(IDirect3DSurface9 *pSurface, const D3DLOCKED_RECT *pLockedRect, cons
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -136,7 +274,7 @@ _getMapInfo(IDirect3DTexture9 *pTexture, UINT Level, const D3DLOCKED_RECT *pLock
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -166,7 +304,7 @@ _getMapInfo(IDirect3DCubeTexture9 *pTexture, D3DCUBEMAP_FACES FaceType, UINT Lev
         Height = Desc.Height;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedRect->Pitch);
+    MappedSize = _getLockSize(Desc.Format, pRect, Width, Height, pLockedRect->Pitch);
 }
 
 
@@ -197,7 +335,7 @@ _getMapInfo(IDirect3DVolume9 *pVolume, const D3DLOCKED_BOX *pLockedVolume, const
         Depth  = Desc.Depth;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
+    MappedSize = _getLockSize(Desc.Format, pBox, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
 }
 
 
@@ -228,7 +366,7 @@ _getMapInfo(IDirect3DVolumeTexture9 *pTexture, UINT Level, const D3DLOCKED_BOX *
         Depth  = Desc.Depth;
     }
 
-    MappedSize = _getLockSize(Desc.Format, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
+    MappedSize = _getLockSize(Desc.Format, pBox, Width, Height, pLockedVolume->RowPitch, Depth, pLockedVolume->SlicePitch);
 }
 
 
diff --git a/helpers/d3dcommonsize.hpp b/helpers/d3dcommonsize.hpp
index 4a7344bb..2ebbbc48 100644
--- a/helpers/d3dcommonsize.hpp
+++ b/helpers/d3dcommonsize.hpp
@@ -114,8 +114,11 @@ _shaderSize(const DWORD *pFunction)
 }
 
 
+static inline void
+_getFormatSize(D3DFORMAT Format, size_t & BlockSize, UINT & BlockWidth, UINT & BlockHeight);
+
 static inline size_t
-_getLockSize(D3DFORMAT Format, UINT Width, UINT Height, INT RowPitch, UINT Depth = 1, INT SlicePitch = 0) {
+_getLockSize(D3DFORMAT Format, bool Partial, UINT Width, UINT Height, INT RowPitch, UINT Depth = 1, INT SlicePitch = 0) {
     if (Width == 0 || Height == 0 || Depth == 0) {
         return 0;
     }
@@ -125,59 +128,39 @@ _getLockSize(D3DFORMAT Format, UINT Width, UINT Height, INT RowPitch, UINT Depth
         return 0;
     }
 
-    if (SlicePitch < 0) {
-        os::log("apitrace: warning: %s: negative slice pitch %i\n", __FUNCTION__, SlicePitch);
-        return 0;
-    }
-
-    switch ((DWORD)Format) {
-    case D3DFMT_DXT1:
-    case D3DFMT_DXT2:
-    case D3DFMT_DXT3:
-    case D3DFMT_DXT4:
-    case D3DFMT_DXT5:
-        Width  = (Width  + 3) / 4;
-        Height = (Height + 3) / 4;
-        break;
-
-#if DIRECT3D_VERSION >= 0x900
-    case D3DFMT_ATI1N:
-    case D3DFMT_ATI2N:
-        /*
-         * Because these are unsupported formats, RowPitch is not set to the
-         * number of bytes between row of blocks, but instead in such way that
-         * Height * RowPitch will match the expected size.
-         */
-        break;
-#endif /* DIRECT3D_VERSION >= 0x900 */
-
-    case D3DFMT_UYVY:
-    case D3DFMT_YUY2:
-#if DIRECT3D_VERSION >= 0x900
-    case D3DFMT_R8G8_B8G8:
-    case D3DFMT_G8R8_G8B8:
-#endif /* DIRECT3D_VERSION >= 0x900 */
-        Width = (Width + 1) / 2;
-        break;
-
-#if DIRECT3D_VERSION >= 0x900
-    case D3DFMT_NV12:
-    case D3DFMT_YV12:
-        return (Height + ((Height + 1) / 2)) * RowPitch;
-
-    case D3DFMT_NULL:
-        return 0;
-#endif /* DIRECT3D_VERSION >= 0x900 */
-
-    default:
-        break;
+    size_t size;
+    if (Format == MAKEFOURCC('N','V','1','2') ||
+        Format == MAKEFOURCC('Y','V','1','2')) {
+        // Planar YUV
+        size = (Height + (Height + 1)/2) * RowPitch;
+    } else {
+        size = Height * RowPitch;
+
+        if (Partial || Height == 1) {
+            // Must take pixel size in consideration
+
+            size_t BlockSize;
+            UINT BlockWidth;
+            UINT BlockHeight;
+            _getFormatSize(Format, BlockSize, BlockWidth, BlockHeight);
+
+            if (BlockWidth && BlockHeight) {
+                Width  = (Width  + BlockWidth  - 1) / BlockWidth;
+                Height = (Height + BlockHeight - 1) / BlockHeight;
+                size = (Width * BlockSize + 7)/ 8;
+                if (Height > 1) {
+                    size += (Height - 1) * RowPitch;
+                }
+            }
+        }
     }
 
-    (void)Width;
-
-    size_t size = Height * RowPitch;
-
     if (Depth > 1) {
+        if (SlicePitch < 0) {
+            os::log("apitrace: warning: %s: negative slice pitch %i\n", __FUNCTION__, SlicePitch);
+            return 0;
+        }
+
         size += (Depth - 1) * SlicePitch;
     }