mesa: fix SwapBytes handling in numerous places

In a number of places the SwapBytes handling didn't handle cases with
GL_(UN)PACK_ALIGNMENT set and 7 byte width cases aligned to 8 bytes.

This adds a common routine to swap bytes a 2D image and uses this
code in:

texture storage
texture get
readpixels
swrast drawpixels.

[airlied: updated with Brian's nitpicks].

Cc: "11.0" <mesa-stable@lists.freedesktop.org>
Reviewed-by: Brian Paul <brianp@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

src/mesa/main/image.c

@@ -49,8 +49,8 @@
  * \param src the array with the source data we want to byte-swap.
  * \param n number of words.
  */
-void
+static void
-_mesa_swap2_copy( GLushort *dst, GLushort *src, GLuint n )
+swap2_copy( GLushort *dst, GLushort *src, GLuint n )
 {
    GLuint i;
    for (i = 0; i < n; i++) {
@@ -58,7 +58,11 @@ _mesa_swap2_copy( GLushort *dst, GLushort *src, GLuint n )
    }
 }
 
-
+void
+_mesa_swap2(GLushort *p, GLuint n)
+{
+   swap2_copy(p, p, n);
+}
 
 /*
  * Flip the order of the 4 bytes in each word in the given array (src) and
@@ -69,8 +73,8 @@ _mesa_swap2_copy( GLushort *dst, GLushort *src, GLuint n )
  * \param src the array with the source data we want to byte-swap.
  * \param n number of words.
  */
-void
+static void
-_mesa_swap4_copy( GLuint *dst, GLuint *src, GLuint n )
+swap4_copy( GLuint *dst, GLuint *src, GLuint n )
 {
    GLuint i, a, b;
    for (i = 0; i < n; i++) {
@@ -83,6 +87,11 @@ _mesa_swap4_copy( GLuint *dst, GLuint *src, GLuint n )
    }
 }
 
+void
+_mesa_swap4(GLuint *p, GLuint n)
+{
+   swap4_copy(p, p, n);
+}
 
 /**
  * Return the byte offset of a specific pixel in an image (1D, 2D or 3D).
@@ -958,3 +967,42 @@ _mesa_clip_blit(struct gl_context *ctx,
 
    return GL_TRUE;
 }
+
+/**
+ * Swap the bytes in a 2D image.
+ *
+ * using the packing information this swaps the bytes
+ * according to the format and type of data being input.
+ * It takes into a/c various packing parameters like
+ * Alignment and RowLength.
+ */
+void
+_mesa_swap_bytes_2d_image(GLenum format, GLenum type,
+                          const struct gl_pixelstore_attrib *packing,
+                          GLsizei width, GLsizei height,
+                          GLvoid *dst, const GLvoid *src)
+{
+   GLint swapSize = _mesa_sizeof_packed_type(type);
+
+   assert(packing->SwapBytes);
+
+   if (swapSize == 2 || swapSize == 4) {
+      int swapsPerPixel = _mesa_bytes_per_pixel(format, type) / swapSize;
+      int stride = _mesa_image_row_stride(packing, width, format, type);
+      int row;
+      uint8_t *dstrow;
+      const uint8_t *srcrow;
+      assert(swapsPerPixel > 0);
+      assert(_mesa_bytes_per_pixel(format, type) % swapSize == 0);
+      dstrow = dst;
+      srcrow = src;
+      for (row = 0; row < height; row++) {
+         if (swapSize == 2)
+            swap2_copy((GLushort *)dstrow, (GLushort *)srcrow, width * swapsPerPixel);
+         else if (swapSize == 4)
+            swap4_copy((GLuint *)dstrow, (GLuint *)srcrow, width * swapsPerPixel);
+         dstrow += stride;
+         srcrow += stride;
+      }
+   }
+}

src/mesa/main/image.h

@@ -35,22 +35,11 @@ struct gl_pixelstore_attrib;
 struct gl_framebuffer;
 
 extern void
-_mesa_swap2_copy(GLushort *dst, GLushort *src, GLuint n);
+_mesa_swap2(GLushort *p, GLuint n);
 
 extern void
-_mesa_swap4_copy(GLuint *dst, GLuint *src, GLuint n);
+_mesa_swap4(GLuint *p, GLuint n);
 
-static inline void
-_mesa_swap2(GLushort *p, GLuint n)
-{
-   _mesa_swap2_copy(p, p, n);
-}
-
-static inline void
-_mesa_swap4(GLuint *p, GLuint n)
-{
-   _mesa_swap4_copy(p, p, n);
-}
 
 extern GLintptr
 _mesa_image_offset( GLuint dimensions,
@@ -146,5 +135,10 @@ _mesa_clip_blit(struct gl_context *ctx,
                 GLint *srcX0, GLint *srcY0, GLint *srcX1, GLint *srcY1,
                 GLint *dstX0, GLint *dstY0, GLint *dstX1, GLint *dstY1);
 
+void
+_mesa_swap_bytes_2d_image(GLenum format, GLenum type,
+                          const struct gl_pixelstore_attrib *packing,
+                          GLsizei width, GLsizei height,
+                          GLvoid *dst, const GLvoid *src);
 
 #endif

src/mesa/main/readpix.c

@@ -613,15 +613,8 @@ read_rgba_pixels( struct gl_context *ctx,
 done_swap:
    /* Handle byte swapping if required */
    if (packing->SwapBytes) {
-      GLint swapSize = _mesa_sizeof_packed_type(type);
+      _mesa_swap_bytes_2d_image(format, type, packing,
-      if (swapSize == 2 || swapSize == 4) {
+                                width, height, dst, dst);
-         int swapsPerPixel = _mesa_bytes_per_pixel(format, type) / swapSize;
-         assert(_mesa_bytes_per_pixel(format, type) % swapSize == 0);
-         if (swapSize == 2)
-            _mesa_swap2((GLushort *) dst, width * height * swapsPerPixel);
-         else if (swapSize == 4)
-            _mesa_swap4((GLuint *) dst, width * height * swapsPerPixel);
-      }
    }
 
 done_unmap:

src/mesa/main/texgetimage.c

@@ -557,17 +557,9 @@ get_tex_rgba_uncompressed(struct gl_context *ctx, GLuint dimensions,
 
    do_swap:
       /* Handle byte swapping if required */
-      if (ctx->Pack.SwapBytes) {
+      if (ctx->Pack.SwapBytes)
-         GLint swapSize = _mesa_sizeof_packed_type(type);
+         _mesa_swap_bytes_2d_image(format, type, &ctx->Pack,
-         if (swapSize == 2 || swapSize == 4) {
+                                   width, height, dest, dest);
-            int swapsPerPixel = _mesa_bytes_per_pixel(format, type) / swapSize;
-            assert(_mesa_bytes_per_pixel(format, type) % swapSize == 0);
-            if (swapSize == 2)
-               _mesa_swap2((GLushort *) dest, width * height * swapsPerPixel);
-            else if (swapSize == 4)
-               _mesa_swap4((GLuint *) dest, width * height * swapsPerPixel);
-         }
-      }
 
       /* Unmap the src texture buffer */
       ctx->Driver.UnmapTextureImage(ctx, texImage, zoffset + img);

src/mesa/main/texstore.c

@@ -727,19 +727,25 @@ texstore_rgba(TEXSTORE_PARAMS)
        */
       GLint swapSize = _mesa_sizeof_packed_type(srcType);
       if (swapSize == 2 || swapSize == 4) {
-         int bytesPerPixel = _mesa_bytes_per_pixel(srcFormat, srcType);
+         int imageStride = _mesa_image_image_stride(srcPacking, srcWidth, srcHeight, srcFormat, srcType);
-         int swapsPerPixel = bytesPerPixel / swapSize;
+         int bufferSize = imageStride * srcDepth;
-         int elementCount = srcWidth * srcHeight * srcDepth;
+         int layer;
-         assert(bytesPerPixel % swapSize == 0);
+         const uint8_t *src;
-         tempImage = malloc(elementCount * bytesPerPixel);
+         uint8_t *dst;
+
+         tempImage = malloc(bufferSize);
          if (!tempImage)
             return GL_FALSE;
-         if (swapSize == 2)
+         src = srcAddr;
-            _mesa_swap2_copy(tempImage, (GLushort *) srcAddr,
+         dst = tempImage;
-                             elementCount * swapsPerPixel);
+         for (layer = 0; layer < srcDepth; layer++) {
-         else
+            _mesa_swap_bytes_2d_image(srcFormat, srcType,
-            _mesa_swap4_copy(tempImage, (GLuint *) srcAddr,
+                                      srcPacking,
-                             elementCount * swapsPerPixel);
+                                      srcWidth, srcHeight,
+                                      dst, src);
+            src += imageStride;
+            dst += imageStride;
+         }
          srcAddr = tempImage;
       }
    }

src/mesa/swrast/s_drawpix.c

@@ -481,17 +481,17 @@ draw_rgba_pixels( struct gl_context *ctx, GLint x, GLint y,
           */
          GLint swapSize = _mesa_sizeof_packed_type(type);
          if (swapSize == 2 || swapSize == 4) {
-            int components = _mesa_components_in_format(format);
+            int imageStride = _mesa_image_image_stride(unpack, width, height, format, type);
-            int elementCount = width * height * components;
+
-            tempImage = malloc(elementCount * swapSize);
+            tempImage = malloc(imageStride);
             if (!tempImage) {
                _mesa_error(ctx, GL_OUT_OF_MEMORY, "glDrawPixels");
                return;
             }
-            if (swapSize == 2)
+
-               _mesa_swap2_copy(tempImage, (GLushort *) pixels, elementCount);
+            _mesa_swap_bytes_2d_image(format, type, unpack,
-            else
+                                      width, height, tempImage, pixels);
-               _mesa_swap4_copy(tempImage, (GLuint *) pixels, elementCount);
+
             pixels = tempImage;
          }
       }