From 2437ae80e5066dec9fe52f56b016bf136d7cea06 Mon Sep 17 00:00:00 2001 From: Søren Sandmann Pedersen Date: Thu, 22 Dec 2011 11:37:26 -0500 Subject: Reject trapezoids where top (botttom) is above (below) the edges When a trapezoid has a top/bottom that is above/below the left/right edges, degenerate trapezoids become possible. For example the edge could be very short and close to horizontal. If the bottom edge is far below the bottom point of such a short edge, the result is that the lower right corner of the trapezoid will be extremely far to the left. This kind of trapezoid causes overflows in the rasterization code, so change pixman_trapezoid_valid() to reject them. --- pixman/pixman.h | 12 ++++++++---- test/composite-traps-test.c | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pixman/pixman.h b/pixman/pixman.h index 18d9513..20ff496 100644 --- a/pixman/pixman.h +++ b/pixman/pixman.h @@ -908,10 +908,14 @@ struct pixman_triangle }; /* whether 't' is a well defined not obviously empty trapezoid */ -#define pixman_trapezoid_valid(t) \ - ((t)->left.p1.y != (t)->left.p2.y && \ - (t)->right.p1.y != (t)->right.p2.y && \ - (int) ((t)->bottom - (t)->top) > 0) +#define pixman_trapezoid_valid(t) \ + ((t)->left.p1.y != (t)->left.p2.y && \ + (t)->right.p1.y != (t)->right.p2.y && \ + (int) ((t)->bottom - (t)->top) > 0 && \ + (t)->bottom <= (t)->left.p2.y && \ + (t)->bottom <= (t)->right.p2.y && \ + (t)->top >= (t)->left.p1.y && \ + (t)->top >= (t)->right.p1.y) struct pixman_span_fix { diff --git a/test/composite-traps-test.c b/test/composite-traps-test.c index ff03b50..de518d8 100644 --- a/test/composite-traps-test.c +++ b/test/composite-traps-test.c @@ -251,6 +251,6 @@ test_composite (int testnum, int main (int argc, const char *argv[]) { - return fuzzer_test_main("composite traps", 40000, 0xE3112106, + return fuzzer_test_main("composite traps", 40000, 0x4346479C, test_composite, argc, argv); } -- cgit v1.2.3