x86emu: Correctly handle 0x66 prefix for some instructions

(Sorry for double posting)

I repost this patch because I havn't got any replies from maintainers
since I posted the initial patch back in March.

Some instructions are not emulated correctly by x86emu when they
are prefixed by the 0x66 opcode.
I've identified problems in the emulation of these intructions: ret,
enter, leave, iret and some forms of call.

Most of the time, the problem is that these instructions should push or
pop 32-bit values to/from the stack, instead of 16bit, when they are
prefixed by the 0x66 special opcode.

The SeaBIOS project aims to produce a complete legacy BIOS
implementation as well as a VGA option ROM, entirely written in C and
using the GCC compiler.

In 16bit code produced by the GCC compiler, the 0x66 prefix is used
almost everywhere. This patch is necessary to allow the SeaBIOS VGA
option ROM to function with Xorg when using the vesa driver.

SeaBIOS currently use postprocessing on the ROM assembly output to
replace the affected instruction with alternative unaffected instructions.
This is obviously not very elegant, and this fix in x86emu would be
more appropriate.

v2: - Decrement BP instead of EBP in accordance with the Intel Manual
    - Assign EIP instead of IP when poping the return address from the
    stack in 32-bit operand size mode in ret_far_IMM, ret_far, and iret
    - When poping EFLAGS from the stack in iret in 32-bit operand size
    mode, apply some mask to preserve Read-only flags.

v3: - Rebase

Reviewed-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Julian Pidancet <julian.pidancet@gmail.com>
(cherry picked from commit 59b618227ebd024e57720aaaea17596953f5b80e)

1 files changed, 142 insertions, 50 deletions

diff --git a/hw/xfree86/x86emu/ops.c b/hw/xfree86/x86emu/ops.c
index 853792e7e..551dca78f 100644
--- a/hw/xfree86/x86emu/ops.c
+++ b/hw/xfree86/x86emu/ops.c
@@ -8949,7 +8949,11 @@ x86emuOp_ret_near_IMM(u8 X86EMU_UNUSED(op1))
     DECODE_PRINTF2("%x\n", imm);
     RETURN_TRACE("RET", M.x86.saved_cs, M.x86.saved_ip);
     TRACE_AND_STEP();
-    M.x86.R_IP = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EIP = pop_long();
+    } else {
+        M.x86.R_IP = pop_word();
+    }
     M.x86.R_SP += imm;
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
@@ -8966,7 +8970,11 @@ x86emuOp_ret_near(u8 X86EMU_UNUSED(op1))
     DECODE_PRINTF("RET\n");
     RETURN_TRACE("RET", M.x86.saved_cs, M.x86.saved_ip);
     TRACE_AND_STEP();
-    M.x86.R_IP = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EIP = pop_long();
+    } else {
+        M.x86.R_IP = pop_word();
+    }
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
 }
@@ -9259,8 +9267,13 @@ x86emuOp_enter(u8 X86EMU_UNUSED(op1))
     frame_pointer = M.x86.R_SP;
     if (nesting > 0) {
         for (i = 1; i < nesting; i++) {
-            M.x86.R_BP -= 2;
-            push_word(fetch_data_word_abs(M.x86.R_SS, M.x86.R_BP));
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                M.x86.R_BP -= 4;
+                push_long(fetch_data_long_abs(M.x86.R_SS, M.x86.R_BP));
+            } else {
+                M.x86.R_BP -= 2;
+                push_word(fetch_data_word_abs(M.x86.R_SS, M.x86.R_BP));
+            }
         }
         push_word(frame_pointer);
     }
@@ -9281,7 +9294,11 @@ x86emuOp_leave(u8 X86EMU_UNUSED(op1))
     DECODE_PRINTF("LEAVE\n");
     TRACE_AND_STEP();
     M.x86.R_SP = M.x86.R_BP;
-    M.x86.R_BP = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EBP = pop_long();
+    } else {
+        M.x86.R_BP = pop_word();
+    }
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
 }
@@ -9301,8 +9318,13 @@ x86emuOp_ret_far_IMM(u8 X86EMU_UNUSED(op1))
     DECODE_PRINTF2("%x\n", imm);
     RETURN_TRACE("RETF", M.x86.saved_cs, M.x86.saved_ip);
     TRACE_AND_STEP();
-    M.x86.R_IP = pop_word();
-    M.x86.R_CS = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EIP = pop_long();
+        M.x86.R_CS = pop_long() & 0xffff;
+    } else {
+        M.x86.R_IP = pop_word();
+        M.x86.R_CS = pop_word();
+    }
     M.x86.R_SP += imm;
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
@@ -9319,8 +9341,13 @@ x86emuOp_ret_far(u8 X86EMU_UNUSED(op1))
     DECODE_PRINTF("RETF\n");
     RETURN_TRACE("RETF", M.x86.saved_cs, M.x86.saved_ip);
     TRACE_AND_STEP();
-    M.x86.R_IP = pop_word();
-    M.x86.R_CS = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EIP = pop_long();
+        M.x86.R_CS = pop_long() & 0xffff;
+    } else {
+        M.x86.R_IP = pop_word();
+        M.x86.R_CS = pop_word();
+    }
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
 }
@@ -9421,9 +9448,15 @@ x86emuOp_iret(u8 X86EMU_UNUSED(op1))
 
     TRACE_AND_STEP();
 
-    M.x86.R_IP = pop_word();
-    M.x86.R_CS = pop_word();
-    M.x86.R_FLG = pop_word();
+    if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+        M.x86.R_EIP = pop_long();
+        M.x86.R_CS = pop_long() & 0xffff;
+        M.x86.R_EFLG = (pop_long() & 0x257FD5) | (M.x86.R_EFLG & 0x1A0000);
+    } else {
+        M.x86.R_IP = pop_word();
+        M.x86.R_CS = pop_word();
+        M.x86.R_FLG = pop_word();
+    }
     DECODE_CLEAR_SEGOVR();
     END_OF_INSTR();
 }
@@ -11740,19 +11773,36 @@ x86emuOp_opcFF_word_RM(u8 X86EMU_UNUSED(op1))
             }
             break;
         case 2:                /* call word ptr ... */
-            destval = fetch_data_word(destoffset);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 3:                /* call far ptr ... */
-            destval = fetch_data_word(destoffset);
-            destval2 = fetch_data_word(destoffset + 2);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_CS);
-            M.x86.R_CS = destval2;
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                destval2 = fetch_data_word(destoffset + 4);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                destval2 = fetch_data_word(destoffset + 2);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 4:                /* jmp word ptr ... */
             destval = fetch_data_word(destoffset);
@@ -11825,19 +11875,36 @@ x86emuOp_opcFF_word_RM(u8 X86EMU_UNUSED(op1))
             }
             break;
         case 2:                /* call word ptr ... */
-            destval = fetch_data_word(destoffset);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 3:                /* call far ptr ... */
-            destval = fetch_data_word(destoffset);
-            destval2 = fetch_data_word(destoffset + 2);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_CS);
-            M.x86.R_CS = destval2;
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                destval2 = fetch_data_word(destoffset + 4);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                destval2 = fetch_data_word(destoffset + 2);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 4:                /* jmp word ptr ... */
             destval = fetch_data_word(destoffset);
@@ -11910,19 +11977,36 @@ x86emuOp_opcFF_word_RM(u8 X86EMU_UNUSED(op1))
             }
             break;
         case 2:                /* call word ptr ... */
-            destval = fetch_data_word(destoffset);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 3:                /* call far ptr ... */
-            destval = fetch_data_word(destoffset);
-            destval2 = fetch_data_word(destoffset + 2);
-            TRACE_AND_STEP();
-            push_word(M.x86.R_CS);
-            M.x86.R_CS = destval2;
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = destval;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destval = fetch_data_long(destoffset);
+                destval2 = fetch_data_word(destoffset + 4);
+                TRACE_AND_STEP();
+                push_long(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = destval;
+            } else {
+                destval = fetch_data_word(destoffset);
+                destval2 = fetch_data_word(destoffset + 2);
+                TRACE_AND_STEP();
+                push_word(M.x86.R_CS);
+                M.x86.R_CS = destval2;
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = destval;
+            }
             break;
         case 4:                /* jmp word ptr ... */
             destval = fetch_data_word(destoffset);
@@ -11993,11 +12077,19 @@ x86emuOp_opcFF_word_RM(u8 X86EMU_UNUSED(op1))
             }
             break;
         case 2:                /* call word ptr ... */
-            destreg = DECODE_RM_WORD_REGISTER(rl);
-            DECODE_PRINTF("\n");
-            TRACE_AND_STEP();
-            push_word(M.x86.R_IP);
-            M.x86.R_IP = *destreg;
+            if (M.x86.mode & SYSMODE_PREFIX_DATA) {
+                destreg = DECODE_RM_LONG_REGISTER(rl);
+                DECODE_PRINTF("\n");
+                TRACE_AND_STEP();
+                push_long(M.x86.R_EIP);
+                M.x86.R_EIP = *destreg;
+            } else {
+                destreg = DECODE_RM_WORD_REGISTER(rl);
+                DECODE_PRINTF("\n");
+                TRACE_AND_STEP();
+                push_word(M.x86.R_IP);
+                M.x86.R_IP = *destreg;
+            }
             break;
         case 3:                /* jmp far ptr ... */
             DECODE_PRINTF("OPERATION UNDEFINED 0XFF \n");