asynchronous io port support (introduced in revision 3 of qxl device)

Fixes same issue in RHBZ#700134, but for a windows guest. Requires a revision 3
pci device, that will be introduced with qemu patches. If the revision is 2 the
old behavior is maintained, namely using the non asynchronous io ports.

qxl revision 3 (QXL_REVISION_V10) gains support for async io operations for

 UPDATE_AREA
 CREATE_PRIMARY
 DESTROY_PRIMARY
 DESTROY_SURFACE_WAIT (not used currently, just exported)
 DESTROY_SURFACES

use that if the revision is 3 or higher.

Async io ports let qemu complete the io without blocking on spice, and issue an
interrupt when the operation started by the io write is complete. This also means
less time when the qemu global mutex is held.

7 files changed, 140 insertions, 35 deletions

diff --git a/display/driver.c b/display/driver.c
index 2c88cc5..d13c6bc 100644
--- a/display/driver.c
+++ b/display/driver.c
@@ -32,7 +32,6 @@
 #include "winddi.h"
 #include "devioctl.h"
 #include "ntddvdeo.h"
-#include "ioaccess.h"
 
 #include "qxldd.h"
 #include "utils.h"
@@ -112,7 +111,7 @@ void DebugPrintV(PDev *pdev, const char *message, va_list ap)
         _snprintf(pdev->log_buf, QXL_LOG_BUF_SIZE, QXLDD_DEBUG_PREFIX);
         _vsnprintf(pdev->log_buf + strlen(QXLDD_DEBUG_PREFIX),
                    QXL_LOG_BUF_SIZE - strlen(QXLDD_DEBUG_PREFIX), message, ap);
-        WRITE_PORT_UCHAR(pdev->log_port, 0);
+        sync_io(pdev, pdev->log_port, 0);
         EngReleaseSemaphore(pdev->Res->print_sem);
     } else {
         EngDebugPrint(QXLDD_DEBUG_PREFIX, (PCHAR)message, ap);
@@ -572,19 +571,19 @@ static VOID CreatePrimarySurface(PDev *pdev, UINT32 depth, UINT32 format,
     pdev->primary_surface_create->flags = 0;
     pdev->primary_surface_create->type = QXL_SURF_TYPE_PRIMARY;
 
-    WRITE_PORT_UCHAR(pdev->create_primary_port, 0);
+    async_io(pdev, ASYNCABLE_CREATE_PRIMARY, 0);
 }
 
 static void DestroyPrimarySurface(PDev *pdev)
 {
     HideMouse(pdev);
-    WRITE_PORT_UCHAR(pdev->destroy_primary_port, 0);
+    async_io(pdev, ASYNCABLE_DESTROY_PRIMARY, 0);
 }
 
 static void DestroyAllSurfaces(PDev *pdev)
 {
     HideMouse(pdev);
-    WRITE_PORT_UCHAR(pdev->destroy_all_surfaces_port, 0);
+    async_io(pdev, ASYNCABLE_DESTROY_ALL_SURFACES, 0);
 }
 
 BOOL SetHardwareMode(PDev *pdev)
@@ -622,7 +621,7 @@ static VOID UpdateMainSlot(PDev *pdev, MemSlot *slot)
 
 static void RemoveVRamSlot(PDev *pdev)
 {
-    WRITE_PORT_UCHAR(pdev->memslot_del_port, pdev->vram_mem_slot);
+    sync_io(pdev, pdev->memslot_del_port, pdev->vram_mem_slot);
     pdev->vram_slot_initialized = FALSE;
 }
 
@@ -642,7 +641,7 @@ static BOOLEAN CreateVRamSlot(PDev *pdev)
     *pdev->ram_slot_start = pdev->fb_phys;
     *pdev->ram_slot_end = pdev->fb_phys + pdev->fb_size;
 
-    WRITE_PORT_UCHAR(pdev->memslot_add_port, slot_id);
+    async_io(pdev, ASYNCABLE_MEMSLOT_ADD, slot_id);
 
     pdev->vram_mem_slot = slot_id;
 
@@ -696,9 +695,24 @@ static BOOL PrepareHardware(PDev *pdev)
     pdev->notify_cmd_port = dev_info.notify_cmd_port;
     pdev->notify_cursor_port = dev_info.notify_cursor_port;
     pdev->notify_oom_port = dev_info.notify_oom_port;
+
+    pdev->asyncable[ASYNCABLE_UPDATE_AREA][ASYNC] = dev_info.update_area_async_port;
+    pdev->asyncable[ASYNCABLE_UPDATE_AREA][SYNC] = dev_info.update_area_port;
+    pdev->asyncable[ASYNCABLE_MEMSLOT_ADD][ASYNC] = dev_info.memslot_add_async_port;
+    pdev->asyncable[ASYNCABLE_MEMSLOT_ADD][SYNC] = dev_info.memslot_add_port;
+    pdev->asyncable[ASYNCABLE_CREATE_PRIMARY][ASYNC] = dev_info.create_primary_async_port;
+    pdev->asyncable[ASYNCABLE_CREATE_PRIMARY][SYNC] = dev_info.create_primary_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_PRIMARY][ASYNC] = dev_info.destroy_primary_async_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_PRIMARY][SYNC] = dev_info.destroy_primary_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_SURFACE][ASYNC] = dev_info.destroy_surface_async_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_SURFACE][SYNC] = dev_info.destroy_surface_wait_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_ALL_SURFACES][ASYNC] = dev_info.destroy_all_surfaces_async_port;
+    pdev->asyncable[ASYNCABLE_DESTROY_ALL_SURFACES][SYNC] = dev_info.destroy_all_surfaces_port;
+
     pdev->display_event = dev_info.display_event;
     pdev->cursor_event = dev_info.cursor_event;
     pdev->sleep_event = dev_info.sleep_event;
+    pdev->io_cmd_event = dev_info.io_cmd_event;
 #if (WINVER < 0x0501)
     pdev->WaitForEvent = dev_info.WaitForEvent;
 #endif
@@ -709,7 +723,6 @@ static BOOL PrepareHardware(PDev *pdev)
 
     pdev->dev_update_id = dev_info.update_id;
 
-    pdev->update_area_port = dev_info.update_area_port;
     pdev->update_area = dev_info.update_area;
     pdev->update_surface = dev_info.update_surface;
 
@@ -754,14 +767,8 @@ static BOOL PrepareHardware(PDev *pdev)
     pdev->fb_size = video_mem_Info.FrameBufferLength;
     pdev->fb_phys = dev_info.fb_phys;
 
-    pdev->destroy_surface_wait_port = dev_info.destroy_surface_wait_port;
-    pdev->destroy_all_surfaces_port = dev_info.destroy_all_surfaces_port;
-    pdev->memslot_add_port = dev_info.memslot_add_port;
     pdev->memslot_del_port = dev_info.memslot_del_port;
 
-    pdev->create_primary_port = dev_info.create_primary_port;
-    pdev->destroy_primary_port = dev_info.destroy_primary_port;
-
     pdev->primary_memory_start = dev_info.surface0_area;
     pdev->primary_memory_size = dev_info.surface0_area_size;
 
diff --git a/display/quic.c b/display/quic.c
index 5dc66d4..ee12fab 100644
--- a/display/quic.c
+++ b/display/quic.c
@@ -35,7 +35,6 @@
 #include "winddi.h"
 #include "devioctl.h"
 #include "ntddvdeo.h"
-#include "ioaccess.h"
 
 #include "qxldd.h"
 #include "utils.h"
diff --git a/display/qxldd.h b/display/qxldd.h
index fcfa752..fcfba29 100644
--- a/display/qxldd.h
+++ b/display/qxldd.h
@@ -30,6 +30,7 @@
 #include "windef.h"
 #include "wingdi.h"
 #include "winddi.h"
+#include "ioaccess.h"
 #include "qxl_driver.h"
 #include "mspace.h"
 #if (WINVER < 0x0501)
@@ -158,6 +159,23 @@ enum {
     NUM_MSPACES,
 };
 
+enum {
+    SYNC = 0,
+    ASYNC = 1
+};
+
+typedef enum {
+    ASYNCABLE_UPDATE_AREA = 0,
+    ASYNCABLE_MEMSLOT_ADD,
+    ASYNCABLE_CREATE_PRIMARY,
+    ASYNCABLE_DESTROY_PRIMARY,
+    ASYNCABLE_DESTROY_SURFACE,
+    ASYNCABLE_DESTROY_ALL_SURFACES,
+
+    ASYNCABLE_COUNT
+} asyncable_t;
+
+
 typedef struct PDev PDev;
 
 typedef struct DrawArea {
@@ -267,6 +285,7 @@ typedef struct PDev {
     PEVENT display_event;
     PEVENT cursor_event;
     PEVENT sleep_event;
+    PEVENT io_cmd_event;
 
     PUCHAR log_port;
     UINT8 *log_buf;
@@ -288,7 +307,6 @@ typedef struct PDev {
 
     UINT32 *dev_update_id;
 
-    PUCHAR update_area_port;
     QXLRect *update_area;
     UINT32 *update_surface;
 
@@ -302,12 +320,9 @@ typedef struct PDev {
     PQXLWaitForEvent WaitForEvent;
 #endif
 
-    PUCHAR create_primary_port;
-    PUCHAR destroy_primary_port;
-    PUCHAR destroy_surface_wait_port;
-    PUCHAR memslot_add_port;
+    PUCHAR asyncable[ASYNCABLE_COUNT][2];
+    HSEMAPHORE io_sem;
     PUCHAR memslot_del_port;
-    PUCHAR destroy_all_surfaces_port;
 
     UINT8* primary_memory_start;
     UINT32 primary_memory_size;
@@ -398,4 +413,50 @@ static _inline RingItem *RingGetTail(PDev *pdev, Ring *ring)
     return ret;
 }
 
+#if (WINVER < 0x0501)
+#define WAIT_FOR_EVENT(pdev, event, timeout) (pdev)->WaitForEvent(event, timeout)
+#else
+#define WAIT_FOR_EVENT(pdev, event, timeout) EngWaitForSingleObject(event, timeout)
+#endif
+
+/* Write to an ioport. For some operations we support a new port that returns
+ * immediatly, and completion is signaled by an interrupt that sets io_cmd_event.
+ * If the pci_revision is >= QXL_REVISION_STABLE_V10, we support it, else do
+ * a regular ioport write.
+ */
+static _inline void async_io(PDev *pdev, asyncable_t op, UCHAR val)
+{
+    if (pdev->pci_revision >= QXL_REVISION_STABLE_V10) {
+        EngAcquireSemaphore(pdev->io_sem);
+        WRITE_PORT_UCHAR(pdev->asyncable[op][ASYNC], val);
+        WAIT_FOR_EVENT(pdev, pdev->io_cmd_event, NULL);
+        EngReleaseSemaphore(pdev->io_sem);
+        DEBUG_PRINT((pdev, 3, "finished async %d\n", (int)op));
+    } else {
+        if (pdev->asyncable[op][SYNC] == NULL) {
+            DEBUG_PRINT((pdev, 0, "ERROR: trying calling sync io on NULL port %d\n", op));
+        } else {
+            EngAcquireSemaphore(pdev->io_sem);
+            WRITE_PORT_UCHAR(pdev->asyncable[op][SYNC], val);
+            EngReleaseSemaphore(pdev->io_sem);
+        }
+    }
+}
+
+/*
+ * Before the introduction of QXL_IO_*_ASYNC all io writes would return
+ * only when their function was complete. Since qemu would only allow
+ * a single outstanding io operation between all vcpu threads, they were
+ * also protected from simultaneous calls between different vcpus.
+ *
+ * With the introduction of _ASYNC we need to explicitly lock between different
+ * threads running on different vcpus, this is what this helper accomplishes.
+ */
+static _inline void sync_io(PDev *pdev, PUCHAR port, UCHAR val)
+{
+    EngAcquireSemaphore(pdev->io_sem);
+    WRITE_PORT_UCHAR(port, val);
+    EngReleaseSemaphore(pdev->io_sem);
+}
+
 #endif
diff --git a/display/res.c b/display/res.c
index c69b600..abda771 100644
--- a/display/res.c
+++ b/display/res.c
@@ -21,9 +21,9 @@
 
 #include <ddraw.h>
 #include <dxmini.h>
+#include "qxldd.h"
 #include "os_dep.h"
 #include "res.h"
-#include "ioaccess.h"
 #include "utils.h"
 #include "mspace.h"
 #include "quic.h"
@@ -33,12 +33,6 @@
 #include "devioctl.h"
 #include "ntddvdeo.h"
 
-#if (WINVER < 0x0501)
-#define WAIT_FOR_EVENT(pdev, event, timeout) (pdev)->WaitForEvent(event, timeout)
-#else
-#define WAIT_FOR_EVENT(pdev, event, timeout) EngWaitForSingleObject(event, timeout)
-#endif
-
 static _inline QXLPHYSICAL PA(PDev *pdev, PVOID virt, UINT8 slot_id)
 {
     PMemSlot *p_slot = &pdev->mem_slots[slot_id];
@@ -79,7 +73,7 @@ static BOOL SetClip(PDev *pdev, CLIPOBJ *clip, QXLDrawable *drawable);
     int notify;                                         \
     SPICE_RING_PUSH(pdev->cmd_ring, notify);            \
     if (notify) {                                       \
-        WRITE_PORT_UCHAR(pdev->notify_cmd_port, 0);     \
+        sync_io(pdev, pdev->notify_cmd_port, 0);     \
     }                                                   \
 } while (0);
 
@@ -87,7 +81,7 @@ static BOOL SetClip(PDev *pdev, CLIPOBJ *clip, QXLDrawable *drawable);
     int notify;                                         \
     SPICE_RING_PUSH(pdev->cursor_ring, notify);         \
     if (notify) {                                       \
-        WRITE_PORT_UCHAR(pdev->notify_cursor_port, 0);  \
+        sync_io(pdev, pdev->notify_cursor_port, 0);  \
     }                                                   \
 } while (0);
 
@@ -238,7 +232,7 @@ static void WaitForReleaseRing(PDev* pdev)
             if (!SPICE_RING_IS_EMPTY(pdev->release_ring)) {
                 break;
             }
-            WRITE_PORT_UCHAR(pdev->notify_oom_port, 0);
+            sync_io(pdev, pdev->notify_oom_port, 0);
         }
         SPICE_RING_CONS_WAIT(pdev->release_ring, wait);
 
@@ -263,7 +257,7 @@ static void WaitForReleaseRing(PDev* pdev)
                          pdev->Res->num_cursor_pages));
 #endif
             //oom
-            WRITE_PORT_UCHAR(pdev->notify_oom_port, 0);
+            sync_io(pdev, pdev->notify_oom_port, 0);
         }
     }
     DEBUG_PRINT((pdev, 16, "%s: 0x%lx, done\n", __FUNCTION__, pdev));
@@ -2538,7 +2532,7 @@ void UpdateArea(PDev *pdev, RECTL *area, UINT32 surface_id)
     DEBUG_PRINT((pdev, 12, "%s\n", __FUNCTION__));
     CopyRect(pdev->update_area, area);
     *pdev->update_surface = surface_id;
-    WRITE_PORT_UCHAR(pdev->update_area_port, 0);
+    async_io(pdev, ASYNCABLE_UPDATE_AREA, 0);
 }
 
 #endif
@@ -3273,6 +3267,13 @@ BOOL ResInit(PDev *pdev)
     }
     pdev->quic_data = usr_data;
     pdev->quic_data_sem = EngCreateSemaphore();
+    if (!pdev->quic_data_sem) {
+        PANIC(pdev, "quic_data_sem creation failed\n");
+    }
+    pdev->io_sem = EngCreateSemaphore();
+    if (!pdev->io_sem) {
+        PANIC(pdev, "io_sem creation failed\n");
+    }
 
     return TRUE;
 }
diff --git a/display/surface.c b/display/surface.c
index 0a93abf..4458bc6 100644
--- a/display/surface.c
+++ b/display/surface.c
@@ -32,7 +32,6 @@
 #include "winddi.h"
 #include "devioctl.h"
 #include "ntddvdeo.h"
-#include "ioaccess.h"
 
 #include "qxldd.h"
 #include "utils.h"
diff --git a/include/qxl_driver.h b/include/qxl_driver.h
index 9827a13..bff417e 100644
--- a/include/qxl_driver.h
+++ b/include/qxl_driver.h
@@ -54,9 +54,16 @@ typedef struct QXLDriverInfo {
     PUCHAR notify_cmd_port;
     PUCHAR notify_cursor_port;
     PUCHAR notify_oom_port;
+    PUCHAR update_area_async_port;
+    PUCHAR memslot_add_async_port;
+    PUCHAR create_primary_async_port;
+    PUCHAR destroy_primary_async_port;
+    PUCHAR destroy_surface_async_port;
+    PUCHAR destroy_all_surfaces_async_port;
     PEVENT display_event;
     PEVENT cursor_event;
     PEVENT sleep_event;
+    PEVENT io_cmd_event;
 
     UINT32 num_pages;
     void *io_pages_virt;
diff --git a/miniport/qxl.c b/miniport/qxl.c
index f3ee090..e3e515c 100644
--- a/miniport/qxl.c
+++ b/miniport/qxl.c
@@ -86,6 +86,7 @@ typedef struct QXLExtension {
     PEVENT display_event;
     PEVENT cursor_event;
     PEVENT sleep_event;
+    PEVENT io_cmd_event;
 
     MemSlot *mem_slots;
 
@@ -520,6 +521,10 @@ void DevExternsionCleanup(QXLExtension *dev)
         VideoPortDeleteEvent(dev, dev->display_event);
     }
 
+    if (dev->io_cmd_event) {
+        VideoPortDeleteEvent(dev, dev->io_cmd_event);
+    }
+
     if (dev->rom) {
         VideoPortUnmapMemory(dev, dev->rom, NULL);
     }
@@ -551,6 +556,7 @@ VP_STATUS FindAdapter(PVOID dev_extension,
     PEVENT display_event = NULL;
     PEVENT cursor_event = NULL;
     PEVENT sleep_event = NULL;
+    PEVENT io_cmd_event = NULL;
 #if (WINVER >= 0x0501)
     VPOSVERSIONINFO  sys_info;
 #endif
@@ -602,9 +608,19 @@ VP_STATUS FindAdapter(PVOID dev_extension,
         return status;
     }
 
+    if ((status = VideoPortCreateEvent(dev_ext, 0, NULL, &io_cmd_event)) != NO_ERROR) {
+        DEBUG_PRINT((0,  "%s: create io_cmd event failed %lu\n",
+                     __FUNCTION__, status));
+        VideoPortDeleteEvent(dev_ext, sleep_event);
+        VideoPortDeleteEvent(dev_ext, display_event);
+        VideoPortDeleteEvent(dev_ext, cursor_event);
+        return status;
+    }
+
     dev_ext->display_event = display_event;
     dev_ext->cursor_event = cursor_event;
     dev_ext->sleep_event = sleep_event;
+    dev_ext->io_cmd_event = io_cmd_event;
 
     if ((status = Prob(dev_ext, conf_info, ranges, QXL_PCI_RANGES)) != NO_ERROR ||
         (status = InitIO(dev_ext, &ranges[QXL_IO_RANGE_INDEX])) != NO_ERROR ||
@@ -948,12 +964,23 @@ BOOLEAN StartIO(PVOID dev_extension, PVIDEO_REQUEST_PACKET packet)
             driver_info->display_event = dev_ext->display_event;
             driver_info->cursor_event = dev_ext->cursor_event;
             driver_info->sleep_event = dev_ext->sleep_event;
+            driver_info->io_cmd_event = dev_ext->io_cmd_event;
             driver_info->cmd_ring = &dev_ext->ram_header->cmd_ring;
             driver_info->cursor_ring = &dev_ext->ram_header->cursor_ring;
             driver_info->release_ring = &dev_ext->ram_header->release_ring;
             driver_info->notify_cmd_port = dev_ext->io_port + QXL_IO_NOTIFY_CMD;
             driver_info->notify_cursor_port = dev_ext->io_port + QXL_IO_NOTIFY_CURSOR;
             driver_info->notify_oom_port = dev_ext->io_port + QXL_IO_NOTIFY_OOM;
+            driver_info->update_area_async_port = dev_ext->io_port + QXL_IO_UPDATE_AREA_ASYNC;
+            driver_info->memslot_add_async_port = dev_ext->io_port + QXL_IO_MEMSLOT_ADD_ASYNC;
+            driver_info->create_primary_async_port =
+                dev_ext->io_port + QXL_IO_CREATE_PRIMARY_ASYNC;
+            driver_info->destroy_primary_async_port =
+                dev_ext->io_port + QXL_IO_DESTROY_PRIMARY_ASYNC;
+            driver_info->destroy_surface_async_port =
+                dev_ext->io_port + QXL_IO_DESTROY_SURFACE_ASYNC;
+            driver_info->destroy_all_surfaces_async_port =
+                dev_ext->io_port + QXL_IO_DESTROY_ALL_SURFACES_ASYNC;
 
             driver_info->log_port = dev_ext->io_port + QXL_IO_LOG;
             driver_info->log_buf = dev_ext->ram_header->log_buf;
@@ -1023,9 +1050,13 @@ VOID InterruptCallback(PVOID dev_extension, PVOID Context)
 
     if (pending & QXL_INTERRUPT_DISPLAY) {
         VideoPortSetEvent(dev_ext, dev_ext->display_event);
-    } if (pending & QXL_INTERRUPT_CURSOR) {
+    }
+    if (pending & QXL_INTERRUPT_CURSOR) {
         VideoPortSetEvent(dev_ext, dev_ext->cursor_event);
     }
+    if (pending & QXL_INTERRUPT_IO_CMD) {
+        VideoPortSetEvent(dev_ext, dev_ext->io_cmd_event);
+    }
 
     dev_ext->ram_header->int_mask = ~0;
     VideoPortWritePortUchar((PUCHAR)dev_ext->io_base + QXL_IO_UPDATE_IRQ, 0);