VMCI: make guest driver work with virtual IOMMU

Right now we use vmalloc()/alloc_page() for our guest queuepair pages
(and bitmap page) and then do a virt_to_phys() before passing them down
to the device.  That's not going to work if DMA remapping is enabled,
since the IOMMU has no idea about the mappings.  Switch to
pci_alloc_consistent() instead.  We still allocate each page
individually, since there's no guarantee that we'll get a contiguous
block of physical for an entire queuepair (especially since we allow up
to 128 MiB!).

Also made split between guest and host in the kernelIf struct much
clearer.  Now it's obvious which fields are which.

Finally, drop the PINNED flag from Linux.  That was only ever used by
vVol-Filter, and that's been moved over vPageChannel, which doesn't use
queuepairs but has custom rings.

Once this is in I'll port it to upstream.  I considered doing upstream
only, but that would mean pre-3.9 guests would be unable to use
IOMMU support.  We should really freeze the in-house driver after
this...

Signed-off-by: Dmitry Torokhov <dtor@vmware.com>

4 files changed, 127 insertions, 142 deletions

diff --git a/open-vm-tools/modules/linux/vmci/common/vmciQPair.c b/open-vm-tools/modules/linux/vmci/common/vmciQPair.c
index 4116fc91..dbc64922 100644
--- a/open-vm-tools/modules/linux/vmci/common/vmciQPair.c
+++ b/open-vm-tools/modules/linux/vmci/common/vmciQPair.c
@@ -478,12 +478,7 @@ vmci_qpair_alloc(VMCIQPair **qpair,            // OUT
    }
 
    if ((flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) && !vmkernel) {
-#if defined(linux)
-      if (VMCI_ROUTE_AS_GUEST != route)
-#endif // linux
-      {
-         return VMCI_ERROR_INVALID_ARGS;
-      }
+      return VMCI_ERROR_INVALID_ARGS;
    }
 
    if (flags & VMCI_QPFLAG_PINNED) {
diff --git a/open-vm-tools/modules/linux/vmci/linux/driver.c b/open-vm-tools/modules/linux/vmci/linux/driver.c
index 82738e43..c9519ca2 100644
--- a/open-vm-tools/modules/linux/vmci/linux/driver.c
+++ b/open-vm-tools/modules/linux/vmci/linux/driver.c
@@ -150,6 +150,14 @@ static void process_bitmap(unsigned long data);
 #  define VMCI_DISABLE_MSIX   1
 #endif
 
+/*
+ * Needed by other components of this module.  It's okay to have one global
+ * instance of this because there can only ever be one VMCI device.  Our
+ * virtual hardware enforces this.
+ */
+
+struct pci_dev *vmci_pdev;
+
 static vmci_device vmci_dev;
 static compat_mod_param_bool vmci_disable_host = 0;
 static compat_mod_param_bool vmci_disable_guest = 0;
@@ -175,7 +183,8 @@ static uint32 data_buffer_size = VMCI_MAX_DG_SIZE;
  * and register a page with the device.
  */
 
-static uint8 *notification_bitmap = NULL;
+static uint8 *notification_bitmap;
+static dma_addr_t notification_base;
 
 
 /*
@@ -1792,7 +1801,8 @@ vmci_probe_device(struct pci_dev *pdev,           // IN: vmci PCI device
     */
    if (capabilities & VMCI_CAPS_NOTIFICATIONS) {
       capabilities = VMCI_CAPS_DATAGRAM;
-      notification_bitmap = vmalloc(PAGE_SIZE);
+      notification_bitmap = pci_alloc_consistent(pdev, PAGE_SIZE,
+                                                 &notification_base);
       if (notification_bitmap == NULL) {
          printk(KERN_ERR "VMCI device unable to allocate notification bitmap.\n");
       } else {
@@ -1823,8 +1833,7 @@ vmci_probe_device(struct pci_dev *pdev,           // IN: vmci PCI device
     * used
     */
    if (capabilities & VMCI_CAPS_NOTIFICATIONS) {
-      unsigned long bitmapPPN;
-      bitmapPPN = page_to_pfn(vmalloc_to_page(notification_bitmap));
+      unsigned long bitmapPPN = notification_base >> PAGE_SHIFT;
       if (!VMCI_RegisterNotificationBitmap(bitmapPPN)) {
          printk(KERN_ERR "VMCI device unable to register notification bitmap "
                 "with PPN 0x%x.\n", (uint32)bitmapPPN);
@@ -1840,6 +1849,7 @@ vmci_probe_device(struct pci_dev *pdev,           // IN: vmci PCI device
    /* Enable device. */
    vmci_dev.enabled = TRUE;
    pci_set_drvdata(pdev, &vmci_dev);
+   vmci_pdev = pdev;
 
    /*
     * We do global initialization here because we need datagrams
@@ -1938,7 +1948,8 @@ vmci_probe_device(struct pci_dev *pdev,           // IN: vmci PCI device
    compat_mutex_unlock(&vmci_dev.lock);
  release:
    if (notification_bitmap) {
-      vfree(notification_bitmap);
+      pci_free_consistent(pdev, PAGE_SIZE, notification_bitmap,
+                          notification_base);
       notification_bitmap = NULL;
    }
    release_region(ioaddr, ioaddr_size);
@@ -1975,6 +1986,7 @@ vmci_remove_device(struct pci_dev* pdev)
 
    VMCIQPGuestEndpoints_Exit();
    VMCIUtil_Exit();
+   vmci_pdev = NULL;
 
    compat_mutex_lock(&dev->lock);
 
@@ -2008,7 +2020,8 @@ vmci_remove_device(struct pci_dev* pdev)
        * device, so we can safely free it here.
        */
 
-      vfree(notification_bitmap);
+      pci_free_consistent(pdev, PAGE_SIZE, notification_bitmap,
+                          notification_base);
       notification_bitmap = NULL;
    }
 
diff --git a/open-vm-tools/modules/linux/vmci/linux/vmciKernelIf.c b/open-vm-tools/modules/linux/vmci/linux/vmciKernelIf.c
index fc262e07..4e72c33f 100644
--- a/open-vm-tools/modules/linux/vmci/linux/vmciKernelIf.c
+++ b/open-vm-tools/modules/linux/vmci/linux/vmciKernelIf.c
@@ -67,14 +67,20 @@
  */
 
 struct VMCIQueueKernelIf {
-   struct page **page;
-   struct page **headerPage;
-   void *va;
-   VMCIMutex __mutex;
-   VMCIMutex *mutex;
-   Bool host;
-   Bool isDataMapped;
-   size_t numPages;
+   VMCIMutex __mutex;               /* Protects the queue. */
+   VMCIMutex *mutex;                /* Shared by producer/consumer queues. */
+   size_t numPages;                 /* Number of pages incl. header. */
+   Bool host;                       /* Host or guest? */
+   union {
+      struct {
+         dma_addr_t *pas;           /* Physical addresses. */
+         void **vas;                /* Virtual addresses. */
+      } g;                          /* Guest. */
+      struct {
+         struct page **headerPage;  /* Guest queue header pages. */
+         struct page **page;        /* Guest queue pages. */
+      } h;                          /* Host. */
+   } u;
 };
 
 typedef struct VMCIDelayedWorkInfo {
@@ -83,6 +89,8 @@ typedef struct VMCIDelayedWorkInfo {
    void *data;
 } VMCIDelayedWorkInfo;
 
+extern struct pci_dev *vmci_pdev;
+
 
 /*
  *-----------------------------------------------------------------------------
@@ -868,14 +876,9 @@ VMCIMutex_Release(VMCIMutex *mutex) // IN:
  *
  * VMCI_AllocQueue --
  *
- *      Allocates kernel VA space of specified size, plus space for the
- *      queue structure/kernel interface and the queue header.  Allocates
- *      physical pages for the queue data pages.
- *
- *      PAGE m:      VMCIQueueHeader (VMCIQueue->qHeader)
- *      PAGE m+1:    VMCIQueue
- *      PAGE m+1+q:  VMCIQueueKernelIf (VMCIQueue->kernelIf)
- *      PAGE n-size: Data pages (VMCIQueue->kernelIf->page[])
+ *      Allocates kernel queue pages of specified size with IOMMU mappings,
+ *      plus space for the queue structure/kernel interface and the queue
+ *      header.
  *
  * Results:
  *      Pointer to the queue on success, NULL otherwise.
@@ -890,14 +893,13 @@ void *
 VMCI_AllocQueue(uint64 size,  // IN: size of queue (not including header)
                 uint32 flags) // IN: queuepair flags
 {
-   uint64 i;
+   size_t i;
    VMCIQueue *queue;
-   VMCIQueueHeader *qHeader;
-   const uint64 numDataPages = CEILING(size, PAGE_SIZE);
-   const uint queueSize =
-      PAGE_SIZE +
-      sizeof *queue + sizeof *(queue->kernelIf) +
-      numDataPages * sizeof *(queue->kernelIf->page);
+   const size_t numPages = CEILING(size, PAGE_SIZE) + 1;
+   const size_t pasSize = numPages * sizeof *queue->kernelIf->u.g.pas;
+   const size_t vasSize = numPages * sizeof *queue->kernelIf->u.g.vas;
+   const size_t queueSize =
+      sizeof *queue + sizeof *(queue->kernelIf) + pasSize + vasSize;
 
    /*
     * Size should be enforced by VMCIQPair_Alloc(), double-check here.
@@ -911,55 +913,33 @@ VMCI_AllocQueue(uint64 size,  // IN: size of queue (not including header)
       return NULL;
    }
 
-   /*
-    * If pinning is requested then double-check the size of the queue.
-    * VMCIQPair_Alloc() will do this for the total queuepair size.
-    */
-
-   if ((flags & VMCI_QPFLAG_PINNED) && size > VMCI_MAX_PINNED_QP_MEMORY) {
-      return NULL;
-   }
-
-   qHeader = (VMCIQueueHeader *)vmalloc(queueSize);
-   if (!qHeader) {
+   queue = vmalloc(queueSize);
+   if (!queue) {
       return NULL;
    }
 
-   queue = (VMCIQueue *)((uint8 *)qHeader + PAGE_SIZE);
-   queue->qHeader = qHeader;
+   queue->qHeader = NULL;
    queue->savedHeader = NULL;
-   queue->kernelIf = (VMCIQueueKernelIf *)((uint8 *)queue + sizeof *queue);
-   queue->kernelIf->headerPage = NULL; // Unused in guest.
-   queue->kernelIf->page = (struct page **)((uint8 *)queue->kernelIf +
-                                            sizeof *(queue->kernelIf));
-   queue->kernelIf->va = NULL;
+   queue->kernelIf = (VMCIQueueKernelIf *)(queue + 1);
+   queue->kernelIf->mutex = NULL;
+   queue->kernelIf->numPages = numPages;
+   queue->kernelIf->u.g.pas = (dma_addr_t *)(queue->kernelIf + 1);
+   queue->kernelIf->u.g.vas =
+      (void **)((uint8 *)queue->kernelIf->u.g.pas + pasSize);
    queue->kernelIf->host = FALSE;
-   queue->kernelIf->isDataMapped = FALSE;
 
-   for (i = 0; i < numDataPages; i++) {
-      queue->kernelIf->page[i] = alloc_pages(GFP_KERNEL, 0);
-      if (!queue->kernelIf->page[i]) {
-         VMCI_FreeQueue(queue, i * PAGE_SIZE);
+   for (i = 0; i < numPages; i++) {
+      queue->kernelIf->u.g.vas[i] =
+         pci_alloc_consistent(vmci_pdev, PAGE_SIZE,
+                              &queue->kernelIf->u.g.pas[i]);
+      if (!queue->kernelIf->u.g.vas[i]) {
+         VMCI_FreeQueue(queue, i * PAGE_SIZE); /* Size excl. the header. */
          return NULL;
       }
    }
 
-   /*
-    * alloc_pages() returns pinned PAs, but we need a permanent mapping to VA
-    * if the caller has requested pinned queuepairs.  Map all of them into
-    * kernel VA now, for the lifetime of the queue.  The page VAs will be
-    * contiguous.
-    */
-
-   if (flags & VMCI_QPFLAG_PINNED) {
-      queue->kernelIf->va = vmap(queue->kernelIf->page, numDataPages, VM_MAP,
-                                 PAGE_KERNEL);
-      if (NULL == queue->kernelIf->va) {
-         VMCI_FreeQueue(queue, numDataPages * PAGE_SIZE);
-         return NULL;
-      }
-      queue->kernelIf->isDataMapped = TRUE;
-   }
+   /* Queue header is the first page. */
+   queue->qHeader = queue->kernelIf->u.g.vas[0];
 
    return (void *)queue;
 }
@@ -991,18 +971,13 @@ VMCI_FreeQueue(void *q,     // IN:
    if (queue) {
       uint64 i;
 
-      if (queue->kernelIf->isDataMapped) {
-         ASSERT(queue->kernelIf->va);
-         vunmap(queue->kernelIf->va);
-         queue->kernelIf->va = NULL;
-      }
-
-      ASSERT(NULL == queue->kernelIf->va);
-
-      for (i = 0; i < CEILING(size, PAGE_SIZE); i++) {
-         __free_page(queue->kernelIf->page[i]);
+      /* Given size does not include header, so add in a page here. */
+      for (i = 0; i < CEILING(size, PAGE_SIZE) + 1; i++) {
+         pci_free_consistent(vmci_pdev, PAGE_SIZE,
+                             queue->kernelIf->u.g.vas[i],
+                             queue->kernelIf->u.g.pas[i]);
       }
-      vfree(queue->qHeader);
+      vfree(queue);
    }
 }
 
@@ -1063,33 +1038,29 @@ VMCI_AllocPPNSet(void *prodQ,            // IN:
       return VMCI_ERROR_NO_MEM;
    }
 
-   producePPNs[0] = page_to_pfn(vmalloc_to_page(produceQ->qHeader));
-   for (i = 1; i < numProducePages; i++) {
+   for (i = 0; i < numProducePages; i++) {
       unsigned long pfn;
 
-      producePPNs[i] = pfn = page_to_pfn(produceQ->kernelIf->page[i - 1]);
+      producePPNs[i] = pfn = produceQ->kernelIf->u.g.pas[i] >> PAGE_SHIFT;
 
       /*
        * Fail allocation if PFN isn't supported by hypervisor.
        */
 
-      if (sizeof pfn > sizeof *producePPNs &&
-          pfn != producePPNs[i]) {
+      if (sizeof pfn > sizeof *producePPNs && pfn != producePPNs[i]) {
          goto ppnError;
       }
    }
-   consumePPNs[0] = page_to_pfn(vmalloc_to_page(consumeQ->qHeader));
-   for (i = 1; i < numConsumePages; i++) {
+   for (i = 0; i < numConsumePages; i++) {
       unsigned long pfn;
 
-      consumePPNs[i] = pfn = page_to_pfn(consumeQ->kernelIf->page[i - 1]);
+      consumePPNs[i] = pfn = consumeQ->kernelIf->u.g.pas[i] >> PAGE_SHIFT;
 
       /*
        * Fail allocation if PFN isn't supported by hypervisor.
        */
 
-      if (sizeof pfn > sizeof *consumePPNs &&
-          pfn != consumePPNs[i]) {
+      if (sizeof pfn > sizeof *consumePPNs && pfn != consumePPNs[i]) {
          goto ppnError;
       }
    }
@@ -1205,15 +1176,15 @@ __VMCIMemcpyToQueue(VMCIQueue *queue,   // OUT:
    size_t bytesCopied = 0;
 
    while (bytesCopied < size) {
-      uint64 pageIndex = (queueOffset + bytesCopied) / PAGE_SIZE;
-      size_t pageOffset = (queueOffset + bytesCopied) & (PAGE_SIZE - 1);
+      const uint64 pageIndex = (queueOffset + bytesCopied) / PAGE_SIZE;
+      const size_t pageOffset = (queueOffset + bytesCopied) & (PAGE_SIZE - 1);
       void *va;
       size_t toCopy;
 
-      if (kernelIf->isDataMapped) {
-         va = (void *)((uint8 *)kernelIf->va + (pageIndex * PAGE_SIZE));
+      if (kernelIf->host) {
+         va = kmap(kernelIf->u.h.page[pageIndex]);
       } else {
-         va = kmap(kernelIf->page[pageIndex]);
+         va = kernelIf->u.g.vas[pageIndex + 1]; /* Skip header. */
       }
 
       ASSERT(va);
@@ -1231,8 +1202,8 @@ __VMCIMemcpyToQueue(VMCIQueue *queue,   // OUT:
          /* The iovec will track bytesCopied internally. */
          err = memcpy_fromiovec((uint8 *)va + pageOffset, iov, toCopy);
          if (err != 0) {
-            if (!kernelIf->isDataMapped) {
-               kunmap(kernelIf->page[pageIndex]);
+            if (kernelIf->host) {
+               kunmap(kernelIf->u.h.page[pageIndex]);
             }
             return VMCI_ERROR_INVALID_ARGS;
          }
@@ -1241,8 +1212,8 @@ __VMCIMemcpyToQueue(VMCIQueue *queue,   // OUT:
       }
 
       bytesCopied += toCopy;
-      if (!kernelIf->isDataMapped) {
-         kunmap(kernelIf->page[pageIndex]);
+      if (kernelIf->host) {
+         kunmap(kernelIf->u.h.page[pageIndex]);
       }
    }
 
@@ -1280,15 +1251,15 @@ __VMCIMemcpyFromQueue(void *dest,             // OUT:
    size_t bytesCopied = 0;
 
    while (bytesCopied < size) {
-      uint64 pageIndex = (queueOffset + bytesCopied) / PAGE_SIZE;
-      size_t pageOffset = (queueOffset + bytesCopied) & (PAGE_SIZE - 1);
+      const uint64 pageIndex = (queueOffset + bytesCopied) / PAGE_SIZE;
+      const size_t pageOffset = (queueOffset + bytesCopied) & (PAGE_SIZE - 1);
       void *va;
       size_t toCopy;
 
-      if (kernelIf->isDataMapped) {
-         va = (void *)((uint8 *)kernelIf->va + (pageIndex * PAGE_SIZE));
+      if (kernelIf->host) {
+         va = kmap(kernelIf->u.h.page[pageIndex]);
       } else {
-         va = kmap(kernelIf->page[pageIndex]);
+         va = kernelIf->u.g.vas[pageIndex + 1]; /* Skip header. */
       }
 
       ASSERT(va);
@@ -1306,8 +1277,8 @@ __VMCIMemcpyFromQueue(void *dest,             // OUT:
          /* The iovec will track bytesCopied internally. */
          err = memcpy_toiovec(iov, (uint8 *)va + pageOffset, toCopy);
          if (err != 0) {
-            if (!kernelIf->isDataMapped) {
-               kunmap(kernelIf->page[pageIndex]);
+            if (kernelIf->host) {
+               kunmap(kernelIf->u.h.page[pageIndex]);
             }
             return VMCI_ERROR_INVALID_ARGS;
          }
@@ -1316,8 +1287,8 @@ __VMCIMemcpyFromQueue(void *dest,             // OUT:
       }
 
       bytesCopied += toCopy;
-      if (!kernelIf->isDataMapped) {
-         kunmap(kernelIf->page[pageIndex]);
+      if (kernelIf->host) {
+         kunmap(kernelIf->u.h.page[pageIndex]);
       }
    }
 
@@ -1584,22 +1555,22 @@ VMCIHost_AllocQueue(uint64 size) // IN:
    VMCIQueue *queue;
    const size_t numPages = CEILING(size, PAGE_SIZE) + 1;
    const size_t queueSize = sizeof *queue + sizeof *(queue->kernelIf);
-   const size_t queuePageSize = numPages * sizeof *queue->kernelIf->page;
+   const size_t queuePageSize = numPages * sizeof *queue->kernelIf->u.h.page;
 
    queue = VMCI_AllocKernelMem(queueSize + queuePageSize, VMCI_MEMORY_NORMAL);
    if (queue) {
       queue->qHeader = NULL;
       queue->savedHeader = NULL;
-      queue->kernelIf = (VMCIQueueKernelIf *)((uint8 *)queue + sizeof *queue);
+      queue->kernelIf = (VMCIQueueKernelIf *)(queue + 1);
       queue->kernelIf->host = TRUE;
       queue->kernelIf->mutex = NULL;
       queue->kernelIf->numPages = numPages;
-      queue->kernelIf->headerPage = (struct page **)((uint8*)queue + queueSize);
-      queue->kernelIf->page = &queue->kernelIf->headerPage[1];
-      memset(queue->kernelIf->headerPage, 0,
-             sizeof *queue->kernelIf->headerPage * queue->kernelIf->numPages);
-      queue->kernelIf->va = NULL;
-      queue->kernelIf->isDataMapped = FALSE;
+      queue->kernelIf->u.h.headerPage =
+         (struct page **)((uint8*)queue + queueSize);
+      queue->kernelIf->u.h.page = &queue->kernelIf->u.h.headerPage[1];
+      memset(queue->kernelIf->u.h.headerPage, 0,
+             (sizeof *queue->kernelIf->u.h.headerPage *
+              queue->kernelIf->numPages));
    }
 
    return queue;
@@ -1893,7 +1864,8 @@ VMCIHost_RegisterUserMemory(QueuePairPageStore *pageStore,  // IN
    VA64 produceUVA;
    VA64 consumeUVA;
 
-   ASSERT(produceQ->kernelIf->headerPage && consumeQ->kernelIf->headerPage);
+   ASSERT(produceQ->kernelIf->u.h.headerPage &&
+          consumeQ->kernelIf->u.h.headerPage);
 
    /*
     * The new style and the old style mapping only differs in that we either
@@ -1933,12 +1905,16 @@ VMCIHost_UnregisterUserMemory(VMCIQueue *produceQ,         // IN/OUT
    ASSERT(consumeQ->kernelIf);
    ASSERT(!produceQ->qHeader && !consumeQ->qHeader);
 
-   VMCIReleasePages(produceQ->kernelIf->headerPage, produceQ->kernelIf->numPages, TRUE);
-   memset(produceQ->kernelIf->headerPage, 0,
-          sizeof *produceQ->kernelIf->headerPage * produceQ->kernelIf->numPages);
-   VMCIReleasePages(consumeQ->kernelIf->headerPage, consumeQ->kernelIf->numPages, TRUE);
-   memset(consumeQ->kernelIf->headerPage, 0,
-          sizeof *consumeQ->kernelIf->headerPage * consumeQ->kernelIf->numPages);
+   VMCIReleasePages(produceQ->kernelIf->u.h.headerPage,
+                    produceQ->kernelIf->numPages, TRUE);
+   memset(produceQ->kernelIf->u.h.headerPage, 0,
+          (sizeof *produceQ->kernelIf->u.h.headerPage *
+           produceQ->kernelIf->numPages));
+   VMCIReleasePages(consumeQ->kernelIf->u.h.headerPage,
+                    consumeQ->kernelIf->numPages, TRUE);
+   memset(consumeQ->kernelIf->u.h.headerPage, 0,
+          (sizeof *consumeQ->kernelIf->u.h.headerPage *
+           consumeQ->kernelIf->numPages));
 }
 
 
@@ -1976,15 +1952,16 @@ VMCIHost_MapQueues(VMCIQueue *produceQ,  // IN/OUT
          return VMCI_ERROR_QUEUEPAIR_MISMATCH;
       }
 
-      if (produceQ->kernelIf->headerPage == NULL ||
-          *produceQ->kernelIf->headerPage == NULL) {
+      if (produceQ->kernelIf->u.h.headerPage == NULL ||
+          *produceQ->kernelIf->u.h.headerPage == NULL) {
          return VMCI_ERROR_UNAVAILABLE;
       }
 
-      ASSERT(*produceQ->kernelIf->headerPage && *consumeQ->kernelIf->headerPage);
+      ASSERT(*produceQ->kernelIf->u.h.headerPage &&
+             *consumeQ->kernelIf->u.h.headerPage);
 
-      headers[0] = *produceQ->kernelIf->headerPage;
-      headers[1] = *consumeQ->kernelIf->headerPage;
+      headers[0] = *produceQ->kernelIf->u.h.headerPage;
+      headers[1] = *consumeQ->kernelIf->u.h.headerPage;
 
       produceQ->qHeader = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
       if (produceQ->qHeader != NULL) {
@@ -2074,11 +2051,11 @@ VMCIHost_GetUserMemory(VA64 produceUVA,       // IN
                            (VA)produceUVA,
                            produceQ->kernelIf->numPages,
                            1, 0,
-                           produceQ->kernelIf->headerPage,
+                           produceQ->kernelIf->u.h.headerPage,
                            NULL);
    if (retval < produceQ->kernelIf->numPages) {
       Log("get_user_pages(produce) failed (retval=%d)\n", retval);
-      VMCIReleasePages(produceQ->kernelIf->headerPage, retval, FALSE);
+      VMCIReleasePages(produceQ->kernelIf->u.h.headerPage, retval, FALSE);
       err = VMCI_ERROR_NO_MEM;
       goto out;
    }
@@ -2088,12 +2065,12 @@ VMCIHost_GetUserMemory(VA64 produceUVA,       // IN
                            (VA)consumeUVA,
                            consumeQ->kernelIf->numPages,
                            1, 0,
-                           consumeQ->kernelIf->headerPage,
+                           consumeQ->kernelIf->u.h.headerPage,
                            NULL);
    if (retval < consumeQ->kernelIf->numPages) {
       Log("get_user_pages(consume) failed (retval=%d)\n", retval);
-      VMCIReleasePages(consumeQ->kernelIf->headerPage, retval, FALSE);
-      VMCIReleasePages(produceQ->kernelIf->headerPage,
+      VMCIReleasePages(consumeQ->kernelIf->u.h.headerPage, retval, FALSE);
+      VMCIReleasePages(produceQ->kernelIf->u.h.headerPage,
                        produceQ->kernelIf->numPages, FALSE);
       err = VMCI_ERROR_NO_MEM;
    }
@@ -2126,7 +2103,7 @@ void
 VMCIHost_ReleaseUserMemory(VMCIQueue *produceQ,  // IN/OUT
                            VMCIQueue *consumeQ)  // IN/OUT
 {
-   ASSERT(produceQ->kernelIf->headerPage);
+   ASSERT(produceQ->kernelIf->u.h.headerPage);
 
    VMCIHost_UnregisterUserMemory(produceQ, consumeQ);
 }
diff --git a/open-vm-tools/modules/linux/vmci/linux/vmci_version.h b/open-vm-tools/modules/linux/vmci/linux/vmci_version.h
index 80e19bb5..0cf941e7 100644
--- a/open-vm-tools/modules/linux/vmci/linux/vmci_version.h
+++ b/open-vm-tools/modules/linux/vmci/linux/vmci_version.h
@@ -25,8 +25,8 @@
 #ifndef _VMCI_VERSION_H_
 #define _VMCI_VERSION_H_
 
-#define VMCI_DRIVER_VERSION          9.5.18.0
-#define VMCI_DRIVER_VERSION_COMMAS   9,5,18,0
-#define VMCI_DRIVER_VERSION_STRING   "9.5.18.0"
+#define VMCI_DRIVER_VERSION          9.5.19.0
+#define VMCI_DRIVER_VERSION_COMMAS   9,5,19,0
+#define VMCI_DRIVER_VERSION_STRING   "9.5.19.0"
 
 #endif /* _VMCI_VERSION_H_ */