Once licensed, switching to a different feature requires a system reboot, notify user about the same.

Details:
On licensed NMOS setup, if user restarts nvidia-gridd service and
selected 'GRID Virtual Apps' license edition. There is mismatch in
licensed feature type enabled in the driver and feature type read from
gridd.conf/UI control. In this change, notify user that license features
are still enabled in the driver even if license is released and hence restart is
required. Set appropriate license status message to indicate the same.

Fix:
Added NV_CTRL_ATTR_NVML_GPU_GRID_LICENSABLE_FEATURES attribute to get feature code
of the currently licensed feature.
Used the existing NVML api nvmlDeviceGetGridLicensableFeatures to fetch this feature code.
And then query this licensed feature code in nvidia-settings.
Setting the restart required status message when there is mismatch in
the queried licensed feature type and the feature type read from gridd.conf/UI.

Testing done:
Verified on NMOS, correct license status message is displayed.

5 files changed, 158 insertions, 16 deletions

diff --git a/src/gtk+-2.x/ctkgridlicense.c b/src/gtk+-2.x/ctkgridlicense.c
index f41ca29..e594f47 100644
--- a/src/gtk+-2.x/ctkgridlicense.c
+++ b/src/gtk+-2.x/ctkgridlicense.c
@@ -119,6 +119,10 @@ static gboolean allow_digits(GtkWidget *widget, GdkEvent *event, gpointer user_d
 static gboolean enable_disable_ui_controls(GtkWidget *widget, GdkEvent *event, gpointer user_data);
 static void update_gui_from_griddconfig(gpointer user_data);
 static gboolean licenseStateQueryFailed = FALSE;
+static void get_licensed_feature_code(gpointer user_data);
+static gboolean is_restart_required(gpointer user_data);
+static gboolean isStateLicensed = FALSE;
+int64_t licensedFeatureCode = NV_GRID_LICENSE_FEATURE_TYPE_VAPP;
 
 GType ctk_manage_grid_license_get_type(void)
 {
@@ -750,9 +754,11 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
     int licenseState = NV_GRID_UNLICENSED;
     int griddFeatureType = ctk_manage_grid_license->feature_type;
 
-    /* Send license state request */
-    if (!(send_message_to_gridd(ctk_manage_grid_license, LICENSE_STATE_REQUEST,
-                                &licenseState))) {
+    /* Send license state and feature type request */
+    if ((!(send_message_to_gridd(ctk_manage_grid_license, LICENSE_STATE_REQUEST,
+                                 &licenseState))) ||
+        (!(send_message_to_gridd(ctk_manage_grid_license, LICENSE_FEATURE_TYPE_REQUEST,
+                                 &griddFeatureType)))) {
         licenseStatusMessage = "Unable to query license state information "
                                "from the NVIDIA GRID "
                                "licensing daemon.\n"
@@ -769,8 +775,10 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
         gtk_widget_set_sensitive(ctk_manage_grid_license->btn_apply, FALSE);
         gtk_widget_set_sensitive(ctk_manage_grid_license->btn_cancel, FALSE);
         /* Disable toggle buttons */
-        gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_qdws, FALSE);
-        gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_vapp, FALSE);
+        if (ctk_manage_grid_license->radio_btn_qdws && ctk_manage_grid_license->radio_btn_vapp) {
+            gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_qdws, FALSE);
+            gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_vapp, FALSE);
+        }
 
         licenseStateQueryFailed = TRUE;
         return ret;
@@ -783,10 +791,16 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
         licenseStateQueryFailed = FALSE;
     }
 
+    /* Set the license feature type fetched from nvidia-gridd.*/
+    ctk_manage_grid_license->gridd_feature_type = griddFeatureType;
+
     if (licenseState == NV_GRID_UNLICENSED) {
         switch (ctk_manage_grid_license->feature_type) {
             case NV_GRID_LICENSE_FEATURE_TYPE_VAPP:
                 licenseStatus = NV_GRID_UNLICENSED_VAPP;
+                if (is_restart_required(ctk_manage_grid_license)) {
+                    licenseStatus = NV_GRID_LICENSE_RESTART_REQUIRED_VAPP;
+                }
                 break;
             case NV_GRID_LICENSE_FEATURE_TYPE_QDWS:
                 licenseStatus = NV_GRID_UNLICENSED_QDWS_SELECTED;
@@ -797,16 +811,11 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
             default:
                 break;
         }
+        if (ctk_manage_grid_license->feature_type != NV_GRID_LICENSE_FEATURE_TYPE_VGPU) {
+            isStateLicensed = FALSE;
+        }
     }
     else {
-        /* Send license feature type request */
-        if (!(send_message_to_gridd(ctk_manage_grid_license, LICENSE_FEATURE_TYPE_REQUEST,
-                                    &griddFeatureType))) {
-            ret = FALSE;
-        }
-        /* Set the license feature type fetched from nvidia-gridd.*/
-        ctk_manage_grid_license->gridd_feature_type = griddFeatureType;
-
         if ((licenseState == NV_GRID_LICENSED) ||
             (licenseState == NV_GRID_LICENSE_RENEWING) ||
             (licenseState == NV_GRID_LICENSE_RENEW_FAILED)) {
@@ -833,6 +842,11 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
                 default:
                     break;
             }
+            if ((ctk_manage_grid_license->feature_type != NV_GRID_LICENSE_FEATURE_TYPE_VGPU) &&
+                (isStateLicensed == FALSE)) {
+                get_licensed_feature_code(ctk_manage_grid_license);
+                isStateLicensed = TRUE;
+            }
         }
         else if (licenseState == NV_GRID_LICENSE_REQUESTING) {
             switch (ctk_manage_grid_license->feature_type) {
@@ -961,6 +975,9 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
               "Your system is currently licensed for Quadro Virtual Datacenter "
               "Workstation.";
           break;
+    case NV_GRID_LICENSE_RESTART_REQUIRED_VAPP:
+          licenseStatusMessage = "Restart your system for GRID Virtual Apps.";
+          break;
     default:
           licenseStatusMessage = "Your system does not have a valid GPU license.\n"
               "Enter license server details and apply.";
@@ -972,7 +989,22 @@ static gboolean update_manage_grid_license_state_info(gpointer user_data)
     return ret;
 }
 
+/*
+ * is_restart_required() - Check if restart is required for new feature to be enabled.
+ */
+static gboolean is_restart_required(gpointer user_data)
+{
+    CtkManageGridLicense *ctk_manage_grid_license = CTK_MANAGE_GRID_LICENSE(user_data);
+    gboolean ret = FALSE;
 
+    /* Once licensed, system reboot required if there is mismatch between feature type
+        fetched from nvidia-gridd and feature code of the feature that is licensed on this system. */
+    if ((licensedFeatureCode) &&
+        (licensedFeatureCode != ctk_manage_grid_license->gridd_feature_type)) {
+        ret = TRUE;
+    }
+    return ret;
+}
 
 /*
  * apply_clicked() - Called when the user clicks on the "Apply" button.
@@ -1060,6 +1092,33 @@ static void apply_clicked(GtkWidget *widget, gpointer user_data)
 }
 
 /*
+ * get_licensed_feature_code() - Get the feature code of the feature that is licensed on this system.
+ */
+static void get_licensed_feature_code(gpointer user_data)
+{
+    CtkManageGridLicense *ctk_manage_grid_license = CTK_MANAGE_GRID_LICENSE(user_data);
+    nvmlGridLicensableFeatures_t *gridLicensableFeatures;
+    gint ret, i;
+
+    ret = NvCtrlNvmlGetGridLicenseAttributes(ctk_manage_grid_license->target,
+                                             NV_CTRL_ATTR_NVML_GPU_GRID_LICENSABLE_FEATURES,
+                                             &gridLicensableFeatures);
+    if (ret != NvCtrlSuccess) {
+        return;
+    }
+
+    for (i = 0; i < gridLicensableFeatures->licensableFeaturesCount; i++)
+    {
+        if (gridLicensableFeatures->gridLicensableFeatures[i].featureState != 0)
+        {
+            licensedFeatureCode = gridLicensableFeatures->gridLicensableFeatures[i].featureCode;
+            break;
+        }
+    }
+    nvfree(gridLicensableFeatures);
+}
+
+/*
  * cancel_clicked() - Called when the user clicks on the "Cancel" button.
  */
 
@@ -1127,8 +1186,10 @@ static void update_gui_from_griddconfig(gpointer user_data)
         gtk_widget_set_sensitive(ctk_manage_grid_license->txt_server_address, TRUE);
         gtk_widget_set_sensitive(ctk_manage_grid_license->txt_server_port, TRUE);
         /* Enable toggle buttons. */
-        gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_qdws, TRUE);
-        gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_vapp, TRUE);
+        if (ctk_manage_grid_license->radio_btn_qdws && ctk_manage_grid_license->radio_btn_vapp) {
+            gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_qdws, TRUE);
+            gtk_widget_set_sensitive(ctk_manage_grid_license->radio_btn_vapp, TRUE);
+        }
 
         textBoxServerStr = gtk_entry_get_text(GTK_ENTRY(ctk_manage_grid_license->txt_server_address));
         /* Enable/Disable Secondary server address/port textboxes if Primary server address textbox string is empty. */
@@ -1499,6 +1560,7 @@ GtkWidget* ctk_manage_grid_license_new(CtrlTarget *target,
     ctk_manage_grid_license->license_edition_state = mode;
     ctk_manage_grid_license->dbusData = dbusData;
     ctk_manage_grid_license->feature_type = 0;
+    ctk_manage_grid_license->target = target;
 
     /* set container properties for the CtkManageGridLicense widget */
 
@@ -1722,6 +1784,10 @@ GtkWidget* ctk_manage_grid_license_new(CtrlTarget *target,
     /* Update GUI with information from the nvidia-gridd config file */
     update_gui_from_griddconfig(ctk_manage_grid_license);
 
+    if (ctk_manage_grid_license->feature_type != NV_GRID_LICENSE_FEATURE_TYPE_VGPU) {
+        get_licensed_feature_code(ctk_manage_grid_license);
+    }
+
     /* Set the license feature type fetched from nvidia-gridd */
     ctk_manage_grid_license->gridd_feature_type = ctk_manage_grid_license->feature_type;
 
diff --git a/src/gtk+-2.x/ctkgridlicense.h b/src/gtk+-2.x/ctkgridlicense.h
index c2705e8..a79ee35 100644
--- a/src/gtk+-2.x/ctkgridlicense.h
+++ b/src/gtk+-2.x/ctkgridlicense.h
@@ -62,6 +62,7 @@ struct _CtkManageGridLicense
     GtkWidget* box_server_info;
 
     DbusData *dbusData;
+    CtrlTarget *target;
     gint license_edition_state;
     gint feature_type;                      // Feature type from UI/gridd.conf.
     int gridd_feature_type;                 // Feature type fetched from nvidia-gridd.
@@ -84,6 +85,7 @@ typedef enum
     NV_GRID_LICENSE_EXPIRED_VGPU,
     NV_GRID_LICENSE_EXPIRED_QDWS,
     NV_GRID_LICENSE_RESTART_REQUIRED,
+    NV_GRID_LICENSE_RESTART_REQUIRED_VAPP,
 } licenseStatus;
 
 struct _CtkManageGridLicenseClass
diff --git a/src/libXNVCtrlAttributes/NvCtrlAttributes.h b/src/libXNVCtrlAttributes/NvCtrlAttributes.h
index 03c9a58..434aee8 100644
--- a/src/libXNVCtrlAttributes/NvCtrlAttributes.h
+++ b/src/libXNVCtrlAttributes/NvCtrlAttributes.h
@@ -272,7 +272,9 @@ typedef enum {
 #define NV_CTRL_ATTR_NVML_GPU_GRID_LICENSE_SUPPORTED_FALSE                                 0
 #define NV_CTRL_ATTR_NVML_GPU_GRID_LICENSE_SUPPORTED_TRUE                                  1
 
-#define NV_CTRL_ATTR_NVML_LAST_ATTRIBUTE (NV_CTRL_ATTR_NVML_GPU_GRID_LICENSE_SUPPORTED)
+#define NV_CTRL_ATTR_NVML_GPU_GRID_LICENSABLE_FEATURES          (NV_CTRL_ATTR_NVML_BASE + 2)
+
+#define NV_CTRL_ATTR_NVML_LAST_ATTRIBUTE (NV_CTRL_ATTR_NVML_GPU_GRID_LICENSABLE_FEATURES)
 
 #define NV_CTRL_ATTR_LAST_ATTRIBUTE \
         (NV_CTRL_ATTR_NVML_LAST_ATTRIBUTE)
diff --git a/src/libXNVCtrlAttributes/NvCtrlAttributesNvml.c b/src/libXNVCtrlAttributes/NvCtrlAttributesNvml.c
index 9662d38..5eb40d2 100644
--- a/src/libXNVCtrlAttributes/NvCtrlAttributesNvml.c
+++ b/src/libXNVCtrlAttributes/NvCtrlAttributesNvml.c
@@ -895,6 +895,54 @@ static ReturnStatus NvCtrlNvmlGetGPUAttribute(const CtrlTarget *ctrl_target,
     return NvCtrlNotSupported;
 }
 
+static ReturnStatus NvCtrlNvmlGetGridLicensableFeatures(const CtrlTarget *ctrl_target,
+                                                    int attr, nvmlGridLicensableFeatures_t **val)
+{
+    const NvCtrlAttributePrivateHandle *h = getPrivateHandleConst(ctrl_target);
+    const NvCtrlNvmlAttributes *nvml;
+    nvmlDevice_t device;
+    nvmlReturn_t ret;
+
+    if ((h == NULL) || (h->nvml == NULL)) {
+        return NvCtrlBadHandle;
+    }
+
+    nvml = h->nvml;
+
+    ret = nvml->lib.deviceGetHandleByIndex(nvml->deviceIdx, &device);
+        if (ret == NVML_SUCCESS) {
+        switch (attr) {
+            case NV_CTRL_ATTR_NVML_GPU_GRID_LICENSABLE_FEATURES:
+                if (nvml->lib.deviceGetGridLicensableFeatures) {
+                    nvmlGridLicensableFeatures_t *gridLicensableFeatures;
+                    gridLicensableFeatures = (nvmlGridLicensableFeatures_t *)nvalloc(sizeof(nvmlGridLicensableFeatures_t));
+                    ret = nvml->lib.deviceGetGridLicensableFeatures(device,
+                                                                    gridLicensableFeatures);
+                    if (ret == NVML_SUCCESS) {
+                        *val = gridLicensableFeatures;
+                        return NvCtrlSuccess;
+                    }
+                } else {
+                    /* return NvCtrlNotSupported against older driver */
+                    ret = NVML_ERROR_FUNCTION_NOT_FOUND;
+                }
+
+                break;
+
+            default:
+                /* Did we forget to handle a GPU integer attribute? */
+                nv_warning_msg("Unhandled integer attribute %s (%d) of GPU "
+                               "(%d)", INT_ATTRIBUTE_NAME(attr), attr,
+                               NvCtrlGetTargetId(ctrl_target));
+                return NvCtrlNotSupported;
+        }
+    }
+
+    /* An NVML error occurred */
+    printNvmlError(ret);
+    return NvCtrlNotSupported;
+}
+
 #ifdef NVML_EXPERIMENTAL
 
 static int getThermalCoolerId(const NvCtrlAttributePrivateHandle *h,
@@ -1072,6 +1120,28 @@ ReturnStatus NvCtrlNvmlGetAttribute(const CtrlTarget *ctrl_target,
 }
 
 
+ReturnStatus NvCtrlNvmlGetGridLicenseAttributes(const CtrlTarget *ctrl_target,
+                                                int attr, nvmlGridLicensableFeatures_t **val)
+{
+    if (NvmlMissing(ctrl_target)) {
+        return NvCtrlMissingExtension;
+    }
+
+    /*
+     * This should't be reached for target types that are not handled through
+     * NVML (Keep TARGET_TYPE_IS_NVML_COMPATIBLE in NvCtrlAttributesPrivate.h up
+     * to date).
+     */
+    assert(TARGET_TYPE_IS_NVML_COMPATIBLE(NvCtrlGetTargetType(ctrl_target)));
+
+    if (NvCtrlGetTargetType(ctrl_target) == GPU_TARGET) {
+        return NvCtrlNvmlGetGridLicensableFeatures(ctrl_target, attr, val);
+    }
+    else {
+            return NvCtrlBadHandle;
+    }
+}
+
 
 /*
  * Set NVML Attribute Values
diff --git a/src/libXNVCtrlAttributes/NvCtrlAttributesPrivate.h b/src/libXNVCtrlAttributes/NvCtrlAttributesPrivate.h
index ec1101b..2a13861 100644
--- a/src/libXNVCtrlAttributes/NvCtrlAttributesPrivate.h
+++ b/src/libXNVCtrlAttributes/NvCtrlAttributesPrivate.h
@@ -427,6 +427,8 @@ ReturnStatus NvCtrlNvmlSetStringAttribute(CtrlTarget *ctrl_target,
                                           int attr, const char *ptr);
 ReturnStatus NvCtrlNvmlGetAttribute(const CtrlTarget *ctrl_target,
                                     int attr, int64_t *val);
+ReturnStatus NvCtrlNvmlGetGridLicenseAttributes(const CtrlTarget *ctrl_target,
+                                                int attr, nvmlGridLicensableFeatures_t **val);
 ReturnStatus NvCtrlNvmlSetAttribute(CtrlTarget *ctrl_target, int attr,
                                     int index, int val);
 ReturnStatus