add support for VK_EXT_debug_report for intels anv driver

To use VK_EXT_debug_report we need to move the instance creation
inside the OMP threading otherwise we end up sharing the same
instance and printing the shader info out multiple times.

We condition everything related to VK_EXT_debug_report to check
if the vendor is not AMD so that other vulkan drivers can use
VK_EXT_debug_report with vkpipeline-db in future too.

Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>

1 files changed, 176 insertions, 112 deletions

diff --git a/run.c b/run.c
index 4f21040..5afacc0 100644
--- a/run.c
+++ b/run.c
@@ -53,6 +53,7 @@ enum vendors vendor;
 
 int max_threads;
 const char **current_pipeline_names;
+VkDebugReportCallbackEXT *msgCallbacks;
 static pthread_mutex_t printf_mutex;
 
 #define sigputs(str) write(STDERR_FILENO, str, strlen(str))
@@ -79,6 +80,30 @@ abort_handler(int signo)
 }
 #pragma GCC diagnostic pop
 
+static VkBool32
+callback(VkDebugReportFlagsEXT flags,
+         VkDebugReportObjectTypeEXT objectType,
+         uint64_t object,
+         size_t location,
+         int32_t messageCode,
+         const char* pLayerPrefix,
+         const char* pMessage,
+         void* pUserData)
+{
+    assert(flags == VK_DEBUG_REPORT_DEBUG_BIT_EXT);
+
+    (void) objectType;
+    (void) object;
+    (void) location;
+    (void) messageCode;
+    (void) pLayerPrefix;
+
+    const char *shader_name = pUserData;
+    printf("%s - %s\n", shader_name, pMessage);
+
+    return VK_FALSE;
+}
+
 /* Pipeline tests. */
 static unsigned pipeline_test_size = 1 << 15; /* next-pow-2(num pipelines in db) */
 static unsigned pipeline_test_length;
@@ -117,6 +142,8 @@ gather_pipeline_test(const char *fpath, const struct stat *sb, int typeflag)
 
 /* Shader stats */
 static PFN_vkGetShaderInfoAMD vkGetShaderInfo = VK_NULL_HANDLE;
+static PFN_vkCreateDebugReportCallbackEXT createDebugReportCallback = VK_NULL_HANDLE;
+static PFN_vkDestroyDebugReportCallbackEXT destroyDebugReportCallback = VK_NULL_HANDLE;
 
 struct shader_stats
 {
@@ -297,7 +324,7 @@ create_compute_pipeline(VkDevice device, struct pipeline_info *info,
                                     NULL, pipeline);
 }
 static int
-create_pipeline(VkDevice device, const char *pipeline_name,
+create_pipeline(VkDevice device,  VkInstance instance, const char *pipeline_name,
                 struct pipeline_info *info)
 {
     VkPipelineLayout layout = VK_NULL_HANDLE;
@@ -343,6 +370,23 @@ create_pipeline(VkDevice device, const char *pipeline_name,
         goto fail;
     }
 
+    VkDebugReportCallbackCreateInfoEXT callbackInfo = {};
+    callbackInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CALLBACK_CREATE_INFO_EXT;
+    callbackInfo.pNext = NULL;
+    callbackInfo.flags = VK_DEBUG_REPORT_DEBUG_BIT_EXT;
+    callbackInfo.pfnCallback = callback;
+    callbackInfo.pUserData = (char *) pipeline_name;
+
+    if (vendor != VENDOR_AMD) {
+        result = createDebugReportCallback(instance, &callbackInfo,
+                                           NULL, &msgCallbacks[omp_get_thread_num()]);
+
+        if (unlikely(result != VK_SUCCESS)) {
+            ret = -1;
+            goto fail;
+        }
+    }
+
     /* Graphics/Compute pipeline. */
     if (info->bindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS) {
         result = create_graphics_pipeline(device, info, layout, &pipeline);
@@ -357,44 +401,35 @@ create_pipeline(VkDevice device, const char *pipeline_name,
     }
 
     /* Shader stats. */
-    for (uint32_t i = 0; i < info->stageCount; i++) {
-        VkPipelineShaderStageCreateInfo *pCreateInfo = &info->pShaderStagesInfo[i];
-        VkShaderStageFlagBits stage = pCreateInfo->stage;
+    if (vendor == VENDOR_AMD) {
+        for (uint32_t i = 0; i < info->stageCount; i++) {
+            VkPipelineShaderStageCreateInfo *pCreateInfo = &info->pShaderStagesInfo[i];
+            VkShaderStageFlagBits stage = pCreateInfo->stage;
 
-        if (!is_shader_stage_valid(device, pipeline, stage))
-            continue;
+            if (!is_shader_stage_valid(device, pipeline, stage))
+                continue;
 
-        char *shader_info = NULL;
-        ret = get_shader_info(device, pipeline, stage, &shader_info);
-        if (unlikely(ret < 0)) {
-            fprintf(stderr, "Failed to get shader info!\n");
-            goto fail;
-        }
+            char *shader_info = NULL;
+            ret = get_shader_info(device, pipeline, stage, &shader_info);
+            if (unlikely(ret < 0)) {
+                fprintf(stderr, "Failed to get shader info!\n");
+                goto fail;
+            }
 
-        if (vendor == VENDOR_AMD) {
             struct shader_stats stats = {};
             if (unlikely(amd_parse_shader_stats(shader_info, &stats) < 0)) {
                 fprintf(stderr, "Failed to parse AMD shader statistics!\n");
             } else {
                 amd_print_shader_stats(pipeline_name, stage, &stats);
             }
-        } else {
-            char *line = NULL;
 
-            pthread_mutex_lock(&printf_mutex);
-
-            line = strtok(shader_info, "\r\n");
-            while (line != NULL) {
-                printf("%s - %s\n",
-                       current_pipeline_names[omp_get_thread_num()],
-                       line);
-                line = strtok(NULL, "\r\n");
-            }
-
-            pthread_mutex_unlock(&printf_mutex);
+            free(shader_info);
         }
+    }
 
-        free(shader_info);
+    if (vendor != VENDOR_AMD) {
+        destroyDebugReportCallback(instance,
+                                   msgCallbacks[omp_get_thread_num()], NULL);
     }
 
 fail:
@@ -424,7 +459,8 @@ free_pipeline(struct pipeline_info *pipeline)
 }
 
 static int
-run(VkDevice device, const char *pipeline_name, const char *data, off_t size)
+run(VkDevice device, VkInstance instance, const char *pipeline_name,
+    const char *data, off_t size)
 {
     struct pipeline_info *pipeline;
     struct blob_reader metadata;
@@ -441,7 +477,7 @@ run(VkDevice device, const char *pipeline_name, const char *data, off_t size)
         return -1;
     }
 
-    ret = create_pipeline(device, pipeline_name, pipeline);
+    ret = create_pipeline(device, instance, pipeline_name, pipeline);
     if (unlikely(ret < 0)) {
         fprintf(stderr, "Failed to create pipeline!\n");
         goto fail;
@@ -463,15 +499,10 @@ print_usage(const char *prog_name)
 
 int main(int argc, char **argv)
 {
-    const char *extensionNames[] = { "VK_AMD_shader_info" };
-    VkQueueFamilyProperties queue_family;
-    VkPhysicalDevice *physical_devices;
-    uint32_t device_count;
-    uint32_t queue_count = 1;
-    VkInstance instance;
-    VkResult result;
-    int ret = 0;
+    const char *shader_info_ext[] = { "VK_AMD_shader_info" };
+    const char *debug_report_ext[] = { "VK_EXT_debug_report" };
     int opt;
+    int ret = 0;
 
     pthread_mutex_init(&printf_mutex, NULL);
 
@@ -496,76 +527,6 @@ int main(int argc, char **argv)
     }
 
     /**
-     * Instance creation.
-     */
-    VkApplicationInfo appInfo = {};
-    appInfo.pApplicationName = "vkpipeline-db";
-    VkInstanceCreateInfo instanceCreateInfo = {};
-    instanceCreateInfo.pApplicationInfo = &appInfo;
-    instanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
-
-    result = vkCreateInstance(&instanceCreateInfo, NULL, &instance);
-    if (unlikely(result != VK_SUCCESS)) {
-        fprintf(stderr, "Failed to create instance (%d).\n", result);
-        return -1;
-    }
-
-    /**
-     * Device creation.
-     */
-    /* Get number of devices. */
-    result = vkEnumeratePhysicalDevices(instance, &device_count, NULL);
-    fprintf(stderr, "Number of devices: %d\n", device_count);
-
-    physical_devices = malloc(sizeof(*physical_devices) * device_count);
-
-    /* Get physical devices. */
-    result = vkEnumeratePhysicalDevices(instance, &device_count,
-                                        physical_devices);
-    if (unlikely(result != VK_SUCCESS)) {
-        fprintf(stderr, "Failed to enumerate physical devices (%d).\n", result);
-        ret = -1;
-        goto fail_device;
-    }
-
-    VkPhysicalDeviceProperties device_properties;
-    vkGetPhysicalDeviceProperties(physical_devices[0], &device_properties);
-    vendor = device_properties.vendorID;
-    fprintf(stderr, "GPU: %s\n", device_properties.deviceName);
-
-    /* Get queue properties. */
-    vkGetPhysicalDeviceQueueFamilyProperties(physical_devices[0], &queue_count,
-                                             &queue_family);
-    assert(queue_family.queueFlags & VK_QUEUE_GRAPHICS_BIT);
-
-    /* Create logical device. */
-    VkDevice device;
-    VkDeviceQueueCreateInfo queueCreateInfo = {};
-    queueCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
-    queueCreateInfo.queueFamilyIndex = 0;
-    queueCreateInfo.queueCount = 1;
-    VkDeviceCreateInfo deviceCreateInfo = {};
-    deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
-    deviceCreateInfo.queueCreateInfoCount = 1;
-    deviceCreateInfo.pQueueCreateInfos = &queueCreateInfo;
-    deviceCreateInfo.enabledExtensionCount = 1;
-    deviceCreateInfo.ppEnabledExtensionNames = extensionNames;
-
-    result = vkCreateDevice(physical_devices[0], &deviceCreateInfo,
-                            NULL, &device);
-    if (unlikely(result != VK_SUCCESS)) {
-        if (result == VK_ERROR_EXTENSION_NOT_PRESENT)
-            fprintf(stderr, "VK_AMD_shader_info is required!\n");
-        fprintf(stderr, "Failed to create device (%d).\n", result);
-        ret = -1;
-        goto fail_device;
-    }
-
-    vkGetShaderInfo =
-        (PFN_vkGetShaderInfoAMD)vkGetDeviceProcAddr(device,
-                                                    "vkGetShaderInfoAMD");
-
-    /**
      * Runner.
      */
     /* Gather all pipeline tests. */
@@ -576,6 +537,7 @@ int main(int argc, char **argv)
 
     unsigned *pipline_counts = calloc(max_threads, sizeof(unsigned));
     current_pipeline_names = calloc(max_threads, sizeof(const char *));
+    msgCallbacks = calloc(max_threads, sizeof(VkDebugReportCallbackEXT));
     omp_set_num_threads(max_threads);
 
     if (signal(SIGABRT, abort_handler) == SIG_ERR)
@@ -583,8 +545,107 @@ int main(int argc, char **argv)
     if (signal(SIGSEGV, abort_handler) == SIG_ERR)
         fprintf(stderr, "WARNING: could not install SIGSEGV handler.\n");
 
+    setenv("ANV_ENABLE_PIPELINE_CACHE", "false", 1);
+
     #pragma omp parallel if(max_threads > 1 && pipeline_test_length > max_threads)
     {
+        VkInstance instance;
+        VkQueueFamilyProperties queue_family;
+        VkPhysicalDevice *physical_devices;
+        uint32_t device_count;
+        uint32_t queue_count = 1;
+        VkResult result;
+
+        /**
+         * Instance creation.
+         */
+        VkApplicationInfo appInfo = {};
+        appInfo.pApplicationName = "vkpipeline-db";
+        VkInstanceCreateInfo instanceCreateInfo = {};
+        instanceCreateInfo.pApplicationInfo = &appInfo;
+        instanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
+        if (vendor != VENDOR_AMD) {
+            instanceCreateInfo.enabledExtensionCount = 1;
+            instanceCreateInfo.ppEnabledExtensionNames = debug_report_ext;
+        }
+
+        result = vkCreateInstance(&instanceCreateInfo, NULL, &instance);
+        if (unlikely(result != VK_SUCCESS)) {
+            if (result == VK_ERROR_EXTENSION_NOT_PRESENT)
+                fprintf(stderr, "VK_EXT_debug_report is required!\n");
+            fprintf(stderr, "Failed to create instance (%d).\n", result);
+            ret = -1;
+            goto fail_device;
+        }
+
+        /**
+         * Device creation.
+         */
+        /* Get number of devices. */
+        result = vkEnumeratePhysicalDevices(instance, &device_count, NULL);
+        fprintf(stderr, "Number of devices: %d\n", device_count);
+
+        physical_devices = malloc(sizeof(*physical_devices) * device_count);
+
+        /* Get physical devices. */
+        result = vkEnumeratePhysicalDevices(instance, &device_count,
+                                            physical_devices);
+        if (unlikely(result != VK_SUCCESS)) {
+            fprintf(stderr, "Failed to enumerate physical devices (%d).\n", result);
+            ret = -1;
+            goto fail_device;
+        }
+
+        VkPhysicalDeviceProperties device_properties;
+        vkGetPhysicalDeviceProperties(physical_devices[0], &device_properties);
+        vendor = device_properties.vendorID;
+        fprintf(stderr, "GPU: %s\n", device_properties.deviceName);
+
+        /* Get queue properties. */
+        vkGetPhysicalDeviceQueueFamilyProperties(physical_devices[0], &queue_count,
+                                                 &queue_family);
+        assert(queue_family.queueFlags & VK_QUEUE_GRAPHICS_BIT);
+
+        /* Create logical device. */
+        VkDevice device;
+        VkDeviceQueueCreateInfo queueCreateInfo = {};
+        queueCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
+        queueCreateInfo.queueFamilyIndex = 0;
+        queueCreateInfo.queueCount = 1;
+        VkDeviceCreateInfo deviceCreateInfo = {};
+        deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
+        deviceCreateInfo.queueCreateInfoCount = 1;
+        deviceCreateInfo.pQueueCreateInfos = &queueCreateInfo;
+        if (vendor == VENDOR_AMD) {
+            deviceCreateInfo.enabledExtensionCount = 1;
+            deviceCreateInfo.ppEnabledExtensionNames = shader_info_ext;
+        }
+
+        result = vkCreateDevice(physical_devices[0], &deviceCreateInfo,
+                                NULL, &device);
+        if (unlikely(result != VK_SUCCESS)) {
+            if (result == VK_ERROR_EXTENSION_NOT_PRESENT)
+                fprintf(stderr, "VK_AMD_shader_info is required!\n");
+            fprintf(stderr, "Failed to create device (%d).\n", result);
+            ret = -1;
+            goto fail_device;
+        }
+
+        if (vendor == VENDOR_AMD) {
+            vkGetShaderInfo =
+                (PFN_vkGetShaderInfoAMD)vkGetDeviceProcAddr(device,
+                                                            "vkGetShaderInfoAMD");
+        } else {
+            createDebugReportCallback =
+                (PFN_vkCreateDebugReportCallbackEXT)vkGetInstanceProcAddr(instance,
+                                                                          "vkCreateDebugReportCallbackEXT");
+
+            destroyDebugReportCallback =
+                (PFN_vkDestroyDebugReportCallbackEXT)vkGetInstanceProcAddr(instance,
+                                                                          "vkDestroyDebugReportCallbackEXT");
+        }
+
+
         struct timespec start, end;
         clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
 
@@ -617,7 +678,7 @@ int main(int argc, char **argv)
                 continue;
             }
 
-            if (unlikely(run(device, current_pipeline_name, data, filesize) < 0))
+            if (unlikely(run(device, instance, current_pipeline_name, data, filesize) < 0))
                 continue;
 
             pipline_counts[omp_get_thread_num()]++;
@@ -636,14 +697,17 @@ int main(int argc, char **argv)
                omp_get_thread_num(),
                (end.tv_sec - start.tv_sec) + 10e-9 * (end.tv_nsec - start.tv_nsec),
                pipline_counts[omp_get_thread_num()]);
+
+        vkDestroyDevice(device, NULL);
+fail_device:
+        free(physical_devices);
+        vkDestroyInstance(instance, NULL);
+
     }
 
     free(pipeline_test);
 
-    vkDestroyDevice(device, NULL);
-fail_device:
-    free(physical_devices);
-    vkDestroyInstance(instance, NULL);
+
 
     return ret;
 }