retrace: add support for intel gpu/cpu monitoring

This patch adds support for CPU/GPU monitoring during the execution of a
trace for Intel GPUs. This only works on Linux OS'es, and requires
libpciaccess development library to be built.

The monitoring is controlled by the following options:
 -o output - output tab-separated values into a file
 -p frames - sample cpu/gpu usage for each N frames
 -ps frame - frame to start profiling on
 -pe frame - frame to end profiling on

For example:
eglretrace -p 1500 -ps 300000 -pe 1800000 -o profile.dat
~/intel/traces/glbenchmark.egypt.egl

will profile 'glbenchmark.egypt.egl' trace, starting on frame 300000 and
ending on frame 1800000, sampling CPU and GPU usage for each 1500 frames.
The output will be saved into 'profile.dat'.

To plot the GPU usage from said log file, the following command can be
used:

echo "pl 'profile.dat' u 2:6 w l title 'GPU usage'" | gnuplot -p

Signed-off-by: Eugeni Dodonov <eugeni@dodonov.net>

5 files changed, 336 insertions, 1 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 67507e2..e1423f5 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -518,6 +518,7 @@ set (retrace_sources
     retrace.cpp
     retrace_stdc.cpp
     glws.cpp
+    intel_stats.cpp
 )
 
 add_executable (glretrace
@@ -536,6 +537,9 @@ set_property (
 
 target_link_libraries (glretrace
     common
+    pthread
+    dl
+    pciaccess
 )
 
 if (WIN32)
@@ -578,6 +582,8 @@ if (EGL_FOUND AND NOT WIN32 AND NOT APPLE)
         common
         ${X11_X11_LIB}
         pthread
+        dl
+        pciaccess
     )
 
     install (TARGETS eglretrace RUNTIME DESTINATION bin) 
diff --git a/glretrace_main.cpp b/glretrace_main.cpp
index 3bf9de8..e09c4d4 100644
--- a/glretrace_main.cpp
+++ b/glretrace_main.cpp
@@ -25,6 +25,8 @@
 
 
 #include <string.h>
+#include <iostream>
+#include <fstream>
 
 #include "os_string.hpp"
 #include "image.hpp"
@@ -33,10 +35,16 @@
 #include "glstate.hpp"
 #include "glretrace.hpp"
 
+#include "intel_stats.h"
+
 
 namespace glretrace {
 
 bool double_buffer = true;
+unsigned long long profile = 0;
+char *output = NULL;
+unsigned long long profile_start = 0;
+unsigned long long profile_end = 0;
 bool insideGlBeginEnd = false;
 trace::Parser parser;
 glws::Profile defaultProfile = glws::PROFILE_COMPAT;
@@ -197,6 +205,7 @@ void frame_complete(trace::Call &call) {
 
 
 static void display(void) {
+    int call_count = 0;
     retrace::Retracer retracer;
 
     retracer.addCallbacks(gl_callbacks);
@@ -208,8 +217,54 @@ static void display(void) {
     startTime = os::getTime();
     trace::Call *call;
 
+    std::ofstream output_s;
+    if (output) {
+        output_s.open(output, std::ios::out);
+        if (!output_s) {
+            std::cerr << "Unable to open output file " << output;
+            exit(1);
+        }
+        std::cout << "Will output to " << output << std::endl;
+    }
+
+    std::cout << "Starting at " << profile_start << std::endl;
+
     while ((call = parser.parse_call())) {
         retracer.retrace(*call);
+        if (profile) {
+            if (call->no == profile_start) {
+                m_reset();
+                std::cout << "Profiling: starting at frame " << call->no << std::endl;
+            }
+            if (profile_end > 0 && call->no == profile_end) {
+                std::cout << "Profiling: finished at frame " << call->no << std::endl;
+            }
+            /* Do we have a starting frame */
+            if ( (profile_start == 0 || call->no >= profile_start) && (profile_end == 0 || call->no <= profile_end)) {
+                if (call_count == profile || (profile_end > 0 && call->no == profile_end)) {
+                    Stats stat = m_count();
+                    unsigned long long elapsed = stat.time.tv_sec * 1000000 + stat.time.tv_usec;
+                    unsigned long long user = stat.user.tv_sec * 1000000 + stat.user.tv_usec;
+                    float userp = user * 100. / elapsed;
+                    float gpup = (stat.gpu)?100 - stat.gpuidle * 100. / stat.gpu:0;
+
+                    ((output_s) ? output_s: std::cout) \
+                             << "Call-No: " << call->no \
+                             << "\tTime(us): " << elapsed \
+                             << "\tGPU-%: " << gpup \
+                             << "\tCPU-%: " << userp \
+                             << "\tCPU-time(us): " << user \
+                             << std::endl;
+
+                    output_s.flush();
+
+                    m_reset();
+
+                    call_count = 0;
+                }
+                call_count++;
+            }
+        }
 
         if (!insideGlBeginEnd &&
             drawable && context &&
@@ -256,7 +311,11 @@ static void usage(void) {
         "  -S FREQUENCY snapshot frequency: frame (default), framebuffer, or draw\n"
         "  -v           verbose output\n"
         "  -D CALLNO    dump state at specific call no\n"
-        "  -w           wait on final frame\n";
+        "  -w           wait on final frame\n"
+        "  -o FILE      save profile results to FILE\n"
+        "  -p FRAMES    sample GPU/CPU usage once each FRAMES\n"
+        "  -ps FRAME    start profiling on frame FRAME\n"
+        "  -pe FRAME    end profile on frame FRAME\n";
 }
 
 extern "C"
@@ -287,6 +346,14 @@ int main(int argc, char **argv)
             retrace::verbosity = -2;
         } else if (!strcmp(arg, "-core")) {
             defaultProfile = glws::PROFILE_CORE;
+        } else if (!strcmp(arg, "-o")) {
+            output = strdup(argv[++i]);
+        } else if (!strcmp(arg, "-p")) {
+            profile = atoll(argv[++i]);
+        } else if (!strcmp(arg, "-ps")) {
+            profile_start = atoll(argv[++i]);
+        } else if (!strcmp(arg, "-pe")) {
+            profile_end = atoll(argv[++i]);
         } else if (!strcmp(arg, "-db")) {
             double_buffer = true;
         } else if (!strcmp(arg, "-sb")) {
@@ -329,6 +396,9 @@ int main(int argc, char **argv)
         }
     }
 
+    if (profile)
+        m_init();
+
     glws::init();
     visual = glws::createVisual(double_buffer);
 
diff --git a/intel_stats.cpp b/intel_stats.cpp
new file mode 100644
index 0000000..a3dec5e
--- /dev/null
+++ b/intel_stats.cpp
@@ -0,0 +1,186 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eugeni Dodonov <eugeni.dodonov@intel.com>
+ *
+ */
+
+extern "C" {
+
+#include <sys/resource.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <err.h>
+#include <pciaccess.h>
+
+#include "intel_stats.h"
+
+void *mmio;
+static pthread_t m_thread;
+static pthread_mutex_t m_lock;
+
+static Stats counter1, counter2;
+
+
+static inline uint32_t
+INREG(uint32_t reg)
+{
+	return *(volatile uint32_t *)((volatile char *)mmio + reg);
+}
+
+void
+intel_get_mmio(struct pci_device *pci_dev)
+{
+    uint32_t devid;
+    int mmio_bar;
+    int err;
+
+    devid = pci_dev->device_id;
+    if (IS_GEN2(devid))
+        mmio_bar = 1;
+    else
+        mmio_bar = 0;
+
+    err = pci_device_map_range (pci_dev,
+                    pci_dev->regions[mmio_bar].base_addr,
+                    pci_dev->regions[mmio_bar].size,
+                    PCI_DEV_MAP_FLAG_WRITABLE,
+                    &mmio);
+
+    if (err != 0) {
+        fprintf(stderr, "Couldn't map MMIO region: %s\n",
+            strerror(err));
+        exit(1);
+    }
+}
+
+struct pci_device *
+intel_get_pci_device(void)
+{
+	struct pci_device *pci_dev;
+	int err;
+
+	err = pci_system_init();
+	if (err != 0) {
+		fprintf(stderr, "Couldn't initialize PCI system: %s\n",
+			strerror(err));
+		exit(1);
+	}
+
+	/* Grab the graphics card */
+	pci_dev = pci_device_find_by_slot(0, 0, 2, 0);
+	if (pci_dev == NULL)
+		errx(1, "Couldn't find graphics card");
+
+	err = pci_device_probe(pci_dev);
+	if (err != 0) {
+		fprintf(stderr, "Couldn't probe graphics card: %s\n",
+			strerror(err));
+		exit(1);
+	}
+
+	if (pci_dev->vendor_id != 0x8086)
+		errx(1, "Graphics card is non-intel");
+
+	return pci_dev;
+}
+
+void m_init() {
+    intel_get_mmio(intel_get_pci_device());
+
+    pthread_mutex_init(&m_lock, NULL);
+
+    pthread_mutex_lock(&m_lock);
+    /* Initialize starting values */
+    pthread_mutex_unlock(&m_lock);
+
+    /* Run GPU monitoring thread */
+    pthread_create(&m_thread, 0, m_monitor, NULL);
+}
+
+void m_reset() {
+    struct rusage r;
+
+    /* Reset counter1 to counter2 values */
+    pthread_mutex_lock(&m_lock);
+
+    getrusage(RUSAGE_SELF, &r);
+    counter2.user = r.ru_utime;
+    counter2.system = r.ru_stime;
+    gettimeofday(&counter2.time, NULL);
+
+    counter1.user = counter2.user;
+    counter1.system = counter2.system;
+    counter1.gpu = counter2.gpu;
+    counter1.gpuidle = counter2.gpuidle;
+
+    counter1.time = counter2.time;
+    pthread_mutex_unlock(&m_lock);
+}
+
+Stats m_count() {
+    Stats stats;
+    struct rusage r;
+
+    pthread_mutex_lock(&m_lock);
+
+    getrusage(RUSAGE_SELF, &r);
+
+    gettimeofday(&counter2.time, NULL);
+
+    counter2.user = r.ru_utime;
+    counter2.system = r.ru_stime;
+
+    /* Do the calculations */
+    timersub(&counter2.time, &counter1.time, &stats.time);
+    timersub(&counter2.user, &counter1.user, &stats.user);
+    timersub(&counter2.system, &counter1.system, &stats.system);
+    stats.gpu = counter2.gpu - counter1.gpu;
+    stats.gpuidle = counter2.gpuidle - counter1.gpuidle;
+
+    pthread_mutex_unlock(&m_lock);
+
+    return stats;
+}
+
+void *m_monitor( void *data) {
+    while(1) {
+        uint32_t ring_head, ring_tail;
+
+        /* GPU usage */
+        ring_head = INREG(LP_RING + RING_HEAD) & HEAD_ADDR;
+        ring_tail = INREG(LP_RING + RING_TAIL) & TAIL_ADDR;
+
+        pthread_mutex_lock(&m_lock);
+        /* GPU stats */
+        if (ring_tail == ring_head)
+            counter2.gpuidle++;
+        counter2.gpu++;
+        pthread_mutex_unlock(&m_lock);
+        usleep(1000000 / SAMPLES_PER_SEC);
+    }
+    return NULL;
+}
+
+}
diff --git a/intel_stats.h b/intel_stats.h
new file mode 100644
index 0000000..d2accd8
--- /dev/null
+++ b/intel_stats.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eugeni Dodonov <eugeni.dodonov@intel.com>
+ *
+ */
+
+#ifndef _INTEL_STATS_HPP_
+#define _INTEL_STATS_HPP_
+
+extern "C" {
+
+#include <sys/types.h>
+#include <pthread.h>
+#include <pciaccess.h>
+#include <stdio.h>
+#include <string.h>
+
+#define LP_RING     0x2030
+#define RING_HEAD      0x04
+#define HEAD_ADDR           0x001FFFFC
+#define RING_TAIL      0x00
+#define TAIL_ADDR           0x000FFFF8
+#define SAMPLES_PER_SEC     1000000
+
+#define PCI_CHIP_I830_M			0x3577
+#define PCI_CHIP_845_G			0x2562
+#define PCI_CHIP_I855_GM		0x3582
+#define PCI_CHIP_I865_G			0x2572
+
+#define IS_GEN2(devid)      (devid == PCI_CHIP_I830_M || \
+                 devid == PCI_CHIP_845_G || \
+                 devid == PCI_CHIP_I855_GM || \
+                 devid == PCI_CHIP_I865_G)
+
+struct _Stats {
+    struct timeval user;
+    struct timeval system;
+    unsigned long long gpu;
+    unsigned long long gpuidle;
+    struct timeval time;
+};
+typedef struct _Stats Stats;
+
+void m_init();
+void m_reset();
+void *m_monitor(void *data);
+Stats m_count();
+
+}
+
+#endif /* _INTEL_STATS_HPP_ */
diff --git a/retrace.cpp b/retrace.cpp
index 412c73b..dcb4e43 100644
--- a/retrace.cpp
+++ b/retrace.cpp
@@ -29,6 +29,7 @@
 
 #include "retrace.hpp"
 
+#include "intel_stats.h"
 
 namespace retrace {