Bring some of the virtual-memory-regions

Tracking user memory by querying virtual memory subsystem is not reboust
enough for master, but works in many cases, yielding much smaller and
efficient traces.

This change brings the ability of retracing traces generated by the
virtual-memory-regions branch.

It also brings more efficient tracing of glFlushMappedBufferRange calls.

The trace file version is bumped as a result.

12 files changed, 350 insertions, 22 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e01da79..db3d9ab 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -416,6 +416,7 @@ add_executable (glretrace
     glstate.cpp
     glstate_params.cpp
     retrace.cpp
+    retrace_stdc.cpp
     glws.cpp
     ${glws_os}
     ${CMAKE_CURRENT_BINARY_DIR}/glproc.hpp
diff --git a/common/trace_format.hpp b/common/trace_format.hpp
index a8ee5eb..cdabeb7 100644
--- a/common/trace_format.hpp
+++ b/common/trace_format.hpp
@@ -72,7 +72,36 @@
 
 namespace Trace {
 
-#define TRACE_VERSION 1
+/*
+ * Trace file version number.
+ *
+ * We keep backwards compatability reading old traces, i.e., it should always be
+ * possible to parse and retrace old trace files.
+ *
+ * So the trace version number refers not only to changes in the binary format
+ * representation, but also semantic changes in the way certain functions
+ * should be retraced.
+ *
+ * Writing/editing old traces will not be supported however.  An older version
+ * of apitrace should be used in such circunstances.
+ *
+ * Changelog:
+ *
+ * - version 0:
+ *   - initial implementation
+ *
+ * - version 1:
+ *   - support for GL user arrays -- a blob is provided whenever an user memory
+ *   is referred (whereas before calls that operate wit user memory instead of
+ *   VBOs should be ignore)
+ *
+ * - version 2:
+ *   - malloc/free memory calls -- allow to pass user memory as malloc memory
+ *   as opposed to blobs
+ *   - glFlushMappedBufferRange will emit a memcpy only for the flushed range
+ *   (whereas previously it would emit a memcpy for the whole mapped range)
+ */
+#define TRACE_VERSION 2
 
 enum Event {
     EVENT_ENTER = 0,
diff --git a/common/trace_local_writer.cpp b/common/trace_local_writer.cpp
index e625f2e..e560e49 100644
--- a/common/trace_local_writer.cpp
+++ b/common/trace_local_writer.cpp
@@ -39,6 +39,19 @@
 namespace Trace {
 
 
+static const char *memcpy_args[3] = {"dest", "src", "n"};
+const FunctionSig memcpy_sig = {0, "memcpy", 3, memcpy_args};
+
+static const char *malloc_args[1] = {"size"};
+const FunctionSig malloc_sig = {1, "malloc", 1, malloc_args};
+
+static const char *free_args[1] = {"ptr"};
+const FunctionSig free_sig = {2, "free", 1, free_args};
+
+static const char *realloc_args[2] = {"ptr", "size"};
+const FunctionSig realloc_sig = {3, "realloc", 2, realloc_args};
+
+
 static void exceptionCallback(void)
 {
     localWriter.flush();
diff --git a/common/trace_writer.hpp b/common/trace_writer.hpp
index dfb76b2..50d4bbf 100644
--- a/common/trace_writer.hpp
+++ b/common/trace_writer.hpp
@@ -105,6 +105,11 @@ namespace Trace {
 
     };
 
+    extern const FunctionSig memcpy_sig;
+    extern const FunctionSig malloc_sig;
+    extern const FunctionSig free_sig;
+    extern const FunctionSig realloc_sig;
+
     /**
      * A specialized Writer class, mean to trace the current process.
      *
diff --git a/glretrace.py b/glretrace.py
index 2eb6f4a..e80fedb 100644
--- a/glretrace.py
+++ b/glretrace.py
@@ -150,6 +150,20 @@ class GlRetracer(Retracer):
         'glReadnPixelsARB',
     ])
 
+    map_function_names = set([
+        'glMapBuffer',
+        'glMapBufferARB',
+        'glMapBufferRange',
+        'glMapNamedBufferEXT',
+        'glMapNamedBufferRangeEXT'
+    ])
+
+    unmap_function_names = set([
+        'glUnmapBuffer',
+        'glUnmapBufferARB',
+        'glUnmapNamedBufferEXT',
+    ])
+
     def retrace_function_body(self, function):
         is_array_pointer = function.name in self.array_pointer_function_names
         is_draw_array = function.name in self.draw_array_function_names
@@ -286,7 +300,7 @@ class GlRetracer(Retracer):
                 print r'             retrace::warning(call) << infoLog << "\n";'
                 print r'             delete [] infoLog;'
                 print r'        }'
-            if function.name in ('glMapBuffer', 'glMapBufferARB', 'glMapBufferRange', 'glMapNamedBufferEXT', 'glMapNamedBufferRangeEXT'):
+            if function.name in self.map_function_names:
                 print r'        if (!__result) {'
                 print r'             retrace::warning(call) << "failed to map buffer\n";'
                 print r'        }'
@@ -303,9 +317,39 @@ class GlRetracer(Retracer):
                 print r'    }'
             print '    }'
 
+            # Update buffer mappings
+            if function.name in self.map_function_names:
+                print r'        if (__result) {'
+                print r'            unsigned long long __address = call.ret->toUIntPtr();'
+                if 'BufferRange' not in function.name:
+                    print r'            GLint length = 0;'
+                    if function.name == 'glMapBuffer':
+                        print r'            glGetBufferParameteriv(target, GL_BUFFER_SIZE, &length);'
+                    elif function.name == 'glMapBufferARB':
+                        print r'            glGetBufferParameterivARB(target, GL_BUFFER_SIZE_ARB, &length);'
+                    elif function.name == 'glMapNamedBufferEXT':
+                        print r'            glGetNamedBufferParameterivEXT(buffer, GL_BUFFER_SIZE, &length);'
+                    else:
+                        assert False
+                print r'             retrace::addRegion(__address, __result, length);'
+                print r'        }'
+            if function.name in self.unmap_function_names:
+                print r'        GLvoid *ptr = NULL;'
+                if function.name == 'glUnmapBuffer':
+                    print r'            glGetBufferPointerv(target, GL_BUFFER_MAP_POINTER, &ptr);'
+                elif function.name == 'glUnmapBufferARB':
+                    print r'            glGetBufferPointervARB(target, GL_BUFFER_MAP_POINTER_ARB, &ptr);'
+                elif function.name == 'glUnmapNamedBufferEXT':
+                    print r'            glGetNamedBufferPointervEXT(buffer, GL_BUFFER_MAP_POINTER, &ptr);'
+                else:
+                    assert False
+                print r'        if (ptr) {'
+                print r'            retrace::delRegionByPointer(ptr);'
+                print r'        }'
+
     def extract_arg(self, function, arg, arg_type, lvalue, rvalue):
         if function.name in self.array_pointer_function_names and arg.name == 'pointer':
-            print '    %s = static_cast<%s>(%s.toPointer(true));' % (lvalue, arg_type, rvalue)
+            print '    %s = static_cast<%s>(retrace::toPointer(%s, true));' % (lvalue, arg_type, rvalue)
             return
 
         if function.name in self.draw_elements_function_names and arg.name == 'indices' or\
@@ -351,6 +395,5 @@ if __name__ == '__main__':
 
 '''
     api = glapi.glapi
-    api.add_function(glapi.memcpy)
     retracer = GlRetracer()
     retracer.retrace_api(api)
diff --git a/gltrace.py b/gltrace.py
index f3b82df..e5be57d 100644
--- a/gltrace.py
+++ b/gltrace.py
@@ -244,9 +244,6 @@ class GlTracer(Tracer):
         print '}'
         print
 
-        # Generate memcpy's signature
-        self.trace_function_decl(glapi.memcpy)
-
         # Generate a helper function to determine whether a parameter name
         # refers to a symbolic value or not
         print 'static bool'
@@ -442,14 +439,13 @@ class GlTracer(Tracer):
             self.emit_memcpy('mapping->map', 'mapping->map', 'mapping->length')
             print '    }'
         if function.name in ('glFlushMappedBufferRange', 'glFlushMappedBufferRangeAPPLE'):
-            # TODO: avoid copying [0, offset] bytes
             print '    struct buffer_mapping *mapping = get_buffer_mapping(target);'
             print '    if (mapping) {'
             if function.name.endswith('APPLE'):
                  print '        GLsizeiptr length = size;'
                  print '        mapping->explicit_flush = true;'
             print '        //assert(offset + length <= mapping->length);'
-            self.emit_memcpy('mapping->map', 'mapping->map', 'offset + length')
+            self.emit_memcpy('(char *)mapping->map + offset', '(const char *)mapping->map + offset', 'length')
             print '    }'
         # FIXME: glFlushMappedNamedBufferRangeEXT
 
@@ -531,7 +527,7 @@ class GlTracer(Tracer):
         Tracer.dispatch_function(self, function)
 
     def emit_memcpy(self, dest, src, length):
-        print '        unsigned __call = Trace::localWriter.beginEnter(&__memcpy_sig);'
+        print '        unsigned __call = Trace::localWriter.beginEnter(&Trace::memcpy_sig);'
         print '        Trace::localWriter.beginArg(0);'
         print '        Trace::localWriter.writeOpaque(%s);' % dest
         print '        Trace::localWriter.endArg();'
diff --git a/retrace.hpp b/retrace.hpp
index 300f007..d66c64d 100644
--- a/retrace.hpp
+++ b/retrace.hpp
@@ -80,6 +80,16 @@ public:
 };
 
 
+void
+addRegion(unsigned long long address, void *buffer, unsigned long long size);
+
+void
+delRegionByPointer(void *ptr);
+
+void *
+toPointer(Trace::Value &value, bool bind = false);
+
+
 /**
  * Output verbosity when retracing files.
  */
@@ -108,6 +118,9 @@ struct stringComparer {
 };
 
 
+extern const Entry stdc_callbacks[];
+
+
 class Retracer
 {
     typedef std::map<const char *, Callback, stringComparer> Map;
@@ -116,7 +129,9 @@ class Retracer
     std::vector<Callback> callbacks;
 
 public:
-    Retracer() {}
+    Retracer() {
+        addCallbacks(stdc_callbacks);
+    }
 
     virtual ~Retracer() {}
 
diff --git a/retrace.py b/retrace.py
index 1df3415..90a4414 100644
--- a/retrace.py
+++ b/retrace.py
@@ -116,7 +116,7 @@ class OpaqueValueExtractor(ValueExtractor):
     in the context of handles.'''
 
     def visit_opaque(self, opaque, lvalue, rvalue):
-        print '    %s = static_cast<%s>((%s).toPointer());' % (lvalue, opaque, rvalue)
+        print '    %s = static_cast<%s>(retrace::toPointer(%s));' % (lvalue, opaque, rvalue)
 
 
 class ValueWrapper(stdapi.Visitor):
@@ -227,8 +227,7 @@ class Retracer:
             try:
                 ValueWrapper().visit(function.type, lvalue, rvalue)
             except NotImplementedError:
-                success = False
-                print '    // FIXME: result'
+                print '    // XXX: result'
         if not success:
             if function.name[-1].islower():
                 sys.stderr.write('warning: unsupported %s call\n' % function.name)
diff --git a/retrace_stdc.cpp b/retrace_stdc.cpp
new file mode 100644
index 0000000..feb06b2
--- /dev/null
+++ b/retrace_stdc.cpp
@@ -0,0 +1,232 @@
+/**************************************************************************
+ *
+ * Copyright 2011 Jose Fonseca
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ **************************************************************************/
+
+
+#include <assert.h>
+
+#include <string.h>
+
+#include "glproc.hpp"
+
+
+
+#include "trace_parser.hpp"
+#include "retrace.hpp"
+
+
+namespace retrace {
+
+struct Region
+{
+    void *buffer;
+    unsigned long long size;
+};
+
+typedef std::map<unsigned long long, Region> RegionMap;
+static RegionMap regionMap;
+
+// Iterator to the first region that contains the address
+static RegionMap::iterator
+lowerBound(unsigned long long address) {
+    RegionMap::iterator it = regionMap.lower_bound(address);
+
+    while (it != regionMap.begin() &&
+           it != regionMap.end() &&
+           it->first + it->second. size > address) {
+        --it;
+    }
+
+    return it;
+}
+
+// Iterator to the first region that not contains the address
+static RegionMap::iterator
+upperBound(unsigned long long address) {
+    RegionMap::iterator it = regionMap.upper_bound(address);
+
+    while (it != regionMap.end() &&
+           it->first + it->second.size > address) {
+        ++it;
+    }
+
+    return it;
+}
+
+void
+addRegion(unsigned long long address, void *buffer, unsigned long long size)
+{
+    // Forget all regions that intersect this new one.
+    if (0) {
+        RegionMap::iterator start = lowerBound(address);
+        if (start != regionMap.end()) {
+            RegionMap::iterator stop = upperBound(address + size);
+            regionMap.erase(start, stop);
+        }
+    }
+
+    assert(buffer);
+
+    Region region;
+    region.buffer = buffer;
+    region.size = size;
+
+    regionMap[address] = region;
+}
+
+static RegionMap::iterator
+lookupRegion(unsigned long long address) {
+    RegionMap::iterator it = regionMap.lower_bound(address);
+
+    if (it == regionMap.end() ||
+        it->first > address) {
+        if (it == regionMap.begin()) {
+            return regionMap.end();
+        } else {
+            --it;
+        }
+    }
+
+    assert(it->first <= address);
+    assert(it->first + it->second.size >= address);
+    return it;
+}
+
+void
+delRegion(unsigned long long address) {
+    RegionMap::iterator it = lookupRegion(address);
+    if (it != regionMap.end()) {
+        regionMap.erase(it);
+    } else {
+        assert(0);
+    }
+}
+
+
+void
+delRegionByPointer(void *ptr) {
+    RegionMap::iterator it = regionMap.begin();
+    while (it != regionMap.end()) {
+        if (it->second.buffer == ptr) {
+            regionMap.erase(it);
+            return;
+        }
+    }
+    assert(0);
+}
+
+void *
+lookupAddress(unsigned long long address) {
+    RegionMap::iterator it = lookupRegion(address);
+    if (it != regionMap.end()) {
+        unsigned long long offset = address - it->first;
+        assert(offset < it->second.size);
+        return (char *)it->second.buffer + offset;
+    }
+
+    if (address >= 0x00400000) {
+        std::cerr << "warning: could not translate address 0x" << std::hex << address << std::dec << "\n";
+    }
+
+    return (void *)(uintptr_t)address;
+}
+
+
+class Translator : protected Trace::Visitor
+{
+protected:
+    bool bind;
+
+    void *result;
+
+    void visit(Trace::Null *) {
+        result = NULL;
+    }
+
+    void visit(Trace::Blob *blob) {
+        result = blob->toPointer(bind);
+    }
+
+    void visit(Trace::Pointer *p) {
+        result = lookupAddress(p->value);
+    }
+
+public:
+    Translator(bool _bind) :
+        bind(_bind),
+        result(NULL)
+    {}
+
+    void * operator() (Trace::Value *node) {
+        _visit(node);
+        return result;
+    }
+};
+
+
+void *
+toPointer(Trace::Value &value, bool bind) {
+    return Translator(bind) (&value);
+}
+
+
+static void retrace_malloc(Trace::Call &call) {
+    size_t size = call.arg(0).toUInt();
+    unsigned long long address = call.ret->toUIntPtr();
+
+    if (!address) {
+        return;
+    }
+
+    void *buffer = malloc(size);
+    if (!buffer) {
+        std::cerr << "error: failed to allocated " << size << " bytes.";
+        return;
+    }
+
+    addRegion(address, buffer, size);
+}
+
+
+static void retrace_memcpy(Trace::Call &call) {
+    void * dest = toPointer(call.arg(0));
+    void * src  = toPointer(call.arg(1));
+    size_t n    = call.arg(2).toUInt();
+
+    if (!dest || !src || !n) {
+        return;
+    }
+
+    memcpy(dest, src, n);
+}
+
+
+const retrace::Entry stdc_callbacks[] = {
+    {"malloc", &retrace_malloc},
+    {"memcpy", &retrace_memcpy},
+    {NULL, NULL}
+};
+
+
+} /* retrace */
diff --git a/specs/glapi.py b/specs/glapi.py
index 5c18630..b54a4bd 100644
--- a/specs/glapi.py
+++ b/specs/glapi.py
@@ -2804,9 +2804,3 @@ glapi.add_functions([
     # GL_WIN_swap_hint
     GlFunction(Void, "glAddSwapHintRectWIN", [(GLint, "x"), (GLint, "y"), (GLsizei, "width"), (GLsizei, "height")]),
 ])
-
-
-# memcpy's prototype.  We don't really want to trace all memcpy calls -- just
-# emit a few fake memcpy calls --, which is why the prototype is not together
-# with the rest.
-memcpy = Function(Void, "memcpy", [(GLmap, "dest"), (Blob(Const(Void), "n"), "src"), (SizeT, "n")])
diff --git a/specs/gltypes.py b/specs/gltypes.py
index c8e3f09..1d9206b 100644
--- a/specs/gltypes.py
+++ b/specs/gltypes.py
@@ -82,7 +82,7 @@ GLrenderbuffer = Handle("renderbuffer", GLuint)
 GLfragmentShaderATI = Handle("fragmentShaderATI", GLuint)
 GLarray = Handle("array", GLuint)
 GLregion = Handle("region", GLuint)
-GLmap = Handle("map", GLpointer)
+GLmap = GLpointer
 GLpipeline = Handle("pipeline", GLuint)
 GLsampler = Handle("sampler", GLuint)
 GLfeedback = Handle("feedback", GLuint)
diff --git a/specs/stdapi.py b/specs/stdapi.py
index fa371cf..a92b9ba 100644
--- a/specs/stdapi.py
+++ b/specs/stdapi.py
@@ -220,7 +220,8 @@ class Arg:
 
 class Function:
 
-    __id = 0
+    # 0-3 are reserved to memcpy, malloc, free, and realloc
+    __id = 4
 
     def __init__(self, type, name, args, call = '', fail = None, sideeffects=True):
         self.id = Function.__id