red_worker: support releasing drawables that are referenced by the pipebitmap-crop.mem-control

Replace drawables that are referenced by the pipe, with rendered images.
It can be used as a last resort, when the qxl ram is full, and after
the glz dictionary and the current tree have been emptied.

2 files changed, 249 insertions, 0 deletions

diff --git a/server/red_worker.c b/server/red_worker.c
index 6cecfef..8cbeb01 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -1131,6 +1131,8 @@ static void cursor_channel_client_release_item_after_push(CursorChannelClient *c
                                                           PipeItem *item);
 
 static void red_push_monitors_config(DisplayChannelClient *dcc);
+static void add_clip_rects(QRegion *rgn, SpiceClipRects *data);
+static void flush_all_surfaces(RedWorker *worker);
 
 /*
  * Macros to make iterating over stuff easier
@@ -1982,6 +1984,241 @@ static uint32_t red_free_current_qxl_drawables(RedWorker *worker,
     return size_diff;
 }
 
+/*
+ * Collecting for each surface the region that is modified by the drawables
+ * that are currently present in the pipe.
+ * Then, we can remove all the corresponding pipe items (this will unref the drawables),
+ * and replace them by pipe items of type PIPE_ITEM_TYPE_IMAGE.
+ */
+typedef struct SurfacePipeDrawData {
+    uint32_t surface_id;
+    PipeItem *last_pipe_item;
+    QRegion dirty_region;
+    DisplayChannelClient *dcc;
+} SurfacePipeDrawData;
+
+static gint surface_pipe_draw_data_compare(gconstpointer a, gconstpointer b)
+{
+    SurfacePipeDrawData *a_data = (SurfacePipeDrawData *)a;
+    SurfacePipeDrawData *b_data = (SurfacePipeDrawData *)b;
+
+    return a_data->surface_id == b_data->surface_id ? 0 : a_data->surface_id - b_data->surface_id;
+}
+
+static void surface_pipe_draw_data_destroy(gpointer data)
+{
+    SurfacePipeDrawData *surface_data = data;
+
+    if (surface_data->last_pipe_item) {
+        red_channel_client_pipe_remove_and_release(&surface_data->dcc->common.base,
+                                                   surface_data->last_pipe_item);
+    }
+    region_destroy(&surface_data->dirty_region);
+    free(surface_data);
+}
+
+static int drawable_is_stream_frame(Drawable *drawable, RedChannelClient *rcc)
+{
+    return drawable->stream ||
+           (drawable->sized_stream &&
+            red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_SIZED_STREAM));
+}
+
+/*
+ *  Returns the SurfacePipeDrawData that corresponds to the surface-id.
+ *  If there isn't such in the list, creates an empty one and inserts it to the list.
+ */
+static SurfacePipeDrawData *surfaces_pipe_draw_data_list_get_by_id(GList **list,
+                                                                   uint32_t surface_id,
+                                                                   DisplayChannelClient *dcc)
+{
+    GList *surface_data_iter;
+    SurfacePipeDrawData dummy_surface_data;
+    SurfacePipeDrawData *surface_data;
+
+    dummy_surface_data.surface_id = surface_id;
+    surface_data_iter = g_list_find_custom(*list,
+                                           &dummy_surface_data,
+                                           surface_pipe_draw_data_compare);
+    if (surface_data_iter) {
+        surface_data = surface_data_iter->data;
+        spice_assert(surface_data);
+    } else {
+        surface_data = spice_malloc(sizeof(SurfacePipeDrawData));
+        surface_data->surface_id = surface_id;
+        surface_data->dcc = dcc;
+        surface_data->last_pipe_item = NULL;
+        region_init(&surface_data->dirty_region);
+        *list = g_list_prepend(*list, surface_data);
+    }
+    return surface_data;
+}
+
+static void surface_pipe_draw_data_list_add_draw_info(GList **surfaces_data_list,
+                                                      uint32_t surface_id,
+                                                      SpiceRect *bbox,
+                                                      SpiceClip *clip,
+                                                      PipeItem *pipe_item,
+                                                      DisplayChannelClient *dcc)
+{
+    SurfacePipeDrawData *surface_data;
+
+    surface_data = surfaces_pipe_draw_data_list_get_by_id(surfaces_data_list,
+                                                          surface_id,
+                                                          dcc);
+    spice_debug("start");
+    if (clip && clip->type == SPICE_CLIP_TYPE_RECTS) {
+        spice_debug("clip");
+        QRegion draw_region, clip_region;
+
+        region_init(&draw_region);
+        region_init(&clip_region);
+        region_add(&draw_region, bbox);
+        add_clip_rects(&clip_region, clip->rects);
+        region_and(&draw_region, &clip_region);
+        region_or(&surface_data->dirty_region, &draw_region);
+        region_destroy(&clip_region);
+        region_destroy(&draw_region);
+    } else {
+        region_add(&surface_data->dirty_region, bbox);
+    }
+    if (surface_data->last_pipe_item) {
+        red_channel_client_pipe_remove_and_release(&dcc->common.base,
+                                                   surface_data->last_pipe_item);
+    }
+    surface_data->last_pipe_item = pipe_item;
+    spice_debug("end");
+}
+
+static void surface_pipe_draw_data_list_destroy_surface(uint32_t surface_id,
+                                                        GList **surfaces_data_list)
+{
+    GList *surface_data_iter;
+    SurfacePipeDrawData dummy_surface_data;
+
+    dummy_surface_data.surface_id = surface_id;
+    surface_data_iter = g_list_find_custom(*surfaces_data_list,
+                                            &dummy_surface_data,
+                                            surface_pipe_draw_data_compare);
+    if (surface_data_iter) {
+        spice_debug("surface-id=%u", surface_id);
+        surface_pipe_draw_data_destroy(surface_data_iter->data);
+        *surfaces_data_list = g_list_delete_link(*surfaces_data_list, surface_data_iter);
+    }
+}
+
+static void red_display_replace_surface_draw_data_with_image(gpointer data,
+                                                             gpointer user_data)
+{
+    SurfacePipeDrawData *surface_data = data;
+    DisplayChannelClient *dcc = user_data;
+    SpiceRect *rects;
+    uint32_t num_rects;
+    uint32_t i;
+
+    spice_debug("start surface-id=%d", surface_data->surface_id);
+    rects = region_dup_rects(&surface_data->dirty_region, &num_rects);
+    for (i = 0; i < num_rects; i++) {
+        spice_debug("image box (%d, %d) (%d, %d) %dx%d",
+                    rects[i].left, rects[i].top,
+                    rects[i].right, rects[i].bottom,
+                    rects[i].right - rects[i].left,
+                    rects[i].bottom - rects[i].top);
+        red_add_surface_area_image(dcc,
+                                   surface_data->surface_id,
+                                   &rects[i],
+                                   surface_data->last_pipe_item,
+                                   FALSE);
+    }
+    red_channel_client_pipe_remove_and_release(&dcc->common.base,
+                                               surface_data->last_pipe_item);
+    surface_data->last_pipe_item = NULL;
+    free(rects);
+    spice_debug("end");
+}
+
+static void red_display_free_pipe_qxl_drawables(DisplayChannelClient *dcc)
+{
+    Ring *pipe_ring;
+    RingItem *item, *next_item;
+    GList *surfaces_draw_data_list = NULL;
+
+    spice_debug(NULL);
+    /* render everything. However, in practice, red_display_free_pipe_qxl_drawables
+     * should be called only as a last resort, after everything has already
+     * been rendered */
+    flush_all_surfaces(dcc->common.worker);
+
+    pipe_ring = &dcc->common.base.pipe;
+
+    RING_FOREACH_REVERSED_SAFE(item, next_item, pipe_ring) {
+        PipeItem *pipe_item = (PipeItem *)item;
+
+        switch (pipe_item->type) {
+        case PIPE_ITEM_TYPE_DRAW: {
+             DrawablePipeItem *dpi;
+
+            dpi = SPICE_CONTAINEROF(pipe_item, DrawablePipeItem, dpi_pipe_item);
+            if (!drawable_is_stream_frame(dpi->drawable, &dcc->common.base)) {
+                Drawable *drawable = dpi->drawable;
+
+                surface_pipe_draw_data_list_add_draw_info(&surfaces_draw_data_list,
+                                                          drawable->surface_id,
+                                                          &drawable->red_drawable->bbox,
+                                                          &drawable->red_drawable->clip,
+                                                          pipe_item,
+                                                          dcc);
+            }
+            break;
+        }
+        case PIPE_ITEM_TYPE_IMAGE:
+        {
+            ImageItem *image_item = (ImageItem *)pipe_item;
+            SpiceRect bbox;
+
+            bbox.left = image_item->pos.x;
+            bbox.top = image_item->pos.y;
+            bbox.right = image_item->pos.x + image_item->width;
+            bbox.bottom = image_item->pos.y + image_item->height;
+
+           surface_pipe_draw_data_list_add_draw_info(&surfaces_draw_data_list,
+                                                     image_item->surface_id,
+                                                     &bbox,
+                                                     NULL,
+                                                     pipe_item,
+                                                     dcc);
+           break;
+        }
+        case PIPE_ITEM_TYPE_UPGRADE:
+        {
+            Drawable *drawable = ((UpgradeItem *)pipe_item)->drawable;
+
+            surface_pipe_draw_data_list_add_draw_info(&surfaces_draw_data_list,
+                                                      drawable->surface_id,
+                                                      &drawable->red_drawable->bbox,
+                                                      &drawable->red_drawable->clip,
+                                                      pipe_item,
+                                                      dcc);
+            break;
+        }
+        case PIPE_ITEM_TYPE_DESTROY_SURFACE: {
+            SurfaceDestroyItem *destroy_item = SPICE_CONTAINEROF(pipe_item, SurfaceDestroyItem,
+                                                                    pipe_item);
+            surface_pipe_draw_data_list_destroy_surface(destroy_item->surface_destroy.surface_id,
+                                                        &surfaces_draw_data_list);
+            break;
+        }
+        default:
+            break;
+        }
+    }
+
+    g_list_foreach(surfaces_draw_data_list,
+                   red_display_replace_surface_draw_data_with_image,
+                   dcc);
+    g_list_free_full(surfaces_draw_data_list, surface_pipe_draw_data_destroy);
+}
+
 static void red_free_qxl_resources(RedWorker *worker, uint32_t target_release_size)
 {
     DisplayChannelClient *dcc;
@@ -2012,6 +2249,18 @@ static void red_free_qxl_resources(RedWorker *worker, uint32_t target_release_si
         pthread_rwlock_unlock(&dcc->glz_dict->encode_lock);
     }
 
+    if (released_size < target_release_size) {
+        WORKER_FOREACH_DCC_SAFE(worker, item, next, dcc) {
+            uint32_t start_qxl_size = worker->qxl_resources.held_size;
+            if (released_size >= target_release_size) {
+                break;
+            }
+            red_display_free_pipe_qxl_drawables(dcc);
+            spice_debug("dcc=%p pipe-release-size=%u", dcc, start_qxl_size - worker->qxl_resources.held_size);
+            spice_assert(worker->qxl_resources.held_size <= start_qxl_size);
+            released_size += start_qxl_size - worker->qxl_resources.held_size;
+        }
+    }
     red_flush_released_qxl_resources(worker);
     spice_debug("total-release-size=%u", released_size);
 }
diff --git a/spice-common b/spice-common
-Subproject 61954eb47d0a64c8252cededca1fbc392da843f
+Subproject 3888f2b4788525490cbc385dadbfe6e90ec6248