drm_hwcomposer: Fix mixed layer composition

Fix cases when mixed layer composition require non-device layer in the middle:

'''
Layer z_order - SF type    - validated type before - validated type fixed
            0 - DEVICE     - CLIENT                - CLIENT
            1 - DEVICE     - DEVICE                - CLIENT
            2 - DEVICE     - DEVICE                - CLIENT
            3 - SOLIDCOLOR - CLIENT                - CLIENT
            4 - DEVICE     - DEVICE                - DEVICE
'''

In such composition SF will merge layers 0 and 3 and hwcomposer will
merge <SF>,1,2,4 that results incorrect merging order.

Issue was observed on the rcar3 (imagination importer), db845c and allwinner H3
(Generic importer) platforms.
Reproduces with compositions that requires 'cursor' or 'dim' layers.
How to reproduce:
 1. Connect USB mouse when on home screen, you should see mouse cursor
    under icons (Tested with Launcher3QuickStep desktop)
 2. Go to Settings -> WIFI -> Connect to the AP, then you should see
    password dialog under AP list.

Solution:
1. Mark intermediate layers as CLIENT to ensure CLIENT section is in range
from bottom layer to most top CLIENT layer.

2. Use this layer composition to validate if DRM can handle it.

Signed-off-by: Roman Stratiienko <roman.stratiienko@globallogic.com>

1 files changed, 20 insertions, 39 deletions

diff --git a/drmhwctwo.cpp b/drmhwctwo.cpp
index 814d8f7..c419758 100644
--- a/drmhwctwo.cpp
+++ b/drmhwctwo.cpp
@@ -573,17 +573,7 @@ HWC2::Error DrmHwcTwo::HwcDisplay::CreateComposition(bool test) {
   uint32_t client_z_order = UINT32_MAX;
   std::map<uint32_t, DrmHwcTwo::HwcLayer *> z_map;
   for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_) {
-    HWC2::Composition comp_type;
-    if (test) {
-      comp_type = l.second.sf_type();
-      if (comp_type == HWC2::Composition::Device) {
-        if (!importer_->CanImportBuffer(l.second.buffer()))
-          comp_type = HWC2::Composition::Client;
-      }
-    } else
-      comp_type = l.second.validated_type();
-
-    switch (comp_type) {
+    switch (l.second.validated_type()) {
       case HWC2::Composition::Device:
         z_map.emplace(std::make_pair(l.second.z_order(), &l.second));
         break;
@@ -798,22 +788,6 @@ HWC2::Error DrmHwcTwo::HwcDisplay::ValidateDisplay(uint32_t *num_types,
   *num_types = 0;
   *num_requests = 0;
   size_t avail_planes = primary_planes_.size() + overlay_planes_.size();
-  bool comp_failed = false;
-
-  HWC2::Error ret;
-
-  for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_)
-    l.second.set_validated_type(HWC2::Composition::Invalid);
-
-  ret = CreateComposition(true);
-  if (ret != HWC2::Error::None)
-    comp_failed = true;
-
-  std::map<uint32_t, DrmHwcTwo::HwcLayer *, std::greater<int>> z_map;
-  for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_) {
-    if (l.second.sf_type() == HWC2::Composition::Device)
-      z_map.emplace(std::make_pair(l.second.z_order(), &l.second));
-  }
 
   /*
    * If more layers then planes, save one plane
@@ -822,22 +796,29 @@ HWC2::Error DrmHwcTwo::HwcDisplay::ValidateDisplay(uint32_t *num_types,
   if (avail_planes < layers_.size())
     avail_planes--;
 
-  for (std::pair<const uint32_t, DrmHwcTwo::HwcLayer *> &l : z_map) {
-    if (comp_failed || !avail_planes--)
-      break;
-    if (importer_->CanImportBuffer(l.second->buffer()))
-      l.second->set_validated_type(HWC2::Composition::Device);
-  }
+  std::map<uint32_t, DrmHwcTwo::HwcLayer *, std::greater<int>> z_map;
+  for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_)
+    z_map.emplace(std::make_pair(l.second.z_order(), &l.second));
 
-  for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_) {
-    DrmHwcTwo::HwcLayer &layer = l.second;
-    // We can only handle layers of Device type, send everything else to SF
-    if (layer.sf_type() != HWC2::Composition::Device ||
-        layer.validated_type() != HWC2::Composition::Device) {
-      layer.set_validated_type(HWC2::Composition::Client);
+  bool gpu_block = false;
+  for (std::pair<const uint32_t, DrmHwcTwo::HwcLayer *> &l : z_map) {
+    if (gpu_block || avail_planes == 0 ||
+        l.second->sf_type() != HWC2::Composition::Device ||
+        !importer_->CanImportBuffer(l.second->buffer())) {
+      gpu_block = true;
       ++*num_types;
+    } else {
+      avail_planes--;
     }
+
+    l.second->set_validated_type(gpu_block ? HWC2::Composition::Client
+                                           : HWC2::Composition::Device);
   }
+
+  if (CreateComposition(true) != HWC2::Error::None)
+    for (std::pair<const hwc2_layer_t, DrmHwcTwo::HwcLayer> &l : layers_)
+      l.second.set_validated_type(HWC2::Composition::Client);
+
   return *num_types ? HWC2::Error::HasChanges : HWC2::Error::None;
 }