User params reset issue of icc profile strings

If the user params end up getting set during a vmreclaim
while we are in a swapped icc profile situation when
processing a softmask we were ending up freeing profiles
that should not have been freed.   This issue was masked
by a check that was disallowing profiling settings in
the manager once they were already set.  Also discovered
issues with a corner case where we have a softmask that
includes a pattern with a softmask.

3 files changed, 81 insertions, 57 deletions

diff --git a/gs/base/gdevp14.c b/gs/base/gdevp14.c
index 06b148a56..f715bd1a2 100644
--- a/gs/base/gdevp14.c
+++ b/gs/base/gdevp14.c
@@ -7305,9 +7305,22 @@ pdf14_increment_smask_color(gs_imager_state * pis, gx_device * dev)
     gsicc_smask_t *smask_profiles = pis->icc_manager->smask_profiles;
     int k;
 
-    /* See if we have profiles already in place */
+    /* See if we have profiles already in place.   Note we also have to
+       worry about a corner case where this device does not have a
+       smaskcolor stucture to store the profiles AND the profiles were
+       already swapped out in the icc_manager.  This can occur when we
+       pushed a transparency mask and then inside the mask we have a pattern
+       which also has a transparency mask.   The state of the icc_manager
+       is that it already has done the swap and there is no need to fool
+       with any of this while dealing with the soft mask within the pattern */
+    if (pdev->smaskcolor == NULL && pis->icc_manager->smask_profiles != NULL &&
+        pis->icc_manager->smask_profiles->swapped) {
+            return 0;
+    }
     if (pdev->smaskcolor != NULL) {
         pdev->smaskcolor->ref_count++;
+        if_debug1(gs_debug_flag_icc,"[icc] Increment smask color now %d\n",
+                  pdev->smaskcolor->ref_count); 
     } else {
         /* Allocate and swap out the current profiles.  The softmask
            profiles should already be in place */
@@ -7328,6 +7341,9 @@ pdf14_increment_smask_color(gs_imager_state * pis, gx_device * dev)
         pis->icc_manager->default_gray = smask_profiles->smask_gray;
         pis->icc_manager->default_rgb = smask_profiles->smask_rgb;
         pis->icc_manager->default_cmyk = smask_profiles->smask_cmyk;
+        pis->icc_manager->smask_profiles->swapped = true;
+        if_debug0(gs_debug_flag_icc,
+                  "[icc] Initial creation of smask color. Ref count 1\n");
         pdev->smaskcolor->ref_count = 1;
         /* We also need to update the profile that is currently in the
            color spaces of the graphic state.  Otherwise this can be
@@ -7362,12 +7378,8 @@ pdf14_increment_smask_color(gs_imager_state * pis, gx_device * dev)
 
                             break;
                     }
-                    if (profile != pcs->cmm_icc_profile_data) {
-                        rc_decrement(pcs->cmm_icc_profile_data,
-                                     "pdf14_increment_smask_color");
-                        rc_increment(profile);
-                        pcs->cmm_icc_profile_data = profile;
-                    }
+                    rc_assign(pcs->cmm_icc_profile_data, profile, 
+                              "pdf14_increment_smask_color");
                 }
             }
         }
@@ -7383,16 +7395,25 @@ pdf14_decrement_smask_color(gs_imager_state * pis, gx_device * dev)
     gsicc_manager_t *icc_manager = pis->icc_manager;
     int k;
 
+    /* See comment in pdf14_increment_smask_color to understand this one */
+    if (pdev->smaskcolor == NULL && pis->icc_manager->smask_profiles != NULL &&
+        pis->icc_manager->smask_profiles->swapped) {
+            return 0;
+    }
     if (smaskcolor != NULL) {
         smaskcolor->ref_count--;
+        if_debug1(gs_debug_flag_icc,"[icc] Decrement smask color.  Now %d\n",
+                  smaskcolor->ref_count); 
         if (smaskcolor->ref_count == 0) {
+            if_debug0(gs_debug_flag_icc,"[icc] Reset smask color.\n");
             /* Lets return the profiles and clean up */
             /* First see if we need to "reset" the profiles that are in
                the graphic state */
             if (pis->is_gstate) {
                 gs_state *pgs = (gs_state*) pis;
+                if_debug0(gs_debug_flag_icc, "[icc] Reseting graphic state color spaces\n"); 
                 for (k = 0; k < 2; k++) {
-                    gs_color_space *pcs     = pgs->color[k].color_space;
+                    gs_color_space *pcs = pgs->color[k].color_space;
                     cmm_profile_t  *profile = pcs->cmm_icc_profile_data;
                     if (profile != NULL) {
                         switch(profile->data_cs) {
@@ -7421,20 +7442,16 @@ pdf14_decrement_smask_color(gs_imager_state * pis, gx_device * dev)
 
                                 break;
                         }
-                        if (profile != pcs->cmm_icc_profile_data) {
-                            rc_decrement(pcs->cmm_icc_profile_data,
-                                         "pdf14_decrement_smask_color");
-                            rc_increment(profile);
-                            pcs->cmm_icc_profile_data = profile;
-                        }
+                        rc_assign(pcs->cmm_icc_profile_data, profile,
+                                  "pdf14_decrement_smask_color");
                     }
                 }
             }
-
             /* Decrement handled in pdf14_free_smask_color */
             icc_manager->default_gray = smaskcolor->profiles->smask_gray;
             icc_manager->default_rgb = smaskcolor->profiles->smask_rgb;
             icc_manager->default_cmyk = smaskcolor->profiles->smask_cmyk;
+            icc_manager->smask_profiles->swapped = false;
             pdf14_free_smask_color(pdev);
         }
     }
diff --git a/gs/base/gscms.h b/gs/base/gscms.h
index 11e74a9fc..bad3ae685 100644
--- a/gs/base/gscms.h
+++ b/gs/base/gscms.h
@@ -382,11 +382,16 @@ struct gsicc_devicen_s {
     int count;
 };
 
+/* Had to add bool so that we know if things were swapped.
+   The reason is that if we are in a swapped state and 
+   there is a vmreclaim we then end up sending the user 
+   params again and we will find that there is a mismatch */
 typedef struct gsicc_smask_s {
     cmm_profile_t *smask_gray;
     cmm_profile_t *smask_rgb;
     cmm_profile_t *smask_cmyk;
     gs_memory_t *memory;
+    bool swapped;  
 } gsicc_smask_t;
 
 /* The manager object */
diff --git a/gs/base/gsicc_manage.c b/gs/base/gsicc_manage.c
index c473b3989..5c67cb073 100644
--- a/gs/base/gsicc_manage.c
+++ b/gs/base/gsicc_manage.c
@@ -173,6 +173,7 @@ gsicc_new_iccsmask(gs_memory_t *memory)
         result->smask_rgb = NULL;
         result->smask_cmyk = NULL;
         result->memory = memory;
+        result->swapped = false;
     }
     return(result);
 }
@@ -618,41 +619,49 @@ gsicc_set_profile(gsicc_manager_t *icc_manager, const char* pname, int namelen,
     int k;
     gsicc_colorbuffer_t default_space; /* Used to verify that we have the correct type */
 
-    /* For now only let this be set once. We could have this changed dynamically
-       in which case we need to do some deaallocations prior to replacing it */
+    /* We need to check for the smask swapped profile condition.  If we are in
+       that state, then any requests for setting the profile will be ignored.
+       This is valid, since we are in the middle of drawing right now and this
+       only would occur if we are doing a vmreclaim while in the middle of
+       soft mask rendering */
     default_space = gsUNDEFINED;
-    switch(defaulttype) {
-        case DEFAULT_GRAY:
-            manager_default_profile = &(icc_manager->default_gray);
-            default_space = gsGRAY;
-            break;
-        case DEFAULT_RGB:
-            manager_default_profile = &(icc_manager->default_rgb);
-            default_space = gsRGB;
-            break;
-        case DEFAULT_CMYK:
-             manager_default_profile = &(icc_manager->default_cmyk);
-             default_space = gsCMYK;
-             break;
-        case NAMED_TYPE:
-             manager_default_profile = &(icc_manager->device_named);
-             break;
-        case LAB_TYPE:
-             manager_default_profile = &(icc_manager->lab_profile);
-             break;
-        case DEVICEN_TYPE:
-            code = gsicc_new_devicen(icc_manager);
-            if (code == 0) {
-                manager_default_profile =
-                    &(icc_manager->device_n->final->iccprofile);
-            } else {
-                return(code);
-            }
-            break;
-        case DEFAULT_NONE:
-        default:
-            return(0);
-            break;
+    if (icc_manager->smask_profiles !=NULL &&
+        icc_manager->smask_profiles->swapped == true) {
+            return 0;
+    } else {
+        switch(defaulttype) {
+            case DEFAULT_GRAY:
+                manager_default_profile = &(icc_manager->default_gray);
+                default_space = gsGRAY;
+                break;
+            case DEFAULT_RGB:
+                manager_default_profile = &(icc_manager->default_rgb);
+                default_space = gsRGB;
+                break;
+            case DEFAULT_CMYK:
+                 manager_default_profile = &(icc_manager->default_cmyk);
+                 default_space = gsCMYK;
+                 break;
+            case NAMED_TYPE:
+                 manager_default_profile = &(icc_manager->device_named);
+                 break;
+            case LAB_TYPE:
+                 manager_default_profile = &(icc_manager->lab_profile);
+                 break;
+            case DEVICEN_TYPE:
+                code = gsicc_new_devicen(icc_manager);
+                if (code == 0) {
+                    manager_default_profile =
+                        &(icc_manager->device_n->final->iccprofile);
+                } else {
+                    return code;
+                }
+                break;
+            case DEFAULT_NONE:
+            default:
+                return 0;
+                break;
+        }
     }
     /* If it is not NULL then it has already been set. If it is different than
        what we already have then we will want to free it.  Since other imager
@@ -660,13 +669,7 @@ gsicc_set_profile(gsicc_manager_t *icc_manager, const char* pname, int namelen,
        counting.  If it is the same as what we already have then we DONT
        increment, since that is done when the imager state is duplicated.  It
        could be the same, due to a resetting of the user params. To avoid
-       recreating the profile data, we compare the string names. Also, we want
-       to avoid the default blowing away the -s settings for the gray, rgb and
-       cmyk profile following a vmreclaim so test if it is one of those default
-       types */
-    if (defaulttype < DEVICEN_TYPE && (*manager_default_profile) != NULL) {
-        return 0;
-    }
+       recreating the profile data, we compare the string names. */
     if ((*manager_default_profile) != NULL) {
         /* Check if this is what we already have.  Also check if it is the
            output intent profile.  */
@@ -2266,4 +2269,3 @@ gs_setlabicc(const gs_imager_state * pis, gs_param_string * pval)
         return gs_throw(code, "cannot find default lab icc profile");
     return code;
 }
-