support for processing PHOTO data + URIsmeego-1-2

"blob" fields: avoid binary encoding if possible
================================================

This change is meant for the PHOTO value, which can contain both
binary data and plain text URIs. Other binary data fields might also
benefit when their content turns out to be plain text (shorter
encoding).

The change is that base64 encoding is not enforced if all characters
are ASCII and printable. That allows special characters like colon,
comma, and semicolon to appear unchanged in the content.

Regardless whether the check succeeds, the result is guaranteed to
contain only ASCII characters, either because it only contains those
to start with or because of the base64 encoding.
(cherry picked from commit 8d5cce896dcc5dba028d1cfa18f08e31adcc6e73)

scripting: added READ() method
==============================

The READ(filename) method returns the content of the file identified
with "filename". Relative paths are interpreted relative to the current
directory. On failures, an error messages is logged and UNASSIGNED
is returned.

This method is useful for inlining the photo data referenced with
local file:// URIs shortly before sending to a remote peer. SyncEvolution
uses the method in its outgoing vcard script as follows:

Field list:
      <!-- Photo -->
      <field name="PHOTO" type="blob" compare="never" merge="fillempty"/>
      <field name="PHOTO_TYPE" type="string" compare="never" merge="fillempty"/>
      <field name="PHOTO_VALUE" type="string" compare="never" merge="fillempty"/>

Profile:
        <property name="PHOTO" filter="no">
          <value field="PHOTO" conversion="BLOB_B64"/>
          <parameter name="TYPE" default="no" show="yes">
            <value field="PHOTO_TYPE"/>
          </parameter>
          <parameter name="VALUE" default="no" show="yes">
            <value field="PHOTO_VALUE"/>
          </parameter>
        </property>

Script:
      if (PHOTO_VALUE == "uri" &&
          SUBSTR(PHOTO, 0, 7) == "file://") {
          // inline the photo data
          string data;
          data = READ(SUBSTR(PHOTO, 7));
          if (data != UNASSIGNED) {
              PHOTO = data;
              PHOTO_VALUE = "binary";
          }
      }

Test cases for inlining, not inlining because of non-file URI and
failed inling (file not found) were added to SyncEvolution.
(cherry picked from commit e3fdd5ca811f24b2f80e598f9d00d2e134aa85e1)

"blob" fields: Added separate CONVMODE_BLOB_AUTO conversion mode for
fields that should be rendered as B64 only in case they are really
non-printable or non-ASCII
====================================================================

This is an addition to 8d5cce896d ("blob" fields: avoid binary
encoding if possible) to avoid change of behaviour for
CONVMODE_BLOB_B64.  (cherry picked from commit
0fd1d64b6ef8beb666c8fac6d29fb2e19341a7bc)

3 files changed, 79 insertions, 4 deletions

diff --git a/src/sysync/mimedirprofile.cpp b/src/sysync/mimedirprofile.cpp
index 4105d03..c8fe995 100644
--- a/src/sysync/mimedirprofile.cpp
+++ b/src/sysync/mimedirprofile.cpp
@@ -23,6 +23,7 @@
 
 #include "syncagent.h"
 
+#include <ctype.h>
 
 using namespace sysync;
 
@@ -174,6 +175,8 @@ bool TMIMEProfileConfig::getConvMode(cAppCharP aText, sInt16 &aConvMode)
       aConvMode = CONVMODE_MULTIMIX;
     else if (strucmp(aText,"blob_b64",n)==0)
       aConvMode = CONVMODE_BLOB_B64;
+    else if (strucmp(aText,"blob_auto",n)==0)
+      aConvMode = CONVMODE_BLOB_AUTO;
     else if (strucmp(aText,"mailto",n)==0)
       aConvMode = CONVMODE_MAILTO;
     else if (strucmp(aText,"valuetype",n)==0)
@@ -2259,7 +2262,8 @@ sInt16 TMimeDirProfileHandler::generateValue(
           maxSiz = 0; // no size restriction
         bool noTruncate=aItem.getTargetItemType()->getFieldOptions(fid)->notruncate;
         // check for BLOB values
-        if ((aConvDefP->convmode & CONVMODE_MASK)==CONVMODE_BLOB_B64) {
+        sInt16 convmode = aConvDefP->convmode & CONVMODE_MASK;
+        if (convmode==CONVMODE_BLOB_B64 || convmode==CONVMODE_BLOB_AUTO) {
           // no value lists, escaping, enums. Simply set value and encoding
           TItemField *fldP = aItem.getArrayField(fid,aRepOffset,true); // existing array elements only
           if (!fldP) return GENVALUE_EXHAUSTED; // no leaf field - must be exhausted array (fldP==NULL is not possible here for non-arrays)
@@ -2274,8 +2278,24 @@ sInt16 TMimeDirProfileHandler::generateValue(
           }
           // append to existing string
           fldP->appendToString(outval,maxSiz);
-          // force B64 encoding
-          aEncoding=enc_base64;
+          if (convmode==CONVMODE_BLOB_AUTO) {
+            // auto mode: use  B64 encoding only if non-printable or
+            // non-ASCII characters are in the value
+            size_t len = outval.size();
+            for (size_t i = 0; i < len; i++) {
+              char c = outval[i];
+              if (!isascii(c) || !isprint(c)) {
+                aEncoding=enc_base64;
+                break;
+              }
+            }
+          }
+          else {
+            // blob mode: always use B64
+            aEncoding=enc_base64;
+          }
+          // only ASCII in value: either because it contains only
+          // those to start with or because they will be encoded
           aNonASCII=false;
         }
         else {
@@ -3647,7 +3667,10 @@ bool TMimeDirProfileHandler::parseValue(
     // find out if value exists (available in source and target)
     if (isFieldAvailable(aItem,fid)) {
       // parse only if field available in both source and target
-      if ((aConvDefP->convmode & CONVMODE_MASK)==CONVMODE_BLOB_B64) {
+      if (
+        (aConvDefP->convmode & CONVMODE_MASK)==CONVMODE_BLOB_B64 ||
+        (aConvDefP->convmode & CONVMODE_MASK)==CONVMODE_BLOB_AUTO
+      ) {
         // move 1:1 into field
         // - get pointer to leaf field
         TItemField *fldP = aItem.getArrayField(fid,aRepOffset);
diff --git a/src/sysync/mimedirprofile.h b/src/sysync/mimedirprofile.h
index 4db8bdf..ed67fbf 100755
--- a/src/sysync/mimedirprofile.h
+++ b/src/sysync/mimedirprofile.h
@@ -49,6 +49,7 @@ namespace sysync {
 #define CONVMODE_VALUETYPE 14 // automatic VALUE parameter e.g. for timestamp fields that contain a date-only value (VALUE=DATE) or duration (VALUE=DURATION)
 #define CONVMODE_MULTIMIX 15 // special mode for mapping enums to bits (like CONVMODE_BITMAP), but mixed from multiple fields and with option to store as-is (special enum "value" syntax needed)
 #define CONVMODE_FULLVALUETYPE 16 // explicit VALUE parameter, does not assume a default
+#define CONVMODE_BLOB_AUTO 17 // like CONVMODE_BLOB_B64, but if data consists of printable ASCII-chars only, no B64 encoding is used
 
 // derived type modes start here
 #define CONVMODE_MIME_DERIVATES 20
diff --git a/src/sysync/scriptcontext.cpp b/src/sysync/scriptcontext.cpp
index 35dff88..c99d2c2 100755
--- a/src/sysync/scriptcontext.cpp
+++ b/src/sysync/scriptcontext.cpp
@@ -27,6 +27,7 @@
   #include "pcre.h" // for RegEx functions
 #endif
 
+#include <stdio.h>
 
 // script debug messages
 #ifdef SYDEBUG
@@ -869,6 +870,55 @@ public:
     aTermP->setAsInteger(exitcode);
   }; // func_Shellexecute
 
+  // string READ(string file)
+  // reads the file and returns its content or UNASSIGNED in case of failure;
+  // errors are logged
+  static void func_Read(TItemField *&aTermP, TScriptContext *aFuncContextP)
+  {
+    // get params
+    string file;
+    aFuncContextP->getLocalVar(0)->getAsString(file);
+
+    // execute now
+    string content;
+    FILE *in;
+    in = fopen(file.c_str(), "rb");
+    if (in) {
+      long size = fseek(in, 0, SEEK_END);
+      if (size >= 0) {
+        // managed to obtain size, use it to pre-allocate result
+        content.reserve(size);
+        fseek(in, 0, SEEK_SET);
+      } else {
+        // ignore seek error, might not be a plain file
+        clearerr(in);
+      }
+
+      if (!ferror(in)) {
+        char buf[8192];
+        size_t read;
+        while ((read = fread(buf, 1, sizeof(buf), in)) > 0) {
+          content.append(buf, read);
+        }
+      }
+    }
+
+    if (in && !ferror(in)) {
+      // return content as string
+      aTermP->setAsString(content);
+    } else {
+        PLOGDEBUGPRINTFX(aFuncContextP->getDbgLogger(),
+                       DBG_ERROR,(
+                                  "IO error in READ(\"%s\"): %s ",
+                                  file.c_str(),
+                                  strerror(errno)));
+    }
+
+    if (in) {
+      fclose(in);
+    }
+  } // func_Read
+
 
   // string REMOTERULENAME()
   // returns name of the LAST matched remote rule (or subrule), empty if none
@@ -2220,6 +2270,7 @@ const TBuiltInFuncDef BuiltInFuncDefs[] = {
   { "REQUESTMAXTIME", TBuiltinStdFuncs::func_RequestMaxTime, fty_none, 1, param_oneInteger },
   { "REQUESTMINTIME", TBuiltinStdFuncs::func_RequestMinTime, fty_none, 1, param_oneInteger },
   { "SHELLEXECUTE", TBuiltinStdFuncs::func_Shellexecute, fty_integer, 3, param_Shellexecute },
+  { "READ",  TBuiltinStdFuncs::func_Read, fty_string, 1, param_oneString },
   { "SESSIONVAR", TBuiltinStdFuncs::func_SessionVar, fty_none, 1, param_oneString },
   { "SETSESSIONVAR", TBuiltinStdFuncs::func_SetSessionVar, fty_none, 2, param_SetSessionVar },
   { "ABORTSESSION", TBuiltinStdFuncs::func_AbortSession, fty_none, 1, param_oneInteger },