Engine 3.4.0.8: Added rawdataitemtype (passes SyncML data 1:1 to DBAPI)

Implementation: a new pair of .cpp/.h: rawdataitemtype plus making the
new type known to syncappbase.

Notes:
- TRawDataItemType is a direct descendant of TMultiFieldItemType;
  as such, it works DB-wise exactly like all the other known
  datatypes.
- It must <use> a <fieldlist> which must contain one field named
  "ITEMDATA". This field could be any type, but only BLOB or string
  make sense.
- "Parsing" an item consists of copying the SyncML <Data> content
  1:1 (except for <zippedbindata> and <unicodedata> special transforms,
  which still apply when enabled) into the ITEMDATA field.
- "Generating" an item consists of copying the content of ITEMDATA field
  1:1 (except for <zippedbindata> and <unicodedata> special transforms,
  which still apply when enabled) into the SyncML <Data> content.
- "raw" data items can't generate CTCap at this time. This could be
  added by having the DB plugin provide a CTCap list or reading
  a CTCap definition from the config.
- CTCap received is ignored.

Sample Configuration
--------------------

Notes:
- the dummy field is just there to show that datatype
  scripts can be used like in MIMEDIR items
- the essential parameter is the basetype="raw".
- Note that <use> must reference a <fieldlist>, not a <profile>

  <datatypes>

    <!-- list of internal fields representing raw data -->
    <fieldlist name="rawitem">
      <field name="dummy" type="string" compare="never"/>
      <field name="ITEMDATA" type="string" compare="never"/>
    </fieldlist>

    <datatype name="vcard21" basetype="raw">
      <use fieldlist="rawitem"/>
      <typestring>text/x-vcard</typestring>
      <versionstring>2.1</versionstring>

      <incomingscript><![CDATA[
        dummy = "this is a dummy!";
        DEBUGSHOWITEM();
      ]]></incomingscript>

      <outgoingscript><![CDATA[
        DEBUGSHOWITEM();
      ]]></outgoingscript>

    </datatype>

  </datatypes>

and further down the (textdb) datastore:

    <datastore name="rawcontacts" type="plugin">
      <plugin_module>[SDK_textdb]</plugin_module>
      <plugin_datastoreadmin>yes</plugin_datastoreadmin>

      <datacharset>UTF-8</datacharset>
      <datalineends>unix</datalineends>

      <storesyncidentifiers>yes</storesyncidentifiers>
      <resumesupport>yes</resumesupport>
      <resumeitemsupport>yes</resumeitemsupport>

      <!-- Mapping of the fields to the fieldlist "contacts" -->
      <fieldmap fieldlist="rawitem">
        <automap indexasname="true"/>
      </fieldmap>

      <!-- datatypes supported by this datastore -->
      <typesupport>
        <use datatype="vcard21" mode="rw" preferred="yes"/>
      </typesupport>

    </datastore>

5 files changed, 391 insertions, 47 deletions

diff --git a/src/global_options.h b/src/global_options.h
index c077170..62436ac 100644
--- a/src/global_options.h
+++ b/src/global_options.h
@@ -71,8 +71,8 @@
 #endif
 
 #ifndef SYSYNC_BUILDNUMBER
-#define SYSYNC_BUILDNUMBER          7
-#define SYSYNC_BUILDNUMBER_TXT     "7"
+#define SYSYNC_BUILDNUMBER          8
+#define SYSYNC_BUILDNUMBER_TXT     "8"
 #endif
 
 
@@ -174,6 +174,7 @@
 #define MIMEDIR_SUPPORT 1
 #define TEXTTYPE_SUPPORT 1
 #define DATAOBJ_SUPPORT 1
+#define RAWTYPE_SUPPORT 1
 
 // intermediate stuff for v3.x development
 /// @todo
diff --git a/src/sysync/rawdataitemtype.cpp b/src/sysync/rawdataitemtype.cpp
new file mode 100644
index 0000000..ef96ec6
--- /dev/null
+++ b/src/sysync/rawdataitemtype.cpp
@@ -0,0 +1,264 @@
+/*
+ *  File:         rawdataitemtype.cpp
+ *
+ *  Author:       Lukas Zeller (luz@synthesis.ch)
+ *
+ *  TRawDataItemType
+ *    Item type for 1:1 raw items (SyncML payload is exchanged 1:1 with database backend) 
+ *
+ *  Copyright (c) 2010 by Synthesis AG (www.synthesis.ch)
+ *
+ *  2010-05-14 : luz : created
+ *
+ */
+
+
+// includes
+#include "prefix_file.h"
+#include "sysync.h"
+#include "rawdataitemtype.h"
+
+using namespace sysync;
+
+
+namespace sysync {
+
+
+// Config
+// ======
+
+
+// MIMEDir-based datatype config
+
+TRawDataTypeConfig::TRawDataTypeConfig(const char* aName, TConfigElement *aParentElement) :
+  TMultiFieldTypeConfig(aName,aParentElement)
+{
+  clear();
+} // TRawDataTypeConfig::TRawDataTypeConfig
+
+
+TRawDataTypeConfig::~TRawDataTypeConfig()
+{
+  clear();
+} // TRawDataTypeConfig::~TRawDataTypeConfig
+
+
+// init defaults
+void TRawDataTypeConfig::clear(void)
+{
+	// clear FIDs
+  fFidItemData = FID_NOT_SUPPORTED;
+  // clear inherited
+  inherited::clear();
+} // TRawDataTypeConfig::clear
+
+
+// resolve dependencies
+void TRawDataTypeConfig::localResolve(bool aLastPass)
+{
+  if (aLastPass) {
+  	// get FIDs of the fields we directly use
+    // - the "ITEMDATA" field containing the raw item data
+    fFidItemData = getFieldIndex("ITEMDATA",fFieldListP);
+    if (fFidItemData==FID_NOT_SUPPORTED) goto missingfield;
+    // %%% add more FID searches (and checks, if the field is mandatory) here
+  }
+  // resolve inherited
+  inherited::localResolve(aLastPass);
+  return;
+missingfield:
+	SYSYNC_THROW(TConfigParseException("fieldlist for RawDataItem must contain certain predefined fields (like ITEMDATA)!"));	
+} // TRawDataTypeConfig::localResolve
+
+
+
+// create Sync Item Type of appropriate type from config
+TSyncItemType *TRawDataTypeConfig::newSyncItemType(TSyncSession *aSessionP, TSyncDataStore *aDatastoreP)
+{
+  return
+    new TRawDataItemType(
+      aSessionP,
+      this,
+      fTypeName.c_str(),
+      fTypeVersion.c_str(),
+      aDatastoreP,
+      fFieldListP
+    );
+} // TRawDataTypeConfig::newSyncItemType
+
+
+
+#ifdef CONFIGURABLE_TYPE_SUPPORT
+
+// config element parsing
+bool TRawDataTypeConfig::localStartElement(const char *aElementName, const char **aAttributes, sInt32 aLine)
+{
+  // checking the elements
+  // - none known here
+    return TMultiFieldTypeConfig::localStartElement(aElementName,aAttributes,aLine);
+  // ok
+  return true;
+} // TRawDataTypeConfig::localStartElement
+
+#endif
+
+
+/*
+ * Implementation of TRawDataItemType
+ */
+
+
+TRawDataItemType::TRawDataItemType(
+  TSyncSession *aSessionP,
+  TDataTypeConfig *aTypeConfigP,
+  const char *aCTType,
+  const char *aVerCT,
+  TSyncDataStore *aRelatedDatastoreP,
+  TFieldListConfig *aFieldDefinitions // field definitions
+) :
+  TMultiFieldItemType(aSessionP,aTypeConfigP,aCTType,aVerCT,aRelatedDatastoreP,aFieldDefinitions)
+{
+	fCfgP = static_cast<TRawDataTypeConfig *>(aTypeConfigP);
+} // TRawDataItemType::TRawDataItemType
+
+
+TRawDataItemType::~TRawDataItemType()
+{
+} // TRawDataItemType::~TRawDataItemType
+
+
+// create new sync item of proper type and optimization for specified target
+TSyncItem *TRawDataItemType::internalNewSyncItem(TSyncItemType *aTargetItemTypeP, TLocalEngineDS *aLocalDatastoreP)
+{
+  // All RawDataItems are stored in MultiFieldItems (in specific, predefined fields)
+  TMultiFieldItemType *targetitemtypeP;
+  GET_CASTED_PTR(targetitemtypeP,TMultiFieldItemType,aTargetItemTypeP,DEBUGTEXT("TRawDataItemType::internalNewSyncItem with bad-typed target","mdit6"));
+  MP_RETURN_NEW(TMultiFieldItem,DBG_OBJINST,"TMultiFieldItem",TMultiFieldItem(this,targetitemtypeP));
+} // TRawDataItemType::internalNewSyncItem
+
+
+// fill in SyncML data (but leaves IDs empty)
+bool TRawDataItemType::internalFillInData(
+  TSyncItem *aSyncItemP,      // SyncItem to be filled with data
+  SmlItemPtr_t aItemP,        // SyncML toolkit item Data to be converted into SyncItem (may be NULL if no data, in case of Delete or Map)
+  TLocalEngineDS *aLocalDatastoreP, // local datastore
+  TStatusCommand &aStatusCmd  // status command that might be modified in case of error
+)
+{
+  // check type
+  TMultiFieldItem *itemP;
+  GET_CASTED_PTR(itemP,TMultiFieldItem,aSyncItemP,DEBUGTEXT("TRawDataItemType::internalFillInData: incompatible item class","mdit7"));
+  // store data, if any, in predefined ITEMDATA field
+  if (aItemP->data) {
+  	// read data into predefined raw data field
+    TItemField *fldP = itemP->getField(fCfgP->fFidItemData);
+    if (fldP) {
+      // get raw data
+      stringSize sz;
+      cAppCharP data = smlPCDataToCharP(aItemP->data,&sz);
+      // put it into ITEMDATA field
+    	fldP->setAsString(data, sz);
+		}
+  }
+  else {
+    // no data
+    aStatusCmd.setStatusCode(412); // incomplete command
+    ADDDEBUGITEM(aStatusCmd,"No data found in item");
+    return false;
+  }
+  // let ancestor process data as well
+  return TMultiFieldItemType::internalFillInData(aSyncItemP,aItemP,aLocalDatastoreP,aStatusCmd);
+} // TRawDataItemType::internalFillInData
+
+
+// sets data and meta from SyncItem data, but leaves source & target untouched
+bool TRawDataItemType::internalSetItemData(
+  TSyncItem *aSyncItemP,  // the syncitem to be represented as SyncML
+  SmlItemPtr_t aItem,     // item with NULL meta and NULL data
+  TLocalEngineDS *aLocalDatastoreP // local datastore
+)
+{
+  // check type
+  TMultiFieldItem *itemP;
+  GET_CASTED_PTR(itemP,TMultiFieldItem,aSyncItemP,DEBUGTEXT("TRawDataItemType::internalSetItemData: incompatible item class","mdit8"));
+  // let ancestor prepare first
+  if (!TMultiFieldItemType::internalSetItemData(aSyncItemP,aItem,aLocalDatastoreP)) return false;
+  // generate data item from predefined ITEMDATA field
+  TItemField *fldP = itemP->getField(fCfgP->fFidItemData);
+  if (fldP) {
+	  string dataitem;
+  	if (fldP->isBasedOn(fty_blob)) {
+    	// is a BLOB, don't use getAsString as BLOB only returns pseudo-data indicating length of BLOB
+    	((TBlobField *)fldP)->getBlobAsString(dataitem);
+    }
+    else {
+    	// for all other types, just get it as string
+    	fldP->getAsString(dataitem);
+    }
+	  // put data item into opaque/cdata PCData
+  	aItem->data=newPCDataStringX((const uInt8 *)dataitem.c_str(),true,dataitem.size());
+  }
+  // can't go wrong
+  return true;
+} // TRawDataItemType::internalSetItemData
+
+
+
+// generates SyncML-Devinf property list for type
+SmlDevInfCTDataPropListPtr_t TRawDataItemType::newCTDataPropList(TTypeVariantDescriptor aVariantDescriptor)
+{
+	// return supported properties
+  #warning "TODO create this list from some configuration data"
+	return NULL; // %%% for now: none
+}
+
+
+// Analyze CTCap part of devInf
+bool TRawDataItemType::analyzeCTCap(SmlDevInfCTCapPtr_t aCTCapP)
+{
+	// no analysis so far
+  // TODO: maybe add mechanism to capture CTCap here and pass it to the DB backend in a predefined field for analysis
+	return inherited::analyzeCTCap(aCTCapP);
+}
+
+
+
+/// @brief helper to create same-typed instance via base class
+TSyncItemType *TRawDataItemType::newCopyForSameType(
+  TSyncSession *aSessionP,     // the session
+  TSyncDataStore *aDatastoreP  // the datastore
+)
+{
+  // create new itemtype of appropriate derived class type that can handle
+  // this type
+  MP_RETURN_NEW(TRawDataItemType,DBG_OBJINST,"TRawDataItemType",TRawDataItemType(
+    aSessionP,
+    fTypeConfigP,
+    getTypeName(),
+    getTypeVers(),
+    aDatastoreP,
+    fFieldDefinitionsP
+  ));
+} // TRawDataItemType::newCopyForSameType
+
+
+/// @brief copy CTCap derived info from another SyncItemType
+/// @return false if item not compatible
+/// @note required to create remote type variants from ruleMatch type alternatives
+bool TRawDataItemType::copyCTCapInfoFrom(TSyncItemType &aSourceItem)
+{
+  // must be same type as myself or based on the type of myself
+  if (!aSourceItem.isBasedOn(getTypeID()))
+    return false; // not compatible
+  TRawDataItemType *itemTypeP = static_cast<TRawDataItemType *>(&aSourceItem);
+  // all CTCap info we might have is in the field options of MultiFieldItemType
+  return inherited::copyCTCapInfoFrom(aSourceItem);
+} // TRawDataItemType::copyCTCapInfoFrom
+
+
+/* end of TRawDataItemType implementation */
+
+} // namespace sysync
+
+
+// eof
diff --git a/src/sysync/rawdataitemtype.h b/src/sysync/rawdataitemtype.h
new file mode 100644
index 0000000..2804f76
--- /dev/null
+++ b/src/sysync/rawdataitemtype.h
@@ -0,0 +1,116 @@
+/*
+ *  File:         rawdataitemtype.h
+ *
+ *  Author:       Lukas Zeller (luz@synthesis.ch)
+ *
+ *  TRawDataItemType
+ *    Item type for 1:1 raw items (SyncML payload is exchanged 1:1 with database backend) 
+ *
+ *  Copyright (c) 2010 by Synthesis AG (www.synthesis.ch)
+ *
+ *  2010-05-14 : luz : created
+ *
+ */
+
+#ifndef RawDataItemType_H
+#define RawDataItemType_H
+
+// includes
+#include "syncitemtype.h"
+#include "multifielditemtype.h"
+
+
+namespace sysync {
+
+
+// MIME-dir based datatype
+class TRawDataTypeConfig : public TMultiFieldTypeConfig
+{
+  typedef TMultiFieldTypeConfig inherited;
+public:
+  TRawDataTypeConfig(const char *aElementName, TConfigElement *aParentElementP);
+  virtual ~TRawDataTypeConfig();
+  // properties
+  // public functions
+  // - create Sync Item Type of appropriate type from config
+  virtual TSyncItemType *newSyncItemType(TSyncSession *aSessionP, TSyncDataStore *aDatastoreP);
+protected:
+  #ifdef CONFIGURABLE_TYPE_SUPPORT
+  // check config elements
+  virtual bool localStartElement(const char *aElementName, const char **aAttributes, sInt32 aLine);
+  #endif
+  virtual void localResolve(bool aLastPass);
+  virtual void clear();
+public:
+  // FIDs of the predefined fields we need to access
+  sInt16 fFidItemData; // BLOB or string field that will contain item <data>
+	// %%% add more FID vars here
+}; // TRawDataTypeConfig
+
+
+const uInt16 ity_rawdata=90; // must be unique
+
+class TRawDataItemType: public TMultiFieldItemType
+{
+  typedef TMultiFieldItemType inherited;
+  friend class TMimeDirProfileHandler;
+public:
+  // constructor
+  TRawDataItemType(
+    TSyncSession *aSessionP,
+    TDataTypeConfig *aTypeConfigP,
+    const char *aCTType,
+    const char *aVerCT,
+    TSyncDataStore *aRelatedDatastoreP,
+    TFieldListConfig *aFieldDefinitions // field definitions
+  );
+  // destructor
+  virtual ~TRawDataItemType();
+  // access to type
+  virtual uInt16 getTypeID(void) const { return ity_rawdata; };
+  virtual bool isBasedOn(uInt16 aItemTypeID) const { return aItemTypeID==ity_rawdata ? true : inherited::isBasedOn(aItemTypeID); };
+  // differentiation between implemented and just descriptive TSyncTypeItems
+  virtual bool isImplemented(void) { return true; }; // RawItem is an implementati on
+  // helper to create same-typed instance via base class
+  // MUST BE IMPLEMENTED IN ALL DERIVED CLASSES!
+  virtual TSyncItemType *newCopyForSameType(
+    TSyncSession *aSessionP,     // the session
+    TSyncDataStore *aDatastoreP  // the datastore
+  );
+protected:
+  // CTCap parsing/generation
+  // - analyze CTCap for specific type
+  virtual bool analyzeCTCap(SmlDevInfCTCapPtr_t aCTCapP);
+  /// @brief copy CTCap derived info from another SyncItemType
+  virtual bool copyCTCapInfoFrom(TSyncItemType &aSourceItemP);
+  // - obtain property list for type, returns NULL if none available
+  virtual SmlDevInfCTDataPropListPtr_t newCTDataPropList(TTypeVariantDescriptor aVariantDescriptor);
+  // Item data management
+  // - create new sync item of proper type and optimization for specified target
+  virtual TSyncItem *internalNewSyncItem(TSyncItemType *aTargetItemTypeP, TLocalEngineDS *aLocalDatastoreP);
+  // - fill in SyncML data (but leaves IDs empty)
+  virtual bool internalFillInData(
+    TSyncItem *aSyncItemP,      // SyncItem to be filled with data
+    SmlItemPtr_t aItemP,        // SyncML toolkit item Data to be converted into SyncItem (may be NULL if no data, in case of Delete or Map)
+    TLocalEngineDS *aLocalDatastoreP, // local datastore
+    TStatusCommand &aStatusCmd  // status command that might be modified in case of error
+  );
+  // - sets data and meta from SyncItem data, but leaves source & target untouched
+  virtual bool internalSetItemData(
+    TSyncItem *aSyncItemP,  // the syncitem to be represented as SyncML
+    SmlItemPtr_t aItem,     // item with NULL meta and NULL data
+    TLocalEngineDS *aLocalDatastoreP // local datastore
+  );
+private:
+	// convenience casted pointer to my config
+	TRawDataTypeConfig *fCfgP;
+}; // TRawDataItemType
+
+
+
+}	// namespace sysync
+
+#endif	// RawDataItemType_H
+
+// eof
+
diff --git a/src/sysync/syncappbase.cpp b/src/sysync/syncappbase.cpp
index 840bd8a..b0f5c3d 100755
--- a/src/sysync/syncappbase.cpp
+++ b/src/sysync/syncappbase.cpp
@@ -3255,6 +3255,9 @@ bool TRootConfig::parseDatatypesConfig(const char **aAttributes, sInt32 aLine)
 #ifdef TEXTTYPE_SUPPORT
   #include "textitemtype.h"
 #endif
+#ifdef RAWTYPE_SUPPORT
+  #include "rawdataitemtype.h"
+#endif
 #ifdef DATAOBJ_SUPPORT
   #include "dataobjtype.h"
 #endif
@@ -3280,6 +3283,11 @@ TDataTypeConfig *TRootConfig::newDataTypeConfig(const char *aName, const char *a
     return new TTextTypeConfig(aName,aParentP);
   else
   #endif
+  #ifdef RAWTYPE_SUPPORT
+  if (strucmp(aBaseType,"raw")==0)
+    return new TRawDataTypeConfig(aName,aParentP);
+  else
+  #endif
   #ifdef DATAOBJ_SUPPORT
   if (strucmp(aBaseType,"dataobj")==0)
     return new TDataObjConfig(aName,aParentP);
diff --git a/src/sysync/syncitemtype.cpp b/src/sysync/syncitemtype.cpp
index dfb665e..05e6eba 100755
--- a/src/sysync/syncitemtype.cpp
+++ b/src/sysync/syncitemtype.cpp
@@ -754,51 +754,6 @@ SmlItemPtr_t TSyncItemType::newSmlItem(
         smlitemP->data->length = utf16bytestream.size();
         // copy contents
         memcpy((appPointer)smlitemP->data->content,utf16bytestream.c_str(),utf16bytestream.size());
-        /*
-        // get original size
-        MemSize_t origSize = smlitemP->data->length;
-        cAppCharP origData = (cAppCharP)smlitemP->data->content;
-        if (origSize) {
-          // we need roughly twice as many bytes (probably less, but not more)
-          MemSize_t utf16size = origSize*2;
-          MemPtr_t utf16Payload = (MemPtr_t)smlLibMalloc(utf16size+2); // one more in case we detect end of buffer only after a surrogate pair
-          if (utf16Payload) {
-            // now convert
-            uInt32 ucs4;
-            uInt16 utf16,utf16_1;
-            appCharP outP = (appCharP)utf16Payload;
-            cAppCharP inP = (cAppCharP)smlitemP->data->content;
-            while (*inP && inP-origData<origSize && outP-(cAppCharP)utf16Payload<utf16size) {
-              inP=UTF8toUCS4(inP, ucs4);
-              if (ucs4==0) break; // error
-              utf16_1 = UCS4toUTF16(ucs4,utf16);
-              if (fTypeConfigP->fMSBFirst) {
-                // Motorola order
-                if (utf16_1) {
-                  *(outP++) = (utf16_1 >> 8) & 0xFF;
-                  *(outP++) = utf16_1 & 0xFF;
-                }
-                *(outP++) = (utf16 >> 8) & 0xFF;
-                *(outP++) = utf16 & 0xFF;
-              }
-              else {
-                // Intel order
-                if (utf16_1) {
-                  *(outP++) = (utf16_1 >> 8) & 0xFF;
-                  *(outP++) = utf16_1 & 0xFF;
-                }
-                *(outP++) = utf16 & 0xFF;
-                *(outP++) = (utf16 >> 8) & 0xFF;
-              }
-            }
-            // replace contents
-            smlitemP->data->length=(MemPtr_t)outP-utf16Payload;
-            smlitemP->data->content=utf16Payload;
-            // forget original data
-            smlLibFree((void *)origData);
-          }
-        }
-        */
       }
       // compress data if zippedbindata selected in type
       #ifdef ZIPPED_BINDATA_SUPPORT