engine: added <plugin_earlystartdataread> to allow StartDataRead to occur early and possibly force a slow sync by returning 508 error code

The new <plugin_earlystartdataread> flag causes StartDataRead to occur immediately after the admin data has been loaded. This causes the followin change in semantics for StartDataRead() database plugin API call:

- the call can return the special 508 status code to signal the engine that a slow sync must be forced for internal database reasons (such as having lost ability to track changes).

- the ContextSupport("ReadNextItem:allfields") call that was issued (if all fields were requested at all) BEFORE StartDataRead() in the default case will be issued before first call to ReadNextItem(), but AFTER StartDataRead().

4 files changed, 72 insertions, 15 deletions

diff --git a/src/DB_interfaces/api_db/pluginapids.cpp b/src/DB_interfaces/api_db/pluginapids.cpp
index 7f4781b..786d983 100755
--- a/src/DB_interfaces/api_db/pluginapids.cpp
+++ b/src/DB_interfaces/api_db/pluginapids.cpp
@@ -104,6 +104,7 @@ void TPluginDSConfig::clear(void)
   fDBAPIModule_Admin.erase();
   fDBAPIModule_Data.erase();
   fDataModuleAlsoHandlesAdmin=false;
+  fEarlyStartDataRead = false;
   // - default to use all debug flags set (if debug for plugin is enabled at all)
   fPluginDbgMask_Admin=0xFFFF;
   fPluginDbgMask_Data=0xFFFF;
@@ -126,6 +127,8 @@ bool TPluginDSConfig::localStartElement(const char *aElementName, const char **a
     expectMacroString(fDBAPIModule_Data);
   else if (strucmp(aElementName,"plugin_datastoreadmin")==0)
     expectBool(fDataModuleAlsoHandlesAdmin);
+  else if (strucmp(aElementName,"plugin_earlystartdataread")==0)
+    expectBool(fEarlyStartDataRead);
   else if (strucmp(aElementName,"plugin_params")==0)
     expectChildParsing(fPluginParams_Data);
   else if (strucmp(aElementName,"plugin_debugflags")==0)
@@ -944,15 +947,30 @@ bool TPluginApiDS::dsFilteredFetchesFromDB(bool aFilterChanged)
 
 
 
-// read sync set IDs and mod dates (and rest of data if technically unavoidable or
-// requested by aNeedAll)
-localstatus TPluginApiDS::apiReadSyncSet(bool aNeedAll)
+// can return 508 to force a slow sync. Other errors abort the sync
+localstatus TPluginApiDS::apiEarlyDataAccessStart(void)
 {
-  TSyError dberr=LOCERR_OK;
-  #ifdef SYDEBUG
-  string ts1,ts2;
-  #endif
+  TSyError dberr = LOCERR_OK;
+  if (fPluginDSConfigP->fEarlyStartDataRead) {
+    // prepare
+    dberr = apiPrepareReadSyncSet();
+    if (dberr==LOCERR_OK) {
+      // start the reading phase anyway (to make sure call order is always StartRead/EndRead/StartWrite/EndWrite)
+      dberr = fDBApi_Data.StartDataRead(fPreviousToRemoteSyncIdentifier.c_str(),fPreviousSuspendIdentifier.c_str());
+      if (dberr!=LOCERR_OK) {
+        PDEBUGPRINTFX(DBG_ERROR,("apiEarlyDataAccessStart - DBapi::StartDataRead error: %hd",dberr));
+      }
+    }
+  }
+  return dberr;
+}
+
 
+
+// prepare for reading the sync set
+localstatus TPluginApiDS::apiPrepareReadSyncSet(void)
+{
+  TSyError dberr = LOCERR_OK;
   #ifdef BASED_ON_BINFILE_CLIENT
   if (binfileDSActive()) {
     // we need to create the context for the data plugin here, as loadAdminData is not called in BASED_ON_BINFILE_CLIENT case.
@@ -976,11 +994,30 @@ localstatus TPluginApiDS::apiReadSyncSet(bool aNeedAll)
       }
     }
     else if (dberr==LOCERR_NOTIMP)
-      dberr=LOCERR_OK; // we just don't have a data plugin, that's ok, inherited (SQL) will handle data
-    if (dberr!=LOCERR_OK)
-      goto endread;
+      dberr=LOCERR_OK; // we just don't have a data plugin, that's ok
   } // binfile active
   #endif // BASED_ON_BINFILE_CLIENT
+  return dberr;
+}
+
+
+
+
+// read sync set IDs and mod dates (and rest of data if technically unavoidable or
+// requested by aNeedAll)
+localstatus TPluginApiDS::apiReadSyncSet(bool aNeedAll)
+{
+  TSyError dberr=LOCERR_OK;
+  #ifdef SYDEBUG
+  string ts1,ts2;
+  #endif
+
+  if (!fPluginDSConfigP->fEarlyStartDataRead) {
+    // normal sequence, start data read is not called before starting to read the sync set
+    dberr = apiPrepareReadSyncSet();
+    if (dberr!=LOCERR_OK)
+      goto endread;
+  }
   #ifndef SDK_ONLY_SUPPORT
   // only handle here if we are in charge - otherwise let ancestor handle it
   if (!fDBApi_Data.Created())
@@ -1022,11 +1059,13 @@ localstatus TPluginApiDS::apiReadSyncSet(bool aNeedAll)
     ts2.c_str()
   ));
   #endif
-  // start the reading phase anyway (to make sure call order is always StartRead/EndRead/StartWrite/EndWrite)
-  dberr = fDBApi_Data.StartDataRead(fPreviousToRemoteSyncIdentifier.c_str(),fPreviousSuspendIdentifier.c_str());
-  if (dberr!=LOCERR_OK) {
-    PDEBUGPRINTFX(DBG_ERROR,("DBapi::StartDataRead fatal error: %hd",dberr));
-    goto endread;
+  if (!fPluginDSConfigP->fEarlyStartDataRead) {
+    // start the reading phase anyway (to make sure call order is always StartRead/EndRead/StartWrite/EndWrite)
+    dberr = fDBApi_Data.StartDataRead(fPreviousToRemoteSyncIdentifier.c_str(),fPreviousSuspendIdentifier.c_str());
+    if (dberr!=LOCERR_OK) {
+      PDEBUGPRINTFX(DBG_ERROR,("DBapi::StartDataRead fatal error: %hd",dberr));
+      goto endread;
+    }
   }
   // we don't need to load the syncset if we are only refreshing from remote
   // but we also must load it if we can't zap without it on slow refresh, or when we can't retrieve items on non-slow refresh
diff --git a/src/DB_interfaces/api_db/pluginapids.h b/src/DB_interfaces/api_db/pluginapids.h
index 2a2ec9b..4fbfc79 100755
--- a/src/DB_interfaces/api_db/pluginapids.h
+++ b/src/DB_interfaces/api_db/pluginapids.h
@@ -106,6 +106,8 @@ public:
   /// either  fDBAPIModule_Data or fDBAPIModule_Admin to select plugin handling of
   /// either data or admin independently.
   bool fDataModuleAlsoHandlesAdmin;
+  // wants startDataRead() called as early as possible
+  bool fEarlyStartDataRead;
   // - config object for API module
   TDB_Api_Config fDBApiConfig_Data;
   TDB_Api_Config fDBApiConfig_Admin;
@@ -252,6 +254,9 @@ public:
   ///   them separately afterwards).
   #endif // not BINFILE_ALWAYS_ACTIVE
 
+  /// perform early data access start (if datastore requests it by setting fEarlyStartDataRead config flag)
+  virtual localstatus apiEarlyDataAccessStart(void);
+  /// read the sync set
   virtual localstatus apiReadSyncSet(bool aNeedAll);
   /// Zap all data in syncset (note that everything outside the sync set will remain intact)
   virtual localstatus apiZapSyncSet(void);
@@ -314,6 +319,8 @@ public:
 private:
   // - connect data handling part of plugin. Returns LOCERR_NOTIMPL when no data plugin is selected
   TSyError connectDataPlugin(void);
+  // - prepare for reading syncset (is called early when fEarlyStartDataRead is set, otherwise from within apiReadSyncSet)
+  localstatus apiPrepareReadSyncSet(void);
   // - alert possible thread change to plugins
   //   Does not check if API is locked or not, see dsThreadMayChangeNow()
   void ThreadMayChangeNow(void);
diff --git a/src/sysync/customimplds.cpp b/src/sysync/customimplds.cpp
index 3bdf4b3..4b9da0a 100755
--- a/src/sysync/customimplds.cpp
+++ b/src/sysync/customimplds.cpp
@@ -1616,6 +1616,15 @@ localstatus TCustomImplDS::implMakeAdminReady(
       if (!TScriptContext::executeTest(true,fScriptContextP,fConfigP->fAdminReadyScript,fConfigP->getDSFuncTableP(),fAgentP))
         sta=510; // script returns false or fails -> DB error
       #endif
+      if (sta==LOCERR_OK) {
+        // successful so far, now allow for early startDataRead to occur if configured on api level
+        sta = apiEarlyDataAccessStart();
+        if (sta==508) {
+          // special case: the database requests a slow sync for internal reasons (like change tracking disabled)
+          // - force slow sync by removing last anchor
+          fLastRemoteAnchor.erase();
+        }
+      }
     } // if apiLoadAdminData successful
   }
   SYSYNC_CATCH(exception &e)
diff --git a/src/sysync/customimplds.h b/src/sysync/customimplds.h
index f6ef8ee..0e299ad 100755
--- a/src/sysync/customimplds.h
+++ b/src/sysync/customimplds.h
@@ -477,6 +477,8 @@ public:
   virtual localstatus apiSaveAdminData(bool aSessionFinished, bool aSuccessful) = 0;
   #endif // BINFILE_ALWAYS_ACTIVE
 
+  /// allow early data access start (if datastore is configured for it)
+  virtual localstatus apiEarlyDataAccessStart(void) { return LOCERR_OK; /* nop if not overridden */ };
   /// read sync set IDs and mod dates.
   /// @param[in] if set, all data fields are needed, so ReadSyncSet MAY
   ///   read items here already. Note that ReadSyncSet MAY read items here