Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-31019

Changing fCV during initial sync leads to divergent data across replica set members

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major - P3
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: 3.6.0-rc0
    • Component/s: Replication, Storage
    • Labels:
      None
    • Backwards Compatibility:
      Fully Compatible
    • Operating System:
      ALL
    • Steps To Reproduce:
      Hide

      I've only had success in reproducing this issue with the MMAPv1 storage engine, and not with the WiredTiger or EphemeralForTest storage engines; however, it isn't clear to me why this issue would be storage engine-specific though.

      python buildscripts/resmoke.py --suites=no_server repro_server31019.js --storageEngine=mmapv1 --repeat=5
      

      repro_server31019.js

      (function() {
          "use strict";
       
          const verbositySettings = tojson({
              verbosity: 1,
              replication: 2,
              storage: 2,
          });
       
          const rst = new ReplSetTest({
              nodes: 1,
              nodeOptions: {
                  setParameter: {logComponentVerbosity: verbositySettings},
              }
          });
       
          rst.startSet();
          rst.initiate();
       
          const primaryDB = rst.getPrimary().getDB("test");
       
          rst.add({
              setParameter: {
                  "failpoint.initialSyncHangBeforeCopyingDatabases": tojson({mode: "alwaysOn"}),
                  logComponentVerbosity: verbositySettings
              }
          });
       
          // We disallow the secondary node from voting so that the primary's featureCompatibilityVersion
          // can be modified while the secondary node is still waiting to complete its initial sync.
          {
              const replSetConfig = rst.getReplSetConfigFromNode(0);
              replSetConfig.members = rst.getReplSetConfig().members;
              replSetConfig.members[1].priority = 0;
              replSetConfig.members[1].votes = 0;
              ++replSetConfig.version;
              assert.commandWorked(primaryDB.adminCommand({replSetReconfig: replSetConfig}));
          }
       
          // We set the primary's featureCompatibilityVersion to "3.4" and implicitly create a collection
          // without a UUID via an insert operation.
          {
              assert.commandWorked(primaryDB.adminCommand({setFeatureCompatibilityVersion: "3.4"}));
       
              primaryDB.mycoll.drop();
              assert.writeOK(primaryDB.mycoll.insert({_id: "while in fCV=3.4"}));
          }
       
          // Next, we set the primary's featureCompatibilityVersion to "3.6" and drop the collection that
          // was previously created. We then implicitly create another collection of the same name (but
          // with a UUID this time) via an insert operation.
          {
              assert.commandWorked(primaryDB.adminCommand({setFeatureCompatibilityVersion: "3.6"}));
       
              primaryDB.mycoll.drop();
              assert.writeOK(primaryDB.mycoll.insert({_id: "while in fCV=3.6"}));
          }
       
          // Finally, we allow the secondary node to proceed with its initial sync. It should end up with
          // only the document that was inserted into the collection when the primary's
          // featureCompatibilityVersion was "3.6".
          const secondaryDB = rst.getSecondary().getDB("test");
          assert.commandWorked(secondaryDB.adminCommand({
              configureFailPoint: "initialSyncHangBeforeCopyingDatabases",
              mode: "off",
          }));
       
          rst.checkReplicatedDataHashes();
          rst.stopSet();
      })();
      

      Show
      I've only had success in reproducing this issue with the MMAPv1 storage engine, and not with the WiredTiger or EphemeralForTest storage engines; however, it isn't clear to me why this issue would be storage engine-specific though. python buildscripts/resmoke.py --suites=no_server repro_server31019.js --storageEngine=mmapv1 --repeat=5 repro_server31019.js ( function () { "use strict" ;   const verbositySettings = tojson({ verbosity: 1, replication: 2, storage: 2, });   const rst = new ReplSetTest({ nodes: 1, nodeOptions: { setParameter: {logComponentVerbosity: verbositySettings}, } });   rst.startSet(); rst.initiate();   const primaryDB = rst.getPrimary().getDB( "test" );   rst.add({ setParameter: { "failpoint.initialSyncHangBeforeCopyingDatabases" : tojson({mode: "alwaysOn" }), logComponentVerbosity: verbositySettings } });   // We disallow the secondary node from voting so that the primary's featureCompatibilityVersion // can be modified while the secondary node is still waiting to complete its initial sync. { const replSetConfig = rst.getReplSetConfigFromNode(0); replSetConfig.members = rst.getReplSetConfig().members; replSetConfig.members[1].priority = 0; replSetConfig.members[1].votes = 0; ++replSetConfig.version; assert.commandWorked(primaryDB.adminCommand({replSetReconfig: replSetConfig})); }   // We set the primary's featureCompatibilityVersion to "3.4" and implicitly create a collection // without a UUID via an insert operation. { assert.commandWorked(primaryDB.adminCommand({setFeatureCompatibilityVersion: "3.4" }));   primaryDB.mycoll.drop(); assert.writeOK(primaryDB.mycoll.insert({_id: "while in fCV=3.4" })); }   // Next, we set the primary's featureCompatibilityVersion to "3.6" and drop the collection that // was previously created. We then implicitly create another collection of the same name (but // with a UUID this time) via an insert operation. { assert.commandWorked(primaryDB.adminCommand({setFeatureCompatibilityVersion: "3.6" }));   primaryDB.mycoll.drop(); assert.writeOK(primaryDB.mycoll.insert({_id: "while in fCV=3.6" })); }   // Finally, we allow the secondary node to proceed with its initial sync. It should end up with // only the document that was inserted into the collection when the primary's // featureCompatibilityVersion was "3.6". const secondaryDB = rst.getSecondary().getDB( "test" ); assert.commandWorked(secondaryDB.adminCommand({ configureFailPoint: "initialSyncHangBeforeCopyingDatabases" , mode: "off" , }));   rst.checkReplicatedDataHashes(); rst.stopSet(); })();
    • Sprint:
      Repl 2017-10-02, Repl 2017-10-23
    • Linked BF Score:
      0

      Description

      The node performing the initial sync appears to be able to retain the documents that were inserted prior to the collection being dropped and re-created after changing the featureCompatibilityVersion to 3.6. This issue is related to UUIDs and their impact on oplog application, and therefore doesn't affect the 3.2 or 3.4 branches.

      2017-09-10T19:21:06.495-0400 The following documents are missing on the primary:
      2017-09-10T19:21:06.495-0400 {  "_id" : "while in fCV=3.4" }
      ...
      2017-09-10T19:21:06.498-0400 checkReplicatedDataHashes, the primary and secondary have a different hash for the test database: {
      2017-09-10T19:21:06.498-0400 	"master" : {
      2017-09-10T19:21:06.499-0400 		"host" : "hanamizu:20010",
      2017-09-10T19:21:06.499-0400 		"collections" : {
      2017-09-10T19:21:06.499-0400 			"mycoll" : "09aabf5621c57d91db16b98b365d8e65"
      2017-09-10T19:21:06.499-0400 		},
      2017-09-10T19:21:06.499-0400 		"md5" : "2105eeb0b1ec2ade59f08fa1f3f40ba9",
      2017-09-10T19:21:06.499-0400 		"timeMillis" : 0,
      2017-09-10T19:21:06.499-0400 		"ok" : 1,
      2017-09-10T19:21:06.499-0400 		"operationTime" : Timestamp(1505085665, 18)
      2017-09-10T19:21:06.499-0400 	},
      2017-09-10T19:21:06.499-0400 	"slaves" : [
      2017-09-10T19:21:06.499-0400 		{
      2017-09-10T19:21:06.500-0400 			"host" : "hanamizu:20011",
      2017-09-10T19:21:06.500-0400 			"collections" : {
      2017-09-10T19:21:06.500-0400 				"mycoll" : "b8b6211fb0b559d95ae6df5cc4071420"
      2017-09-10T19:21:06.500-0400 			},
      2017-09-10T19:21:06.500-0400 			"md5" : "072bbaef3649d98b3270e6a2a6eac21f",
      2017-09-10T19:21:06.500-0400 			"timeMillis" : 0,
      2017-09-10T19:21:06.500-0400 			"ok" : 1,
      2017-09-10T19:21:06.500-0400 			"operationTime" : Timestamp(1505085665, 18)
      2017-09-10T19:21:06.500-0400 		}
      2017-09-10T19:21:06.500-0400 	]
      2017-09-10T19:21:06.500-0400 }
      

        Attachments

          Issue Links

            Activity

              People

              • Votes:
                0 Vote for this issue
                Watchers:
                6 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: