Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-15890

InitialSync from 2.6 ReplSet to new 2.8 Secondary fatally asserts

XMLWordPrintableJSON

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • 2.8.0-rc0
    • Affects Version/s: None
    • Component/s: Replication
    • Labels:
    • ALL
    • Hide 
      // Multiversion initial sync test.
load("./jstests/multiVersion/libs/multi_rs.js");
load("./jstests/replsets/rslib.js");

var oldVersion = "2.6";
var newVersion = "latest";

var name = "multiversioninitsync";

// *****************************************
// Test A:
// 2.8 Secondary is synced from 2.6 ReplSet.
// *****************************************
var nodes = {n1: {binVersion: oldVersion},
             n2: {binVersion: oldVersion}};

print("Start up a two-node 2.6 replica set.");
var rst = new ReplSetTest({name: name, nodes: nodes});
rst.startSet();
rst.initiate();

// Wait for a primary node.
var primary = rst.getPrimary();

// Insert some data and wait for replication.
for (var i=0; i<25; i++) {
    primary.getDB("foo").foo.insert({_id: i});
}
rst.awaitReplication();

print("Bring up a new node with latest version and add to set.");
rst.add({binVersion: newVersion});
rst.reInitiate();

// Wait for a primary node.
var primary = rst.getPrimary();
var secondaries = rst.getSecondaries();

print("Wait for new node to be synced.");
assert.soon(function() {
    var op1 = getLatestOp(primary);
    var op2 = getLatestOp(secondaries[0]);
    var op3 = getLatestOp(secondaries[1]);

    return friendlyEqual(op1, op2) && friendlyEqual(op1, op3);
});

rst.stopSet();
      Show
      // Multiversion initial sync test. load( "./jstests/multiVersion/libs/multi_rs.js" ); load( "./jstests/replsets/rslib.js" ); var oldVersion = "2.6" ; var newVersion = "latest" ; var name = "multiversioninitsync" ; // ***************************************** // Test A: // 2.8 Secondary is synced from 2.6 ReplSet. // ***************************************** var nodes = {n1: {binVersion: oldVersion}, n2: {binVersion: oldVersion}}; print( "Start up a two-node 2.6 replica set." ); var rst = new ReplSetTest({name: name, nodes: nodes}); rst.startSet(); rst.initiate(); // Wait for a primary node. var primary = rst.getPrimary(); // Insert some data and wait for replication. for ( var i=0; i<25; i++) { primary.getDB( "foo" ).foo.insert({_id: i}); } rst.awaitReplication(); print( "Bring up a new node with latest version and add to set." ); rst.add({binVersion: newVersion}); rst.reInitiate(); // Wait for a primary node. var primary = rst.getPrimary(); var secondaries = rst.getSecondaries(); print( "Wait for new node to be synced." ); assert .soon(function() { var op1 = getLatestOp(primary); var op2 = getLatestOp(secondaries[0]); var op3 = getLatestOp(secondaries[1]); return friendlyEqual(op1, op2) && friendlyEqual(op1, op3); }); rst.stopSet();

      Start up a 2 node 2.6 replica set, and then try to add a 2.8 server to the replica set. This causes a fatal assertion:

       m31001| 2014-10-30T17:11:04.164-0400 [conn31] end connection 10.4.121.229:63101 (4 connections now open)
 m31002| 2014-10-30T17:11:04.164-0400 E REPLSETS [rsSync] initial sync exception: 13106 nextSafe(): { $err: "not master and slaveOk=false", code: 13435 } 1 attempts remaining
 m31002| 2014-10-30T17:11:09.165-0400 I REPLSETS [rsSync] initial sync pending
 m31002| 2014-10-30T17:11:09.165-0400 I REPLSETS syncing from: Falcon.local:31001
 m31001| 2014-10-30T17:11:09.165-0400 [initandlisten] connection accepted from 10.4.121.229:63104 #34 (4 connections now open)
 m31002| 2014-10-30T17:11:09.166-0400 I REPLSETS [rsSync] initial sync drop all databases
 m31002| 2014-10-30T17:11:09.166-0400 I STORAGE  [rsSync] dropAllDatabasesExceptLocal 1
 m31002| 2014-10-30T17:11:09.166-0400 I REPLSETS [rsSync] initial sync clone all databases
 m31002| 2014-10-30T17:11:09.166-0400 I REPLSETS [rsSync] initial sync cloning db: foo
 m31001| 2014-10-30T17:11:09.166-0400 [initandlisten] connection accepted from 10.4.121.229:63105 #35 (5 connections now open)
 m31001| 2014-10-30T17:11:09.167-0400 [conn35] end connection 10.4.121.229:63105 (4 connections now open)
 m31001| 2014-10-30T17:11:09.167-0400 [initandlisten] connection accepted from 10.4.121.229:63106 #36 (5 connections now open)
 m31001| 2014-10-30T17:11:09.167-0400 [conn36] assertion 13435 not master and slaveOk=false ns:foo.system.namespaces query:{}
 m31001| 2014-10-30T17:11:09.167-0400 [conn36] end connection 10.4.121.229:63106 (4 connections now open)
 m31001| 2014-10-30T17:11:09.167-0400 [conn34] end connection 10.4.121.229:63104 (4 connections now open)
 m31002| 2014-10-30T17:11:09.168-0400 E REPLSETS [rsSync] initial sync exception: 13106 nextSafe(): { $err: "not master and slaveOk=false", code: 13435 } 0 attempts remaining
 m31002| 2014-10-30T17:11:14.168-0400 I -        [rsSync] Fatal Assertion 16233
 m31002| 2014-10-30T17:11:14.174-0400 I -        [rsSync] warning: log line attempted (20k) over max size (10k), printing beginning and end ... 
 m31002|  0x10092940a 0x1008b4dcb 0x1008a2ecf 0x10060cafd 0x100620abc 0x10096a9d1 0x7fff8e4ab899 0x7fff8e4ab72a 0x7fff8e4affc9
 m31002| ----- BEGIN BACKTRACE -----
 m31002| {"backtrace":[{"b":"100000000","o":"92940A"},{"b":"100000000","o":"8B4DCB"},{"b":"100000000","o":"8A2ECF"},{"b":"100000000","o":"60CAFD"},{"b":"100000000","o":"620ABC"},{"b":"100000000","o":"96A9D1"},{"b":"7FFF8E4AA000","o":"1899"},{"b":"7FFF8E4AA000","o":"172A"},{"b":"7FFF8E4AA000","o":"5FC9"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "0916af4d9e45fcbe95fe5f43dc6f8902292467cd modules: subscription", "uname" : { "sysname" : "Darwin", "release" : "13.4.0", "version" : "Darwin Kernel Version 13.4.0: Sun Aug 17 19:50:11 PDT 2014; root:xnu-2422.115.4~1/RELEASE_X86_64", "machine" : "x86_64" }, "somap" : [ { "path" : "/Users/Amalia/mongo/mongod", "machType" : 2, "b" : "100000000", "buildId" : "7C557EA6DDFD3DF0993A51A8EBE76D64" }, { "path" : "/usr/lib/libnetsnmpagent.25.dylib", "machType" : 6, "b" : "101C50000", "buildId" : "3A3E90140DC0312B86596D22D893720E" }, { "path" : "/usr/lib/libnetsnmpmibs.25.dylib", "machType" : 6, "b" : "101CA3000", "buildId" : "74EE1FC991C83CEC8676DA97AD6659A1" }, { "path" : "/usr/lib/libnetsnmp.25.dylib", "machType" : 6, "b" : "101D8C000", "buildId" : "BC2E32D680FE3E6896F09895D0E84C66" }, { "path" : "/usr/lib/libsasl2.2.dylib", "machType" : 6, "b" : "7FFF85DEA000", "buildId" : "C8E2571068B6368ABF3E48EC7273177B" }, { "path" : "/System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos", "machType" : 6, "b" : "7FFF8FB6F000", "buildId" : "59427E1137D634C995BBD4438BFACA5B" }, { "path" : "/usr/lib/libssl.0.9.8.dylib", "machType" : 6, "b" : "7FFF89CDB000", "buildId" : "51C844FFD7CD35259ABB84B8DD11D5E4" }, { "path" : "/usr/lib/libcrypto.0.9.8.dylib", "machType" : 6, "b" : "7FFF8D733000", "buildId" : "ED7F386510D4346B8C9CD968EB3B5D35" }, { "path" : "/usr/lib/libSystem.B.dylib", "machType" : 6, "b" : "7FFF87A20000", "buildId" : "E303F2F8A8CF3DF384B3F2D0EE41CCF6" }, { "path" : "/usr/lib/libstdc++.6.dylib", "machType" : 6, "b" : "7FFF87DDF000", "buildId" : "0241E6A4136833BE950BD0A175C41F54" }, { "path" : "/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit", "machType" : 6, "b" : "7FFF8B855000", "buildId" : "057FDBA356D639038C0B849214BF1985" }, { "path" : "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation", "machType" : 6, "b" : "7FFF8412E000", "buildId" : "729BD6DA1F633E72A14826F21EBF52BB" }, { "path" : "/System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices", "machType" : 6, "b" : "7FFF88FC5000", "buildId" : "67A369BA2326383F995B853DAD7BFF43" }, { "path" : "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices", "machType" : 6, "b" : "7FFF85F73000", "buildId" : "F250170A88053731909778CAD04481F0" }, { "path" : "/System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration", "machType" : 6, "b" : "7FFF84B11000", "buildId" : "A4165553770E3D27B21701FC1F852B87" }, { "path" : "/System/Library/PrivateFrameworks/TrustEvaluationAgent.framework/Versions/A/TrustEvaluationAgent", "machType" : 6, "b" : "7FFF8A79F000", "buildId" : "334A82F44AE43719A51186D0B0723E2B" }, { "path" : "/usr/lib/libz.1.dylib", "machType" : 6, "b" : "7FFF8752C000", "buildId" : "4C5FA1A54471398E95432E54D46E63E9" }, { "path" : "/usr/lib/system/libcache.dylib", "machType" : 6, "b" : "7FFF9000E000", "bu .......... th" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJPEG.dylib", "machType" : 6, "b" : "7FFF849E9000", "buildId" : "BE0ED4E1F7FC303886D30456DD173FCB" }, { "path" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libTIFF.dylib", "machType" : 6, "b" : "7FFF8799F000", "buildId" : "FBC5800B7F343755A44E7B37B3E0B32E" }, { "path" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib", "machType" : 6, "b" : "7FFF91E25000", "buildId" : "151BA92C6E7C3B698024FDD1E2C89DD3" }, { "path" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib", "machType" : 6, "b" : "7FFF84A8A000", "buildId" : "7E51DFC3740A3CD398A11EC510A4A055" }, { "path" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJP2.dylib", "machType" : 6, "b" : "7FFF8D3B9000", "buildId" : "BE5FF7655ECE38B5BF5DBE806F5CAD18" }, { "path" : "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libRadiance.dylib", "machType" : 6, "b" : "7FFF8E426000", "buildId" : "461482C9CADB3B36B023597C64AD4B00" }, { "path" : "/usr/lib/libcups.2.dylib", "machType" : 6, "b" : "7FFF89D10000", "buildId" : "36EA435043B43A5C990410685BFDA7D4" }, { "path" : "/System/Library/Frameworks/GSS.framework/Versions/A/GSS", "machType" : 6, "b" : "7FFF8C43A000", "buildId" : "44E914BEB0D03E059451CA9E539AFA52" }, { "path" : "/usr/lib/libresolv.9.dylib", "machType" : 6, "b" : "7FFF86D0D000", "buildId" : "11C2C826F1C639C6B4E86E0C41D4FA95" }, { "path" : "/usr/lib/libiconv.2.dylib", "machType" : 6, "b" : "7FFF87AD0000", "buildId" : "BB44B115AC323877A0EDAEC6232A4563" }, { "path" : "/System/Library/PrivateFrameworks/Heimdal.framework/Versions/A/Heimdal", "machType" : 6, "b" : "7FFF8A739000", "buildId" : "F34D66279F8038238B57DB629307DF87" }, { "path" : "/usr/lib/libheimdal-asn1.dylib", "machType" : 6, "b" : "7FFF8D68E000", "buildId" : "CAE21FFF5763399CB7C5EEBFFEEF2242" }, { "path" : "/System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory", "machType" : 6, "b" : "7FFF851AA000", "buildId" : "F08601E8F7E83222AD056A26003779CF" }, { "path" : "/System/Library/PrivateFrameworks/CommonAuth.framework/Versions/A/CommonAuth", "machType" : 6, "b" : "7FFF91E41000", "buildId" : "32BA436F63193A0BB5D22EB75FF36B5B" }, { "path" : "/System/Library/Frameworks/SecurityFoundation.framework/Versions/A/SecurityFoundation", "machType" : 6, "b" : "7FFF86C2D000", "buildId" : "0FDC8F53104C3938A8525B33C30BAAD5" }, { "path" : "/System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio", "machType" : 6, "b" : "7FFF85384000", "buildId" : "BE13E840FB453BC2BCF5031629754FD5" }, { "path" : "/System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox", "machType" : 6, "b" : "7FFF8E4B2000", "buildId" : "69B273E85A8E3FC7B807C16B657662FE" } ] }}
 m31002|  mongod(_ZN5mongo15printStackTraceERSo+0x3A) [0x10092940a]
 m31002|  mongod(_ZN5mongo10logContextEPKc+0xDB) [0x1008b4dcb]
 m31002|  mongod(_ZN5mongo13fassertFailedEi+0xAF) [0x1008a2ecf]
 m31002|  mongod(_ZN5mongo4repl17syncDoInitialSyncEv+0x151D) [0x10060cafd]
 m31002|  mongod(_ZN5mongo4repl13runSyncThreadEv+0x4EC) [0x100620abc]
 m31002|  mongod(_ZN5boost12_GLOBAL__N_112thread_proxyEPv+0xB1) [0x10096a9d1]
 m31002|  libsystem_pthread.dylib(_pthread_body+0x8A) [0x7fff8e4ab899]
 m31002|  libsystem_pthread.dylib(_pthread_struct_init+0x0) [0x7fff8e4ab72a]
 m31002|  libsystem_pthread.dylib(thread_start+0xD) [0x7fff8e4affc9]
 m31002| -----  END BACKTRACE  -----
 m31002| 2014-10-30T17:11:14.174-0400 I -        [rsSync] 
 m31002| 
 m31002| ***aborting after fassert() failure
 m31002|

            Assignee:
            spencer@mongodb.com Spencer Brody (Inactive)
            Reporter:
            amalia.hawkins@10gen.com Amalia Hawkins
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: