Details
-
Bug
-
Resolution: Won't Fix
-
Minor - P4
-
None
-
None
-
None
-
ALL
Description
overview of steps:
- run a 2.0 cluster with two mongos instances (s0 and s1)
- create a bunch of collections (sharded) with old metadata format
- upgrade everything to 2.2.3
- create some more collections, from s0
- run the mongos 2.4 --upgrade with the failpoint which causes the upgrade to stop in the critical section, so the config servers become out of sync (upgradeV3ToV4CriticalResyncFail) with data: {unclean:true}
- try to modify the metadata from the existing mongos instances, s0 and s1, by doing chunk splits.
var keyfilepath = "/Users/mike/keyfile.key"
|
setVerboseShell(false);
|
|
|
var options = {
|
mongosOptions : {binVersion:MongoRunner.versionIterator(["2.0.0","2.0.0"]),
|
logpath:"/tmp/stest"},
|
configOptions:{binVersion:"2.0.0", logpath:"/tmp/stest-config"},
|
shardOptions:{binVersion:"2.0.0", logpath:"/tmp/sttest-shards"},
|
//rsOptions:{binVersion:allversions, logpath:"/tmp/sttest-rs"},
|
rsOptions:{binVersion:MongoRunner.versionIterator(["2.0.0","2.0.0"]), logpath:"/tmp/sttest-rs", nopreallocj:""},
|
separateConfig:true,
|
sync:true,
|
rs:true
|
}
|
|
|
var st = new ShardingTest({shards:2, mongos:2, other:options})
|
shards = st.s0.getDB("config").shards.find().toArray();
|
|
|
createShardedCollection = function(admin, coll){
|
printjson(admin.runCommand({ enableSharding : coll.getDB() + "" }));
|
printjson(admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }));
|
printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } }));
|
printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : -300 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : -200 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : -100 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : 100 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : 200 } }));
|
printjson(admin.runCommand({ split : coll + "", middle : { _id : 300 } }));
|
}
|
|
|
for(var i=0;i<10;i++){
|
createShardedCollection(st.s0.getDB("admin"), st.s0.getDB("foo" + i).bar)
|
}
|
|
|
st.upgradeCluster("2.2.3")
|
sleep(30000)
|
|
|
for(var i=11;i<20;i++){
|
createShardedCollection(st.s0.getDB("admin"), st.s0.getDB("foo" + i).bar)
|
}
|
|
|
st.stopBalancer();
|
printjson("trying to start up a new mongos - going to fail uncleanly");
|
var newmongos = MongoRunner.runMongos({configdb:st._configDB, binVersion:"g2.4", setParameter:"enableTestCommands=1", upgrade:"", setFailPoint:"upgradeV3ToV4CriticalResyncFail={mode:'alwaysOn',data:{unclean:true}}"})
|
sleep(10000)
|
|
|
|
|
printjson("here are the hashes of hte config servers");
|
var hash0 = st.c0.getDB("config").runCommand({dbhash:1}).md5
|
var hash1 = st.c1.getDB("config").runCommand({dbhash:1}).md5
|
var hash2 = st.c2.getDB("config").runCommand({dbhash:1}).md5
|
print(hash0, hash1, hash2)
|
|
|
|
|
printjson("trying to do a bunch of metadata ops");
|
for(var i=0;i<20;i++){
|
printjson(st.s0.getDB("admin").runCommand({ split : "foo"+i+".bar", middle : { _id : -24 } }));
|
printjson(st.s1.getDB("admin").runCommand({ split : "foo"+i+".bar", middle : { _id : -88 } }));
|
}
|
|
|
hash0 = st.c0.getDB("config").runCommand({dbhash:1}).md5
|
hash1 = st.c1.getDB("config").runCommand({dbhash:1}).md5
|
hash2 = st.c2.getDB("config").runCommand({dbhash:1}).md5
|
print(hash0, hash1, hash2)
|
So one of the mongos instances (s1) correctly refuses to run the split commands, but the other one (s0) will still run the split commands without complaining.
Also, trying to run an additional mongos (any version) at this point, will fail with this message:
m27003| Wed Mar 6 18:49:56.732 [mongosMain] ERROR: config servers not in sync! config servers mikes-MacBook-Pro.local:29000 and mikes-MacBook-Pro.local:29002 differ
|
m27003| chunks: "bd332c8ddd0be0167ce2bf9f56f14b6c" chunks: "1ecde1726b6e799447dcd48314969317"
|
m27003| databases: "01ba08e3f7efafba520270cb16529211" databases: "01ba08e3f7efafba520270cb16529211"
|
However, having the one mongos up which is capable of modifying metadata is dangerous.