From d7b5fbe6f09c66e46f651788e353e8591a9c0c40 Mon Sep 17 00:00:00 2001 From: rstam Date: Fri, 12 Apr 2013 21:02:42 -0400 Subject: [PATCH] CSHARP-724: Major Issue in the Failover scenario: Once a Primary goes Down => The driver will never be able to have a single Primary until the old Primary goes back online --- .../Communication/MongoServerInstance.cs | 53 ++- .../Proxies/DiscoveringMongoServerProxy.cs | 17 + MongoDB.Driver/MongoCollection.cs | 2 +- MongoDB.Driver/MongoServer.cs | 12 +- MongoDB.DriverUnitTests/Jira/CSharp269Tests.cs | 8 +- MongoDB.DriverUnitTests/Jira/CSharp724Tests.cs | 300 +++++++++++++ .../MongoDB.DriverUnitTests.csproj | 499 +++++++++++---------- MongoDB.DriverUnitTests/MongoServerTests.cs | 2 +- 8 files changed, 627 insertions(+), 266 deletions(-) create mode 100644 MongoDB.DriverUnitTests/Jira/CSharp724Tests.cs diff --git a/MongoDB.Driver/Communication/MongoServerInstance.cs b/MongoDB.Driver/Communication/MongoServerInstance.cs index d185bc9..dc93f83 100644 --- a/MongoDB.Driver/Communication/MongoServerInstance.cs +++ b/MongoDB.Driver/Communication/MongoServerInstance.cs @@ -360,13 +360,21 @@ namespace MongoDB.Driver try { Ping(connection); - LookupServerInformation(connection); } catch { // ignore exceptions (if any occured state will already be set to Disconnected) // Console.WriteLine("MongoServerInstance[{0}]: VerifyState failed: {1}.", sequentialId, ex.Message); } + try + { + LookupServerInformation(connection); + } + catch + { + // ignore exceptions (if any occured state will already be set to Disconnected) + // Console.WriteLine("MongoServerInstance[{0}]: VerifyState failed: {1}.", sequentialId, ex.Message); + } } finally { @@ -417,25 +425,42 @@ namespace MongoDB.Driver OnStateChanged(); try - { - var connection = _connectionPool.AcquireConnection(); - try - { - Ping(connection); - LookupServerInformation(connection); + { + var connection = _connectionPool.AcquireConnection(); + try + { + try + { + Ping(connection); + } + catch(Exception ex) + { + //ignore the ping exception + } + + try + { + LookupServerInformation(connection); + } + catch (Exception ex) + { + throw new Exception("Unable to connect to instance", ex); + } + SetState(MongoServerState.Connected); } finally { _connectionPool.ReleaseConnection(connection); } - SetState(MongoServerState.Connected); + } catch (Exception ex) { lock (_serverInstanceLock) { _connectException = ex; - } + } + _connectionPool.Clear(); Interlocked.Exchange(ref _connectException, ex); SetState(MongoServerState.Disconnected); @@ -698,10 +723,18 @@ namespace MongoDB.Driver var connection = _connectionPool.AcquireConnection(_stateVerificationAcquireConnectionOptions); try { - Ping(connection); + try + { + Ping(connection); + } + catch + { + //Ignore internal ping Exception we absolutely want to have the LookupServerInformation to be called + } LookupServerInformation(connection); ThreadPool.QueueUserWorkItem(o => _connectionPool.MaintainPoolSize()); SetState(MongoServerState.Connected); + } finally { diff --git a/MongoDB.Driver/Communication/Proxies/DiscoveringMongoServerProxy.cs b/MongoDB.Driver/Communication/Proxies/DiscoveringMongoServerProxy.cs index 187c91c..9559571 100644 --- a/MongoDB.Driver/Communication/Proxies/DiscoveringMongoServerProxy.cs +++ b/MongoDB.Driver/Communication/Proxies/DiscoveringMongoServerProxy.cs @@ -97,6 +97,23 @@ namespace MongoDB.Driver.Internal } /// + /// Gets the name of the replica set (null if not connected to a replica set). + /// + public string ReplicaSetName + { + get + { + var replicaSetProxy = _serverProxy as ReplicaSetMongoServerProxy; + if (replicaSetProxy != null) + { + return replicaSetProxy.ReplicaSetName; + } + + return null; + } + } + + /// /// Gets the state. /// public MongoServerState State diff --git a/MongoDB.Driver/MongoCollection.cs b/MongoDB.Driver/MongoCollection.cs index 89f753a..25de003 100644 --- a/MongoDB.Driver/MongoCollection.cs +++ b/MongoDB.Driver/MongoCollection.cs @@ -1780,7 +1780,7 @@ namespace MongoDB.Driver resultSerializationOptions, resultSerializer); - var connection = _server.AcquireConnection(ReadPreference.Primary); + var connection = _server.AcquireConnection(readPreference); try { return commandOperation.Execute(connection); diff --git a/MongoDB.Driver/MongoServer.cs b/MongoDB.Driver/MongoServer.cs index b43b287..3235845 100644 --- a/MongoDB.Driver/MongoServer.cs +++ b/MongoDB.Driver/MongoServer.cs @@ -295,10 +295,16 @@ namespace MongoDB.Driver { get { - var instanceManager = _serverProxy as ReplicaSetMongoServerProxy; - if (instanceManager != null) + var replicaSetProxy = _serverProxy as ReplicaSetMongoServerProxy; + if (replicaSetProxy != null) { - return instanceManager.ReplicaSetName; + return replicaSetProxy.ReplicaSetName; + } + + var discoveringProxy = _serverProxy as DiscoveringMongoServerProxy; + if (discoveringProxy != null) + { + return discoveringProxy.ReplicaSetName; } return null; diff --git a/MongoDB.DriverUnitTests/Jira/CSharp269Tests.cs b/MongoDB.DriverUnitTests/Jira/CSharp269Tests.cs index f8049ed..23a9872 100644 --- a/MongoDB.DriverUnitTests/Jira/CSharp269Tests.cs +++ b/MongoDB.DriverUnitTests/Jira/CSharp269Tests.cs @@ -18,7 +18,9 @@ using System.IO; using System.Text; using System.Threading; using MongoDB.Driver; +using MongoDB.Driver.GridFS; using NUnit.Framework; +using MongoDB.Driver.Builders; namespace MongoDB.DriverUnitTests.Jira.CSharp269 { @@ -43,11 +45,13 @@ namespace MongoDB.DriverUnitTests.Jira.CSharp269 [Test] public void TestUploadAndDownload() { + MongoGridFSFileInfo uploadedFileInfo; + var text = "HelloWorld"; var bytes = Encoding.UTF8.GetBytes(text); using (var stream = new MemoryStream(bytes)) { - _database.GridFS.Upload(stream, "HelloWorld.txt"); + uploadedFileInfo = _database.GridFS.Upload(stream, "HelloWorld.txt"); } // use RequestStart so that if we are running this test against a replica set we will bind to a specific secondary @@ -55,7 +59,7 @@ namespace MongoDB.DriverUnitTests.Jira.CSharp269 { // wait for the GridFS file to be replicated before trying to Download it var timeoutAt = DateTime.UtcNow.AddSeconds(30); - while (!_database.GridFS.Exists("HelloWorld.txt")) + while (!_database.GridFS.Exists(Query.EQ("_id", uploadedFileInfo.Id))) { if (DateTime.UtcNow >= timeoutAt) { diff --git a/MongoDB.DriverUnitTests/Jira/CSharp724Tests.cs b/MongoDB.DriverUnitTests/Jira/CSharp724Tests.cs new file mode 100644 index 0000000..687b1ae --- /dev/null +++ b/MongoDB.DriverUnitTests/Jira/CSharp724Tests.cs @@ -0,0 +1,300 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using MongoDB.Bson; +using MongoDB.Driver; +using NUnit.Framework; + +namespace MongoDB.DriverUnitTests.Jira +{ + [TestFixture()] + public class CSharp724Tests + { + /* + * D:\MongoDB\2.4.0\mongodb-win32-x86_64-2008plus-2.4.0\bin>mongod.exe --dbpath D:\MongoDB\2.4.0\mongodb-win32-x86_64-2008plus-2.4.0\bin\data\db\ + */ + + private const string PathToMongoServer = @"D:\MongoDB\2.4.0\mongodb-win32-x86_64-2008plus-2.4.0\bin\"; + private const string ReplicaSetName = @"CSharp724Tests"; + private const string ReplicaSetArgs = @"--replSet " + ReplicaSetName; + + private const string AdditionalOptions = @" -v "; + + private const string MongoServerExeName = @"mongod.exe "; + + private const string firstDataDir = @"data1\db\ "; + private const string secondDataDir = @"data2\db\ "; + private const string thirdDataDir = @"data3\db\ "; + + private const int firstInstancePort = 27017; + private const int secondInstancePort = 27018; + private const int thirdInstancePort = 27019; + + private static List _mongoAddresses = new List + { + new MongoServerAddress("localhost", firstInstancePort), + new MongoServerAddress("localhost", secondInstancePort), + new MongoServerAddress("localhost", thirdInstancePort) + }; + + internal static Dictionary InstancesByPort = new Dictionary(); + + private static Process firstMongoProcess; + private static Process secondMongoProcess; + private static Process thirdMongoProcess; + + + private MongoServer server; + + [SetUp] + public void Setup() + { + //Cleanup data + CleanupMongoDb(PathToMongoServer + firstDataDir); + CleanupMongoDb(PathToMongoServer + secondDataDir); + CleanupMongoDb(PathToMongoServer + thirdDataDir); + + var firstInstanceCmd = ReplicaSetArgs + " --dbpath " + + PathToMongoServer + firstDataDir + "--port " + firstInstancePort + AdditionalOptions; + var secondInstanceCmd = ReplicaSetArgs + " --dbpath " + + PathToMongoServer + secondDataDir + "--port " + secondInstancePort + AdditionalOptions; + var thirdInstanceCmd = ReplicaSetArgs + " --dbpath " + + PathToMongoServer + thirdDataDir + "--port " + thirdInstancePort + AdditionalOptions; + + MongoProcessLaucher.LaunchMongoProcess(out thirdMongoProcess,PathToMongoServer+MongoServerExeName, thirdInstanceCmd, thirdInstancePort, InstancesByPort); + MongoProcessLaucher.LaunchMongoProcess(out firstMongoProcess,PathToMongoServer+MongoServerExeName, firstInstanceCmd, firstInstancePort, InstancesByPort); + MongoProcessLaucher.LaunchMongoProcess(out secondMongoProcess, PathToMongoServer+MongoServerExeName, secondInstanceCmd, secondInstancePort, InstancesByPort); + + foreach (var address in _mongoAddresses) + { + var clientSetup = new MongoClient(new MongoClientSettings() + { + ConnectionMode = ConnectionMode.Direct, + Server = new MongoServerAddress(address.Host, address.Port), + WriteConcern = WriteConcern.Acknowledged, + }); + var setupSrvReplicaSet = clientSetup.GetServer(); + SetupReplicaSet(setupSrvReplicaSet, _mongoAddresses); + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + + foreach (var address in _mongoAddresses) + { + var clientSetup = new MongoClient(new MongoClientSettings() + { + ConnectionMode = ConnectionMode.Direct, + Server = new MongoServerAddress(address.Host, address.Port), + WriteConcern = WriteConcern.Acknowledged, + }); + var setupSrvReplicaSet = clientSetup.GetServer(); + SetupReplicaSet(setupSrvReplicaSet, _mongoAddresses); + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + + + + //wait for the 3 instances to decide who is primary. 30 seconds should be enough + Thread.Sleep(TimeSpan.FromSeconds(10)); + + var client = new MongoClient(new MongoClientSettings() + { + ConnectionMode = ConnectionMode.ReplicaSet, + Servers = new List{ new MongoServerAddress("localhost", firstInstancePort), new MongoServerAddress("localhost", secondInstancePort),new MongoServerAddress("localhost",thirdInstancePort) }, + ReplicaSetName = ReplicaSetName, + WriteConcern = WriteConcern.Acknowledged, + ReadPreference = ReadPreference.Primary, + }); + + server = client.GetServer(); + } + + private void CleanupMongoDb(string pathToCleanup) + { + var info = new DirectoryInfo(pathToCleanup); + if (info.Exists) + { + info.Delete(true); + info.Create(); + } + else + { + info.Create(); + } + } + + private bool SetupReplicaSet(MongoServer srvConfigure, IEnumerable addresses) + { + int membersCount = WaitForLocalMongoToBeReady(srvConfigure); + + var mongoServerAddresses = addresses as MongoServerAddress[] ?? addresses.ToArray(); + if (membersCount == 0) + { + UInt16 serverId = 0; + var replReconfig = new CommandDocument + { + { + "replSetInitiate", new BsonDocument + { + {"_id", ReplicaSetName}, + { + "members", new BsonArray(new [] { + new BsonDocument + { + {"_id", serverId++}, + {"host", string.Format("{0}:{1}", mongoServerAddresses.First().Host, mongoServerAddresses.First().Port)}, + {"arbiterOnly", false} + }}) + // Never give priority 0 as the instance could not vote or initiate a vote + } + } + } + }; + + try + { + var result = srvConfigure.GetDatabase("admin").RunCommand(replReconfig); + } + catch(MongoCommandException ex) + { + return false; + } + } + if (membersCount != mongoServerAddresses.Count()) + { + UInt16 serverId = 0; + var replReconfig = new CommandDocument { + { "replSetReconfig", new BsonDocument { + {"_id", ReplicaSetName}, + {"version", 2 }, + { "members", new BsonArray( + from server in mongoServerAddresses + select new BsonDocument { + {"_id", serverId++}, + { "host", string.Format("{0}:{1}", server.Host, server.Port) }, + {"arbiterOnly", serverId == 3},}) + } } } }; + + try + { + var replReconfigResult = srvConfigure.GetDatabase("admin").RunCommand(replReconfig); + } + catch (MongoCommandException ex) + { + return false; + } + } + return true; + } + + private int WaitForLocalMongoToBeReady(MongoServer srvConfigure) + { + int retries = 0; + while (true) + { + CommandResult res; + try + { + res = srvConfigure.GetDatabase("admin").RunCommand("replSetGetStatus"); + } + catch (Exception ex) + { + if (retries == 3) + throw; + + if (ex.Message.Contains("EMPTYCONFIG")) + { + return 0; + } + if (ex.Message.Contains("Received replSetInitiate - should come online shortly")) + { + Thread.Sleep(TimeSpan.FromSeconds(5)); + continue; + } + + retries++; + // Maybe we got to wait a little before asking something to Mongo + Thread.Sleep(TimeSpan.FromSeconds(10)); + continue; + } + + var members = res.Response["members"].AsBsonArray; + return members.Count; + } + } + + [TearDown] + public void TearDown() + { + try + { + firstMongoProcess.Kill(); + } + catch + { + + } + try + { + secondMongoProcess.Kill(); + } + catch + { + + } + try + { + thirdMongoProcess.Kill(); + } + catch + { + + } + } + + [Test, Timeout(120000)] + public void TestWhenPrimaryGoesDownDriverDetectsTheSwitch() + { + server.Connect(TimeSpan.FromSeconds(10)); + Assert.IsNotNull(server.Primary); + Assert.AreEqual(3, server.Instances.Length); + Assert.AreEqual(1,server.Secondaries.Length); + Assert.AreEqual(1, server.Arbiters.Length); + //detect which instance is primary + var pimaryPort = server.Primary.Address.Port; + var primaryProcess = InstancesByPort[pimaryPort]; + //Simulate Mongo Primary Crash + primaryProcess.Kill(); + + Thread.Sleep(TimeSpan.FromSeconds(5)); + bool isConnected = false; + + while (!isConnected) + { + try + { + server.Connect(TimeSpan.FromSeconds(10)); + isConnected = true; + } + catch + { + //ignore Exceptions + Thread.Sleep(1000); + } + } + + Assert.AreNotEqual(pimaryPort,server.Primary.Address.Port); + } + } + + public static class MongoProcessLaucher + { + public static void LaunchMongoProcess(out Process process, string executable ,string cmdLine, int port, Dictionary instances) + { + process = Process.Start(executable, cmdLine); + instances.Add(port,process); + } + } +} \ No newline at end of file diff --git a/MongoDB.DriverUnitTests/MongoDB.DriverUnitTests.csproj b/MongoDB.DriverUnitTests/MongoDB.DriverUnitTests.csproj index ff307b0..a43c319 100644 --- a/MongoDB.DriverUnitTests/MongoDB.DriverUnitTests.csproj +++ b/MongoDB.DriverUnitTests/MongoDB.DriverUnitTests.csproj @@ -1,256 +1,257 @@ - - - - Debug - AnyCPU - 9.0.30729 - 2.0 - {FBBF0D71-107F-49C4-8858-B3A4DC24AA33} - Library - Properties - MongoDB.DriverUnitTests - MongoDB.DriverUnitTests - v3.5 - 512 - - - 3.5 - - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - AllRules.ruleset - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - AllRules.ruleset - - - false - - - - - - - - False - ..\Tools\NUnit\nunit.framework.dll - - - - - 3.5 - - - - - - 3.5 - - - 3.5 - - - - - - - - - Properties\GlobalAssemblyInfo.cs - - - TestEnvironment.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {0E9A3A2A-49CD-4F6C-847C-DC79B4B65CE6} - MongoDB.Bson - - - {AE5166CD-76B0-4911-BD80-CED9521F37A1} - MongoDB.Driver - - - - - False - .NET Framework 3.5 SP1 Client Profile - false - - - False - .NET Framework 3.5 SP1 - true - - - False - Windows Installer 3.1 - true - - - - - PreserveNewest - - - + + + + Debug + AnyCPU + 9.0.30729 + 2.0 + {FBBF0D71-107F-49C4-8858-B3A4DC24AA33} + Library + Properties + MongoDB.DriverUnitTests + MongoDB.DriverUnitTests + v3.5 + 512 + + + 3.5 + + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + AllRules.ruleset + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + AllRules.ruleset + + + false + + + + + + + + False + ..\Tools\NUnit\nunit.framework.dll + + + + + 3.5 + + + + + + 3.5 + + + 3.5 + + + + + + + + + Properties\GlobalAssemblyInfo.cs + + + TestEnvironment.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {0E9A3A2A-49CD-4F6C-847C-DC79B4B65CE6} + MongoDB.Bson + + + {AE5166CD-76B0-4911-BD80-CED9521F37A1} + MongoDB.Driver + + + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + true + + + False + Windows Installer 3.1 + true + + + + + PreserveNewest + + + + --> \ No newline at end of file diff --git a/MongoDB.DriverUnitTests/MongoServerTests.cs b/MongoDB.DriverUnitTests/MongoServerTests.cs index 9236013..55584ce 100644 --- a/MongoDB.DriverUnitTests/MongoServerTests.cs +++ b/MongoDB.DriverUnitTests/MongoServerTests.cs @@ -218,7 +218,7 @@ namespace MongoDB.DriverUnitTests public void TestReconnect() { _server.Reconnect(); - Assert.AreEqual(MongoServerState.Connected, _server.State); + Assert.IsTrue(_server.State == MongoServerState.Connected || _server.State == MongoServerState.ConnectedToSubset); } [Test] -- 1.8.1.msysgit.1