Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-55278

replSetReconfig ignoring specified version in command

XMLWordPrintableJSON

    • Type: Icon: Question Question
    • Resolution: Works as Designed
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 4.4.4
    • Component/s: Replication
    • Labels:
      None
    • Replication

      There was a cluster on Atlas continuing to fail on rs.reconfig due to the version of the replSetConfig not matching what was expected. Attached mongod logs show that the reconfig command had specified version:6 yet the new config created had much larger version number being incremented by every run. Eventually the version number became too large and version field value of 2147501593 is out of range error was thrown during the reconfig.

      I'm curious if this is an expected behavior or a bug in how we are handling replSetConfig versions.

      {"t":{"$date":"2021-03-16T20:50:19.841+00:00"},"s":"I",  "c":"REPL",     "id":21352,   "ctx":"conn205053","msg":"replSetReconfig admin command received from client","attr":{"newConfig":{"_id":"LibraCluster0-shard-0","version":6,"members":[{"slaveDelay":0,"buildIndexes":true,"host":"libracluster0-shard-00-00-4hdny.mongodb.net:27017","priority":6.5,"votes":1,"arbiterOnly":false,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"_id":0,"horizons":{"PRIVATE":"libracluster0-shard-00-00-pri.4hdny.mongodb.net:27017","PUBLIC":"libracluster0-shard-00-00.4hdny.mongodb.net:27017","MESH":"libracluster0-shard-00-00.4hdny.mesh.mongodb.net:30454"},"hidden":false},{"_id":1,"hidden":false,"arbiterOnly":false,"priority":7.0,"host":"libracluster0-shard-00-01-4hdny.mongodb.net:27017","slaveDelay":0,"horizons":{"PRIVATE":"libracluster0-shard-00-01-pri.4hdny.mongodb.net:27017","PUBLIC":"libracluster0-shard-00-01.4hdny.mongodb.net:27017","MESH":"libracluster0-shard-00-01.4hdny.mesh.mongodb.net:30454"},"tags":{"provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL","nodeType":"ELECTABLE"},"buildIndexes":true,"votes":1},{"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"hidden":false,"horizons":{"MESH":"libracluster0-shard-00-02.4hdny.mesh.mongodb.net:30454","PRIVATE":"libracluster0-shard-00-02-pri.4hdny.mongodb.net:27017","PUBLIC":"libracluster0-shard-00-02.4hdny.mongodb.net:27017"},"_id":2,"votes":1,"arbiterOnly":false,"buildIndexes":true,"slaveDelay":0,"host":"libracluster0-shard-00-02-4hdny.mongodb.net:27017","priority":7.0}],"settings":{"replicaSetId":{"$oid":"604fda5b42a6090a243ffdaa"},"heartbeatTimeoutSecs":10,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"catchUpTimeoutMillis":-1,"chainingAllowed":false,"electionTimeoutMillis":10000.0,"catchUpTakeoverDelayMillis":30000,"heartbeatIntervalMillis":2000},"protocolVersion":1}}}
{"t":{"$date":"2021-03-16T20:50:20.053+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"192.168.248.67:36324","connectionId":205110,"connectionCount":41}}
{"t":{"$date":"2021-03-16T20:50:20.088+00:00"},"s":"I",  "c":"REPL",     "id":21353,   "ctx":"conn205053","msg":"replSetReconfig config object parses ok","attr":{"numMembers":3}}
{"t":{"$date":"2021-03-16T20:50:20.088+00:00"},"s":"I",  "c":"REPL",     "id":51814,   "ctx":"conn205053","msg":"Persisting new config to disk"}
{"t":{"$date":"2021-03-16T20:50:20.091+00:00"},"s":"I",  "c":"REPL",     "id":21392,   "ctx":"conn205053","msg":"New replica set config in use","attr":{"config":{"_id":"LibraCluster0-shard-0","version":811109984,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":0,"host":"libracluster0-shard-00-00-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":6.5,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"horizons":{"MESH":"libracluster0-shard-00-00.4hdny.mesh.mongodb.net:30454","PRIVATE":"libracluster0-shard-00-00-pri.4hdny.mongodb.net:27017","PUBLIC":"libracluster0-shard-00-00.4hdny.mongodb.net:27017"},"slaveDelay":0,"votes":1},{"_id":1,"host":"libracluster0-shard-00-01-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":7.0,"tags":{"provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL","nodeType":"ELECTABLE"},"horizons":{"MESH":"libracluster0-shard-00-01.4hdny.mesh.mongodb.net:30454","PUBLIC":"libracluster0-shard-00-01.4hdny.mongodb.net:27017","PRIVATE":"libracluster0-shard-00-01-pri.4hdny.mongodb.net:27017"},"slaveDelay":0,"votes":1},{"_id":2,"host":"libracluster0-shard-00-02-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":7.0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"horizons":{"PUBLIC":"libracluster0-shard-00-02.4hdny.mongodb.net:27017","PRIVATE":"libracluster0-shard-00-02-pri.4hdny.mongodb.net:27017","MESH":"libracluster0-shard-00-02.4hdny.mesh.mongodb.net:30454"},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":false,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"604fda5b42a6090a243ffdaa"}}}}}

      Out of range:

      {"t":{"$date":"2021-03-17T19:55:00.542+00:00"},"s":"E",  "c":"REPL",     "id":21420,   "ctx":"conn266426","msg":"replSetReconfig error while validating new config","attr":{"error":{"code":2,"codeName":"BadValue","errmsg":"version field value of 2147511677 is out of range"},"newConfig":{"_id":"LibraCluster0-shard-0","version":2147511677,"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"members":[{"_id":0,"host":"libracluster0-shard-00-00-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":6.5,"tags":{"provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL","nodeType":"ELECTABLE"},"horizons":{"MESH":"libracluster0-shard-00-00.4hdny.mesh.mongodb.net:30454","PUBLIC":"libracluster0-shard-00-00.4hdny.mongodb.net:27017","PRIVATE":"libracluster0-shard-00-00-pri.4hdny.mongodb.net:27017"},"slaveDelay":0,"votes":1},{"_id":1,"host":"libracluster0-shard-00-01-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":7.0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"horizons":{"MESH":"libracluster0-shard-00-01.4hdny.mesh.mongodb.net:30454","PUBLIC":"libracluster0-shard-00-01.4hdny.mongodb.net:27017","PRIVATE":"libracluster0-shard-00-01-pri.4hdny.mongodb.net:27017"},"slaveDelay":0,"votes":1},{"_id":2,"host":"libracluster0-shard-00-02-4hdny.mongodb.net:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":7.0,"tags":{"nodeType":"ELECTABLE","provider":"AWS","region":"US_EAST_1","workloadType":"OPERATIONAL"},"horizons":{"MESH":"libracluster0-shard-00-02.4hdny.mesh.mongodb.net:30454","PUBLIC":"libracluster0-shard-00-02.4hdny.mongodb.net:27017","PRIVATE":"libracluster0-shard-00-02-pri.4hdny.mongodb.net:27017"},"slaveDelay":0,"votes":1}],"settings":{"chainingAllowed":false,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":{"$oid":"604fda5b42a6090a243ffdaa"}}}}}

        1. libracluster0-shard-00-02-4hdny.mongodb.net_2021-03-16T20_31_09_2021-03-16T21_00_00_mongodb.log
          8.65 MB
        2. mongodb.log
          76.93 MB

            Assignee:
            backlog-server-repl [DO NOT USE] Backlog - Replication Team
            Reporter:
            yuta.arai@mongodb.com Yuta Arai
            Votes:
            1 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: