Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-49918

mongodump/restore does not respect order

    XMLWordPrintableJSON

Details

    • Bug
    • Status: Closed
    • Major - P3
    • Resolution: Works as Designed
    • 4.0.19, 3.6.19, 4.2.8
    • None
    • Storage, Tools
    • None
    • ALL
    • Hide

      db.dumpCapped.drop()
      db.createCollection( "dumpCapped", { capped: true, size:    50000 } )
      for(x=0;x<=10000;x++){
      	db.dumpCapped.insert({"x":x})
      }
      opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:1})
      { "_id" : ObjectId("5f1de6c78e567968a23bd09c"), "x" : 8481 }
      opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:-1})
      { "_id" : ObjectId("5f1de6c88e567968a23bd68b"), "x" : 10000 }
      opsmanager:PRIMARY>
       
      Verify count and min/max:
      opsmanager:PRIMARY> db.dumpCapped.count()
      1520
      opsmanager:PRIMARY> db.dumpCapped.aggregate(
      ...    [
      ...      {
      ...        $group:
      ...          {
      ...            _id: null,
      ...            minX: { $min: "$x" }
      ...          }
      ...      }
      ...    ]
      ... )
      { "_id" : null, "minX" : 8481 }
      opsmanager:PRIMARY> db.dumpCapped.aggregate(
      ...    [
      ...      {
      ...        $group:
      ...          {
      ...            _id: null,
      ...            maxX: { $max: "$x" }
      ...          }
      ...      }
      ...    ]
      ... )
      { "_id" : null, "maxX" : 10000 }
      opsmanager:PRIMARY>
       
      Dump:
      mongodump -h localhost:27017 -d ipc -c dumpCapped  -u ${USER} -p ${PW} --authenticationDatabase admin --out /data/backup/
      2020-07-26T22:28:38.012+0200	writing ipc.dumpCapped to
      2020-07-26T22:28:38.030+0200	done dumping ipc.dumpCapped (1520 documents)
       
      db.dumpCapped.drop()
      //recreate capped collection (even with same size):
      db.createCollection( "dumpCapped", { capped: true, size:    50000 } )
       
      mongorestore -h localhost:27017 -d ipc -c dumpCapped  -u ${USER} -p ${PW} --authenticationDatabase admin /data/backup/ipc/dumpCapped.bson
      2020-07-26T22:30:32.071+0200	checking for collection data in /data/backup/ipc/dumpCapped.bson
      2020-07-26T22:30:32.080+0200	reading metadata for ipc.dumpCapped from /data/backup/ipc/dumpCapped.metadata.json
      2020-07-26T22:30:32.081+0200	restoring ipc.dumpCapped from /data/backup/ipc/dumpCapped.bson
      2020-07-26T22:30:32.202+0200	no indexes to restore
      2020-07-26T22:30:32.202+0200	finished restoring ipc.dumpCapped (1520 documents)
      2020-07-26T22:30:32.202+0200	done
       
      //first inserted document is OK
      opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:1})
      { "_id" : ObjectId("5f1de8936b4a00b5fc18dd5e"), "x" : 8481 }
      //last inserted document is NOT OK:
      opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:-1})
      { "_id" : ObjectId("5f1de8936b4a00b5fc18e150"), "x" : 9491 }
       
      //verify count and in/max are OK:
      opsmanager:PRIMARY> db.dumpCapped.count()
      1520
      opsmanager:PRIMARY> db.dumpCapped.aggregate(
      ... ... ...    [
      ... ... ...      {
      ... ... ...        $group:
      ... ... ...          {
      ... ... ...            _id: null,
      ... ... ...            minX: { $min: "$x" }
      ... ... ...          }
      ... ... ...      }
      ... ... ...    ]
      ... ... ... )
      { "_id" : null, "minX" : 8481 }
      opsmanager:PRIMARY> db.dumpCapped.aggregate(
      ... ... ...    [
      ... ... ...      {
      ... ... ...        $group:
      ... ... ...          {
      ... ... ...            _id: null,
      ... ... ...            maxX: { $max: "$x" }
      ... ... ...          }
      ... ... ...      }
      ... ... ...    ]
      ... ... ... )
      { "_id" : null, "maxX" : 10000 }
      

      Show
      db.dumpCapped.drop() db.createCollection( "dumpCapped", { capped: true, size: 50000 } ) for(x=0;x<=10000;x++){ db.dumpCapped.insert({"x":x}) } opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:1}) { "_id" : ObjectId("5f1de6c78e567968a23bd09c"), "x" : 8481 } opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:-1}) { "_id" : ObjectId("5f1de6c88e567968a23bd68b"), "x" : 10000 } opsmanager:PRIMARY>   Verify count and min/max: opsmanager:PRIMARY> db.dumpCapped.count() 1520 opsmanager:PRIMARY> db.dumpCapped.aggregate( ... [ ... { ... $group: ... { ... _id: null, ... minX: { $min: "$x" } ... } ... } ... ] ... ) { "_id" : null, "minX" : 8481 } opsmanager:PRIMARY> db.dumpCapped.aggregate( ... [ ... { ... $group: ... { ... _id: null, ... maxX: { $max: "$x" } ... } ... } ... ] ... ) { "_id" : null, "maxX" : 10000 } opsmanager:PRIMARY>   Dump: mongodump -h localhost:27017 -d ipc -c dumpCapped -u ${USER} -p ${PW} --authenticationDatabase admin --out /data/backup/ 2020-07-26T22:28:38.012+0200 writing ipc.dumpCapped to 2020-07-26T22:28:38.030+0200 done dumping ipc.dumpCapped (1520 documents)   db.dumpCapped.drop() //recreate capped collection (even with same size): db.createCollection( "dumpCapped", { capped: true, size: 50000 } )   mongorestore -h localhost:27017 -d ipc -c dumpCapped -u ${USER} -p ${PW} --authenticationDatabase admin /data/backup/ipc/dumpCapped.bson 2020-07-26T22:30:32.071+0200 checking for collection data in /data/backup/ipc/dumpCapped.bson 2020-07-26T22:30:32.080+0200 reading metadata for ipc.dumpCapped from /data/backup/ipc/dumpCapped.metadata.json 2020-07-26T22:30:32.081+0200 restoring ipc.dumpCapped from /data/backup/ipc/dumpCapped.bson 2020-07-26T22:30:32.202+0200 no indexes to restore 2020-07-26T22:30:32.202+0200 finished restoring ipc.dumpCapped (1520 documents) 2020-07-26T22:30:32.202+0200 done   //first inserted document is OK opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:1}) { "_id" : ObjectId("5f1de8936b4a00b5fc18dd5e"), "x" : 8481 } //last inserted document is NOT OK: opsmanager:PRIMARY> db.dumpCapped.find().limit(1).sort({$natural:-1}) { "_id" : ObjectId("5f1de8936b4a00b5fc18e150"), "x" : 9491 }   //verify count and in/max are OK: opsmanager:PRIMARY> db.dumpCapped.count() 1520 opsmanager:PRIMARY> db.dumpCapped.aggregate( ... ... ... [ ... ... ... { ... ... ... $group: ... ... ... { ... ... ... _id: null, ... ... ... minX: { $min: "$x" } ... ... ... } ... ... ... } ... ... ... ] ... ... ... ) { "_id" : null, "minX" : 8481 } opsmanager:PRIMARY> db.dumpCapped.aggregate( ... ... ... [ ... ... ... { ... ... ... $group: ... ... ... { ... ... ... _id: null, ... ... ... maxX: { $max: "$x" } ... ... ... } ... ... ... } ... ... ... ] ... ... ... ) { "_id" : null, "maxX" : 10000 }

    Description

      Since capped collections (except the oplog) can't be resized (see feature request SERVER-49917), we had to dump the capped collection, drop it, recreate it with a new size and restore it.
      We observed that the insertion order of the old capped collection was not respected when restoring the dump into the newly created capped collection.
      This is a huge problem because the insertion order defines the order of the time to live within the capped collection (FIFO).

      Attachments

        Issue Links

          Activity

            People

              dmitry.agranat@mongodb.com Dmitry Agranat
              kay.agahd@idealo.de Kay Agahd
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: