Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-6417

Fix not restoring tombstone in rollback to stable

    • Type: Icon: Bug Bug
    • Resolution: Fixed
    • Priority: Icon: Major - P3 Major - P3
    • WT10.0.0, 4.4.0-rc11, 4.7.0
    • Affects Version/s: None
    • Component/s: None
    • Labels:
      None
    • 8
    • Storage - Ra 2020-06-15, Storage - Ra 2020-06-29

      background_unique_indexes failed once after the wiredtiger drop:

      commit 816ba22112096f783ecbfd1f76b386942d5d9ee7
      Author: Luke Chen <luke.chen@mongodb.com>
      Date:   Tue Jun 9 15:58:09 2020 +1000
      
          Import wiredtiger: bf1d78126c2ff448b868f199aef22b6c528dd45d from branch mongodb-4.4
      
          ref: 4eb9f719ee..bf1d78126c
          for: 4.4.0-rc9
      
          WT-6353       Print out failing values and value history of prealloc
          WT-6375       Test to verify RTS fixes the inconsistent checkpoint data
          WT-6379       Split Evergreen stress test job into multiple shorter running jobs
          WT-6381       RTS to remove hs record that is appended to the update chain
          WT-6382       Use history store btree when searching
      
      
      
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-13] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 324.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 324.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":324,"x":324,"iter":6},"o2":{"_id":324},"ts":{"$timestamp":{"t":1591693960,"i":583}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.629Z"},"v":2,"stmtId":324,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":582}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-5] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 312.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 312.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":312,"x":312,"iter":6},"o2":{"_id":312},"ts":{"$timestamp":{"t":1591693960,"i":571}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.621Z"},"v":2,"stmtId":312,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":570}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-8] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 313.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 313.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":313,"x":313,"iter":6},"o2":{"_id":313},"ts":{"$timestamp":{"t":1591693960,"i":572}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.622Z"},"v":2,"stmtId":313,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":571}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.020+00:00 F  REPL     21238   [ReplWriterWorker-6] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 314.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 314.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":314,"x":314,"iter":6},"o2":{"_id":314},"ts":{"$timestamp":{"t":1591693960,"i":573}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.623Z"},"v":2,"stmtId":314,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":572}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-4] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 333.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 333.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":333,"x":333,"iter":6},"o2":{"_id":333},"ts":{"$timestamp":{"t":1591693960,"i":592}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.635Z"},"v":2,"stmtId":333,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":591}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-7] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 316.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 316.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":316,"x":316,"iter":6},"o2":{"_id":316},"ts":{"$timestamp":{"t":1591693960,"i":575}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.625Z"},"v":2,"stmtId":316,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":574}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.019+00:00 F  REPL     21238   [ReplWriterWorker-2] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 308.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 308.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":308,"x":308,"iter":6},"o2":{"_id":308},"ts":{"$timestamp":{"t":1591693960,"i":567}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.619Z"},"v":2,"stmtId":308,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":566}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.020+00:00 F  REPL     21238   [ReplWriterWorker-0] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 325.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 325.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":325,"x":325,"iter":6},"o2":{"_id":325},"ts":{"$timestamp":{"t":1591693960,"i":584}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.630Z"},"v":2,"stmtId":325,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":583}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.020+00:00 F  REPL     21238   [ReplWriterWorker-3] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 307.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 307.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":307,"x":307,"iter":6},"o2":{"_id":307},"ts":{"$timestamp":{"t":1591693960,"i":566}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.618Z"},"v":2,"stmtId":307,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":565}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.021+00:00 F  REPL     21238   [ReplWriterWorker-1] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 319.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 319.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":319,"x":319,"iter":6},"o2":{"_id":319},"ts":{"$timestamp":{"t":1591693960,"i":578}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.626Z"},"v":2,"stmtId":319,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":577}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.021+00:00 F  REPL     21238   [ReplWriterWorker-14] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 315.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 315.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":315,"x":315,"iter":6},"o2":{"_id":315},"ts":{"$timestamp":{"t":1591693960,"i":574}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.624Z"},"v":2,"stmtId":315,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":573}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.022+00:00 F  REPL     21238   [ReplWriterWorker-11] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 309.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 309.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":309,"x":309,"iter":6},"o2":{"_id":309},"ts":{"$timestamp":{"t":1591693960,"i":568}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.620Z"},"v":2,"stmtId":309,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":567}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.022+00:00 F  REPL     21238   [ReplWriterWorker-9] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 310.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 310.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":310,"x":310,"iter":6},"o2":{"_id":310},"ts":{"$timestamp":{"t":1591693960,"i":569}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.620Z"},"v":2,"stmtId":310,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":568}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.022+00:00 F  REPL     21238   [ReplWriterWorker-12] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 306.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 306.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":306,"x":306,"iter":6},"o2":{"_id":306},"ts":{"$timestamp":{"t":1591693960,"i":565}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.618Z"},"v":2,"stmtId":306,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":564}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.023+00:00 F  REPL     21238   [ReplWriterWorker-10] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 328.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 328.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":328,"x":328,"iter":6},"o2":{"_id":328},"ts":{"$timestamp":{"t":1591693960,"i":587}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.633Z"},"v":2,"stmtId":328,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":586}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.023+00:00 F  REPL     21238   [ReplWriterWorker-15] "Writer worker caught exception","attr":{"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 334.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 334.0 }","oplogEntry":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":334,"x":334,"iter":6},"o2":{"_id":334},"ts":{"$timestamp":{"t":1591693960,"i":593}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.636Z"},"v":2,"stmtId":334,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":592}},"t":3}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.023+00:00 F  REPL     21235   [initandlisten] "Failed to apply batch of operations","attr":{"numOperationsInBatch":695,"firstOperation":{"lsid":{"id":{"$uuid":"bb410aac-04d4-4066-afeb-c00a3e478d89"},"uid":{"$binary":{"base64":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","subType":"0"}}},"txnNumber":13,"op":"u","ns":"background_unique_indexes.test","ui":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"o":{"_id":306,"x":306,"iter":6},"o2":{"_id":306},"ts":{"$timestamp":{"t":1591693960,"i":565}},"t":3,"wall":{"$date":"2020-06-09T09:12:40.618Z"},"v":2,"stmtId":306,"prevOpTime":{"ts":{"$timestamp":{"t":1591693960,"i":564}},"t":3}},"lastOperation":{"op":"i","ns":"config.system.indexBuilds","ui":{"$uuid":"89db71f5-9ac2-4820-90cf-2267d75223f2"},"o":{"_id":{"$uuid":"54bbc277-4105-441f-be72-a570e0ed1a1b"},"collectionUUID":{"$uuid":"bd2120ea-5453-4edf-9cfd-837ed12c5376"},"commitQuorum":"votingMembers","indexNames":["x_1"]},"ts":{"$timestamp":{"t":1591693961,"i":148}},"t":3,"wall":{"$date":"2020-06-09T09:12:41.154Z"},"v":2},"failedWriterThread":0,"error":"DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 333.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 333.0 }"}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.028+00:00 F  REPL     21570   [initandlisten] "Caught exception during replication recovery","attr":{"error":{"code":11000,"codeName":"DuplicateKey","errmsg":"E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 333.0 }","keyPattern":{"_id":1},"keyValue":{"_id":333}}}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.028+00:00 F  CONTROL  4757800 [initandlisten] "Writing fatal message","attr":{"message":"terminate() called. An exception is active; attempting to gather more information"}
      [ReplicaSetFixture:job1:node1] | 2020-06-09T09:12:57.028+00:00 F  CONTROL  4757800 [initandlisten] "Writing fatal message","attr":{"message":"DBException::toString(): DuplicateKey{ keyPattern: { _id: 1 }, keyValue: { _id: 333.0 } }: E11000 duplicate key error collection: background_unique_indexes.test index: _id_ dup key: { _id: 333.0 }\nActual exception type: mongo::error_details::ExceptionForImpl<(mongo::ErrorCodes::Error)11000, mongo::AssertionException>\n"}
      

            Assignee:
            chenhao.qu@mongodb.com Chenhao Qu
            Reporter:
            chenhao.qu@mongodb.com Chenhao Qu
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: