Details
-
Bug
-
Resolution: Done
-
Major - P3
-
None
-
None
-
None
-
Storage Execution
-
ALL
Description
Under heavy insert load on a 2-node replica set, WT eviction appears to hang on the secondary.
Situation was both nodes running on a 24-cpu box, 32GB RAM, pushing 16 threads with vectored writes of 100 tiny documents at a time
Per Michael, this seems to be related to a snapshot pinning down the memory
- 24.71% mongod mongod [.] __rec_txn_read
|
- __rec_txn_read
|
- 99.65% __rec_row_leaf_insert
|
__wt_reconcile
|
__wt_evict
|
__wt_evict_page
|
__wt_page_in_func
|
__wt_row_search
|
__wt_btcur_insert
|
__curfile_insert
|
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
|
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
|
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
|
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
|
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
|
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
|
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
|
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
|
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
|
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
|
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
|
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
|
mongo::ThreadPool::_consumeTasks()
|
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
|
0x7f957762ea40
|
- 22.88% mongod mongod [.] __wt_row_modify
|
- __wt_row_modify
|
- 99.42% __split_multi_inmem
|
__wt_split_rewrite
|
__wt_evict
|
__wt_evict_page
|
__wt_page_in_func
|
__wt_row_search
|
__wt_btcur_insert
|
__curfile_insert
|
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
|
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
|
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
|
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
|
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
|
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
|
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
|
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
|
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
|
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
|
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
|
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
|
mongo::ThreadPool::_consumeTasks()
|
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
|
0x7f957762ea40
|
+ 0.58% __wt_split_rewrite
|
- 21.43% mongod mongod [.] __wt_row_search
|
- __wt_row_search
|
- 99.71% __split_multi_inmem
|
__wt_split_rewrite
|
__wt_evict
|
__wt_evict_page
|
__wt_page_in_func
|
__wt_row_search
|
__wt_btcur_insert
|
__curfile_insert
|
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
|
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
|
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
|
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
|
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
|
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
|
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
|
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
|
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
|
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
|
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
|
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
|
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
|
mongo::ThreadPool::_consumeTasks()
|
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
|
Attachments
Issue Links
- is duplicated by
-
SERVER-20201 Performance drop over time when running with enableReplSnapshotThread=true under insert only workload
-
- Closed
-