-
Type:
Bug
-
Resolution: Done
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: WiredTiger
-
Labels:None
-
Storage Execution
-
ALL
Under heavy insert load on a 2-node replica set, WT eviction appears to hang on the secondary.
Situation was both nodes running on a 24-cpu box, 32GB RAM, pushing 16 threads with vectored writes of 100 tiny documents at a time
Per Michael, this seems to be related to a snapshot pinning down the memory
- 24.71% mongod mongod [.] __rec_txn_read
- __rec_txn_read
- 99.65% __rec_row_leaf_insert
__wt_reconcile
__wt_evict
__wt_evict_page
__wt_page_in_func
__wt_row_search
__wt_btcur_insert
__curfile_insert
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
mongo::ThreadPool::_consumeTasks()
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
0x7f957762ea40
- 22.88% mongod mongod [.] __wt_row_modify
- __wt_row_modify
- 99.42% __split_multi_inmem
__wt_split_rewrite
__wt_evict
__wt_evict_page
__wt_page_in_func
__wt_row_search
__wt_btcur_insert
__curfile_insert
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
mongo::ThreadPool::_consumeTasks()
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
0x7f957762ea40
+ 0.58% __wt_split_rewrite
- 21.43% mongod mongod [.] __wt_row_search
- __wt_row_search
- 99.71% __split_multi_inmem
__wt_split_rewrite
__wt_evict
__wt_evict_page
__wt_page_in_func
__wt_row_search
__wt_btcur_insert
__curfile_insert
mongo::WiredTigerIndexUnique::_insert(__wt_cursor*, mongo::BSONObj const&, mongo::Record
mongo::WiredTigerIndex::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo::R
mongo::IndexAccessMethod::insert(mongo::OperationContext*, mongo::BSONObj const&, mongo:
mongo::IndexCatalog::_indexFilteredRecords(mongo::OperationContext*, mongo::IndexCatalog
mongo::IndexCatalog::_indexRecords(mongo::OperationContext*, mongo::IndexCatalogEntry*,
mongo::IndexCatalog::indexRecords(mongo::OperationContext*, std::vector<mongo::BsonRecor
mongo::Collection::_insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterat
mongo::Collection::insertDocuments(mongo::OperationContext*, __gnu_cxx::__normal_iterato
mongo::Collection::insertDocument(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::applyOperation_inlock(mongo::OperationContext*, mongo::Database*, mongo::BS
std::_Function_handler<mongo::Status (mongo::OperationContext*, mongo::Database*, mongo:
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool,
mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, bool)
mongo::repl::multiSyncApply(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> >
mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*)
mongo::ThreadPool::_consumeTasks()
mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::string const&)
- is duplicated by
-
-
- Closed
-