|
Thanks for uploading the stack traces david.hows. From the first set of traces I can see that one thread is creating an index:
#2 0x0000000001bbae6f in __wt_spin_lock (session=0x3cfd900, t=<optimized out>) at src/third_party/wiredtiger/src/include/mutex.i:159
|
#3 __wt_session_create (session=0x3cfd900, uri=0x1ce447d8 "table:index-10731-8723464202028817165", config=0xdc9a5b8 "type=file,internal_page_max=16k,leaf_page_max=16k,checksum=on,prefix_compression=true,block_compressor=,,,,key_format=u,value_format=u,app_metadata=(formatVersion=6,infoObj={ \"v\" : 1, \"key\" : { \"x\" : "...) at src/third_party/wiredtiger/src/session/session_api.c:395
|
#4 0x0000000001bbb0de in __session_create (wt_session=0x3cfd900, uri=<optimized out>, config=0xdc9a5b8 "type=file,internal_page_max=16k,leaf_page_max=16k,checksum=on,prefix_compression=true,block_compressor=,,,,key_format=u,value_format=u,app_metadata=(formatVersion=6,infoObj={ \"v\" : 1, \"key\" : { \"x\" : "...) at src/third_party/wiredtiger/src/session/session_api.c:442
|
#5 0x00000000011e6f23 in mongo::WiredTigerIndex::Create (txn=<optimized out>, uri="table:index-10731-8723464202028817165", config="type=file,internal_page_max=16k,leaf_page_max=16k,checksum=on,prefix_compression=true,block_compressor=,,,,key_format=u,value_format=u,app_metadata=(formatVersion=6,infoObj={ \"v\" : 1, \"key\" : { \"x\" : "...) at src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp:210
|
#6 0x00000000011f32da in mongo::WiredTigerKVEngine::createSortedDataInterface (this=<optimized out>, opCtx=0x22c0cee0, ident=..., desc=<optimized out>) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:420
|
#7 0x0000000001126b15 in mongo::KVCollectionCatalogEntry::prepareForIndexBuild (this=0x1ce44f00, txn=0x22c0cee0, spec=0x2c7fdb80) at src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp:133
|
#8 0x0000000000ad5eef in mongo::IndexCatalog::IndexBuildBlock::init (this=0x2b88b660) at src/mongo/db/catalog/index_catalog.cpp:360
|
#9 0x0000000000adcb83 in mongo::MultiIndexBlock::init (this=this@entry=0x7f945b8820a0, indexSpecs=std::vector of length 1, capacity 1 = {...}) at src/mongo/db/catalog/index_create.cpp:179
|
#10 0x0000000000b2616a in mongo::CmdCreateIndex::run (this=<optimized out>, txn=0x22c0cee0, dbname="abcd1251", cmdObj=..., options=<optimized out>, errmsg=..., result=...) at src/mongo/db/commands/create_indexes.cpp:224
|
#11 0x0000000000bfd220 in mongo::Command::run (this=this@entry=0x2106ba0 <mongo::cmdCreateIndex>, txn=txn@entry=0x22c0cee0, request=..., replyBuilder=replyBuilder@entry=0x7f945b882a50) at src/mongo/db/dbcommands.cpp:1388
|
Another thread is creating a collection:
#0 0x00007f946500706d in fdatasync () from /usr/lib/libc.so.6
|
#1 0x0000000001b8bfc2 in __wt_handle_sync (fd=14828) at src/third_party/wiredtiger/src/os_posix/os_fsync.c:44
|
#2 0x0000000001b8c3fa in __wt_fsync (session=session@entry=0x3cfdc00, fh=0x28424a20) at src/third_party/wiredtiger/src/os_posix/os_fsync.c:129
|
#3 0x0000000001b06195 in __wt_block_manager_create (session=session@entry=0x3cfdc00, filename=filename@entry=0x2aef1bd5 "collection-10730-8723464202028817165.wt", allocsize=4096) at src/third_party/wiredtiger/src/block/block_open.c:99
|
#4 0x0000000001ba7967 in __create_file (config=<optimized out>, exclusive=false, uri=0x2aef1bd0 "file:collection-10730-8723464202028817165.wt", session=0x3cfdc00) at src/third_party/wiredtiger/src/schema/schema_create.c:104
|
#5 __wt_schema_create (session=session@entry=0x3cfdc00, uri=<optimized out>, uri@entry=0x2aef1bd0 "file:collection-10730-8723464202028817165.wt", config=<optimized out>) at src/third_party/wiredtiger/src/schema/schema_create.c:691
|
#6 0x0000000001ba8595 in __create_colgroup (session=session@entry=0x3cfdc00, name=<optimized out>, exclusive=exclusive@entry=false, config=config@entry=0x28aaa418 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1)") at src/third_party/wiredtiger/src/schema/schema_create.c:266
|
#7 0x0000000001ba7f67 in __create_table (config=0x28aaa418 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1)", exclusive=false, name=0x2c9ceb38 "table:collection-10730-8723464202028817165", session=0x3cfdc00) at src/third_party/wiredtiger/src/schema/schema_create.c:616
|
#8 __wt_schema_create (session=session@entry=0x3cfdc00, uri=<optimized out>, uri@entry=0x2c9ceb38 "table:collection-10730-8723464202028817165", config=config@entry=0x28aaa418 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1)") at src/third_party/wiredtiger/src/schema/schema_create.c:697
|
#9 0x0000000001bbaeab in __wt_session_create (session=0x3cfdc00, uri=0x2c9ceb38 "table:collection-10730-8723464202028817165", config=0x28aaa418 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1)") at src/third_party/wiredtiger/src/session/session_api.c:395
|
#10 0x0000000001bbb0de in __session_create (wt_session=0x3cfdc00, uri=<optimized out>, config=0x28aaa418 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1)") at src/third_party/wiredtiger/src/session/session_api.c:442
|
#11 0x00000000011f2c24 in mongo::WiredTigerKVEngine::createRecordStore (this=<optimized out>, opCtx=<optimized out>, ns=..., ident=..., options=...) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:363
|
#12 0x0000000001129003 in mongo::KVDatabaseCatalogEntry::createCollection (this=0xa3acfa0, txn=0x24b3a530, ns=..., options=..., allocateDefaultSpace=true) at src/mongo/db/storage/kv/kv_database_catalog_entry.cpp:213
|
Another thread is attempting to get size statistics for the listDatabases command:
#2 0x0000000001baa02b in __wt_spin_lock (session=0x3cfd300, t=<optimized out>) at src/third_party/wiredtiger/src/include/mutex.i:159
|
#3 __schema_add_table (tablep=<synthetic pointer>, ok_incomplete=false, namelen=30, name=0x2b8880c9 "index-23236-808695384733862891", session=0x3cfd300) at src/third_party/wiredtiger/src/schema/schema_list.c:26
|
#4 __wt_schema_get_table (session=session@entry=0x3cfd300, name=name@entry=0x2b8880c9 "index-23236-808695384733862891", namelen=30, ok_incomplete=ok_incomplete@entry=false, tablep=tablep@entry=0x7f945b47ddf8) at src/third_party/wiredtiger/src/schema/schema_list.c:98
|
#5 0x0000000001bb60e3 in __wt_curstat_table_init (session=session@entry=0x3cfd300, uri=uri@entry=0x2b8880c3 "table:index-23236-808695384733862891", cfg=0x22e01ca0, cst=0x2b870000) at src/third_party/wiredtiger/src/schema/schema_stat.c:72
|
#6 0x0000000001b6733e in __wt_curstat_init (session=session@entry=0x3cfd300, uri=<optimized out>, uri@entry=0x2b8880b8 "statistics:table:index-23236-808695384733862891", cfg=<optimized out>, cst=<optimized out>) at src/third_party/wiredtiger/src/cursor/cur_stat.c:459
|
#7 0x0000000001b67c2e in __wt_curstat_open (session=session@entry=0x3cfd300, uri=uri@entry=0x2b8880b8 "statistics:table:index-23236-808695384733862891", cfg=cfg@entry=0x7f945b47df70, cursorp=cursorp@entry=0x7f945b47df68) at src/third_party/wiredtiger/src/cursor/cur_stat.c:584
|
#8 0x0000000001bba035 in __wt_open_cursor (session=session@entry=0x3cfd300, uri=<optimized out>, uri@entry=0x2b8880b8 "statistics:table:index-23236-808695384733862891", owner=owner@entry=0x0, cfg=cfg@entry=0x7f945b47df70, cursorp=cursorp@entry=0x7f945b47df68) at src/third_party/wiredtiger/src/session/session_api.c:296
|
#9 0x0000000001bba5c7 in __session_open_cursor (wt_session=0x3cfd300, uri=0x2b8880b8 "statistics:table:index-23236-808695384733862891", to_dup=0x0, config=<optimized out>, cursorp=0x7f945b47e010) at src/third_party/wiredtiger/src/session/session_api.c:359
|
#10 0x0000000001213fa6 in mongo::WiredTigerUtil::getStatisticsValue (session=session@entry=0x3cfd300, uri="statistics:table:index-23236-808695384733862891", config="statistics=(size)", statisticsKey=statisticsKey@entry=2009) at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp:267
|
#11 0x0000000001214c5e in mongo::WiredTigerUtil::getStatisticsValueAs<long> (maximumResultType=9223372036854775807, statisticsKey=2009, config="statistics=(size)", uri="statistics:table:index-23236-808695384733862891", session=0x3cfd300) at src/mongo/db/storage/wiredtiger/wiredtiger_util.h:277
|
#12 mongo::WiredTigerUtil::getStatisticsValueAs<long> (statisticsKey=2009, config="statistics=(size)", uri="statistics:table:index-23236-808695384733862891", session=0x3cfd300) at src/mongo/db/storage/wiredtiger/wiredtiger_util.h:267
|
#13 mongo::WiredTigerUtil::getIdentSize (s=0x3cfd300, uri="table:index-23236-808695384733862891") at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp:299
|
#14 0x00000000011f1eb4 in mongo::WiredTigerKVEngine::getIdentSize (this=0x3b48000, opCtx=<optimized out>, ident=...) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:259
|
#15 0x00000000011276d7 in mongo::KVDatabaseCatalogEntry::sizeOnDisk (this=0xc921c70, opCtx=0x2bb27cd0) at src/mongo/db/storage/kv/kv_database_catalog_entry.cpp:151
|
#16 0x0000000000b675e3 in mongo::CmdListDatabases::run (this=<optimized out>, txn=<optimized out>, dbname=..., jsobj=..., errmsg=..., result=...) at src/mongo/db/commands/list_databases.cpp:107
|
#17 0x0000000000bfd220 in mongo::Command::run (this=this@entry=0x21076a0 <mongo::cmdListDatabases>, txn=txn@entry=0x2bb27cd0, request=..., replyBuilder=replyBuilder@entry=0x7f945b47f0f0) at src/mongo/db/dbcommands.cpp:1388
|
|