-
Type: Bug
-
Resolution: Fixed
-
Priority: Major - P3
-
Affects Version/s: None
-
Component/s: None
-
5
-
Storage Engines 2020-03-23
The concurrency_simultaneous_replication_wiredtiger_cursor_sweeps test is failing with a couple of different symptoms.
The shell command:
python3 \
buildscripts/resmoke.py --suites=concurrency_simultaneous_replication_wiredtiger_cursor_sweeps \
--storageEngine=wiredTiger --continueOnFailure --installDir=dist-test/bin --jobs=1 \
--mongodSetParameters='{enableIndexBuildCommitQuorum: false}' --shuffleMode=on \
--storageEngineCacheSizeGB=1
asserted in wt_rec_upd_select():
311 /* 312 * The checkpoint transaction is special. Make sure we never write metadata updates from a 313 * checkpoint in a concurrent session. 314 */ 315 WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || upd == NULL || 316 upd->txnid == WT_TXN_NONE || upd->txnid != S2C(session)->txn_global.checkpoint_state.id || 317 WT_SESSION_IS_CHECKPOINT(session));
and here's the core dump:
Thread 1 (Thread 0x7f69f005c700 (LWP 48341)): #0 0x00007f6a18260e97 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a18262801 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f6a1a1c026f in __wt_abort (session=0x7f6a0d348090) at src/os_common/os_abort.c:30 #3 0x00007f6a1a1f22b5 in __wt_rec_upd_select (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20, ripcip=0x0, vpack=0x0, upd_select=0x7f69f0057c98) at src/reconcile/rec_visibility.c:315 #4 0x00007f6a1a1e991f in __rec_row_leaf_insert (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20) at src/reconcile/rec_row.c:580 #5 0x00007f6a1a1e86d2 in __wt_rec_row_leaf (session=0x7f6a0d348090, r=0x7f69ba8d1020, pageref=0x7f69c26ca1e0, salvage=0x0) at src/reconcile/rec_row.c:752 #6 0x00007f6a1a1f49a9 in __reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172, page_lockedp=0x7f69f00580d6) at src/reconcile/rec_write.c:176 #7 0x00007f6a1a1f4331 in __wt_reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172) at src/reconcile/rec_write.c:87 #8 0x00007f6a1a177232 in __evict_review (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, evict_flags=4, inmem_splitp=0x7f69f00581bd) at src/evict/evict_page.c:627 #9 0x00007f6a1a1763c1 in __wt_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, previous_state=3 '\003', flags=4) at src/evict/evict_page.c:150 #10 0x00007f6a1a175fcb in __wt_page_release_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=0) at src/evict/evict_page.c:81 #11 0x00007f6a1a095620 in __wt_page_in_func (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:341 #12 0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d348090, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655 #13 0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69ce566720, srch_key=0x7f69ce566820, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425 #14 0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69ce566720, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383 #15 0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69ce566720) at src/btree/bt_cursor.c:587 #16 0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69ce566720) at src/cursor/cur_file.c:200 #17 0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d348090, key=0x7f69b1d74500 "file:collection-2525--8874772165521119671.wt", valuep=0x7f69f00587b8) at src/meta/meta_table.c:310 #18 0x00007f6a1a102bca in __conn_dhandle_config_set (session=0x7f6a0d348090) at src/conn/conn_dhandle.c:50 #19 0x00007f6a1a10285b in __wt_conn_dhandle_open (session=0x7f6a0d348090, cfg=0x0, flags=16) at src/conn/conn_dhandle.c:450 #20 0x00007f6a1a22e407 in __wt_session_get_dhandle (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", checkpoint=0x0, cfg=0x0, flags=16) at src/session/session_dhandle.c:511 #21 0x00007f6a1a201c0b in __create_file (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", exclusive=false, config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,che cksum=on,key_format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:122 #22 0x00007f6a1a200bf4 in __schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key_fo rmat=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:658 #23 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key _format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:688 #24 0x00007f6a1a2015d1 in __create_colgroup (session=0x7f6a0d348090, name=0x7f69c37a81a0 "colgroup:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_valu e_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:254 #25 0x00007f6a1a203074 in __create_table (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=6 4MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:582 #26 0x00007f6a1a200cf9 in __schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on, block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:664 #27 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum= on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:688 #28 0x00007f6a1a215539 in __wt_session_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum =on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:672 #29 0x00007f6a1a2195ea in __session_create (wt_session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum =on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:715 #30 0x000055582f76439e in mongo::WiredTigerKVEngine::createGroupedRecordStore (this=0x7f6a11b0c020, opCtx=0x7f69fcd0c1e0, ns="test3_fsmdb3rename_capped_collection_dbname_droptarget48_1.fsmcoll3", ident="collection-2525--8874772165 521119671", options=..., prefix=...) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:1529 #31 0x00005558303fec4e in mongo::DurableCatalogImpl::createCollection (this=0x7f6a11dac820, opCtx=0x7f69fcd0c1e0, nss=..., options=..., allocateDefaultSpace=true) at src/mongo/db/storage/durable_catalog_impl.cpp:829
For the record, checkpoint is running:
Thread 75 (Thread 0x7f6a05035700 (LWP 48186)): #0 0x00007f6a1833903f in select () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a1a1cbb00 in __wt_sleep (seconds=0, micro_seconds=1000) at src/os_posix/os_sleep.c:30 #2 0x00007f6a1a096ab0 in __wt_spin_backoff (yield_count=0x7f6a05033d58, sleep_usecs=0x7f6a05033d60) at ./src/include/misc.i:165 #3 0x00007f6a1a09584a in __wt_page_in_func (session=0x7f6a0d3498c8, ref=0x7f69c26ca1e0, flags=1040, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:422 #4 0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d3498c8, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655 #5 0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69e1d46020, srch_key=0x7f69e1d46120, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425 #6 0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69e1d46020, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383 #7 0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69e1d46020) at src/btree/bt_cursor.c:587 #8 0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69e1d46020) at src/cursor/cur_file.c:200 #9 0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", valuep=0x7f69da904f40) at src/meta/meta_table.c:310 #10 0x00007f6a1a1b9d40 in __wt_meta_track_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt") at src/meta/meta_track.c:432 #11 0x00007f6a1a1b7492 in __wt_metadata_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", value=0x7f69cebf0420 "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersio n=1,oplogKeyExtractionVersion=1),assert=(commit_timestamp=none,durable_timestamp=none,read_timestamp=none),block_allocation=best,block_compressor=snappy,cache_resident=false,checksum=on,collator=,columns=,dictionary=0,encryption=( keyid=,name=),format=btree,huffman_key=,huffman_value=,id=20,ignore_in_memory_cache_size=false,internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_ke y_max=0,leaf_page_max=32KB,leaf_value_max=64MB,log=(enabled=true),memory_page_image_max=0,memory_page_max=10m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,prefix_compression_min=4,split_deepen_min_child=0,split_dee pen_per_child=0,split_pct=90,value_format=u,version=(major=1,minor=1),checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=6 0522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)") at src/meta/meta_ table.c:216 #12 0x00007f6a1a1b3c54 in __ckpt_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", v=0x7f69c4b9c020 "checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e 219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=60522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11 320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)", use_base=true) at src/meta/meta_ckpt.c:195 #13 0x00007f6a1a1b6366 in __wt_meta_ckptlist_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", ckptbase=0x7f69c49ab020, ckptlsn=0x7f6a050343a8) at src/meta/meta_ckpt.c:790 #14 0x00007f6a1a253d0c in __checkpoint_tree (session=0x7f6a0d3498c8, is_checkpoint=true, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1637 #15 0x00007f6a1a256b95 in __checkpoint_tree_helper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1718 #16 0x00007f6a1a256aa6 in __checkpoint_apply_to_dhandles (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, op=0x7f6a1a256b00 <__checkpoint_tree_helper>) at src/txn/txn_ckpt.c:199 #17 0x00007f6a1a254af0 in __txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:864 #18 0x00007f6a1a253444 in __txn_checkpoint_wrapper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1076 #19 0x00007f6a1a253262 in __wt_txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, waiting=true) at src/txn/txn_ckpt.c:1133 #20 0x00007f6a1a225056 in __session_checkpoint (wt_session=0x7f6a0d3498c8, config=0x55583278ea9c "use_timestamp=true") at src/session/session_api.c:1979 #21 0x000055582f77833b in mongo::WiredTigerKVEngine::WiredTigerCheckpointThread::run (this=0x7f6a0b43bf20) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:549