Details
-
Bug
-
Status: Closed
-
Major - P3
-
Resolution: Fixed
-
None
-
None
-
5
-
Storage Engines 2020-03-23
Description
The concurrency_simultaneous_replication_wiredtiger_cursor_sweeps test is failing with a couple of different symptoms.
The shell command:
python3 \
|
buildscripts/resmoke.py --suites=concurrency_simultaneous_replication_wiredtiger_cursor_sweeps \
|
--storageEngine=wiredTiger --continueOnFailure --installDir=dist-test/bin --jobs=1 \
|
--mongodSetParameters='{enableIndexBuildCommitQuorum: false}' --shuffleMode=on \
|
--storageEngineCacheSizeGB=1
|
asserted in wt_rec_upd_select():
311 /*
|
312 * The checkpoint transaction is special. Make sure we never write metadata updates from a
|
313 * checkpoint in a concurrent session.
|
314 */
|
315 WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || upd == NULL ||
|
316 upd->txnid == WT_TXN_NONE || upd->txnid != S2C(session)->txn_global.checkpoint_state.id ||
|
317 WT_SESSION_IS_CHECKPOINT(session));
|
and here's the core dump:
Thread 1 (Thread 0x7f69f005c700 (LWP 48341)):
|
#0 0x00007f6a18260e97 in raise () from /lib/x86_64-linux-gnu/libc.so.6
|
#1 0x00007f6a18262801 in abort () from /lib/x86_64-linux-gnu/libc.so.6
|
#2 0x00007f6a1a1c026f in __wt_abort (session=0x7f6a0d348090) at src/os_common/os_abort.c:30
|
#3 0x00007f6a1a1f22b5 in __wt_rec_upd_select (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20, ripcip=0x0, vpack=0x0, upd_select=0x7f69f0057c98) at src/reconcile/rec_visibility.c:315
|
#4 0x00007f6a1a1e991f in __rec_row_leaf_insert (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20) at src/reconcile/rec_row.c:580
|
#5 0x00007f6a1a1e86d2 in __wt_rec_row_leaf (session=0x7f6a0d348090, r=0x7f69ba8d1020, pageref=0x7f69c26ca1e0, salvage=0x0) at src/reconcile/rec_row.c:752
|
#6 0x00007f6a1a1f49a9 in __reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172, page_lockedp=0x7f69f00580d6) at src/reconcile/rec_write.c:176
|
#7 0x00007f6a1a1f4331 in __wt_reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172) at src/reconcile/rec_write.c:87
|
#8 0x00007f6a1a177232 in __evict_review (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, evict_flags=4, inmem_splitp=0x7f69f00581bd) at src/evict/evict_page.c:627
|
#9 0x00007f6a1a1763c1 in __wt_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, previous_state=3 '\003', flags=4) at src/evict/evict_page.c:150
|
#10 0x00007f6a1a175fcb in __wt_page_release_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=0) at src/evict/evict_page.c:81
|
#11 0x00007f6a1a095620 in __wt_page_in_func (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:341
|
#12 0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d348090, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655
|
#13 0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69ce566720, srch_key=0x7f69ce566820, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425
|
#14 0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69ce566720, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383
|
#15 0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69ce566720) at src/btree/bt_cursor.c:587
|
#16 0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69ce566720) at src/cursor/cur_file.c:200
|
#17 0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d348090, key=0x7f69b1d74500 "file:collection-2525--8874772165521119671.wt", valuep=0x7f69f00587b8) at src/meta/meta_table.c:310
|
#18 0x00007f6a1a102bca in __conn_dhandle_config_set (session=0x7f6a0d348090) at src/conn/conn_dhandle.c:50
|
#19 0x00007f6a1a10285b in __wt_conn_dhandle_open (session=0x7f6a0d348090, cfg=0x0, flags=16) at src/conn/conn_dhandle.c:450
|
#20 0x00007f6a1a22e407 in __wt_session_get_dhandle (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", checkpoint=0x0, cfg=0x0, flags=16) at src/session/session_dhandle.c:511
|
#21 0x00007f6a1a201c0b in __create_file (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", exclusive=false, config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,che
|
cksum=on,key_format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:122
|
#22 0x00007f6a1a200bf4 in __schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key_fo
|
rmat=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:658
|
#23 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key
|
_format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:688
|
#24 0x00007f6a1a2015d1 in __create_colgroup (session=0x7f6a0d348090, name=0x7f69c37a81a0 "colgroup:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_valu
|
e_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:254
|
#25 0x00007f6a1a203074 in __create_table (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=6
|
4MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:582
|
#26 0x00007f6a1a200cf9 in __schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,
|
block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:664
|
#27 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=
|
on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:688
|
#28 0x00007f6a1a215539 in __wt_session_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum
|
=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:672
|
#29 0x00007f6a1a2195ea in __session_create (wt_session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum
|
=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:715
|
#30 0x000055582f76439e in mongo::WiredTigerKVEngine::createGroupedRecordStore (this=0x7f6a11b0c020, opCtx=0x7f69fcd0c1e0, ns="test3_fsmdb3rename_capped_collection_dbname_droptarget48_1.fsmcoll3", ident="collection-2525--8874772165
|
521119671", options=..., prefix=...) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:1529
|
#31 0x00005558303fec4e in mongo::DurableCatalogImpl::createCollection (this=0x7f6a11dac820, opCtx=0x7f69fcd0c1e0, nss=..., options=..., allocateDefaultSpace=true) at src/mongo/db/storage/durable_catalog_impl.cpp:829
|
For the record, checkpoint is running:
Thread 75 (Thread 0x7f6a05035700 (LWP 48186)):
|
#0 0x00007f6a1833903f in select () from /lib/x86_64-linux-gnu/libc.so.6
|
#1 0x00007f6a1a1cbb00 in __wt_sleep (seconds=0, micro_seconds=1000) at src/os_posix/os_sleep.c:30
|
#2 0x00007f6a1a096ab0 in __wt_spin_backoff (yield_count=0x7f6a05033d58, sleep_usecs=0x7f6a05033d60) at ./src/include/misc.i:165
|
#3 0x00007f6a1a09584a in __wt_page_in_func (session=0x7f6a0d3498c8, ref=0x7f69c26ca1e0, flags=1040, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:422
|
#4 0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d3498c8, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655
|
#5 0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69e1d46020, srch_key=0x7f69e1d46120, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425
|
#6 0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69e1d46020, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383
|
#7 0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69e1d46020) at src/btree/bt_cursor.c:587
|
#8 0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69e1d46020) at src/cursor/cur_file.c:200
|
#9 0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", valuep=0x7f69da904f40) at src/meta/meta_table.c:310
|
#10 0x00007f6a1a1b9d40 in __wt_meta_track_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt") at src/meta/meta_track.c:432
|
#11 0x00007f6a1a1b7492 in __wt_metadata_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", value=0x7f69cebf0420 "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersio
|
n=1,oplogKeyExtractionVersion=1),assert=(commit_timestamp=none,durable_timestamp=none,read_timestamp=none),block_allocation=best,block_compressor=snappy,cache_resident=false,checksum=on,collator=,columns=,dictionary=0,encryption=(
|
keyid=,name=),format=btree,huffman_key=,huffman_value=,id=20,ignore_in_memory_cache_size=false,internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_ke
|
y_max=0,leaf_page_max=32KB,leaf_value_max=64MB,log=(enabled=true),memory_page_image_max=0,memory_page_max=10m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,prefix_compression_min=4,split_deepen_min_child=0,split_dee
|
pen_per_child=0,split_pct=90,value_format=u,version=(major=1,minor=1),checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=6
|
0522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)") at src/meta/meta_
|
table.c:216
|
#12 0x00007f6a1a1b3c54 in __ckpt_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", v=0x7f69c4b9c020 "checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e
|
219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=60522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11
|
320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)", use_base=true) at src/meta/meta_ckpt.c:195
|
#13 0x00007f6a1a1b6366 in __wt_meta_ckptlist_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", ckptbase=0x7f69c49ab020, ckptlsn=0x7f6a050343a8) at src/meta/meta_ckpt.c:790
|
#14 0x00007f6a1a253d0c in __checkpoint_tree (session=0x7f6a0d3498c8, is_checkpoint=true, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1637
|
#15 0x00007f6a1a256b95 in __checkpoint_tree_helper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1718
|
#16 0x00007f6a1a256aa6 in __checkpoint_apply_to_dhandles (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, op=0x7f6a1a256b00 <__checkpoint_tree_helper>) at src/txn/txn_ckpt.c:199
|
#17 0x00007f6a1a254af0 in __txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:864
|
#18 0x00007f6a1a253444 in __txn_checkpoint_wrapper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1076
|
#19 0x00007f6a1a253262 in __wt_txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, waiting=true) at src/txn/txn_ckpt.c:1133
|
#20 0x00007f6a1a225056 in __session_checkpoint (wt_session=0x7f6a0d3498c8, config=0x55583278ea9c "use_timestamp=true") at src/session/session_api.c:1979
|
#21 0x000055582f77833b in mongo::WiredTigerKVEngine::WiredTigerCheckpointThread::run (this=0x7f6a0b43bf20) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:549
|