Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-5822

Don't evict metadata updates from a running checkpoint

    • 5
    • Storage Engines 2020-03-23

      The concurrency_simultaneous_replication_wiredtiger_cursor_sweeps test is failing with a couple of different symptoms.

      The shell command:

      python3 \
             buildscripts/resmoke.py --suites=concurrency_simultaneous_replication_wiredtiger_cursor_sweeps \
             --storageEngine=wiredTiger --continueOnFailure --installDir=dist-test/bin --jobs=1 \
             --mongodSetParameters='{enableIndexBuildCommitQuorum: false}' --shuffleMode=on \
             --storageEngineCacheSizeGB=1
      

      asserted in wt_rec_upd_select():

      311	    /*
      312	     * The checkpoint transaction is special. Make sure we never write metadata updates from a
      313	     * checkpoint in a concurrent session.
      314	     */
      315	    WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || upd == NULL ||
      316	        upd->txnid == WT_TXN_NONE || upd->txnid != S2C(session)->txn_global.checkpoint_state.id ||
      317	        WT_SESSION_IS_CHECKPOINT(session));
      

      and here's the core dump:

      Thread 1 (Thread 0x7f69f005c700 (LWP 48341)):
      #0  0x00007f6a18260e97 in raise () from /lib/x86_64-linux-gnu/libc.so.6
      #1  0x00007f6a18262801 in abort () from /lib/x86_64-linux-gnu/libc.so.6
      #2  0x00007f6a1a1c026f in __wt_abort (session=0x7f6a0d348090) at src/os_common/os_abort.c:30
      #3  0x00007f6a1a1f22b5 in __wt_rec_upd_select (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20, ripcip=0x0, vpack=0x0, upd_select=0x7f69f0057c98) at src/reconcile/rec_visibility.c:315
      #4  0x00007f6a1a1e991f in __rec_row_leaf_insert (session=0x7f6a0d348090, r=0x7f69ba8d1020, ins=0x7f69dd521a20) at src/reconcile/rec_row.c:580
      #5  0x00007f6a1a1e86d2 in __wt_rec_row_leaf (session=0x7f6a0d348090, r=0x7f69ba8d1020, pageref=0x7f69c26ca1e0, salvage=0x0) at src/reconcile/rec_row.c:752
      #6  0x00007f6a1a1f49a9 in __reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172, page_lockedp=0x7f69f00580d6) at src/reconcile/rec_write.c:176
      #7  0x00007f6a1a1f4331 in __wt_reconcile (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, salvage=0x0, flags=172) at src/reconcile/rec_write.c:87
      #8  0x00007f6a1a177232 in __evict_review (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, evict_flags=4, inmem_splitp=0x7f69f00581bd) at src/evict/evict_page.c:627
      #9  0x00007f6a1a1763c1 in __wt_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, previous_state=3 '\003', flags=4) at src/evict/evict_page.c:150
      #10 0x00007f6a1a175fcb in __wt_page_release_evict (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=0) at src/evict/evict_page.c:81
      #11 0x00007f6a1a095620 in __wt_page_in_func (session=0x7f6a0d348090, ref=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:341
      #12 0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d348090, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655
      #13 0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69ce566720, srch_key=0x7f69ce566820, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425
      #14 0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69ce566720, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383
      #15 0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69ce566720) at src/btree/bt_cursor.c:587
      #16 0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69ce566720) at src/cursor/cur_file.c:200
      #17 0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d348090, key=0x7f69b1d74500 "file:collection-2525--8874772165521119671.wt", valuep=0x7f69f00587b8) at src/meta/meta_table.c:310
      #18 0x00007f6a1a102bca in __conn_dhandle_config_set (session=0x7f6a0d348090) at src/conn/conn_dhandle.c:50
      #19 0x00007f6a1a10285b in __wt_conn_dhandle_open (session=0x7f6a0d348090, cfg=0x0, flags=16) at src/conn/conn_dhandle.c:450
      #20 0x00007f6a1a22e407 in __wt_session_get_dhandle (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", checkpoint=0x0, cfg=0x0, flags=16) at src/session/session_dhandle.c:511
      #21 0x00007f6a1a201c0b in __create_file (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", exclusive=false, config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,che
      cksum=on,key_format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:122
      #22 0x00007f6a1a200bf4 in __schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key_fo
      rmat=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:658
      #23 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69d0990760 "file:collection-2525--8874772165521119671.wt", config=0x7f69c1d93120 "app_metadata=(formatVersion=1),block_compressor=snappy,checksum=on,key
      _format=q,leaf_value_max=64MB,log=(enabled=false),memory_page_max=10m,split_pct=90,type=file,value_format=u") at src/schema/schema_create.c:688
      #24 0x00007f6a1a2015d1 in __create_colgroup (session=0x7f6a0d348090, name=0x7f69c37a81a0 "colgroup:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_valu
      e_max=64MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:254
      #25 0x00007f6a1a203074 in __create_table (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", exclusive=false, config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=6
      4MB,checksum=on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:582
      #26 0x00007f6a1a200cf9 in __schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=on,
      block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:664
      #27 0x00007f6a1a200a55 in __wt_schema_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum=
      on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/schema/schema_create.c:688
      #28 0x00007f6a1a215539 in __wt_session_create (session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum
      =on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:672
      #29 0x00007f6a1a2195ea in __session_create (wt_session=0x7f6a0d348090, uri=0x7f69ce252920 "table:collection-2525--8874772165521119671", config=0x7f69c1d93020 "type=file,memory_page_max=10m,split_pct=90,leaf_value_max=64MB,checksum
      =on,block_compressor=snappy,,key_format=q,value_format=u,app_metadata=(formatVersion=1),log=(enabled=false)") at src/session/session_api.c:715
      #30 0x000055582f76439e in mongo::WiredTigerKVEngine::createGroupedRecordStore (this=0x7f6a11b0c020, opCtx=0x7f69fcd0c1e0, ns="test3_fsmdb3rename_capped_collection_dbname_droptarget48_1.fsmcoll3", ident="collection-2525--8874772165
      521119671", options=..., prefix=...) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:1529
      #31 0x00005558303fec4e in mongo::DurableCatalogImpl::createCollection (this=0x7f6a11dac820, opCtx=0x7f69fcd0c1e0, nss=..., options=..., allocateDefaultSpace=true) at src/mongo/db/storage/durable_catalog_impl.cpp:829
      

      For the record, checkpoint is running:

      Thread 75 (Thread 0x7f6a05035700 (LWP 48186)):
      #0  0x00007f6a1833903f in select () from /lib/x86_64-linux-gnu/libc.so.6
      #1  0x00007f6a1a1cbb00 in __wt_sleep (seconds=0, micro_seconds=1000) at src/os_posix/os_sleep.c:30
      #2  0x00007f6a1a096ab0 in __wt_spin_backoff (yield_count=0x7f6a05033d58, sleep_usecs=0x7f6a05033d60) at ./src/include/misc.i:165
      #3  0x00007f6a1a09584a in __wt_page_in_func (session=0x7f6a0d3498c8, ref=0x7f69c26ca1e0, flags=1040, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at src/btree/bt_read.c:422
      #4  0x00007f6a1a0e3d08 in __wt_page_swap_func (session=0x7f6a0d3498c8, held=0x7f6a11b57138, want=0x7f69c26ca1e0, flags=1024, func=0x7f6a1a276eb4 "__wt_row_search", line=425) at ./src/include/btree.i:1655
      #5  0x00007f6a1a0e2668 in __wt_row_search (cbt=0x7f69e1d46020, srch_key=0x7f69e1d46120, insert=false, leaf=0x0, leaf_safe=false, leaf_foundp=0x0) at src/btree/row_srch.c:425
      #6  0x00007f6a1a06649f in __cursor_row_search (cbt=0x7f69e1d46020, insert=false, leaf=0x0, leaf_foundp=0x0) at src/btree/bt_cursor.c:383
      #7  0x00007f6a1a066881 in __wt_btcur_search (cbt=0x7f69e1d46020) at src/btree/bt_cursor.c:587
      #8  0x00007f6a1a12b5fb in __curfile_search (cursor=0x7f69e1d46020) at src/cursor/cur_file.c:200
      #9  0x00007f6a1a1b6fed in __wt_metadata_search (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", valuep=0x7f69da904f40) at src/meta/meta_table.c:310
      #10 0x00007f6a1a1b9d40 in __wt_meta_track_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt") at src/meta/meta_track.c:432
      #11 0x00007f6a1a1b7492 in __wt_metadata_update (session=0x7f6a0d3498c8, key=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", value=0x7f69cebf0420 "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersio
      n=1,oplogKeyExtractionVersion=1),assert=(commit_timestamp=none,durable_timestamp=none,read_timestamp=none),block_allocation=best,block_compressor=snappy,cache_resident=false,checksum=on,collator=,columns=,dictionary=0,encryption=(
      keyid=,name=),format=btree,huffman_key=,huffman_value=,id=20,ignore_in_memory_cache_size=false,internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_ke
      y_max=0,leaf_page_max=32KB,leaf_value_max=64MB,log=(enabled=true),memory_page_image_max=0,memory_page_max=10m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,prefix_compression_min=4,split_deepen_min_child=0,split_dee
      pen_per_child=0,split_pct=90,value_format=u,version=(major=1,minor=1),checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=6
      0522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)") at src/meta/meta_
      table.c:216
      #12 0x00007f6a1a1b3c54 in __ckpt_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", v=0x7f69c4b9c020 "checkpoint=(WiredTigerCheckpoint.18=(addr=\"01e219f881e48a23bfb5e219f981e4de33dd90e
      219fa81e4aaf67dab808080e403a39fc0e4039b4fc0\",order=18,time=1583954549,size=60522496,newest_durable_ts=6803032943355756552,oldest_start_ts=6803029434367475720,oldest_start_txn=174,newest_stop_ts=-1,newest_stop_txn=-11,write_gen=11
      320)),checkpoint_backup_info=,checkpoint_lsn=(3,63520000)", use_base=true) at src/meta/meta_ckpt.c:195
      #13 0x00007f6a1a1b6366 in __wt_meta_ckptlist_set (session=0x7f6a0d3498c8, fname=0x7f69ceb46f20 "file:collection-16--8874772165521119671.wt", ckptbase=0x7f69c49ab020, ckptlsn=0x7f6a050343a8) at src/meta/meta_ckpt.c:790
      #14 0x00007f6a1a253d0c in __checkpoint_tree (session=0x7f6a0d3498c8, is_checkpoint=true, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1637
      #15 0x00007f6a1a256b95 in __checkpoint_tree_helper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1718
      #16 0x00007f6a1a256aa6 in __checkpoint_apply_to_dhandles (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, op=0x7f6a1a256b00 <__checkpoint_tree_helper>) at src/txn/txn_ckpt.c:199
      #17 0x00007f6a1a254af0 in __txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:864
      #18 0x00007f6a1a253444 in __txn_checkpoint_wrapper (session=0x7f6a0d3498c8, cfg=0x7f6a05034680) at src/txn/txn_ckpt.c:1076
      #19 0x00007f6a1a253262 in __wt_txn_checkpoint (session=0x7f6a0d3498c8, cfg=0x7f6a05034680, waiting=true) at src/txn/txn_ckpt.c:1133
      #20 0x00007f6a1a225056 in __session_checkpoint (wt_session=0x7f6a0d3498c8, config=0x55583278ea9c "use_timestamp=true") at src/session/session_api.c:1979
      #21 0x000055582f77833b in mongo::WiredTigerKVEngine::WiredTigerCheckpointThread::run (this=0x7f6a0b43bf20) at src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:549
      

            Assignee:
            chenhao.qu@mongodb.com Chenhao Qu
            Reporter:
            keith.bostic@mongodb.com Keith Bostic (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: