Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-13485

Fix the checkpoint failure with evict generation drain timed out

    • StorEng - Defined Pipeline

      format-stress-sanitizer-ppc-test-2 on rhel8-ppc

      Host: rhel81-ppc-16.pic.build.10gen.cc
      Project: wiredtiger
      Commit: cf768a3b

      Task Logs:

      format-stress-sanitizer-ppc-test-2 task_log

      Logs:

          [1724824779:492662][16474:0x7fff74f9f170], t, file:T00007.wt, WT_SESSION.checkpoint: [WT_VERB_GENERATION][ERROR]: evict generation drain timed out
          [1724824779:492711][16474:0x7fff74f9f170], t, file:T00007.wt, WT_SESSION.checkpoint: [WT_VERB_DEFAULT][ERROR]: __gen_drain_callback, 216: WiredTiger assertion failed: '0'. Expression returned false
          [1724824779:492719][16474:0x7fff74f9f170], t, file:T00007.wt, WT_SESSION.checkpoint: [WT_VERB_DEFAULT][ERROR]: __wt_abort, 28: aborting WiredTiger library
          [1724824775:177746][13986:0x7fff96fdf170], t, file:T00002.wt, WT_SESSION.checkpoint: [WT_VERB_GENERATION][ERROR]: evict generation drain timed out
          [1724824775:177809][13986:0x7fff96fdf170], t, file:T00002.wt, WT_SESSION.checkpoint: [WT_VERB_DEFAULT][ERROR]: __gen_drain_callback, 216: WiredTiger assertion failed: '0'. Expression returned false
          [1724824775:177816][13986:0x7fff96fdf170], t, file:T00002.wt, WT_SESSION.checkpoint: [WT_VERB_DEFAULT][ERROR]: __wt_abort, 28: aborting WiredTiger library
      

      logs

      format-stress-sanitizer-ppc-test-2 task_log

      Logs:

      #0  0x00007fffa7934488 in raise () from /lib64/libc.so.6
      #0  0x00007fffa7934488 in raise () from /lib64/libc.so.6
      #1  0x00007fffa791450c in abort () from /lib64/libc.so.6
      #2  0x00007fffa7e1c7a0 in __wt_abort (session=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/os_common/os_abort.c:30
      #3  0x00007fffa7ed3a98 in __gen_drain_callback (session=0x10037cee2d0, array_session=0x10037cebed0, exit_walkp=0x7fff74f9cfb0, cookiep=0x7fff74f9d020) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:216
      #4  0x00007fffa7ebcdc0 in __wt_session_array_walk (session=0x10037cee2d0, walk_func=0x7fffa7ed3684 <__gen_drain_callback>, skip_internal=<optimized out>, cookiep=0x7fff74f9d020) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/session/session_helper.c:59
      #5  0x00007fffa7ed34f4 in __gen_drain (session=0x10037cee2d0, which=<optimized out>, generation=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:238
      #6  0x00007fffa7ed3c5c in __wt_gen_next_drain (session=<optimized out>, which=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:101
      #7  0x00007fffa7cb06cc in __wt_sync_file (session=0x10037cee2d0, syncop=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/btree/bt_sync.c:237
      #8  0x00007fffa7f0ccf8 in __checkpoint_tree (session=session@entry=0x10037cee2d0, is_checkpoint=is_checkpoint@entry=true, cfg=cfg@entry=0x7fff74f9e620) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:2418
      #9  0x00007fffa7f0d240 in __checkpoint_tree_helper (session=0x10037cee2d0, cfg=0x7fff74f9e620) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:2546
      #10 0x00007fffa7f04e0c in __checkpoint_apply_to_dhandles (session=session@entry=0x10037cee2d0, cfg=cfg@entry=0x7fff74f9e620, op=op@entry=0x7fffa7f0d1b8 <__checkpoint_tree_helper>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:377
      #11 0x00007fffa7f0e2ac in __txn_checkpoint (session=session@entry=0x10037cee2d0, cfg=0x7fff74f9e620) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1271
      #12 0x00007fffa7f0f8a0 in __txn_checkpoint_wrapper (session=session@entry=0x10037cee2d0, cfg=cfg@entry=0x7fff74f9e620) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1568
      #13 0x00007fffa7f0feb8 in __wt_txn_checkpoint (session=0x10037cee2d0, cfg=0x7fff74f9e620, waiting=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1646
      #14 0x00007fffa7eaf3c4 in __session_checkpoint (wt_session=0x10037cee2d0, config=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/session/session_api.c:2444
      #15 0x0000000010006d78 in checkpoint (arg=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/test/format/checkpoint.c:144
      #16 0x00007fffa7b78ba0 in start_thread () from /lib64/libpthread.so.0
      #17 0x00007fffa7a21398 in clone () from /lib64/libc.so.6
      #0  0x00007fffa69e4488 in raise () from /lib64/libc.so.6
      #0  0x00007fffa69e4488 in raise () from /lib64/libc.so.6
      #1  0x00007fffa69c450c in abort () from /lib64/libc.so.6
      #2  0x00007fffa6ecc7a0 in __wt_abort (session=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/os_common/os_abort.c:30
      #3  0x00007fffa6f83a98 in __gen_drain_callback (session=0x100256b1fa0, array_session=0x100256adda0, exit_walkp=0x7fff96fdd010, cookiep=0x7fff96fdd080) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:216
      #4  0x00007fffa6f6cdc0 in __wt_session_array_walk (session=0x100256b1fa0, walk_func=0x7fffa6f83684 <__gen_drain_callback>, skip_internal=<optimized out>, cookiep=0x7fff96fdd080) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/session/session_helper.c:59
      #5  0x00007fffa6f834f4 in __gen_drain (session=0x100256b1fa0, which=<optimized out>, generation=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:238
      #6  0x00007fffa6f83c5c in __wt_gen_next_drain (session=<optimized out>, which=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/support/generation.c:101
      #7  0x00007fffa6d606cc in __wt_sync_file (session=0x100256b1fa0, syncop=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/btree/bt_sync.c:237
      #8  0x00007fffa6fbccf8 in __checkpoint_tree (session=session@entry=0x100256b1fa0, is_checkpoint=is_checkpoint@entry=true, cfg=cfg@entry=0x7fff96fde680) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:2418
      #9  0x00007fffa6fbd240 in __checkpoint_tree_helper (session=0x100256b1fa0, cfg=0x7fff96fde680) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:2546
      #10 0x00007fffa6fb4e0c in __checkpoint_apply_to_dhandles (session=session@entry=0x100256b1fa0, cfg=cfg@entry=0x7fff96fde680, op=op@entry=0x7fffa6fbd1b8 <__checkpoint_tree_helper>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:377
      #11 0x00007fffa6fbe2ac in __txn_checkpoint (session=session@entry=0x100256b1fa0, cfg=0x7fff96fde680) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1271
      #12 0x00007fffa6fbf8a0 in __txn_checkpoint_wrapper (session=session@entry=0x100256b1fa0, cfg=cfg@entry=0x7fff96fde680) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1568
      #13 0x00007fffa6fbfeb8 in __wt_txn_checkpoint (session=0x100256b1fa0, cfg=0x7fff96fde680, waiting=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/txn/txn_ckpt.c:1646
      #14 0x00007fffa6f5f3c4 in __session_checkpoint (wt_session=0x100256b1fa0, config=<optimized out>) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/session/session_api.c:2444
      #15 0x00007fffa6db9968 in __ckpt_server (arg=0x100256b1fa0) at /data/mci/c395138cb58afa989956ea6856d6770f/wiredtiger/src/conn/conn_ckpt.c:100
      #16 0x00007fffa6c28ba0 in start_thread () from /lib64/libpthread.so.0
      #17 0x00007fffa6ad1398 in clone () from /lib64/libc.so.6
      

      logs

            Assignee:
            backlog-server-storage-engines [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            Xgen-BuildBaron-User xgen-buildbaron-user
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated: