Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-1851

test/format hung in LSM tree.

    • Type: Icon: Task Task
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • WT2.6.0
    • Affects Version/s: None
    • Component/s: None
    • Labels:

      Configured with a 30MB cache and 3 LSM threads (two workers). The call stacks are:

      Thread 16 (Thread 0x7f184dfd4700 (LWP 3206)):
      #0  0x00000034676e15e3 in select () from /lib64/libc.so.6
      WT-1  0x00000000004442d3 in __wt_sleep (seconds=0, micro_seconds=67000)
          at ../src/os_posix/os_sleep.c:23
      WT-2  0x000000000042bf2f in __evict_pass (session=0x13908e0)
          at ../src/evict/evict_lru.c:549
      WT-3  0x000000000042b029 in __evict_server (arg=0x13908e0)
          at ../src/evict/evict_lru.c:168
      WT-4  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-5  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 15 (Thread 0x7f184d5d3700 (LWP 3207)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x13913e0, cond=0x13d7fc0, 
          usecs=10000000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x00000000004216fa in __sweep_server (arg=0x13913e0)
          at ../src/conn/conn_sweep.c:171
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 14 (Thread 0x7f184b7d0700 (LWP 3208)):
      #0  0x00000034676e15e3 in select () from /lib64/libc.so.6
      WT-1  0x00000000004442d3 in __wt_sleep (seconds=0, micro_seconds=10000)
          at ../src/os_posix/os_sleep.c:23
      WT-2  0x0000000000437063 in __lsm_manager_run_server (session=0x1391960)
          at ../src/lsm/lsm_manager.c:423
      WT-3  0x00000000004374a9 in __lsm_worker_manager (arg=0x142baf0)
          at ../src/lsm/lsm_manager.c:514
      WT-4  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-5  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 13 (Thread 0x7f184cbd2700 (LWP 3209)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1391c20, cond=0x142c530, 
          usecs=10000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000043d388 in __lsm_worker (arg=0x142bb18)
          at ../src/lsm/lsm_worker.c:158
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 12 (Thread 0x7f184c1d1700 (LWP 3210)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1391ee0, cond=0x142c530, 
          usecs=10000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000043d388 in __lsm_worker (arg=0x142bb40)
      ---Type <return> to continue, or q <return> to quit---
          at ../src/lsm/lsm_worker.c:158
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 11 (Thread 0x7f183bfff700 (LWP 3415)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1390ba0, cond=0x142eef0, 
          usecs=10000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042baa2 in __evict_worker (arg=0x13d7690)
          at ../src/evict/evict_lru.c:405
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 10 (Thread 0x7f18499cd700 (LWP 3416)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1390e60, cond=0x142eef0, 
          usecs=10000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042baa2 in __evict_worker (arg=0x13d76b0)
          at ../src/evict/evict_lru.c:405
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 9 (Thread 0x7f184adcf700 (LWP 3418)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x13958a0, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x13958a0, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x000000000048a9cc in __wt_cache_full_check (session=0x13958a0)
          at ../src/include/cache.i:197
      WT-4  0x000000000048bc75 in __wt_page_in_func (session=0x13958a0, 
          ref=0x7f17b0067060, flags=0, file=0x6a005d "../src/btree/col_srch.c", 
          line=96) at ../src/btree/bt_page.c:84
      WT-5  0x00000000004a4504 in __wt_page_swap_func (session=0x13958a0, 
          held=0x7f17b00d8ac8, want=0x7f17b0067060, flags=0, 
          file=0x6a005d "../src/btree/col_srch.c", line=96)
          at ../src/include/btree.i:1153
      WT-6  0x00000000004a4d27 in __wt_col_search (session=0x13958a0, recno=164121, 
          leaf=0x0, cbt=0x7f17d426b0e0) at ../src/btree/col_srch.c:96
      WT-7  0x00000000004f0468 in __cursor_col_search (session=0x13958a0, 
          cbt=0x7f17d426b0e0) at ../src/btree/bt_cursor.c:225
      WT-8  0x00000000004f07d1 in __wt_btcur_search (cbt=0x7f17d426b0e0)
          at ../src/btree/bt_cursor.c:310
      WT-9  0x00000000004ba77e in __curfile_search (cursor=0x7f17d426b0e0)
          at ../src/cursor/cur_file.c:200
      WT-10 0x00000000004e7043 in __wt_bloom_hash_get (bloom=0x7f17d41b5590, 
      ---Type <return> to continue, or q <return> to quit---
          bhash=0x7f184adcebd0) at ../src/bloom/bloom.c:284
      WT-11 0x00000000004d1b1d in __clsm_lookup (clsm=0x7f17d4066c40, 
          value=0x7f17d4066d40) at ../src/lsm/lsm_cursor.c:1036
      WT-12 0x00000000004d2071 in __clsm_search (cursor=0x7f17d4066c40)
          at ../src/lsm/lsm_cursor.c:1097
      WT-13 0x000000000041084a in read_row (cursor=0x7f17d4066c40, key=0x7f184adcedf0, 
          keyno=72507) at ../../../test/format/ops.c:608
      WT-14 0x0000000000410385 in ops (arg=0x1f85af0) at ../../../test/format/ops.c:482
      WT-15 0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-16 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 8 (Thread 0x7f184a3ce700 (LWP 3419)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x13916a0, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x13916a0, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x00000000004cde98 in __wt_cache_full_check (session=0x13916a0)
          at ../src/include/cache.i:197
      WT-4  0x00000000004ce8cd in __cursor_enter (session=0x13916a0)
          at ../src/include/cursor.i:63
      WT-5  0x00000000004cf23c in __clsm_enter (clsm=0x7f17ac1f8e10, reset=1, update=0)
          at ../src/lsm/lsm_cursor.c:243
      WT-6  0x00000000004d204e in __clsm_search (cursor=0x7f17ac1f8e10)
          at ../src/lsm/lsm_cursor.c:1095
      WT-7  0x000000000041084a in read_row (cursor=0x7f17ac1f8e10, key=0x7f184a3cddf0, 
          keyno=94056) at ../../../test/format/ops.c:608
      WT-8  0x0000000000410385 in ops (arg=0x1f85b48) at ../../../test/format/ops.c:482
      WT-9  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-10 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 7 (Thread 0x7f1848fcc700 (LWP 3420)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x13955e0, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x13955e0, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x00000000004cde98 in __wt_cache_full_check (session=0x13955e0)
          at ../src/include/cache.i:197
      WT-4  0x00000000004ce8cd in __cursor_enter (session=0x13955e0)
          at ../src/include/cursor.i:63
      WT-5  0x00000000004cf23c in __clsm_enter (clsm=0x7f184404ed40, reset=1, update=0)
          at ../src/lsm/lsm_cursor.c:243
      WT-6  0x00000000004d204e in __clsm_search (cursor=0x7f184404ed40)
          at ../src/lsm/lsm_cursor.c:1095
      WT-7  0x000000000041084a in read_row (cursor=0x7f184404ed40, key=0x7f1848fcbdf0, 
          keyno=24905) at ../../../test/format/ops.c:608
      ---Type <return> to continue, or q <return> to quit---
      WT-8  0x0000000000410385 in ops (arg=0x1f85ba0) at ../../../test/format/ops.c:482
      WT-9  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-10 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 6 (Thread 0x7f183b5fe700 (LWP 3421)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1395060, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x1395060, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x00000000004cde98 in __wt_cache_full_check (session=0x1395060)
          at ../src/include/cache.i:197
      WT-4  0x00000000004ce8cd in __cursor_enter (session=0x1395060)
          at ../src/include/cursor.i:63
      WT-5  0x00000000004cf23c in __clsm_enter (clsm=0x7f182c1a1830, reset=1, update=0)
          at ../src/lsm/lsm_cursor.c:243
      WT-6  0x00000000004d204e in __clsm_search (cursor=0x7f182c1a1830)
          at ../src/lsm/lsm_cursor.c:1095
      WT-7  0x000000000041084a in read_row (cursor=0x7f182c1a1830, key=0x7f183b5fddf0, 
          keyno=64180) at ../../../test/format/ops.c:608
      WT-8  0x0000000000410385 in ops (arg=0x1f85bf8) at ../../../test/format/ops.c:482
      WT-9  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-10 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 5 (Thread 0x7f183abfd700 (LWP 3422)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1395e20, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x1395e20, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x00000000004cde98 in __wt_cache_full_check (session=0x1395e20)
          at ../src/include/cache.i:197
      WT-4  0x00000000004ce19e in __wt_txn_idle_cache_check (session=0x1395e20)
          at ../src/include/txn.i:261
      WT-5  0x00000000004ce213 in __wt_txn_id_check (session=0x1395e20)
          at ../src/include/txn.i:284
      WT-6  0x00000000004ceed5 in __clsm_enter (clsm=0x7f17a84a25e0, reset=0, update=1)
          at ../src/lsm/lsm_cursor.c:186
      WT-7  0x00000000004d2f37 in __clsm_insert (cursor=0x7f17a84a25e0)
          at ../src/lsm/lsm_cursor.c:1324
      WT-8  0x0000000000411747 in row_insert (cursor=0x7f17a84a25e0, 
          key=0x7f183abfcdf0, value=0x7f183abfcdc0, keyno=58844)
          at ../../../test/format/ops.c:976
      WT-9  0x0000000000410170 in ops (arg=0x1f85c50) at ../../../test/format/ops.c:418
      WT-10 0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-11 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      ---Type <return> to continue, or q <return> to quit---
      Thread 4 (Thread 0x7f183a1fc700 (LWP 3423)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1395320, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x1395320, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x00000000004cde98 in __wt_cache_full_check (session=0x1395320)
          at ../src/include/cache.i:197
      WT-4  0x00000000004ce19e in __wt_txn_idle_cache_check (session=0x1395320)
          at ../src/include/txn.i:261
      WT-5  0x00000000004ce213 in __wt_txn_id_check (session=0x1395320)
          at ../src/include/txn.i:284
      WT-6  0x00000000004ceed5 in __clsm_enter (clsm=0x7f17e81ab970, reset=0, update=1)
          at ../src/lsm/lsm_cursor.c:186
      WT-7  0x00000000004d2f37 in __clsm_insert (cursor=0x7f17e81ab970)
          at ../src/lsm/lsm_cursor.c:1324
      WT-8  0x0000000000411747 in row_insert (cursor=0x7f17e81ab970, 
          key=0x7f183a1fbdf0, value=0x7f183a1fbdc0, keyno=76841)
          at ../../../test/format/ops.c:976
      WT-9  0x0000000000410170 in ops (arg=0x1f85ca8) at ../../../test/format/ops.c:418
      WT-10 0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-11 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 3 (Thread 0x7f18397fb700 (LWP 3424)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1395b60, cond=0x142eef0, 
          usecs=100000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042db16 in __wt_cache_wait (session=0x1395b60, full=100)
          at ../src/evict/evict_lru.c:1499
      WT-3  0x000000000048a9cc in __wt_cache_full_check (session=0x1395b60)
          at ../src/include/cache.i:197
      WT-4  0x000000000048bc75 in __wt_page_in_func (session=0x1395b60, 
          ref=0x7f17b08d9810, flags=0, file=0x6a005d "../src/btree/col_srch.c", 
          line=96) at ../src/btree/bt_page.c:84
      WT-5  0x00000000004a4504 in __wt_page_swap_func (session=0x1395b60, 
          held=0x7f17b091e0c8, want=0x7f17b08d9810, flags=0, 
          file=0x6a005d "../src/btree/col_srch.c", line=96)
          at ../src/include/btree.i:1153
      WT-6  0x00000000004a4d27 in __wt_col_search (session=0x1395b60, recno=49731, 
          leaf=0x0, cbt=0x7f17ec175c80) at ../src/btree/col_srch.c:96
      WT-7  0x00000000004f0468 in __cursor_col_search (session=0x1395b60, 
          cbt=0x7f17ec175c80) at ../src/btree/bt_cursor.c:225
      WT-8  0x00000000004f07d1 in __wt_btcur_search (cbt=0x7f17ec175c80)
          at ../src/btree/bt_cursor.c:310
      WT-9  0x00000000004ba77e in __curfile_search (cursor=0x7f17ec175c80)
          at ../src/cursor/cur_file.c:200
      WT-10 0x00000000004e7043 in __wt_bloom_hash_get (bloom=0x7f17ec2d18f0, 
      ---Type <return> to continue, or q <return> to quit---
          bhash=0x7f18397fabd0) at ../src/bloom/bloom.c:284
      WT-11 0x00000000004d1b1d in __clsm_lookup (clsm=0x7f17ec290050, 
          value=0x7f17ec290150) at ../src/lsm/lsm_cursor.c:1036
      WT-12 0x00000000004d2071 in __clsm_search (cursor=0x7f17ec290050)
          at ../src/lsm/lsm_cursor.c:1097
      WT-13 0x000000000041084a in read_row (cursor=0x7f17ec290050, key=0x7f18397fadf0, 
          keyno=62505) at ../../../test/format/ops.c:608
      WT-14 0x0000000000410385 in ops (arg=0x1f85d00) at ../../../test/format/ops.c:482
      WT-15 0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-16 0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 2 (Thread 0x7f1838dfa700 (LWP 24482)):
      #0  0x0000003467a0b98e in pthread_cond_timedwait@@GLIBC_2.3.2 ()
         from /lib64/libpthread.so.0
      WT-1  0x0000000000442418 in __wt_cond_wait (session=0x1391120, cond=0x142eef0, 
          usecs=10000) at ../src/os_posix/os_mtx_cond.c:79
      WT-2  0x000000000042baa2 in __evict_worker (arg=0x13d76d0)
          at ../src/evict/evict_lru.c:405
      WT-3  0x0000003467a079d1 in start_thread () from /lib64/libpthread.so.0
      WT-4  0x00000034676e8b6d in clone () from /lib64/libc.so.6
      
      Thread 1 (Thread 0x7f184e9e9720 (LWP 2624)):
      #0  0x00000034676accdd in nanosleep () from /lib64/libc.so.6
      WT-1  0x00000034676e1e54 in usleep () from /lib64/libc.so.6
      WT-2  0x000000000040f8b7 in wts_ops (lastrun=1) at ../../../test/format/ops.c:168
      WT-3  0x0000000000412937 in main (argc=0, argv=0x7fffeeadd3f0)
          at ../../../test/format/t.c:202
      

      The configuration file is:

      ############################################
      #  RUN PARAMETERS
      ############################################
      abort=0
      auto_throttle=1
      firstfit=0
      bitcnt=8
      bloom=1
      bloom_bit_count=60
      bloom_hash_count=28
      bloom_oldest=0
      cache=30
      checkpoints=1
      checksum=uncompressed
      chunk_size=1
      compaction=0
      compression=zlib-noraw
      data_extend=0
      data_source=lsm
      delete_pct=6
      dictionary=0
      evict_max=4
      file_type=row-store
      backups=0
      huffman_key=0
      huffman_value=0
      insert_pct=56
      internal_key_truncation=1
      internal_page_max=17
      isolation=read-uncommitted
      key_gap=4
      key_max=65
      key_min=25
      leak_memory=0
      leaf_page_max=9
      logging=0
      logging_archive=0
      logging_prealloc=1
      lsm_worker_threads=3
      merge_max=5
      mmap=0
      ops=100000
      prefix_compression=1
      prefix_compression_min=7
      repeat_data_pct=49
      reverse=0
      rows=100000
      runs=100
      split_pct=43
      statistics=0
      statistics_server=0
      threads=7
      timer=20
      value_max=1765
      value_min=11
      wiredtiger_config=
      write_pct=46
      ############################################
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            alexander.gorrod@mongodb.com Alexander Gorrod
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

              Created:
              Updated:
              Resolved: