This Jenkins job has been hung on ocelot for over 3 days:
http://build.wiredtiger.com:8080/job/wiredtiger-test-race-condition-stress-sanitizer/29863/
It is hung while trying to dump the cache on a "format running more than 15 minutes" error. Here are the stacks. Looks like a deadlock. I have not debugged other than getting the stacks.
(gdb) thread apply all bt
Thread 22 (Thread 0x7fedaf9f9700 (LWP 33139)):
#0 0x00007fedb2c5abd3 in select () at ../sysdeps/unix/syscall-template.S:81
#1 0x000000000066d66e in __wt_sleep (seconds=0, micro_seconds=1000) at ../src/os_posix/os_sleep.c:32
#2 0x00000000005f1eff in __evict_lock_handle_list (session=0x7fedb44e2d40) at ../src/evict/evict_lru.c:54
#3 0x00000000005e6285 in __evict_server (session=0x7fedb44e2d40, did_work=0x7fedaf9f8be0) at ../src/evict/evict_lru.c:396
#4 0x00000000005e58ff in __wt_evict_thread_run (session=0x7fedb44e2d40, thread=0x607000001130) at ../src/evict/evict_lru.c:290
#5 0x00000000007badc4 in __thread_run (arg=0x607000001130) at ../src/support/thread_group.c:31
#6 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedaf9f9700) at pthread_create.c:308
#7 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 21 (Thread 0x7fedaf1f8700 (LWP 33140)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e31e8, cond=0x60c000000100, usecs=10000, run_func=0x782930 <__write_blocked>,
signalled=0x7fedaf1f7460) at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000780286 in __wt_cond_wait (session=0x7fedb44e31e8, cond=0x60c000000100, usecs=10000, run_func=0x782930 <__write_blocked>)
at ../src/include/misc.i:19
#3 0x0000000000782284 in __wt_writelock (session=0x7fedb44e31e8, l=0x62c000000940) at ../src/support/mtx_rw.c:437
#4 0x00000000005a9ed6 in __sweep_remove_handles (session=0x7fedb44e31e8) at ../src/conn/conn_sweep.c:248
#5 0x00000000005a6eae in __sweep_server (arg=0x7fedb44e31e8) at ../src/conn/conn_sweep.c:352
#6 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedaf1f8700) at pthread_create.c:308
#7 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 20 (Thread 0x7fedae9f7700 (LWP 33141)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000798) at pthread_mutex_lock.c:64
#3 0x00000000007e20a0 in __wt_spin_lock (session=0x7fedb44e3690, t=0x62c000000798) at ../src/include/mutex.i:173
#4 0x00000000007d465c in __wt_spin_lock_track (session=0x7fedb44e3690, t=0x62c000000798) at ../src/include/mutex.i:323
#5 0x00000000007d3517 in __wt_txn_checkpoint (session=0x7fedb44e3690, cfg=0x7fedae9f6d20, waiting=true) at ../src/txn/txn_ckpt.c:1170
#6 0x00000000007403da in __session_checkpoint (wt_session=0x7fedb44e3690, config=0x0) at ../src/session/session_api.c:1980
#7 0x00000000009cbcaf in __ckpt_server (arg=0x7fedb44e3690) at ../src/conn/conn_ckpt.c:106
#8 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedae9f7700) at pthread_create.c:308
#9 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 19 (Thread 0x7fedae1f6700 (LWP 33142)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e3fe0, cond=0x60c000000040, usecs=10000, run_func=0x780500 <__read_blocked>,
signalled=0x7fedae1f5560) at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000780286 in __wt_cond_wait (session=0x7fedb44e3fe0, cond=0x60c000000040, usecs=10000, run_func=0x780500 <__read_blocked>)
at ../src/include/misc.i:19
#3 0x000000000077fb51 in __wt_readlock (session=0x7fedb44e3fe0, l=0x62c000000940) at ../src/support/mtx_rw.c:259
#4 0x000000000061d6c2 in __lsm_manager_run_server (session=0x7fedb44e3fe0) at ../src/lsm/lsm_manager.c:397
#5 0x00000000006189cf in __lsm_worker_manager (arg=0x62c000007168) at ../src/lsm/lsm_manager.c:499
#6 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedae1f6700) at pthread_create.c:308
#7 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 18 (Thread 0x7fedad9f5700 (LWP 33143)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e4488, cond=0x60c0000004c0, usecs=10000, run_func=0x0, signalled=0x7fedad9f4bc0)
at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000636b76 in __wt_cond_wait (session=0x7fedb44e4488, cond=0x60c0000004c0, usecs=10000, run_func=0x0) at ../src/include/misc.i:19
#3 0x0000000000635a7a in __lsm_worker (arg=0x62c000007198) at ../src/lsm/lsm_worker.c:171
#4 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedad9f5700) at pthread_create.c:308
#5 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 17 (Thread 0x7fedad1f4700 (LWP 33144)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b357c0 in __wt_spin_lock (session=0x7fedb44e4930, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000b32f7c in __wt_spin_lock_track (session=0x7fedb44e4930, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b3ace9 in __lsm_drop_file (session=0x7fedb44e4930, uri=0x603000b736d0 "file:wt-000038.lsm") at ../src/lsm/lsm_work_unit.c:685
#6 0x0000000000b38cbf in __lsm_free_chunks (session=0x7fedb44e4930, lsm_tree=0x615000000300) at ../src/lsm/lsm_work_unit.c:758
#7 0x0000000000b386b1 in __wt_lsm_free_chunks (session=0x7fedb44e4930, lsm_tree=0x615000000300) at ../src/lsm/lsm_work_unit.c:845
#8 0x00000000006366f8 in __lsm_worker_general_op (session=0x7fedb44e4930, cookie=0x62c0000071c8, completed=0x7fedad1f3cf0) at ../src/lsm/lsm_worker.c:86
#9 0x00000000006353ab in __lsm_worker (arg=0x62c0000071c8) at ../src/lsm/lsm_worker.c:135
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedad1f4700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 16 (Thread 0x7fedac9f3700 (LWP 33145)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e4dd8, cond=0x60c0000001c0, usecs=10000, run_func=0x780500 <__read_blocked>,
signalled=0x7fedac9f19a0) at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000780286 in __wt_cond_wait (session=0x7fedb44e4dd8, cond=0x60c0000001c0, usecs=10000, run_func=0x780500 <__read_blocked>)
at ../src/include/misc.i:19
#3 0x000000000077fb51 in __wt_readlock (session=0x7fedb44e4dd8, l=0x62c000006e28) at ../src/support/mtx_rw.c:259
#4 0x0000000000b744b1 in __wt_block_truncate (session=0x7fedb44e4dd8, block=0x61c0002ca880, len=384000) at ../src/block/block_write.c:46
#5 0x0000000000bd3aa8 in __wt_block_checkpoint_unload (session=0x7fedb44e4dd8, block=0x61c0002ca880, checkpoint=false) at ../src/block/block_ckpt.c:181
#6 0x0000000000b653e0 in __bm_checkpoint_unload (bm=0x61200001b940, session=0x7fedb44e4dd8) at ../src/block/block_mgr.c:178
#7 0x000000000087f230 in __wt_btree_close (session=0x7fedb44e4dd8) at ../src/btree/bt_handle.c:283
#8 0x000000000058cce5 in __wt_conn_dhandle_close (session=0x7fedb44e4dd8, final=false, mark_dead=true) at ../src/conn/conn_dhandle.c:356
#9 0x0000000000761143 in __wt_session_release_dhandle (session=0x7fedb44e4dd8) at ../src/session/session_dhandle.c:290
#10 0x0000000000b3734a in __lsm_discard_handle (session=0x7fedb44e4dd8, uri=0x6030007214e0 "file:wt-000042.lsm", checkpoint=0x0)
at ../src/lsm/lsm_work_unit.c:654
#11 0x0000000000b35c16 in __wt_lsm_checkpoint_chunk (session=0x7fedb44e4dd8, lsm_tree=0x615000000300, chunk=0x60e005dd6a60) at ../src/lsm/lsm_work_unit.c:386
#12 0x0000000000636496 in __lsm_worker_general_op (session=0x7fedb44e4dd8, cookie=0x62c0000071f8, completed=0x7fedac9f2cf0) at ../src/lsm/lsm_worker.c:79
#13 0x00000000006353ab in __lsm_worker (arg=0x62c0000071f8) at ../src/lsm/lsm_worker.c:135
#14 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedac9f3700) at pthread_create.c:308
#15 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 15 (Thread 0x7fedac1f2700 (LWP 33146)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44e9d00, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44e9d00, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000012240, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x614000012240) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x614000012240, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x613000000040, cursor=0x614000012240, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000040) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedac1f2700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 14 (Thread 0x7fedab9f1700 (LWP 33147)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ea1a8, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ea1a8, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000021440, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x614000021440) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x614000021440, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x613000000200, cursor=0x614000021440, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000200) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fedab9f1700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 13 (Thread 0x7fed9c474700 (LWP 33148)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ea650, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ea650, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000031e40, reset=false, update=true) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b08f30 in __clsm_update (cursor=0x614000031e40) at ../src/lsm/lsm_cursor.c:1613
#7 0x0000000000532322 in row_update (tinfo=0x6130000003c0, cursor=0x614000031e40, positioned=false) at ../../../test/format/ops.c:1806
#8 0x0000000000528dce in ops (arg=0x6130000003c0) at ../../../test/format/ops.c:1059
#9 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9c474700) at pthread_create.c:308
#10 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 12 (Thread 0x7fed9bc73700 (LWP 33149)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44eaaf8, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44eaaf8, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000042240, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000afe3b5 in __clsm_next (cursor=0x614000042240) at ../src/lsm/lsm_cursor.c:941
#7 0x000000000052af8d in read_op (cursor=0x614000042240, op=NEXT, exactp=0x0) at ../../../test/format/ops.c:297
#8 0x000000000053374f in nextprev (tinfo=0x613000000580, cursor=0x614000042240, next=true) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000580) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9bc73700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 11 (Thread 0x7fed9b472700 (LWP 33150)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44eafa0, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44eafa0, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000051840, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000afe3b5 in __clsm_next (cursor=0x614000051840) at ../src/lsm/lsm_cursor.c:941
#7 0x000000000052af8d in read_op (cursor=0x614000051840, op=NEXT, exactp=0x0) at ../../../test/format/ops.c:297
#8 0x000000000053374f in nextprev (tinfo=0x613000000740, cursor=0x614000051840, next=true) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000740) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9b472700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 10 (Thread 0x7fed9ac71700 (LWP 33151)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44eb8f0, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44eb8f0, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000071240, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x614000071240) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x614000071240, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x613000000900, cursor=0x614000071240, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000900) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9ac71700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 9 (Thread 0x7fed9a470700 (LWP 33152)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44eb448, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44eb448, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000061a40, reset=false, update=true) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b08f30 in __clsm_update (cursor=0x614000061a40) at ../src/lsm/lsm_cursor.c:1613
#7 0x0000000000532322 in row_update (tinfo=0x613000000ac0, cursor=0x614000061a40, positioned=false) at ../../../test/format/ops.c:1806
#8 0x0000000000528dce in ops (arg=0x613000000ac0) at ../../../test/format/ops.c:1059
#9 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9a470700) at pthread_create.c:308
#10 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 8 (Thread 0x7fed99c6f700 (LWP 33153)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ebd98, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ebd98, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000081240, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x614000081240) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x614000081240, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x613000000c80, cursor=0x614000081240, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000c80) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed99c6f700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 7 (Thread 0x7fed9946e700 (LWP 33154)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ec240, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ec240, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x614000091c40, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000afe3b5 in __clsm_next (cursor=0x614000091c40) at ../src/lsm/lsm_cursor.c:941
#7 0x000000000052af8d in read_op (cursor=0x614000091c40, op=NEXT, exactp=0x0) at ../../../test/format/ops.c:297
#8 0x000000000053374f in nextprev (tinfo=0x613000000e40, cursor=0x614000091c40, next=true) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000000e40) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9946e700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 6 (Thread 0x7fed98c6d700 (LWP 33155)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ec6e8, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ec6e8, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x6140000a1a40, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000afe3b5 in __clsm_next (cursor=0x6140000a1a40) at ../src/lsm/lsm_cursor.c:941
#7 0x000000000052af8d in read_op (cursor=0x6140000a1a40, op=NEXT, exactp=0x0) at ../../../test/format/ops.c:297
#8 0x000000000053374f in nextprev (tinfo=0x613000001000, cursor=0x6140000a1a40, next=true) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000001000) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed98c6d700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 5 (Thread 0x7fed9846c700 (LWP 33156)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ecb90, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ecb90, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x6140000b2040, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x6140000b2040) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x6140000b2040, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x6130000011c0, cursor=0x6140000b2040, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x6130000011c0) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9846c700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 4 (Thread 0x7fed97c6b700 (LWP 33157)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ed038, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ed038, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x6140000c1e40, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000afe3b5 in __clsm_next (cursor=0x6140000c1e40) at ../src/lsm/lsm_cursor.c:941
#7 0x000000000052af8d in read_op (cursor=0x6140000c1e40, op=NEXT, exactp=0x0) at ../../../test/format/ops.c:297
#8 0x000000000053374f in nextprev (tinfo=0x613000001380, cursor=0x6140000c1e40, next=true) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000001380) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed97c6b700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 3 (Thread 0x7fed9746a700 (LWP 33158)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007fedb3a7ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fedb3a7ec08 in __GI___pthread_mutex_lock (mutex=0x62c000000898) at pthread_mutex_lock.c:64
#3 0x0000000000b0ff50 in __wt_spin_lock (session=0x7fedb44ed4e0, t=0x62c000000898) at ../src/include/mutex.i:173
#4 0x0000000000af938c in __wt_spin_lock_track (session=0x7fedb44ed4e0, t=0x62c000000898) at ../src/include/mutex.i:323
#5 0x0000000000b11b54 in __clsm_enter (clsm=0x6140000d1240, reset=false, update=false) at ../src/lsm/lsm_cursor.c:262
#6 0x0000000000b004d5 in __clsm_prev (cursor=0x6140000d1240) at ../src/lsm/lsm_cursor.c:1104
#7 0x000000000052afe5 in read_op (cursor=0x6140000d1240, op=PREV, exactp=0x0) at ../../../test/format/ops.c:301
#8 0x000000000053374f in nextprev (tinfo=0x613000001540, cursor=0x6140000d1240, next=false) at ../../../test/format/ops.c:1340
#9 0x0000000000529046 in ops (arg=0x613000001540) at ../../../test/format/ops.c:1083
#10 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed9746a700) at pthread_create.c:308
#11 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 2 (Thread 0x7fed96c69700 (LWP 33159)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e3b38, cond=0x60c000000280, usecs=10000, run_func=0x782930 <__write_blocked>,
signalled=0x7fed96c679c0) at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000780286 in __wt_cond_wait (session=0x7fedb44e3b38, cond=0x60c000000280, usecs=10000, run_func=0x782930 <__write_blocked>)
at ../src/include/misc.i:19
#3 0x0000000000782284 in __wt_writelock (session=0x7fedb44e3b38, l=0x62c000006e28) at ../src/support/mtx_rw.c:437
#4 0x00000000009de459 in __backup_start (session=0x7fedb44e3b38, cb=0x614000120040, is_dup=false, cfg=0x7fed96c68b40) at ../src/cursor/cur_backup.c:268
#5 0x00000000009d993c in __wt_curbackup_open (session=0x7fedb44e3b38, uri=0xc03500 <.str.10> "backup:", other=0x0, cfg=0x7fed96c68b40, cursorp=0x7fed96c68b20)
at ../src/cursor/cur_backup.c:173
#6 0x00000000007095cb in __session_open_cursor_int (session=0x7fedb44e3b38, uri=0xc03500 <.str.10> "backup:", owner=0x0, other=0x0, cfg=0x7fed96c68b40,
cursorp=0x7fed96c68b20) at ../src/session/session_api.c:495
#7 0x000000000070d3d6 in __session_open_cursor (wt_session=0x7fedb44e3b38, uri=0xc03500 <.str.10> "backup:", to_dup=0x0, config=0x0, cursorp=0x7fed96c68c80)
at ../src/session/session_api.c:616
#8 0x0000000000515503 in backup (arg=0x0) at ../../../test/format/backup.c:156
#9 0x00007fedb3a7cdc5 in start_thread (arg=0x7fed96c69700) at pthread_create.c:308
#10 0x00007fedb2c6376d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 1 (Thread 0x7fedb467b7c0 (LWP 33122)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1 0x000000000066c4df in __wt_cond_wait_signal (session=0x7fedb44e0800, cond=0x60c000000040, usecs=10000, run_func=0x780500 <__read_blocked>,
signalled=0x7fffb01a6ee0) at ../src/os_posix/os_mtx_cond.c:122
#2 0x0000000000780286 in __wt_cond_wait (session=0x7fedb44e0800, cond=0x60c000000040, usecs=10000, run_func=0x780500 <__read_blocked>)
at ../src/include/misc.i:19
#3 0x000000000077fb51 in __wt_readlock (session=0x7fedb44e0800, l=0x62c000000940) at ../src/support/mtx_rw.c:259
#4 0x00000000005efefd in __wt_verbose_dump_cache (session=0x7fedb44e0800) at ../src/evict/evict_lru.c:2704
#5 0x0000000000567d6d in __conn_debug_info (wt_conn=0x62c000000200, config=0xc0b940 <.str.15> "cache") at ../src/conn/conn_api.c:1176
#6 0x0000000000524ab1 in wts_ops (lastrun=0) at ../../../test/format/ops.c:246
#7 0x000000000053cbe4 in main (argc=4, argv=0x7fffb01a83e8) at ../../../test/format/t.c:209
- is caused by
-
WT-4561 WiredTiger LSM chunk drop can race with hot backups
-
- Closed
-