Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-732

Deadlock in stress testing

    • Type: Icon: Task Task
    • Resolution: Done
    • WT2.1.1
    • Affects Version/s: None
    • Component/s: None

      http://mjc.homeunix.org:8180/job/wiredtiger-test-format-stress/2636/console

      Here is the CONFIG:

      ############################################
      #  RUN PARAMETERS
      ############################################
      # bitcnt not applicable to this run
      cache=84
      compression=none
      data_extend=0
      data_source=table
      delete_pct=36
      dictionary=0
      file_type=variable-length column-store
      hot_backups=0
      # huffman_key not applicable to this run
      huffman_value=0
      insert_pct=39
      internal_key_truncation=0
      internal_page_max=11
      key_gap=13
      # key_max not applicable to this run
      # key_min not applicable to this run
      leaf_page_max=15
      ops=100000
      # prefix not applicable to this run
      repeat_data_pct=50
      reverse=0
      rows=100000
      runs=100
      split_pct=45
      statistics=1
      threads=26
      value_max=3872
      value_min=6
      # wiredtiger_config not applicable to this run
      write_pct=90
      ############################################
      

      Checkpoint (thread 6) is holding the schema lock, blocking most other things.

      It is waiting on ref in state WT_REF_EVICT_WALK, but that isn't part of the btree handle's eviction stack. It is looping at bt_walk.c:329.

      Full details:

      Thread 11 (Thread 0x7f4664bff700 (LWP 35029)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x0000000000439e20 in __session_close (wt_session=0x7f469ac27698,
          config=0x0) at ../src/session/session_api.c:76
      WT-3  0x000000000040fd58 in ops (arg=0x7f468f114690)
          at ../../../test/format/ops.c:425
      WT-4  0x0000003fea607d15 in start_thread (arg=0x7f4664bff700)
          at pthread_create.c:308
      WT-5  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 10 (Thread 0x7f467cffb700 (LWP 35018)):
      #0  pthread_rwlock_wrlock ()
          at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S:85
      WT-1  0x000000000040f69d in ops (arg=0x7f468f114320)
          at ../../../test/format/ops.c:245
      WT-2  0x0000003fea607d15 in start_thread (arg=0x7f467cffb700)
          at pthread_create.c:308
      WT-3  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 9 (Thread 0x7f4679bfb700 (LWP 35023)):
      #0  pthread_rwlock_wrlock ()
          at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S:85
      WT-1  0x000000000040f69d in ops (arg=0x7f468f1144b0)
          at ../../../test/format/ops.c:245
      WT-2  0x0000003fea607d15 in start_thread (arg=0x7f4679bfb700)
          at pthread_create.c:308
      WT-3  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 8 (Thread 0x7f46407ff700 (LWP 35033)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x000000000043d787 in __wt_session_get_btree (session=0x7f469ac25618,
          uri=0x7f4692ec7400 "file:wt.wt", checkpoint=0x0, cfg=0x7f46407fecc0,
          flags=0) at ../src/session/session_dhandle.c:318
      WT-3  0x000000000043d3ba in __wt_session_get_btree_ckpt (session=0x7f469ac25618,
          uri=0x7f4692ec7400 "file:wt.wt", cfg=0x7f46407fecc0, flags=0)
          at ../src/session/session_dhandle.c:192
      WT-4  0x0000000000479cbb in __wt_curfile_open (session=0x7f469ac25618,
          uri=0x7f4692ec7400 "file:wt.wt", owner=0x0, cfg=0x7f46407fecc0,
          cursorp=0x7f46407fece0) at ../src/cursor/cur_file.c:447
      WT-5  0x000000000043a41b in __wt_open_cursor (session=0x7f469ac25618,
          uri=0x7f4692ec7400 "file:wt.wt", owner=0x0, cfg=0x7f46407fecc0,
          cursorp=0x7f46407fece0) at ../src/session/session_api.c:187
      WT-6  0x000000000047f040 in __wt_curtable_open (session=0x7f469ac25618,
          uri=0x7f469ac18100 "table:wt", cfg=0x7f46407fecc0, cursorp=0x7f46407fece0)
          at ../src/cursor/cur_table.c:719
      WT-7  0x000000000043a552 in __wt_open_cursor (session=0x7f469ac25618,
          uri=0x7f469ac18100 "table:wt", owner=0x0, cfg=0x7f46407fecc0,
          cursorp=0x7f46407fece0) at ../src/session/session_api.c:197
      WT-8  0x000000000043a851 in __session_open_cursor (wt_session=0x7f469ac25618,
          uri=0x7f469ac18100 "table:wt", to_dup=0x0, config=0x631dc0 "overwrite",
          cursorp=0x7f46407fedf8) at ../src/session/session_api.c:241
      WT-9  0x000000000040f52a in ops (arg=0x7f468f1147d0)
          at ../../../test/format/ops.c:216
      WT-10 0x0000003fea607d15 in start_thread (arg=0x7f46407ff700)
          at pthread_create.c:308
      WT-11 0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 7 (Thread 0x7f466e3ff700 (LWP 35024)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x0000000000439e20 in __session_close (wt_session=0x7f469ac27eb8,
          config=0x0) at ../src/session/session_api.c:76
      WT-3  0x000000000040fd58 in ops (arg=0x7f468f114500)
          at ../../../test/format/ops.c:425
      WT-4  0x0000003fea607d15 in start_thread (arg=0x7f466e3ff700)
          at pthread_create.c:308
      WT-5  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 6 (Thread 0x7f466bbfa700 (LWP 35028)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x000000000045c876 in __wt_tree_walk (session=0x7f469ac286d8,
          pagep=0x7f466bbf9930, flags=19) at ../src/btree/bt_walk.c:342
      WT-3  0x0000000000449480 in __wt_sync_file (session=0x7f469ac286d8, syncop=16)
          at ../src/btree/bt_evict.c:576
      WT-4  0x0000000000457b7a in __wt_bt_cache_op (session=0x7f469ac286d8,
          ckptbase=0x7f46826d8000, op=16) at ../src/btree/bt_sync.c:64
      WT-5  0x0000000000444e62 in __checkpoint_worker (session=0x7f469ac286d8,
          cfg=0x7f466bbf9cd0, is_checkpoint=1) at ../src/txn/txn_ckpt.c:713
      WT-6  0x000000000044503d in __wt_checkpoint (session=0x7f469ac286d8,
          cfg=0x7f466bbf9cd0) at ../src/txn/txn_ckpt.c:760
      WT-7  0x0000000000489102 in __wt_meta_btree_apply (session=0x7f469ac286d8,
          func=0x444fd5 <__wt_checkpoint>, cfg=0x7f466bbf9cd0)
          at ../src/meta/meta_apply.c:45
      WT-8  0x0000000000443e1b in __checkpoint_apply (session=0x7f469ac286d8,
          cfg=0x7f466bbf9cd0, op=0x444fd5 <__wt_checkpoint>)
          at ../src/txn/txn_ckpt.c:129
      WT-9  0x0000000000444026 in __wt_txn_checkpoint (session=0x7f469ac286d8,
          cfg=0x7f466bbf9cd0) at ../src/txn/txn_ckpt.c:226
      WT-10 0x000000000043caf3 in __session_checkpoint (wt_session=0x7f469ac286d8,
          config=0x7f466bbf9d40 "name=thread-21") at ../src/session/session_api.c:781
      WT-11 0x000000000040f6d5 in ops (arg=0x7f468f114640)
          at ../../../test/format/ops.c:250
      WT-12 0x0000003fea607d15 in start_thread (arg=0x7f466bbfa700)
          at pthread_create.c:308
      WT-13 0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 5 (Thread 0x7f466c3fb700 (LWP 35027)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x000000000043cacf in __session_checkpoint (wt_session=0x7f469ac25a28,
          config=0x0) at ../src/session/session_api.c:781
      WT-3  0x000000000040f6d5 in ops (arg=0x7f468f1145f0)
      ---Type <return> to continue, or q <return> to quit---
          at ../../../test/format/ops.c:250
      WT-4  0x0000003fea607d15 in start_thread (arg=0x7f466c3fb700)
          at pthread_create.c:308
      WT-5  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 4 (Thread 0x7f46893ff700 (LWP 35012)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x0000000000434a7e in __wt_schema_get_table (session=0x7f469ac25c30,
          name=0x7f469ac18106 "wt", namelen=2, ok_incomplete=0,
          tablep=0x7f46893feb58) at ../src/schema/schema_list.c:87
      WT-3  0x000000000047efe4 in __wt_curtable_open (session=0x7f469ac25c30,
          uri=0x7f469ac18100 "table:wt", cfg=0x7f46893fecc0, cursorp=0x7f46893fece0)
          at ../src/cursor/cur_table.c:715
      WT-4  0x000000000043a552 in __wt_open_cursor (session=0x7f469ac25c30,
          uri=0x7f469ac18100 "table:wt", owner=0x0, cfg=0x7f46893fecc0,
          cursorp=0x7f46893fece0) at ../src/session/session_api.c:197
      WT-5  0x000000000043a851 in __session_open_cursor (wt_session=0x7f469ac25c30,
          uri=0x7f469ac18100 "table:wt", to_dup=0x0, config=0x631dc0 "overwrite",
          cursorp=0x7f46893fedf8) at ../src/session/session_api.c:241
      WT-6  0x000000000040f52a in ops (arg=0x7f468f114140)
          at ../../../test/format/ops.c:216
      WT-7  0x0000003fea607d15 in start_thread (arg=0x7f46893ff700)
          at pthread_create.c:308
      WT-8  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 3 (Thread 0x7f469abff700 (LWP 34839)):
      #0  pthread_cond_timedwait@@GLIBC_2.3.2 ()
          at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:217
      WT-1  0x0000000000429b01 in __wt_cond_wait (session=0x7f469ac25208,
          cond=0x7f4680a97240, usecs=100000) at ../src/os_posix/os_mtx.c:75
      WT-2  0x0000000000448765 in __wt_cache_evict_server (arg=0x7f469ac25208)
          at ../src/btree/bt_evict.c:168
      WT-3  0x0000003fea607d15 in start_thread (arg=0x7f469abff700)
          at pthread_create.c:308
      WT-4  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 2 (Thread 0x7f463fffe700 (LWP 35008)):
      #0  0x0000003fe9ad9167 in sched_yield ()
          at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x000000000042b649 in __wt_yield () at ../src/os_posix/os_yield.c:17
      WT-2  0x0000000000439e20 in __session_close (wt_session=0x7f469ac25820,
          config=0x0) at ../src/session/session_api.c:76
      WT-3  0x000000000040fd58 in ops (arg=0x7f468f114000)
          at ../../../test/format/ops.c:425
      WT-4  0x0000003fea607d15 in start_thread (arg=0x7f463fffe700)
          at pthread_create.c:308
      WT-5  0x0000003fe9af253d in clone ()
          at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
      
      Thread 1 (Thread 0x7f469b525780 (LWP 104334)):
      #0  0x0000003fe9abb4fd in nanosleep () at ../sysdeps/unix/syscall-template.S:81
      WT-1  0x0000003fe9aebfe4 in usleep (useconds=<optimized out>)
          at ../sysdeps/unix/sysv/linux/usleep.c:32
      WT-2  0x000000000040f21d in wts_ops () at ../../../test/format/ops.c:119
      WT-3  0x0000000000411939 in main (argc=0, argv=0x7fffe7db7280)
          at ../../../test/format/t.c:150
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            michael.cahill@mongodb.com Michael Cahill (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

              Created:
              Updated:
              Resolved: