Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-9156

Fix use-after-free failure in v4.2

      Running format stress configurations with ASAN in the 4.2 branch produces occasional use-after-free errors. 

      Here's the ASAN report:

      ==26062==ERROR: AddressSanitizer: heap-use-after-free on address 0x606000a6db70 at pc 0x000000ae02f6 bp 0x7f0db39abaf0 sp 0x7f0db39abae8
      READ of size 1 at 0x606000a6db70 thread T17
          #0 0xae02f5 in __wt_ref_info /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1089:24
          #1 0xadfd5c in __ref_is_leaf /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:90:5
          #2 0xad906f in __tree_walk_skip_count_callback /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:598:35
          #3 0xad82a3 in __tree_walk_internal /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:473:17
          #4 0xad8df5 in __wt_tree_walk_skip /home/ubuntu/src/4.2/build_posix/../src/btree/bt_walk.c:622:9
          #5 0xa21fae in __wt_btcur_next_random /home/ubuntu/src/4.2/build_posix/../src/btree/bt_random.c:581:9
          #6 0x60ae6f in __wt_curfile_next_random /home/ubuntu/src/4.2/build_posix/../src/cursor/cur_file.c:120:5
          #7 0x5204b6 in random_kv /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/random.c:73:27
          #8 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
          #9 0x7f0dbbe6f71e in clone /build/glibc-S9d2JN/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      
      
      0x606000a6db70 is located 48 bytes inside of 56-byte region [0x606000a6db40,0x606000a6db78)
      freed by thread T19 here:
          #0 0x4c3502 in free /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:124:3
          #1 0x7334e8 in __wt_free_int /home/ubuntu/src/4.2/build_posix/../src/os_common/os_alloc.c:303:5
          #2 0x7b0452 in __wt_ref_addr_free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:657:9
          #3 0x7ae621 in __wt_ref_block_free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1131:5
          #4 0x79d6c8 in __rec_write_wrapup /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:2226:9
          #5 0x78d221 in __reconcile /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:216:28
          #6 0x78bae5 in __wt_reconcile /home/ubuntu/src/4.2/build_posix/../src/reconcile/rec_write.c:103:11
          #7 0xabdc87 in __wt_sync_file /home/ubuntu/src/4.2/build_posix/../src/btree/bt_sync.c:311:13
          #8 0x8cb0a5 in __checkpoint_tree /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1626:9
          #9 0x8d4582 in __checkpoint_tree_helper /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1734:11
          #10 0x8d433c in __checkpoint_apply /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:197:9
          #11 0x8cdfb8 in __txn_checkpoint /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:848:5
          #12 0x8c9275 in __txn_checkpoint_wrapper /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1041:11
          #13 0x8c8b7f in __wt_txn_checkpoint /home/ubuntu/src/4.2/build_posix/../src/txn/txn_ckpt.c:1097:9
          #14 0x81fbff in __session_checkpoint /home/ubuntu/src/4.2/build_posix/../src/session/session_api.c:1956:11
          #15 0x4fd06c in checkpoint /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/checkpoint.c:110:9
          #16 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
      
      
      previously allocated by thread T4 here:
          #0 0x4c3a7a in calloc /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:155:3
          #1 0x731c6a in __wt_calloc /home/ubuntu/src/4.2/build_posix/../src/os_common/os_alloc.c:50:14
          #2 0x69cbd0 in __evict_page_dirty_update /home/ubuntu/src/4.2/build_posix/../src/evict/evict_page.c:397:13
          #3 0x697771 in __wt_evict /home/ubuntu/src/4.2/build_posix/../src/evict/evict_page.c:196:9
          #4 0x677b96 in __evict_page /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:2251:5
          #5 0x670846 in __evict_lru_pages /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:1106:20
          #6 0x67b5e6 in __evict_pass /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:707:13
          #7 0x66fb9c in __evict_server /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:376:5
          #8 0x66f308 in __wt_evict_thread_run /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:288:15
          #9 0x8a89d3 in __thread_run /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:31:9
          #10 0x7f0dbcfb86da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
      
      
      Thread T17 created by T0 here:
          #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3
          #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5
          #2 0x50d14b in operations /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/ops.c:203:9
          #3 0x52a059 in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:290:13
          #4 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310
      
      
      Thread T19 created by T0 here:
          #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3
          #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5
          #2 0x50d3dc in operations /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/ops.c:218:9
          #3 0x52a059 in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:290:13
          #4 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310
      
      
      Thread T4 created by T0 here:
          #0 0x4ac19d in pthread_create /data/mci/7cb6c3b42484980c0d68ddf179e14841/toolchain-builder/tmp/build-llvm.sh-6rk/llvm/projects/compiler-rt/lib/asan/asan_interceptors.cc:210:3
          #1 0x752925 in __wt_thread_create /home/ubuntu/src/4.2/build_posix/../src/os_posix/os_thread.c:28:5
          #2 0x8a5ff7 in __thread_group_resize /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:204:9
          #3 0x8a6b33 in __wt_thread_group_create /home/ubuntu/src/4.2/build_posix/../src/support/thread_group.c:288:5
          #4 0x671ebe in __wt_evict_create /home/ubuntu/src/4.2/build_posix/../src/evict/evict_lru.c:477:5
          #5 0x5dc351 in __wt_connection_workers /home/ubuntu/src/4.2/build_posix/../src/conn/conn_open.c:231:5
          #6 0x598454 in wiredtiger_open /home/ubuntu/src/4.2/build_posix/../src/conn/conn_api.c:2720:5
          #7 0x534a77 in wts_open /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/wts.c:306:5
          #8 0x529bcf in main /home/ubuntu/src/4.2/build_posix/test/format/../../../test/format/t.c:280:13
          #9 0x7f0dbbd6fbf6 in __libc_start_main /build/glibc-S9d2JN/glibc-2.27/csu/../csu/libc-start.c:310
      
      
      SUMMARY: AddressSanitizer: heap-use-after-free /home/ubuntu/src/4.2/build_posix/../src/include/btree.i:1089:24 in __wt_ref_info
      Shadow bytes around the buggy address:
        0x0c0c80145b10: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa
        0x0c0c80145b20: 00 00 00 00 00 00 00 fa fa fa fa fa fd fd fd fd
        0x0c0c80145b30: fd fd fd fa fa fa fa fa fd fd fd fd fd fd fd fa
        0x0c0c80145b40: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa
        0x0c0c80145b50: fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa
      =>0x0c0c80145b60: fa fa fa fa fa fa fa fa fd fd fd fd fd fd[fd]fa
        0x0c0c80145b70: fa fa fa fa 00 00 00 00 00 00 00 fa fa fa fa fa
        0x0c0c80145b80: fd fd fd fd fd fd fd fa fa fa fa fa 00 00 00 00
        0x0c0c80145b90: 00 00 00 fa fa fa fa fa 00 00 00 00 00 00 00 fa
        0x0c0c80145ba0: fa fa fa fa 00 00 00 00 00 00 00 00 fa fa fa fa
        0x0c0c80145bb0: fd fd fd fd fd fd fd fa fa fa fa fa 00 00 00 00
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07 
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
        Shadow gap:              cc
      ==26062==ABORTING
      AddressSanitizer:DEADLYSIGNAL
      AddressSanitizer:DEADLYSIGNAL
      AddressSanitizer:DEADLYSIGNAL
      AddressSanitizer:DEADLYSIGNAL

      Here's the failing config:

      ############################################
      #  RUN PARAMETERS: V2
      ############################################
      assert.commit_timestamp=0
      assert.read_timestamp=0
      backup=0
      backup.incremental=off
      backup.incr_granularity=2120
      btree.bitcnt=8
      btree.compression=snappy
      btree.dictionary=0
      btree.huffman_key=0
      btree.huffman_value=0
      btree.internal_key_truncation=1
      btree.internal_page_max=13
      btree.key_gap=11
      btree.key_max=65
      btree.key_min=30
      btree.leaf_page_max=11
      btree.memory_page_max=6
      btree.prefix_compression=1
      btree.prefix_compression_min=0
      btree.repeat_data_pct=79
      btree.reverse=0
      btree.split_pct=78
      btree.value_max=1421
      btree.value_min=18
      cache=93
      cache.evict_max=1
      cache.minimum=20
      checkpoint=on
      checkpoint.log_size=36
      checkpoint.wait=16
      disk.checksum=uncompressed
      disk.data_extend=0
      disk.direct_io=0
      disk.encryption=rotn-7
      disk.firstfit=0
      disk.mmap=1
      disk.mmap_all=0
      format.abort=0
      format.independent_thread_rng=0
      format.major_timeout=0
      logging=1
      logging.archive=0
      logging.compression=snappy
      logging.file_max=216011
      logging.prealloc=0
      lsm.auto_throttle=1
      lsm.bloom=1
      lsm.bloom_bit_count=41
      lsm.bloom_hash_count=20
      lsm.bloom_oldest=0
      lsm.chunk_size=10
      lsm.merge_max=12
      lsm.worker_threads=3
      ops.alter=0
      ops.compaction=0
      ops.pct.delete=30
      ops.pct.insert=61
      ops.pct.modify=1
      ops.pct.read=4
      ops.pct.write=4
      ops.prepare=0
      ops.random_cursor=1
      ops.rebalance=1
      ops.salvage=1
      ops.truncate=1
      ops.verify=1
      quiet=1
      runs=1
      runs.in_memory=0
      runs.ops=0
      runs.rows=1000000
      runs.source=file
      runs.threads=4
      runs.timer=6
      runs.type=row-store
      statistics=0
      statistics.server=0
      stress.aggressive_sweep=0
      stress.checkpoint=0
      stress.hs_sweep=0
      stress.split_1=0
      stress.split_2=0
      stress.split_3=0
      stress.split_4=0
      stress.split_5=0
      stress.split_6=0
      stress.split_7=0
      stress.split_8=0
      transaction.frequency=100
      transaction.isolation=snapshot
      transaction.timestamps=1
      wiredtiger.config=
      wiredtiger.rwlock=1
      wiredtiger.leak_memory=0
      ############################################

            Assignee:
            keith.smith@mongodb.com Keith Smith
            Reporter:
            keith.smith@mongodb.com Keith Smith
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: