Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-3617

lookaside with timestamps: heap-use-after-free

    • Type: Icon: Bug Bug
    • Resolution: Fixed
    • Priority: Icon: Major - P3 Major - P3
    • WT3.0.0
    • Affects Version/s: None
    • Component/s: None
    • None
    • Storage 2017-10-23

      lookaside with timestamps: heap-use-after-free

      It looks like the truncation of the update list can race with transaction commit when re-building a page in-memory.

      Build #17238 origin/wt-3435-lookaside-with-timestamps (Oct 2, 2017 6:51:42 AM)
      http://build.wiredtiger.com:8080/job/wiredtiger-test-format-stress-sanitizer/17238/

      ==17403==ERROR: AddressSanitizer: heap-use-after-free on address 0x60600004413c at pc 0x00000072e149 bp 0x7f29a4082e90 sp 0x7f29a4082e88
      READ of size 1 at 0x60600004413c thread T15
          #0 0x72e148 in __wt_txn_commit /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/txn/txn.c:684:19
          #1 0x6ce0a6 in __session_commit_transaction /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/session/session_api.c:1451:9
          #2 0x529381 in commit_transaction /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:498:3
          #3 0x527a2f in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:949:4
          #4 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
          #5 0x7f29b337176c in __clone (/lib64/libc.so.6+0xf776c)
      
      0x60600004413c is located 28 bytes inside of 53-byte region [0x606000044120,0x606000044155)
      freed by thread T15 here:
          #0 0x4df510 in __interceptor_free /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:47
          #1 0x607d19 in __wt_free_int /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:327:2
          #2 0x87827d in __wt_update_obsolete_free /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:363:3
          #3 0x8118e5 in __split_multi_inmem /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:1496:5
          #4 0x81266b in __wt_split_rewrite /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:2242:2
          #5 0x5cedf2 in __evict_page_dirty_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:340:4
          #6 0x5cc5a9 in __wt_evict /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:186:3
          #7 0x5b20e3 in __evict_page /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2235:2
          #8 0x5b007c in __wt_cache_eviction_worker /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2334:17
          #9 0xa111d8 in __wt_cache_eviction_check /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cache.i:398:10
          #10 0xa108c6 in __cursor_enter /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:152:3
          #11 0xa00a8c in __cursor_func_init /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:343:4
          #12 0xa032f6 in __wt_btcur_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:731:8
          #13 0x8d574a in __curfile_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:240:2
          #14 0x52aa50 in row_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1653:16
          #15 0x5266da in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:778:11
          #16 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
      
      previously allocated by thread T15 here:
          #0 0x4dfa0d in calloc /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:74
          #1 0x606572 in __wt_calloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:52:11
          #2 0x875af6 in __wt_update_alloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:280:3
          #3 0x873cc9 in __wt_row_modify /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:94:4
          #4 0xa12dc7 in __cursor_row_modify_v /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:376:10
          #5 0xa08d0c in __btcur_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1106:9
          #6 0xa0a8c4 in __wt_btcur_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1342:10
          #7 0x8d63a8 in __curfile_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:327:2
          #8 0x52ccc7 in row_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1453:16
          #9 0x5274f4 in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:886:11
          #10 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
      
      Thread T15 created by T0 here:
          #0 0x437361 in __interceptor_pthread_create /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_interceptors.cc:305
          #1 0x61cc30 in __wt_thread_create /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_posix/os_thread.c:30:2
          #2 0x522466 in wts_ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:162:3
          #3 0x533556 in main /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:210:5
          #4 0x7f29b329bb34 in __libc_start_main (/lib64/libc.so.6+0x21b34)
      
      SUMMARY: AddressSanitizer: heap-use-after-free /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/txn/txn.c:684:19 in __wt_txn_commit
      Shadow bytes around the buggy address:
        0x0c0c800007d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c800007e0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c800007f0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c80000800: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
        0x0c0c80000810: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      =>0x0c0c80000820: fa fa fa fa fd fd fd[fd]fd fd fd fa fa fa fa fa
        0x0c0c80000830: fa fa fa fa fa fa fa fa fa fa fa fa fd fd fd fd
        0x0c0c80000840: fd fd fd fd fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c80000850: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c80000860: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x0c0c80000870: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
      ==17403==ABORTING
      

      Here's the CONFIG:

      ############################################
      #  RUN PARAMETERS
      ############################################
      abort=0
      alter=0
      auto_throttle=1
      backups=1
      bitcnt=3
      bloom=1
      bloom_bit_count=63
      bloom_hash_count=13
      bloom_oldest=0
      cache=18
      checkpoints=1
      checksum=uncompressed
      chunk_size=9
      compaction=0
      compression=zlib
      data_extend=0
      data_source=file
      delete_pct=12
      dictionary=0
      direct_io=0
      encryption=none
      evict_max=5
      file_type=row-store
      firstfit=0
      huffman_key=0
      huffman_value=0
      independent_thread_rng=0
      in_memory=0
      insert_pct=13
      internal_key_truncation=1
      internal_page_max=12
      isolation=random
      key_gap=2
      key_max=47
      key_min=25
      leaf_page_max=17
      leak_memory=0
      logging=1
      logging_archive=1
      logging_compression=zlib
      logging_prealloc=1
      long_running_txn=0
      lsm_worker_threads=4
      merge_max=17
      mmap=1
      modify_pct=0
      ops=100000
      prefix_compression=1
      prefix_compression_min=2
      quiet=1
      read_pct=2
      rebalance=1
      repeat_data_pct=64
      reverse=0
      rows=100000
      runs=1
      salvage=1
      split_pct=84
      statistics=0
      statistics_server=0
      threads=11
      timer=360
      transaction_timestamps=0
      transaction-frequency=40
      value_max=3570
      value_min=17
      verify=1
      wiredtiger_config=
      write_pct=73
      ############################################
      

            Assignee:
            michael.cahill@mongodb.com Michael Cahill (Inactive)
            Reporter:
            keith.bostic@mongodb.com Keith Bostic (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: